From 4b693f9718aae7a8bca1c809b59511cf6885e22a Mon Sep 17 00:00:00 2001 From: eunchae Date: Thu, 6 Aug 2026 17:37:31 +0900 Subject: [PATCH 1/8] feat(mmdet): compile decoding parameters into the runner Export wrote .postproc.json -- anchor scales, head convolution layout, normalisation values -- and the runner read it at startup as a third positional argument: run_mmdet [size] Once an architecture is fixed those values never change again. They are constants, and they were being shipped as input, parsed on every run, and carried as a third file that has to stay in step with the second. A .gguf and a .postproc.json from different exports load without complaint and decode wrongly. mmdet_to_pt.py now emits .postproc.h: a generated `mmdet_params()` returning the same values, compiled into the runner alongside the head component and the backbone graph. run_mmdet [size] Deployment is the executable and the weights. Nothing is read at startup and no pair can be mismatched. build_mmdet_cpp.sh takes the header as an optional second argument and otherwise finds *.postproc.h in the generated directory. A second argument that is not a header is still read as an architecture name, so existing invocations keep working. frcnn_to_pt.py and the two-stage runners keep their own sidecar; only the single-stage path changes here. Verified on RetinaNet R18 at 512: detections are byte-identical to the same model run through the previous JSON path. Co-Authored-By: Claude Opus 5 --- tools/build/build_mmdet_cpp.sh | 20 ++++++-- tools/detect/head.h | 12 +++++ tools/frontend/mmdet/mmdet_to_pt.py | 76 ++++++++++++++++++++++++++--- tools/verify/backbone/run_mmdet.cpp | 64 +++++++++--------------- 4 files changed, 121 insertions(+), 51 deletions(-) diff --git a/tools/build/build_mmdet_cpp.sh b/tools/build/build_mmdet_cpp.sh index e5958e9..3b0e1f2 100755 --- a/tools/build/build_mmdet_cpp.sh +++ b/tools/build/build_mmdet_cpp.sh @@ -5,8 +5,10 @@ # head.cpp 는 라이브러리가 아니라 여기서 러너와 함께 컴파일된다 → g2c output/.cpp 를 직접 # 컴파일(arch/ 복사·cli REG 없음). # -# 사용: build_mmdet_cpp.sh [arch_name] +# 사용: build_mmdet_cpp.sh [arch_name] # gen_dir = g2c --output 디렉토리 (예: output/MMDetBackbone) — .cpp/.h/.gguf 있음 +# params.h = mmdet_to_pt.py 가 생성한 .postproc.h (mmdet_params()). 생략하면 +# gen_dir 안의 *.postproc.h 를 찾는다. # arch_name= 클래스명(생략 시 gen_dir 의 *.cpp 에서 자동) # env: VISP_BUILD = libvisioncpp 빌드 디렉토리 (기본: /build) # @@ -17,7 +19,18 @@ DETECT="$V/tools/detect" # 공용 head/decode 부품 (head.cpp/ RUN="$V/tools/verify" # E2E 검증 러너 GEN="${1:?usage: build_mmdet_cpp.sh [arch_name]}" GEN="$(cd "$GEN" && pwd)" -ARCH="${2:-}" +# 2번째 인자는 params 헤더거나(.h) 아키텍처 이름이다 — 확장자로 가른다. +PARAMS="" +ARCH="" +case "${2:-}" in + *.h) PARAMS="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"; ARCH="${3:-}" ;; + "") ;; + *) ARCH="$2" ;; +esac +if [ -z "$PARAMS" ]; then + PARAMS="$(ls "$GEN"/*.postproc.h 2>/dev/null | head -1)" +fi +[ -f "$PARAMS" ] || { echo "params 헤더를 못 찾음 (mmdet_to_pt.py 가 만든 .postproc.h): ${PARAMS:-없음}"; exit 1; } if [ -z "$ARCH" ]; then ARCH="$(basename "$(ls "$GEN"/*.cpp | grep -v run_ | head -1)" .cpp)" fi @@ -31,7 +44,7 @@ FMT_INC="$BUILD/_deps/fmt-src/include" FMT_FLAGS="" [ -f "$FMT_INC/fmt/format.h" ] && FMT_FLAGS="-DVISP_FMT_LIB -I$FMT_INC" -echo "arch=$ARCH gen=$GEN build=$BUILD fmt=${FMT_FLAGS:-fallback}" +echo "arch=$ARCH gen=$GEN params=$PARAMS build=$BUILD fmt=${FMT_FLAGS:-fallback}" INC="$GEN/inc" mkdir -p "$INC/visp/arch" cp "$GEN/$ARCH.h" "$INC/visp/arch/$ARCH.h" @@ -39,6 +52,7 @@ cp "$GEN/$ARCH.h" "$INC/visp/arch/$ARCH.h" # run_mmdet.cpp + head.cpp(러너와 함께 컴파일, 라이브러리 아님) + g2c 백본 output/.cpp g++ -std=c++20 -O2 $FMT_FLAGS \ -DARCH="$ARCH" -DVISP_ARCH_HEADER="\"visp/arch/$ARCH.h\"" \ + -DMMDET_PARAMS_HEADER="\"$PARAMS\"" \ -I"$DETECT" -I"$INC" -I"$V/include" -I"$V/src" \ -I"$V/depend/llama/ggml/include" -I"$V/depend/llama/vendor" \ "$RUN/backbone/run_mmdet.cpp" "$DETECT/head.cpp" "$GEN/$ARCH.cpp" \ diff --git a/tools/detect/head.h b/tools/detect/head.h index 82f6426..871c9fe 100755 --- a/tools/detect/head.h +++ b/tools/detect/head.h @@ -5,6 +5,7 @@ #pragma once #include "visp/ml.h" +#include "visp/postproc.h" // det_params #include #include @@ -24,6 +25,17 @@ struct anchor_head_cfg { bool head_has_norm = false; // 타워에 norm(GN 등) — 이번 PoC(RetinaNet)=false }; +// 러너가 검출기 하나를 돌리는 데 필요한 설정 전부. +// 모델이 정해지면 이 값들은 **상수**다 → mmdet_to_pt.py 가 `.postproc.h` 로 +// `mmdet_params()` 를 생성하고, 러너와 함께 컴파일된다. 런타임에 읽는 파일이 없다. +struct mmdet_cfg { + anchor_head_cfg head; + det_params det; + float img_mean[3] = {0, 0, 0}; + float img_std[3] = {1, 1, 1}; + bool to_rgb = false; +}; + // FPN features(레벨별, cwhn) → 레벨별 raw cls_score / bbox_pred(cwhn). // · cls_out[l] : ne={num_base*num_classes, feat_w, feat_h, 1} // · box_out[l] : ne={num_base*4, feat_w, feat_h, 1} diff --git a/tools/frontend/mmdet/mmdet_to_pt.py b/tools/frontend/mmdet/mmdet_to_pt.py index ffcecd4..d509d44 100755 --- a/tools/frontend/mmdet/mmdet_to_pt.py +++ b/tools/frontend/mmdet/mmdet_to_pt.py @@ -12,7 +12,6 @@ PYTHONPATH=:<이 폴더> g2c --model retinanet_bb.pt --name Retina --output output/retina """ import argparse -import json import os import sys import torch @@ -23,6 +22,70 @@ from mmdet_wrap import MMDetBackbone, build # noqa: E402,F401 (MMDetBackbone: 피클 등록) +def _f(v): + """C++ float 리터럴. repr 로 유효숫자를 보존한다 (0.1 이 0.1f 로 반올림되지 않게).""" + return f"{float(v)!r}f" + + +def _arr(name, values): + return f" c.det.{name} = {{{', '.join(_f(v) for v in values)}}};\n" + + +def emit_params(cfg, config_name): + """decode/anchor config → `mmdet_params()` 를 담은 C++ 헤더. + + JSON 사이드카를 대신한다. 값이 실행 파일 안으로 들어가므로 배포물이 하나 줄고, + gguf 와 설정이 서로 다른 export 에서 나와 짝이 어긋나는 사고가 원천적으로 사라진다. + """ + h = cfg.get("head_type", "raw") + out = [ + "// Generated by mmdet_to_pt.py — do not edit.\n", + f"// source: {config_name}\n", + f"// head_type: {h}\n", + "#pragma once\n\n", + '#include "head.h"\n\n', + "namespace visp {\n\n", + "inline mmdet_cfg mmdet_params() {\n", + " mmdet_cfg c;\n", + ] + if h != "anchor": + out += [ + " // The head in this config was not recognised, so only the backbone is\n", + " // exported and decoding is left to the caller.\n", + " return c;\n}\n\n} // namespace visp\n", + ] + return "".join(out) + + for k in ("stacked_convs", "feat_channels", "num_base", "num_classes"): + if k in cfg: + out.append(f" c.head.{k} = {int(cfg[k])};\n") + for k in ("cls_convs_prefix", "reg_convs_prefix", "cls_head", "reg_head"): + if k in cfg: + out.append(f' c.head.{k} = "{cfg[k]}";\n') + out.append(f" c.head.head_has_norm = {str(bool(cfg.get('head_has_norm', False))).lower()};\n\n") + + out.append(_arr("strides", cfg["strides"])) + out.append(_arr("octave_scales", cfg["octave_scales"])) + out.append(_arr("ratios", cfg["ratios"])) + out.append(f" c.det.octave_base_scale = {_f(cfg.get('octave_base_scale', 4.0))};\n") + out.append(f" c.det.center_offset = {_f(cfg.get('center_offset', 0.0))};\n") + for i, v in enumerate(cfg.get("means", [0.0] * 4)): + out.append(f" c.det.means[{i}] = {_f(v)};\n") + for i, v in enumerate(cfg.get("stds", [1.0] * 4)): + out.append(f" c.det.stds[{i}] = {_f(v)};\n") + out.append(f" c.det.num_classes = {int(cfg.get('num_classes', 80))};\n") + out.append(f" c.det.use_sigmoid = {str(bool(cfg.get('use_sigmoid', True))).lower()};\n\n") + + for i, v in enumerate(cfg.get("img_mean", [0.0] * 3)): + out.append(f" c.img_mean[{i}] = {_f(v)};\n") + for i, v in enumerate(cfg.get("img_std", [1.0] * 3)): + out.append(f" c.img_std[{i}] = {_f(v)};\n") + out.append(f" c.to_rgb = {str(bool(cfg.get('to_rgb', False))).lower()};\n") + + out.append(" return c;\n}\n\n} // namespace visp\n") + return "".join(out) + + def main(argv=None): ap = argparse.ArgumentParser(prog="mmdet_to_pt") ap.add_argument("--config", required=True, help="mmdet config .py") @@ -37,11 +100,12 @@ def main(argv=None): n_head = sum(1 for k in m.state_dict() if k.startswith("bbox_head")) print(f" → saved: {a.out} (state_dict {len(m.state_dict())} tensors, head {n_head} 포함)") - # decode/anchor + head-conv config 사이드카 (vision.cpp mmdet 부품이 읽음) - sidecar = os.path.splitext(a.out)[0] + ".postproc.json" - with open(sidecar, "w") as f: - json.dump(cfg, f, indent=2) - print(f" → sidecar: {sidecar} (head_type={cfg.get('head_type')})") + # decode/anchor + head-conv config → **C++ 함수**로 생성. + # 모델이 정해지면 이 값들은 상수라 런타임에 읽을 이유가 없다 → 러너와 함께 컴파일한다. + header = os.path.splitext(a.out)[0] + ".postproc.h" + with open(header, "w") as f: + f.write(emit_params(cfg, os.path.basename(a.config))) + print(f" → params: {header} (head_type={cfg.get('head_type')})") if __name__ == "__main__": diff --git a/tools/verify/backbone/run_mmdet.cpp b/tools/verify/backbone/run_mmdet.cpp index d12fd5c..0d307a7 100755 --- a/tools/verify/backbone/run_mmdet.cpp +++ b/tools/verify/backbone/run_mmdet.cpp @@ -3,26 +3,25 @@ // 백본 = g2c 가 생성한 output/.cpp (그대로 컴파일) → _forward // head = tools/detect/head.cpp 부품 (러너와 함께 컴파일, 라이브러리 아님) // decode+NMS = src/visp/postproc.cpp detect_anchor (라이브러리) -// cfg = .postproc.json (tools/frontend/mmdet/mmdet_to_pt.py 가 생성) +// cfg = .postproc.h (mmdet_to_pt.py 가 생성한 mmdet_params(), 함께 컴파일) // // 백본을 arch/ 로 복사하거나 cli REG 에 등록하지 않는다 — output/.cpp 를 직접 컴파일해 // libvisioncpp 와 링크(build_mmdet_cpp.sh). run_yolo_cpp 와 동일한 -DARCH 매크로 방식. // // 컴파일: -DARCH=<클래스명> -DVISP_ARCH_HEADER='"/.h"' -// 실행: run_mmdet [size=512] +// 실행: run_mmdet [size=512] #include VISP_ARCH_HEADER // 백본: _forward / _params / _detect_params #include "head.h" // head 부품: anchor_head_forward (같은 폴더) +#include MMDET_PARAMS_HEADER // 생성된 mmdet_params() — 값이 실행 파일 안에 있다 #include "visp/image.h" // image_load (이미지 입력 pre) #include "visp/ml.h" #include "visp/postproc.h" // detect_anchor, preprocess, det_params, detection #include -#include #include #include -#include #include #include #include @@ -52,9 +51,9 @@ static std::vector to_vec(tensor t) { return d; } -// 입력이 이미지(.jpg/.png…)면 preprocess()(resize+normalize+to_rgb, postproc.json 메타)로 텐서 생성. +// 입력이 이미지(.jpg/.png…)면 preprocess()(resize+normalize+to_rgb, 생성된 상수)로 텐서 생성. // .bin 이면 이미 전처리된 CWHN f32 텐서로 간주. → yolo run_yolo_cpp 는 .bin(외부 전처리)만, 여기선 둘 다. -static std::vector load_input(const char* path, int SZ, nlohmann::json const& j) { +static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c) { std::string s(path); auto ext = [&](const char* e) { size_t n = std::strlen(e); @@ -64,10 +63,9 @@ static std::vector load_input(const char* path, int SZ, nlohmann::json co image_data img = image_load(path); int iw = img.extent[0], ih = img.extent[1]; int ic = n_channels(img.format); // stbi_load(...,0)=네이티브 채널수 (JPEG=3, PNG+α=4) - float mean[3] = {0, 0, 0}, sd[3] = {1, 1, 1}; - if (j.contains("img_mean")) { auto v = j["img_mean"].get>(); for (int i = 0; i < 3; ++i) mean[i] = v[i]; } - if (j.contains("img_std")) { auto v = j["img_std"].get>(); for (int i = 0; i < 3; ++i) sd[i] = v[i]; } - bool to_rgb = j.value("to_rgb", false); + float const (&mean)[3] = c.img_mean; + float const (&sd)[3] = c.img_std; + bool to_rgb = c.to_rgb; printf("- preprocess: image %dx%dx%d → %dx%d (mean %.1f,%.1f,%.1f std %.1f,%.1f,%.1f to_rgb=%d)\n", iw, ih, ic, SZ, SZ, mean[0], mean[1], mean[2], sd[0], sd[1], sd[2], (int)to_rgb); return preprocess(img.data.get(), ih, iw, ic, SZ, mean, sd, to_rgb); @@ -76,16 +74,14 @@ static std::vector load_input(const char* path, int SZ, nlohmann::json co } int main(int argc, char** argv) { - if (argc < 5) { - fprintf(stderr, - "usage: %s [size=512]\n", argv[0]); + if (argc < 4) { + fprintf(stderr, "usage: %s [size=512]\n", argv[0]); return 1; } const char* gguf = argv[1]; const char* inp = argv[2]; - const char* jsonp = argv[3]; - const char* outp = argv[4]; - const int SZ = argc > 5 ? atoi(argv[5]) : 512; + const char* outp = argv[3]; + const int SZ = argc > 4 ? atoi(argv[4]) : 512; // 1) 가중치 (백본 + head 전부 이 gguf 에) backend_device backend = backend_init(); @@ -103,33 +99,17 @@ int main(int argc, char** argv) { tensor bb = FWD(m, input, p); ggml_build_forward_expand(graph, bb); - // 3) postproc.json → head-conv cfg + anchor decode cfg - nlohmann::json j; - { std::ifstream jf(jsonp); if (!jf) { fprintf(stderr, "cannot open %s\n", jsonp); return 1; } jf >> j; } - int L = (int)j["strides"].size(); - - anchor_head_cfg hc; - hc.stacked_convs = j.value("stacked_convs", 4); - hc.feat_channels = j.value("feat_channels", 256); - hc.num_base = j.value("num_base", 9); - hc.num_classes = j.value("num_classes", 80); - hc.cls_convs_prefix = j.value("cls_convs_prefix", std::string("bbox_head.cls_convs")); - hc.reg_convs_prefix = j.value("reg_convs_prefix", std::string("bbox_head.reg_convs")); - hc.cls_head = j.value("cls_head", std::string("bbox_head.retina_cls")); - hc.reg_head = j.value("reg_head", std::string("bbox_head.retina_reg")); - - det_params dp; - dp.strides = j["strides"].get>(); - dp.octave_base_scale = j.value("octave_base_scale", 4.0f); - dp.octave_scales = j["octave_scales"].get>(); - dp.ratios = j["ratios"].get>(); - dp.center_offset = j.value("center_offset", 0.0f); - dp.num_classes = j.value("num_classes", 80); - dp.use_sigmoid = j.value("use_sigmoid", true); + // 3) 설정 — 컴파일 시점에 박힌 상수. 런타임에 읽는 파일이 없다. + mmdet_cfg cfg = mmdet_params(); + anchor_head_cfg& hc = cfg.head; + det_params& dp = cfg.det; dp.input_w = SZ; dp.input_h = SZ; - { auto mn = j["means"].get>(); auto sd = j["stds"].get>(); - for (int i = 0; i < 4; ++i) { dp.means[i] = mn[i]; dp.stds[i] = sd[i]; } } + int L = (int)dp.strides.size(); + if (L == 0) { + fprintf(stderr, "no FPN strides — this config's head was not recognised at export\n"); + return 1; + } // 4) 백본 features(out_0..L-1) 를 잡아 head 부품 조립 std::vector feats; @@ -147,7 +127,7 @@ int main(int argc, char** argv) { // 5) 계산 (입력: 이미지면 preprocess, .bin 이면 전처리된 텐서) compute_graph_allocate(graph, backend); - auto in_data = load_input(inp, SZ, j); + auto in_data = load_input(inp, SZ, cfg); if (const char* dp = std::getenv("MMDET_DUMP_PRE")) { // 디버그: 전처리 텐서 덤프 FILE* f = fopen(dp, "wb"); if (f) { fwrite(in_data.data(), sizeof(float), in_data.size(), f); fclose(f); } } From 4ad68a3b91b025107eaed7c968daefc72d6bdde3 Mon Sep 17 00:00:00 2001 From: eunchae Date: Thu, 6 Aug 2026 17:37:31 +0900 Subject: [PATCH 2/8] docs(mmdet): guide for running MMDetection detectors Documents the path the previous commit completes: backbone and neck run as a compiled ggml graph, while the head, decoding and post-processing are C++ assembled from library primitives -- detection heads carry control flow that depends on the data, and tracing records only the path one input happened to take. Covers the export frontend, the interface a compiled backbone module must satisfy, the runner, the components in tools/detect, the decoder API in postproc.h, the two-stage and instance-segmentation variants, ByteTrack, and the fields of the generated parameters. Co-Authored-By: Claude Opus 5 --- README.md | 1 + docs/mmdet-detectors.md | 449 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 docs/mmdet-detectors.md diff --git a/README.md b/README.md index a45382c..599f201 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Based on [ggml](https://github.com/ggml-org/ggml) similar to the [llama.cpp](htt | [**MI-GAN**](#mi-gan) | Inpainting | CPU, Vulkan | | [**ESRGAN**](#real-esrgan) | Super-resolution | CPU, Vulkan | | [**YOLOv9t**](#yolov9t) | Object detection | CPU | +| [**MMDetection** models](docs/mmdet-detectors.md) | Object detection, segmentation, tracking | CPU | | [_Implement a model [**Guide**]_](docs/model-implementation-guide.md) | | | **Backbones:** SWIN (v1), DINO (v2), TinyViT diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md new file mode 100644 index 0000000..bdac70e --- /dev/null +++ b/docs/mmdet-detectors.md @@ -0,0 +1,449 @@ +# Object Detection with MMDetection Models + +This guide describes how to run detectors from +[MMDetection](https://github.com/open-mmlab/mmdetection) with _vision_.cpp. +If you are new to the library, read the [Overview](overview.md) first. + +MMDetection defines hundreds of detectors as compositions of a backbone, a neck and a head. +The backbone and neck are plain feed-forward networks and translate directly into a ggml graph. +The head does not: it contains data-dependent control flow (NMS, dynamic offsets, variable +proposal counts) that cannot be captured by tracing. _vision_.cpp therefore uses a **hybrid** +structure — the feature extractor runs as a compiled ggml graph, and the head, decoding and +post-processing are hand-written C++ built from library primitives. + +``` + image ──▶ backbone + neck (compiled ggml graph) ──▶ FPN features + │ + head │ tools/detect/head.cpp + ▼ + raw cls / bbox + │ + decode + NMS │ visp/postproc.h + ▼ + detections +``` + +Everything below the graph is ordinary CPU code in `visp/postproc.h`, so it is reusable +outside MMDetection: the decoders are parameterised by plain structs, not by framework config +objects. + +## Contents + +- [Prerequisites](#prerequisites) +- [Pipeline](#pipeline) +- [Detection heads](#detection-heads) +- [Post-processing API](#post-processing-api) +- [Two-stage detectors](#two-stage-detectors) +- [Instance segmentation](#instance-segmentation) +- [Multi-object tracking](#multi-object-tracking) +- [Configuration reference](#configuration-reference) + + +## Prerequisites + +- _vision_.cpp built from source — see [Building](../README.md#building). + The runners link against `libvisioncpp` plus the ggml libraries. +- A Python environment with MMDetection installed, for the export step only. + Nothing in the runtime path depends on Python. + +## Pipeline + +Running a detector takes four steps. Steps 1 and 2 happen once per model; steps 3 and 4 are +the deployment path. + +### Step 1 — Export the detector + +`mmdet_to_pt.py` loads an MMDetection config, wraps the detector so that the backbone and neck +form a traceable `nn.Module`, and writes two files. + +```sh +python tools/frontend/mmdet/mmdet_to_pt.py \ + --config /path/to/retinanet_r18_fpn_1x_coco.py \ + --checkpoint retinanet_r18.pth \ + --out backbone.pt \ + --size 512 +``` + +| Option | Description | +| :--- | :--- | +| `--config` | MMDetection config file (`.py`). Required. | +| `--checkpoint` | Weights (`.pth`). If omitted, the config's initialisation is used — useful for shape checks, not for accuracy. | +| `--out` | Output path for the traceable module (`.pt`). Required. | +| `--size` | Square input resolution used for tracing. Default `512`. | + +Outputs: + +`backbone.pt` +: The backbone and neck as a traceable module. The head is kept as an attribute so that its + weights are preserved in the `state_dict`, but it does not participate in the forward pass. + +`backbone.postproc.h` +: Everything the C++ side needs to reconstruct the head and decode its output: anchor + generator settings, bbox coder statistics, head convolution layout, and the pre-processing + normalisation taken from the config's `data_preprocessor` — emitted as a generated + `mmdet_params()` function. These values are constants once the architecture is chosen, so + they are compiled into the runner rather than read at run time. + See [Configuration reference](#configuration-reference). + +The `.pt` file pickles by module name `mmdet_wrap`, so the export directory must be on +`PYTHONPATH` when it is loaded again. + +### Step 2 — Compile the backbone + +`backbone.pt` is a plain PyTorch module and is compiled to a _vision_.cpp arch module by a +PyTorch-to-ggml model compiler. That compiler is outside the scope of this document; what +matters is the interface the generated code must satisfy. + +**Generated files** — for an architecture named `MMDetBackbone`: + +| File | Contents | +| :--- | :--- | +| `MMDetBackbone.h` | Declarations (see below). | +| `MMDetBackbone.cpp` | Builds the ggml graph for backbone and neck. | +| `MMDetBackbone.gguf` | Weights for **both** the backbone and the head, under their original `state_dict` names. | + +**Header contract** — the runner includes the header and reaches the graph through three +macro-expanded names: + +```c++ +namespace visp { + +struct MMDetBackbone_params { /* ... */ }; + +tensor MMDetBackbone_forward(model_ref m, tensor x, MMDetBackbone_params const& p); +MMDetBackbone_params MMDetBackbone_detect_params(model_file const& f); + +} // namespace visp +``` + +**Graph contract** + +- The input tensor is named `x`, type `f32`, shape `{3, size, size, 1}` in ggml `ne` order + (CWHN — channels vary fastest). +- Each FPN level is exposed as a named graph tensor `out_0`, `out_1`, … `out_{L-1}`, + ordered from the finest level to the coarsest. The runner resolves them with + `ggml_graph_get_tensor`, so the names must match exactly. +- Head weights must be present in the GGUF under the names the config uses, for example + `bbox_head.cls_convs.0.conv.weight` and `bbox_head.retina_cls.weight`. The head component + looks them up by prefix. + +If a level cannot be found the runner stops and reports the missing `out_`, which means the +generated graph did not name its outputs as expected. + +### Step 3 — Build the runner + +```sh +bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone +``` + +The script compiles three translation units together, with the generated parameters +included as a header, and links them against `libvisioncpp`: + +- `tools/verify/backbone/run_mmdet.cpp` — the runner, +- `tools/detect/head.cpp` — the head component, +- `output/MMDetBackbone/MMDetBackbone.cpp` — the generated graph, +- `backbone.postproc.h` — the generated parameters. + +`build_mmdet_cpp.sh [params.h] [arch_name]` +: `gen_dir` is the directory holding the generated `.cpp`, `.h` and `.gguf`. The parameters + header is found in `gen_dir` when it is there, and named explicitly otherwise. + `arch_name` defaults to the base name of the `.cpp` found there. + +The library is looked up in `build/`, which is where [Building](../README.md#building) puts it. +If you configured elsewhere, point `VISP_BUILD` at that directory: + +```sh +VISP_BUILD=/path/to/that/directory bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone +``` + +The result is `/run_mmdet`. + +### Step 4 — Run + +```sh +output/MMDetBackbone/run_mmdet \ + output/MMDetBackbone/MMDetBackbone.gguf \ + image.jpg \ + boxes.bin \ + 512 +``` + +``` +run_mmdet [size=512] +``` + +`` +: Weights produced in step 2. + +`` +: An image (`.jpg`, `.jpeg`, `.png`, `.bmp`) or a pre-processed tensor (`.bin`). + Images are resized and normalised in-process using `preprocess()`; the mean, standard + deviation and channel order are compiled in. A `.bin` file is taken as-is and must + contain `3 × size × size` `float32` values in CWHN order. + +`` +: Output path. Each detection is written as six `float32` values: + `x1, y1, x2, y2, score, label`. Coordinates are in input-image pixels. + +`[size]` +: Input resolution. Must match the value passed to `--size` in step 1 and the shape the graph + was compiled for. + +## Detection heads + +Head components take FPN features and produce the raw per-level tensors that the decoders +expect. They are declared in `tools/detect/head.h`. + +### `anchor_head_forward` + +Shared convolution tower followed by classification and regression convolutions — the layout +used by RetinaNet, ATSS, GFL and other anchor-based dense heads. All levels share one set of +weights. + +```c++ +void anchor_head_forward(model_ref m, std::vector const& feats, + anchor_head_cfg const& c, + std::vector& cls_out, std::vector& box_out); +``` + +`anchor_head_cfg` + +| Field | Default | Description | +| :--- | :--- | :--- | +| `stacked_convs` | `4` | Depth of the shared cls/reg tower. | +| `feat_channels` | `256` | Channels inside the tower. | +| `num_base` | `9` | Anchors per location. | +| `num_classes` | `80` | Classification output channels. | +| `cls_convs_prefix` | `bbox_head.cls_convs` | Weight-name prefix of the cls tower. | +| `reg_convs_prefix` | `bbox_head.reg_convs` | Weight-name prefix of the reg tower. | +| `cls_head` | `bbox_head.retina_cls` | Final classification convolution. | +| `reg_head` | `bbox_head.retina_reg` | Final regression convolution. | +| `head_has_norm` | `false` | Whether the tower contains normalisation layers. | + +Output shapes, per level `l`, in ggml `ne` order: + +- `cls_out[l]` — `{num_base * num_classes, feat_w, feat_h, 1}` +- `box_out[l]` — `{num_base * 4, feat_w, feat_h, 1}` + +Feed these to [`detect_anchor`](#detect_anchor). + +### `vfnet_head_forward` + +VFNet's head predicts distances rather than anchor deltas, and refines them with a +star-shaped deformable convolution whose offsets are computed from the first bbox prediction. +That offset computation is exactly the part that cannot be traced, so it is assembled here as +an explicit graph. + +```c++ +void vfnet_head_forward(model_ref m, std::vector const& feats, + vfnet_head_cfg const& c, tensor dcn_base, + std::vector& cls_out, std::vector& box_out); +``` + +`dcn_base` is the fixed 3×3 sampling grid, shape `{18, 1, 1, 1}`, supplied by the caller. The +component computes `offset = star_dcn_offset(bbox_pred) - dcn_base` and applies +`conv_2d_deform`, a library primitive. Per level, `cls_out[l]` is `{num_classes, w, h, 1}` and +`box_out[l]` is `{4, w, h, 1}`. + +`vfnet_head_cfg` adds `gn_groups` (GroupNorm groups in the tower), `strides` (per level, used +to project offsets into feature scale) and `reg_denoms` (per level, `bbox_pred = exp(reg) * +reg_denom`). + +### Adding a head + +1. Add a `_head_forward` function to `tools/detect/head.cpp` that turns FPN features + into raw per-level tensors. Use library primitives (`conv_2d`, `group_norm`, + `conv_2d_deform`); do not add framework-specific code to `src/visp`. +2. Extract the head's structural parameters in `mmdet_wrap.postproc_cfg`; they are emitted + into the generated parameters header. +3. Connect the raw output to the matching decoder in `visp/postproc.h`, or add one if the + decoding scheme is new. + +## Post-processing API + +Declared in `src/visp/postproc.h`, implemented as plain CPU code with no ggml dependency. +All multi-level inputs are per-level flat `float` buffers in CWHN order +(`index = (y * W + x) * C + c`), with the per-level `(feat_h, feat_w)` passed alongside. + +```c++ +struct detection { + float x1, y1, x2, y2; // pixel coordinates + float score; + int label; +}; +``` + +### Pre-processing + +`std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, int out_size, float const mean[3], float const std[3], bool to_rgb, int* out_w = nullptr, int* out_h = nullptr)` +: Resize to `out_size × out_size` and normalise to `(v - mean) / std`, optionally swapping + channel order. Returns a CWHN `float32` tensor ready for the graph input. + +### Dense heads + + +`std::vector detect_anchor(cls_scores, bbox_preds, feat_hw, det_params const& p)` +: Anchor-based decoding: anchor generation, delta decoding, per-level top-k, score + thresholding and NMS. Used by RetinaNet, ATSS, GFL and RPN-style heads. + +`std::vector detect_fcos(cls_scores, bbox_preds, centerness, feat_hw, fcos_params const& p)` +: Anchor-free distance decoding with centerness weighting. + +`std::vector detect_yolox(cls, box, obj, feat_hw, yolox_params const& p)` +: Grid-based decoding with an objectness branch; score is `sigmoid(cls) * sigmoid(obj)`. + +`std::vector detect_detr(float const* cls, float const* bbox, detr_params const& p)` +: Set prediction. Takes query logits and normalised `cxcywh` boxes, applies top-k, and + performs no NMS. Set `use_sigmoid` for Deformable-DETR-style heads. + +`det_params` carries the anchor generator (`strides`, `octave_base_scale`, `octave_scales`, +`ratios`, `center_offset`), the bbox coder (`means`, `stds`), and the test-time thresholds +(`score_thr`, `nms_thr`, `nms_pre`, `max_per_img`). `input_w`/`input_h` clip boxes to the +image. + +### Two-stage components + +`std::vector rpn_proposals(rpn_cls, rpn_bbox, feat_hw, rpn_params const& p)` +: Region proposals from RPN outputs: anchor decode, per-level top-k, NMS across levels. + Returns `M × 4` boxes in image coordinates, `M ≤ max_per_img`. + +`std::vector roi_align(feats, feat_hw, float const* rois, int m, roi_align_params const& p)` +: MMCV-compatible RoIAlign (`aligned = true`, adaptive `sampling_ratio`). Level assignment + follows `clamp(floor(log2(sqrt(w*h) / finest_scale + 1e-6)), 0, L-1)`. + Returns `M × C × out × out` in NCHW order. + +`std::vector detect_roi(float const* scores, float const* bbox_deltas, float const* proposals, int n, roi_params const& p)` +: Final RoI-head decoding: class-wise delta decoding and per-class NMS. `scores` are + post-softmax with background last. Set `class_agnostic` when `bbox_pred` has four columns + instead of `num_classes * 4`. + +### Masks and keypoints + +`std::vector paste_mask(float const* mask_logit, int mh, int mw, detection const& box, float thr = 0.5f, int* out_h = nullptr, int* out_w = nullptr)` +: Sigmoid, resize to the box, threshold. Returns a binary mask covering the box. + +`std::vector decode_keypoints(float const* heatmap, int k, int hm_h, int hm_w, float stride)` +: Per-keypoint argmax over a heatmap; returns `k × 3` as `(x, y, score)`. + +### Building blocks + +`gen_anchors`, `gen_points`, `delta2bbox`, `distance2bbox` and `nms` are exposed individually +for building custom decoders. + +## Two-stage detectors + +RPN proposals and RoIAlign are data-dependent — the number of proposals is not known until the +network has run — so a two-stage detector cannot be a single graph. It is split into two +compiled sub-graphs with host code in between. + +``` + image + │ SubA (backbone + neck + RPN) 14 outputs: P2-P5, rpn_cls×5, rpn_bbox×5 + ▼ + │ rpn_proposals(host) decode + per-level NMS -> 1000 proposals + │ roi_align(host) proposals + P2-P5 -> roi_feat (N,256,7,7) + ▼ + │ SubB (bbox head, Shared2FC) -> cls_score (N,81), bbox_pred (N,320) + ▼ + │ detect_roi(host) softmax + delta decode + per-class NMS + ▼ detections +``` + +Export both sub-graphs with `frcnn_to_pt.py`, compile each, then build and run: + +```sh +python tools/frontend/mmdet/frcnn_to_pt.py \ + --config faster-rcnn_r50_fpn_1x_coco.py --checkpoint frcnn.pth --out /tmp/frcnn +# compile /tmp/frcnn/FRCNN_SubA.pt at 1,3,800,800 and /tmp/frcnn/FRCNN_SubB.pt at 4,256,7,7 + +bash tools/build/build_frcnn_cpp.sh output/FRCNN_SubA output/FRCNN_SubB + +output/FRCNN_SubA/run_frcnn \ + output/FRCNN_SubA/FRCNN_SubA.gguf output/FRCNN_SubB/FRCNN_SubB.gguf \ + /tmp/frcnn/frcnn.json input.bin 800 +``` + +`run_roi_verify` and `run_rpn_verify` check the two host stages in isolation against dumps +from the reference implementation. + +## Instance segmentation + +Mask R-CNN extends the above with a second RoIAlign at output size 14 over the final boxes, +a mask sub-graph, and host-side mask pasting. + +``` + final boxes + │ roi_align(out=14, host) -> mask_feat (M,256,14,14) + │ SubC (mask head FCN) -> mask_logits (M,80,28,28) + │ paste_mask(host) -> per-instance binary masks + ▼ +``` + +```sh +run_maskrcnn [size=800] +``` + +> **Note** +> ggml's `conv_transpose_2d_p0` does not support batching. `run_maskrcnn` therefore evaluates +> the mask sub-graph one RoI at a time. Running it batched leaves only the first RoI correct. + +## Multi-object tracking + +Tracking is state management, not a network — there is nothing to compile. `ByteTracker` +(`src/visp/tracker.h`) keeps track state across frames and is detector-agnostic: it +consumes `std::vector` from any of the decoders above. + +```c++ +ByteTracker tracker; // byte_params overrides thresholds +for (int frame = 0; frame < n; ++frame) { + std::vector dets = /* run the detector */; + std::vector tracks = tracker.track(dets, frame); + // tracks[i].id is stable across frames +} +``` + +Each call performs Kalman prediction over an 8-state `cxcyah` model, two-stage IoU matching +(high-score detections first, then low-score), and track lifecycle management — +`num_tentatives` consecutive matches to confirm a track, `num_frames_retain` frames without a +match to drop it. Passing `frame_id == 0` resets the tracker. + +## Configuration reference + +`mmdet_params()` in `.postproc.h` is generated by the export step and compiled into the +runner. Its fields are grouped here by consumer. + +**Pre-processing** (`c.img_mean`, `c.img_std`, `c.to_rgb`) — used only when the runner is +given an image rather than a `.bin`. + +| Field | Description | +| :--- | :--- | +| `img_mean`, `img_std` | Per-channel normalisation, taken from the config's `data_preprocessor`. | +| `to_rgb` | Whether to swap channel order before normalising. | + +**Head reconstruction** (`c.head`) — maps onto `anchor_head_cfg`. + +| Field | Description | +| :--- | :--- | +| `stacked_convs`, `feat_channels` | Shape of the shared tower. | +| `cls_convs_prefix`, `reg_convs_prefix` | Weight-name prefixes of the towers. | +| `cls_head`, `reg_head` | Names of the final convolutions. | +| `head_has_norm` | Whether the tower contains normalisation layers. | + +**Decoding** (`c.det`) — maps onto `det_params`. + +| Field | Description | +| :--- | :--- | +| `strides` | Stride per FPN level; its length defines the level count `L`. | +| `octave_base_scale`, `octave_scales`, `ratios`, `center_offset` | Anchor generator. | +| `num_base` | Anchors per location, `len(octave_scales) * len(ratios)`. | +| `means`, `stds` | Delta coder statistics. | +| `num_classes`, `use_sigmoid` | Classification output layout and activation. | + +When the config's head is not recognised the generated function returns defaults and leaves +the stride list empty, and the runner stops rather than decoding with meaningless anchors. The +backbone still exports, but decoding must then be supplied by the caller. + +Isolation harnesses for each stage live in `tools/verify/`: `run_vfnet_head` for a dense head, +`run_rpn_verify` and `run_roi_verify` for the two-stage host components, and +`run_bytetrack_verify` for tracking. Each one runs a single stage against a dump from the +reference implementation, which is the fastest way to locate a mismatch. From cbbef3bc13f1d9b9c1c943705b5b710363762b2c Mon Sep 17 00:00:00 2001 From: eunchae Date: Thu, 6 Aug 2026 17:50:19 +0900 Subject: [PATCH 3/8] feat(tools): make detections readable, and rewrite the tools README The runner wrote raw float32 and printed only a count, so the only way to see what a detector produced was to write a script. Two additions: - run_mmdet prints the highest-scoring detections as a table. VISP_PRINT_DETS sets how many, 0 turns it off. The file it writes is unchanged -- still raw float32, still what a reference comparison needs. - tools/verify/draw_boxes.py draws a detections file onto the image it came from, scaling boxes back from the square input the detector ran on. The README is rewritten in English and brought in line with the guide: the parameters are a generated function rather than a sidecar, the runner takes one argument fewer, and the sections on heads, two-stage detectors, segmentation and tracking describe what each part does rather than restating accuracy figures that belong with the harnesses producing them. Co-Authored-By: Claude Opus 5 --- tools/README.md | 272 ++++++++++++++-------------- tools/verify/backbone/run_mmdet.cpp | 22 +++ tools/verify/draw_boxes.py | 93 ++++++++++ 3 files changed, 249 insertions(+), 138 deletions(-) create mode 100644 tools/verify/draw_boxes.py diff --git a/tools/README.md b/tools/README.md index 46c081c..98b4426 100755 --- a/tools/README.md +++ b/tools/README.md @@ -1,184 +1,180 @@ -# vision.cpp / tools — 검출 frontend & 검증 +# tools — detection frontends, components and runners -검출기를 vision.cpp 에서 실행하기 위한 부품을 **기능(function)별로 분리**한다. 프레임워크(mmdet 등) -지식은 `frontend//` 에만 격리하고, 검출 head/decode 부품(`detect/`)과 검증 러너(`verify/`)는 -**프레임워크 중립**으로 둔다 — mmdet 외 프레임워크도 여기 붙일 수 있게. **g2c 코어·libvisioncpp -코어는 건드리지 않는다.** 구조는 **backbone/neck = g2c 생성(그대로) + head = `detect/` C++ 부품** 하이브리드. -검출 head 는 trace 가 안 되므로(NMS·동적 offset 등) ggml 로 억지 변환하지 않고 손코딩 C++ 로 조립한다. +Everything needed to run a detector with _vision_.cpp that does not belong in the library +itself. [docs/mmdet-detectors.md](../docs/mmdet-detectors.md) explains the pipeline end to end; +this file describes how the directory is arranged. -## 폴더 구조 (기능별) +## Layout ``` tools/ - detect/ # 검출 공통 C++ 부품 (프레임워크 무관) — head/decode - head.h head.cpp # anchor_head_forward · vfnet_head_forward … frontend/ - mmdet/ # mmdet 전용: 검출기 → traceable .pt + postproc.json (torch-side) - mmdet_wrap.py mmdet_to_pt.py frcnn_wrap.py frcnn_to_pt.py - # # (다른 프레임워크는 frontend// 로 추가) - verify/ # E2E 검증 러너 (task별) - dense_head/ run_vfnet_head.cpp - roi/ run_frcnn.cpp run_roi_verify.cpp run_rpn_verify.cpp - seg/ run_maskrcnn.cpp - tracking/ run_bytetrack_verify.cpp - backbone/ run_dump.cpp run_mmdet.cpp - build/ # 빌드 스크립트 build_{mmdet,frcnn,maskrcnn}_cpp.sh + mmdet/ MMDetection-specific. The only place that imports mmdet. + mmdet_wrap.py traceable module + config extraction + mmdet_to_pt.py CLI: config -> backbone.pt + .postproc.h + frcnn_wrap.py two-stage / mask sub-graphs + frcnn_to_pt.py CLI for the above + detect/ Framework-neutral head components, compiled into the runner + head.h head.cpp + verify/ Runners and inspection, grouped by task + backbone/ run_mmdet.cpp run_dump.cpp + dense_head/ run_vfnet_head.cpp + roi/ run_frcnn.cpp run_roi_verify.cpp run_rpn_verify.cpp + seg/ run_maskrcnn.cpp + tracking/ run_bytetrack_verify.cpp + draw_boxes.py + build/ Build scripts ``` -- **`detect/head.cpp` 는 라이브러리가 아니라 러너와 함께 컴파일**된다(`build/build_mmdet_cpp.sh`). - libvisioncpp 에 넣지 않는다 = 프레임워크 지식이 코어로 새지 않음. -- head 가 쓰는 `conv_2d`/`group_norm`/`conv_2d_deform` 등은 vision.cpp 라이브러리 프리미티브(무수정). +A detector runs as a hybrid: the backbone and neck are a compiled ggml graph, while the head, +decoding and post-processing are C++ assembled from library primitives. Detection heads carry +control flow that depends on the data — suppression counts that are not known in advance, +deformable offsets derived from an earlier prediction, proposal counts that vary per image — and +tracing records only the path one input happened to take. -## 흐름 (main 의 run_yolo_cpp 러너 패턴) +Two arrangements keep framework knowledge from spreading: -``` -mmdet config - │ ① python mmdet_to_pt.py (전처리 — mmdet 지식은 여기만 → backbone.pt + postproc.json) - ▼ -backbone.pt + .postproc.json - │ ② g2c --model backbone.pt --name (g2c 정식 CLI, 코어 무수정·mmdet 코드 없음) - ▼ -output//{.cpp, .h, .gguf} (백본 forward + 가중치[backbone+head]) - │ ③ build_mmdet_cpp.sh output/ (run_mmdet + head.cpp + output/.cpp 컴파일, libvisioncpp 링크) - ▼ -output//run_mmdet - │ ④ run_mmdet - ├─ _forward : g2c 백본(output/.cpp) → FPN features out_0..L-1 - ├─ C++ head 부품 : head.cpp (anchor / vfnet …) → raw cls/box - └─ detect_anchor … : decode + NMS → 박스 -``` +- **`detect/head.cpp` is not part of `libvisioncpp`.** It is compiled together with the runner, + so detector-specific structure never enters the core library. +- **Decoding lives in the library, not in the frontend.** `detect_anchor`, `roi_align` and + `rpn_proposals` in `src/visp/postproc.h` take numbers, not configuration objects, which is why + they are reusable for detectors that never went through MMDetection. + +## Single-stage detectors -## 파일 (기능별 위치) +```sh +# 1. Export. Writes backbone.pt and backbone.postproc.h. +python tools/frontend/mmdet/mmdet_to_pt.py \ + --config retinanet_r18_fpn_1x_coco.py --checkpoint retinanet_r18.pth \ + --out backbone.pt --size 512 -| 파일 | 역할 | -|---|---| -| `mmdet_wrap.py` | `MMDetBackbone`(backbone+neck features nn.Module, head 는 가중치 유지 attribute) + `postproc_cfg`(anchor/decode + head-conv 구조 + 전처리 메타 추출). **유일한 mmdet 의존 지점.** | -| `mmdet_to_pt.py` | CLI: mmdet config → `backbone.pt` + `.postproc.json`. 피클 모듈명 = `mmdet_wrap` (self-contained import). | -| `head.h` / `head.cpp` | C++ head 부품. `anchor_head_forward`(RetinaNet/ATSS: 공유 cls/reg conv 타워) · `vfnet_head_forward`(VFNet: star deformable offset 계산 + `conv_2d_deform`). 러너와 함께 컴파일. | -| `run_mmdet.cpp` | 러너(제네릭, `-DARCH`). `_forward`(백본) → head 부품 → `detect_anchor`. | -| `run_vfnet_head.cpp` | VFNet head 격리 검증 harness (torch FPN features → head → cls/box 덤프). | -| `build_mmdet_cpp.sh` | `run_mmdet.cpp` + `head.cpp` + `output/.cpp` 를 libvisioncpp 와 컴파일. | +# 2. Compile backbone.pt to a vision.cpp arch module: .cpp, .h, .gguf. +# The interface the generated code must satisfy is in docs/mmdet-detectors.md. -decode+NMS·전처리는 vision.cpp 라이브러리(`src/visp/postproc.{h,cpp}` 의 `detect_anchor`/`preprocess`)를 -그대로 쓴다. `conv_2d_deform`(DCN)도 라이브러리 프리미티브(ggml `conv_2d_deform` 커널 래퍼). +# 3. Build the runner: the generated graph, head.cpp and run_mmdet.cpp together. +bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone -## 사용 예 (RetinaNet r18) +# 4. Run. +output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg boxes.bin 512 +``` -```bash -PY=; G2C=; V=$G2C/vision.cpp -FE=$V/tools/frontend/mmdet # mmdet frontend (torch-side, mmdet 지식 유일 지점) -BUILD=$V/tools/build # 빌드 스크립트 -CFG=/configs/retinanet/retinanet_r18_fpn_1x_coco.py +`backbone.postproc.h` holds a generated `mmdet_params()` — anchor scales, head convolution +layout, normalisation values. Once an architecture is fixed those are constants, so they are +compiled into the runner rather than read at run time, and the deployed set is the executable +and the weights. -# ① mmdet → backbone.pt + postproc.json (frontend/mmdet = mmdet 지식 유일 지점) -PYTHONPATH=$FE $PY $FE/mmdet_to_pt.py --config $CFG --out /tmp/rn.pt --size 512 +`mmdet_wrap.postproc_cfg` is the only code that reads an MMDetection configuration. It also +extracts `img_mean` / `img_std` / `to_rgb` from `data_preprocessor`, so pre-processing is the +library's `preprocess()` driven by extracted values rather than anything hand-written. -# ② g2c 정식 CLI → output/MMDetBackbone/{cpp,h,gguf} (g2c 코어 무수정, .pt 는 generic torch 모듈) -PYTHONPATH=$G2C:$FE $PY -m shared.compile.pipeline --model /tmp/rn.pt --name MMDetBackbone \ - --input-shape 1,3,512,512 --output output/MMDetBackbone +## Looking at the output -# ③ 러너 컴파일 (output/.cpp + verify/backbone/run_mmdet + detect/head.cpp + libvisioncpp) -VISP_BUILD=$V/build bash $BUILD/build_mmdet_cpp.sh output/MMDetBackbone +`run_mmdet` writes detections as raw `float32` — six numbers per box, `x1 y1 x2 y2 score label` +— because that is what comparing against a reference implementation needs. It also prints the +highest-scoring ones: -# ④ 실행 (백본 + C++ head + detect_anchor → 박스). 입력이 이미지면 preprocess() 자동 전처리. -output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf \ - image.jpg /tmp/rn.postproc.json boxes.bin 512 ``` +- detect(anchor): 100 boxes → boxes.bin (x1,y1,x2,y2,score,label f32*6) -pre(전처리)도 손코딩이 아니라 **범용 부품 + config 추출**: `postproc.cpp` 의 `preprocess()` -(resize+normalize+to_rgb, CPU 스칼라) + mmdet_wrap 이 `data_preprocessor` 에서 `img_mean/img_std/ -to_rgb` 자동 추출 → postproc.json. + # x1 y1 x2 y2 score label + 0 459.3 241.1 512.0 263.3 0.837 63 + 1 306.4 69.4 361.2 84.5 0.835 63 + ... (98 more) +``` -## 검증 +`VISP_PRINT_DETS` changes how many are printed, or turns the table off with `0`. -- **RetinaNet r18** (anchor head): C++ head raw cls/box cos 0.999999~1.0, 최종 박스 `predict_by_feat` - 대비 IoU>0.99 매칭 100/100. -- **VFNet r50** (distance + star DCN head): `vfnet_head_forward` (star deformable offset 계산 + - `conv_2d_deform`) 를 torch head 와 비교 → 5레벨 cls/box **cos 0.999998~1.0** (격리 검증 - `run_vfnet_head`). DCN offset 계산이 자동변환 안 되는 부분을 손코딩으로 해결한 사례. +To see the boxes on the image: -## Two-stage (Faster R-CNN) +```sh +python tools/verify/draw_boxes.py image.jpg boxes.bin -o annotated.png -t 0.3 +``` -RPN proposal·RoIAlign 은 데이터 의존(proposal 개수 가변)이라 단일 그래프에 안 들어감 → g2c 로 -**두 subgraph**(SubA/SubB)만 뽑고, 그 사이는 host C++ op 으로 오케스트레이션. +`--size` if the detector ran at something other than 512, `--labels` for a class list other than +COCO 80. -``` -이미지 - │ g2c SubA (backbone+neck+RPN) [frcnn_wrap.FRCNN_SubA → 14 출력] - ▼ P2-P5 + rpn_cls×5 + rpn_bbox×5 - │ rpn_proposals (host) RPN decode + level NMS → 1000 proposals - │ roi_align (host) proposal + P2-P5 → roi_feat (N,256,7,7) - ▼ - │ g2c SubB (bbox_head Shared2FC) [frcnn_wrap.FRCNN_SubB → cls,bbox] - ▼ cls_score(N,81) + bbox_pred(N,320) - │ detect_roi (host) softmax + delta decode + per-class NMS → 박스 - ▼ 최종 박스 -``` +`run_dump` does the same for any generated graph — it prints each output tensor's shape and +writes it as raw `float32`, which is the way to inspect a backbone that has no head attached. -파일: `frcnn_wrap.py`(SubA/SubB + config), `frcnn_to_pt.py`(→ .pt×2 + frcnn.json), -`run_frcnn.cpp`(오케스트레이션 러너), `build_frcnn_cpp.sh`. host op 은 `postproc.cpp` 의 -`rpn_proposals`/`roi_align`/`detect_roi` (라이브러리). 검증 harness: `run_roi_verify.cpp`, -`run_rpn_verify.cpp`. +## Heads -```bash -python tools/frontend/mmdet/frcnn_to_pt.py --config faster-rcnn_r50_fpn_1x_coco.py --checkpoint frcnn.pth --out /tmp/frcnn -g2c --model /tmp/frcnn/FRCNN_SubA.pt --name FRCNN_SubA --input-shape 1,3,800,800 --output output/FRCNN_SubA -g2c --model /tmp/frcnn/FRCNN_SubB.pt --name FRCNN_SubB --input-shape 4,256,7,7 --output output/FRCNN_SubB -bash tools/build/build_frcnn_cpp.sh output/FRCNN_SubA output/FRCNN_SubB -output/FRCNN_SubA/run_frcnn output/FRCNN_SubA/FRCNN_SubA.gguf output/FRCNN_SubB/FRCNN_SubB.gguf \ - /tmp/frcnn/frcnn.json input.bin 800 -``` +`detect/head.h` declares the components that turn FPN features into raw per-level tensors. -**검증 (Faster R-CNN r50, 800, trained, demo.jpg):** -- RoIAlign : torch `bbox_roi_extractor` 대비 **cos 1.0, max|Δ|=7e-07** (1000 proposals) -- RPN proposals : torch `RPNHead.predict_by_feat` 대비 **1000/1000 IoU>0.99** -- E2E 박스 : torch 풀 two-stage 대비 **score>0.3 20/20 매칭(IoU>0.95), score>0.05 48/49** +`anchor_head_forward` +: Shared convolution tower followed by classification and regression convolutions — RetinaNet, + ATSS, GFL and other anchor-based dense heads. The differences between them are values in + `anchor_head_cfg`, not code. -## Instance segmentation (Mask R-CNN) +`vfnet_head_forward` +: Distances rather than anchor deltas, refined by a star-shaped deformable convolution whose + sampling offsets are computed from the first bbox prediction. Those offsets are values + produced during the forward pass, so the component builds that computation explicitly. -Faster R-CNN + mask 분기. 최종 박스에 **2nd RoIAlign(out=14)** → g2c SubC(mask_head FCN: -4×conv + deconv + conv_logits) → `paste_mask`(host). +Adding a head means a new `_head_forward` here using library primitives (`conv_2d`, +`group_norm`, `conv_2d_deform`), the matching parameters emitted by `mmdet_wrap.postproc_cfg`, +and a decoder in `postproc.h` if the decoding scheme is new. + +## Two-stage detectors + +RPN proposals and RoIAlign are data-dependent — the number of proposals is not known until the +network has run — so a two-stage detector cannot be a single graph. ``` -Faster R-CNN → 최종 박스 - │ roi_align(out=14, host) 박스 → mask_feat (M,256,14,14) - │ g2c SubC (mask_head) → mask_logits (M,80,28,28) [frcnn_wrap.MaskRCNN_SubC] - │ paste_mask (host) label mask → sigmoid+resize+threshold → 인스턴스 마스크 - ▼ + image + │ SubA (backbone + neck + RPN) 14 outputs: P2-P5, rpn_cls×5, rpn_bbox×5 + ▼ + │ rpn_proposals (host) decode + per-level NMS -> proposals + │ roi_align (host) proposals + P2-P5 -> roi_feat (N,256,7,7) + ▼ + │ SubB (bbox head, Shared2FC) -> cls_score, bbox_pred + ▼ + │ detect_roi (host) softmax + delta decode + per-class NMS + ▼ detections ``` -파일: `frcnn_wrap.MaskRCNN_SubC`, `run_maskrcnn.cpp`, `build_maskrcnn_cpp.sh`. host op 은 -`postproc.cpp` 의 `roi_align`/`paste_mask`. +```sh +python tools/frontend/mmdet/frcnn_to_pt.py \ + --config faster-rcnn_r50_fpn_1x_coco.py --checkpoint frcnn.pth --out /tmp/frcnn +# compile FRCNN_SubA.pt at 1,3,800,800 and FRCNN_SubB.pt at 4,256,7,7 + +bash tools/build/build_frcnn_cpp.sh output/FRCNN_SubA output/FRCNN_SubB +output/FRCNN_SubA/run_frcnn output/FRCNN_SubA/FRCNN_SubA.gguf \ + output/FRCNN_SubB/FRCNN_SubB.gguf /tmp/frcnn/frcnn.json input.bin 800 +``` -> ⚠️ **ggml `conv_transpose_2d_p0` 는 batch(N>1) 미지원** → run_maskrcnn 은 SubC 를 **roi 별 -> (batch=1)** 로 실행한다. (배치로 돌리면 첫 roi 만 정확: batch0 cos 1.0, batch1+ 깨짐.) +The host operations are `rpn_proposals`, `roi_align` and `detect_roi` in the library. -**검증 (Mask R-CNN r50, 800, trained, demo.jpg):** torch 풀 Mask R-CNN 대비 -- 박스 20/20 매칭, **mask IoU 평균 0.984, IoU>0.9 20/20** (score>0.3); -- score>0.05: mask IoU 평균 0.975, IoU>0.9 42/43. +## Instance segmentation -## Tracking (MOT — ByteTrack) +Mask R-CNN adds a second RoIAlign at output size 14 over the final boxes, a mask sub-graph, and +host-side mask pasting. -검출(그래프)은 그대로 쓰고, 프레임 간 **association**만 host 부품으로 한다. tracking 은 신경망이 -아니라 상태추적 로직 → g2c 변환 대상 없음. 상태 유지 class `ByteTracker`(라이브러리 `tracker.{h,cpp}`). +``` + final boxes -> roi_align(out=14) -> SubC (mask head FCN) -> paste_mask -> instance masks +``` ``` -프레임별 검출 (위 검출기 그대로) - → ByteTracker.track(dets, frame_id) - ① Kalman 예측 (SORT 8-state cxcyah) - ② IoU 2단계 매칭 (high-score 먼저 → low-score) + tentative/confirmed - ③ track 생성/유지/삭제 (num_frames_retain) - → track ID (프레임 간 유지) +run_maskrcnn [size=800] ``` -`ByteTracker` 는 검출기 무관(generic). 검증 harness: `run_bytetrack_verify.cpp`. +> ggml's `conv_transpose_2d_p0` does not support batching, so `run_maskrcnn` evaluates the mask +> sub-graph one RoI at a time. Running it batched leaves only the first RoI correct. + +## Tracking + +Tracking is state management, not a network — there is nothing to compile. `ByteTracker` in +`src/visp/tracker.h` holds track state across frames and is detector-agnostic: it consumes +`std::vector` from any of the decoders. -**검증:** 합성 검출 시퀀스(10 프레임, 5 물체: 등속 이동 + 등장/소멸)를 mmdet `ByteTracker` 와 -동일 입력으로 비교 → **track ID 44/44 완전 일치, 충돌 0** (Kalman·2단계 매칭 복제). +## Isolation harnesses -## 확장 (다른 head) +Each takes one stage and compares it against a dump from the reference implementation, which +locates a mismatch to a single stage rather than to the pipeline as a whole. -- **anchor**(RetinaNet/ATSS, DeltaXYWHBBoxCoder): `anchor_head_forward` 그대로(cls/reg conv 이름 자동 탐지). -- **VFNet**(distance + DCN): `vfnet_head_forward`. offset = star_dcn_offset(bbox_pred) - dcn_base → - `conv_2d_deform`. distance decode 는 `postproc.cpp` 의 `detect_fcos` 연결로 박스화(진행 중). -- **FCOS/DETR/two-stage**: `head.cpp` 에 부품 추가 + `postproc` 의 `detect_fcos`/`detect_detr`/`detect_roi` 연결. +| Harness | Stage | +| :--- | :--- | +| `run_vfnet_head` | A dense head in isolation, from FPN features to raw cls/box | +| `run_rpn_verify` | RPN proposal generation | +| `run_roi_verify` | RoIAlign | +| `run_bytetrack_verify` | Frame-to-frame association | +| `run_dump` | Any generated graph — every output tensor as raw `float32` | diff --git a/tools/verify/backbone/run_mmdet.cpp b/tools/verify/backbone/run_mmdet.cpp index 0d307a7..f141c1e 100755 --- a/tools/verify/backbone/run_mmdet.cpp +++ b/tools/verify/backbone/run_mmdet.cpp @@ -20,7 +20,9 @@ #include +#include #include +#include #include #include #include @@ -155,5 +157,25 @@ int main(int argc, char** argv) { } fclose(f); printf("- detect(anchor): %zu boxes → %s (x1,y1,x2,y2,score,label f32*6)\n", dets.size(), outp); + + // 파일은 대조용 raw f32 다. 사람이 결과를 눈으로 확인할 수 있게 상위 몇 개는 표로도 찍는다 + // (개수는 VISP_PRINT_DETS 로 조절, 0 이면 끄기). + int n_print = 10; + if (const char* e = std::getenv("VISP_PRINT_DETS")) { + n_print = atoi(e); + } + n_print = std::min(n_print, (int)dets.size()); + if (n_print > 0) { + printf("\n %5s %8s %8s %8s %8s %8s %6s\n", "#", "x1", "y1", "x2", "y2", "score", "label"); + for (int i = 0; i < n_print; ++i) { + detection const& d = dets[i]; + printf(" %5d %8.1f %8.1f %8.1f %8.1f %8.3f %6d\n", + i, d.x1, d.y1, d.x2, d.y2, d.score, d.label); + } + if ((int)dets.size() > n_print) { + printf(" %5s (%zu more)\n", "...", dets.size() - n_print); + } + printf("\n"); + } return 0; } diff --git a/tools/verify/draw_boxes.py b/tools/verify/draw_boxes.py new file mode 100644 index 0000000..0bf53a6 --- /dev/null +++ b/tools/verify/draw_boxes.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Draw detections onto the image they came from. + +The runners write detections as raw float32 -- six numbers per box -- because that is what +comparing against a reference implementation needs. This turns such a file into something to +look at. + + python tools/verify/draw_boxes.py image.jpg boxes.bin -o annotated.png + +Boxes are in the coordinate space of the resized square input the detector ran on, so the +script scales them back to the original image. Pass --size if the detector ran at something +other than 512. +""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +# COCO 80. Detectors trained on something else take --labels. +COCO = ( + "person bicycle car motorcycle airplane bus train truck boat traffic_light fire_hydrant " + "stop_sign parking_meter bench bird cat dog horse sheep cow elephant bear zebra giraffe " + "backpack umbrella handbag tie suitcase frisbee skis snowboard sports_ball kite " + "baseball_bat baseball_glove skateboard surfboard tennis_racket bottle wine_glass cup fork " + "knife spoon bowl banana apple sandwich orange broccoli carrot hot_dog pizza donut cake " + "chair couch potted_plant bed dining_table toilet tv laptop mouse remote keyboard " + "cell_phone microwave oven toaster sink refrigerator book clock vase scissors teddy_bear " + "hair_drier toothbrush" +).split() + +# Distinct enough to tell classes apart without a legend. +PALETTE = [ + (230, 60, 60), (60, 160, 230), (70, 190, 110), (240, 160, 40), (170, 100, 220), + (40, 200, 200), (230, 110, 170), (150, 160, 60), (110, 130, 240), (200, 90, 60), +] + + +def load(path: Path) -> np.ndarray: + d = np.fromfile(path, dtype="float32") + if d.size % 6: + raise SystemExit(f"{path}: {d.size} floats is not a multiple of 6") + return d.reshape(-1, 6) + + +def main(argv=None) -> None: + ap = argparse.ArgumentParser(description="Draw raw float32 detections onto an image.") + ap.add_argument("image", type=Path, help="the image the detector was given") + ap.add_argument("boxes", type=Path, help="detections written by the runner (.bin)") + ap.add_argument("-o", "--output", type=Path, default=Path("annotated.png")) + ap.add_argument("-t", "--threshold", type=float, default=0.3, + help="skip detections below this score. Default 0.3") + ap.add_argument("--size", type=int, default=512, + help="square input resolution the detector ran at. Default 512") + ap.add_argument("--labels", type=Path, default=None, + help="class names, one per line. Default: COCO 80") + a = ap.parse_args(argv) + + names = (a.labels.read_text(encoding="utf-8").split() if a.labels else COCO) + img = Image.open(a.image).convert("RGB") + dets = load(a.boxes) + keep = dets[dets[:, 4] >= a.threshold] + + # The detector saw a square of --size; put the boxes back on the original. + sx, sy = img.width / a.size, img.height / a.size + + draw = ImageDraw.Draw(img) + try: + font = ImageFont.load_default(size=max(12, img.height // 45)) + except TypeError: # Pillow < 9.2 has no size argument + font = ImageFont.load_default() + + for x1, y1, x2, y2, score, label in keep: + label = int(label) + colour = PALETTE[label % len(PALETTE)] + box = (x1 * sx, y1 * sy, x2 * sx, y2 * sy) + draw.rectangle(box, outline=colour, width=max(2, img.height // 300)) + + name = names[label] if label < len(names) else str(label) + text = f"{name} {score:.2f}" + tw, th = draw.textbbox((0, 0), text, font=font)[2:] + ty = max(0.0, box[1] - th - 2) + draw.rectangle((box[0], ty, box[0] + tw + 6, ty + th + 4), fill=colour) + draw.text((box[0] + 3, ty + 2), text, fill=(255, 255, 255), font=font) + + img.save(a.output) + print(f" → {a.output} ({len(keep)} of {len(dets)} detections at score >= {a.threshold})") + + +if __name__ == "__main__": + main() From 5997204e7bae483fa766cab9a5883900ff18e58b Mon Sep 17 00:00:00 2001 From: eunchae Date: Thu, 6 Aug 2026 17:55:12 +0900 Subject: [PATCH 4/8] feat(mmdet): draw the detections by default The runner wrote raw float32 and nothing else, so seeing what a detector had found meant writing a script. Every other command-line entry point in this repository produces an image; the detector runner now does too. The output extension decides: run_mmdet model.gguf image.jpg detected.png 512 # boxes drawn on the image run_mmdet model.gguf image.jpg boxes.bin 512 # raw float32 Raw output is unchanged and one extension away, because comparing against a reference implementation needs the numbers, not a picture. tools/detect/draw.h holds the drawing: box outlines, one colour per class, no text. Keeping a font out of the runner costs nothing, because the runner also prints the highest-scoring detections as a table -- the image carries where, the table carries what. VISP_DRAW_THRESHOLD and VISP_PRINT_DETS adjust both. tools/verify/draw_boxes.py stays for drawing a .bin that was kept for comparison, and does add class names and scores as text. Verified on RetinaNet R18 at 512: the .bin path is still byte-identical to the previous output, and the image path draws boxes scaled back to the original resolution. Co-Authored-By: Claude Opus 5 --- docs/mmdet-detectors.md | 60 ++++++++++++++++++++-- tools/README.md | 38 ++++++++------ tools/detect/draw.h | 77 +++++++++++++++++++++++++++++ tools/verify/backbone/run_mmdet.cpp | 73 +++++++++++++++++++++------ 4 files changed, 212 insertions(+), 36 deletions(-) create mode 100644 tools/detect/draw.h diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md index bdac70e..173570e 100644 --- a/docs/mmdet-detectors.md +++ b/docs/mmdet-detectors.md @@ -31,6 +31,7 @@ objects. - [Prerequisites](#prerequisites) - [Pipeline](#pipeline) +- [Output](#output) - [Detection heads](#detection-heads) - [Post-processing API](#post-processing-api) - [Two-stage detectors](#two-stage-detectors) @@ -164,12 +165,12 @@ The result is `/run_mmdet`. output/MMDetBackbone/run_mmdet \ output/MMDetBackbone/MMDetBackbone.gguf \ image.jpg \ - boxes.bin \ + detected.png \ 512 ``` ``` -run_mmdet [size=512] +run_mmdet [size=512] ``` `` @@ -181,14 +182,63 @@ run_mmdet [size=512] deviation and channel order are compiled in. A `.bin` file is taken as-is and must contain `3 × size × size` `float32` values in CWHN order. -`` -: Output path. Each detection is written as six `float32` values: - `x1, y1, x2, y2, score, label`. Coordinates are in input-image pixels. +`` +: Where to write the result. The extension decides what is written: `.bin` gives raw + detections, anything else gives the input image with the boxes drawn on it. `[size]` : Input resolution. Must match the value passed to `--size` in step 1 and the shape the graph was compiled for. +### Output + +The extension of the output path decides what is written. + +```sh +run_mmdet model.gguf image.jpg detected.png 512 # the image, boxes drawn on it +run_mmdet model.gguf image.jpg boxes.bin 512 # raw float32, six numbers per box +``` + +An image is the default, which is what the rest of the command-line tools produce and what a +person looking at a result wants. Raw `float32` is what comparing against a reference +implementation needs, so it stays one extension away rather than being the only option. + +Either way the highest-scoring detections are printed: + +``` +- detect(anchor): 100 boxes, 12 drawn at score >= 0.30 → detected.png + + # x1 y1 x2 y2 score label + 0 459.3 241.1 512.0 263.3 0.837 63 + 1 306.4 69.4 361.2 84.5 0.835 63 + ... (98 more) +``` + +The image carries where, the table carries what. Class is drawn as colour rather than text, +which keeps a font out of the runner. + +| Variable | Effect | +| :--- | :--- | +| `VISP_DRAW_THRESHOLD` | Minimum score to draw. Default `0.3` | +| `VISP_PRINT_DETS` | How many rows to print. `0` turns the table off | + +In the raw form each detection is six `float32` values written back to back: + +``` +x1 y1 x2 y2 score label +``` + +Coordinates are pixels in the square input the detector ran on. There is no header and no +count — the number of detections is the file size divided by 24 bytes. + +```python +import numpy as np +d = np.fromfile("boxes.bin", dtype="float32").reshape(-1, 6) +``` + +`tools/verify/draw_boxes.py` draws such a file afterwards, with class names and scores as text. + + ## Detection heads Head components take FPN features and produce the raw per-level tensors that the decoders diff --git a/tools/README.md b/tools/README.md index 98b4426..eeec448 100755 --- a/tools/README.md +++ b/tools/README.md @@ -55,7 +55,7 @@ python tools/frontend/mmdet/mmdet_to_pt.py \ bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone # 4. Run. -output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg boxes.bin 512 +output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg detected.png 512 ``` `backbone.postproc.h` holds a generated `mmdet_params()` — anchor scales, head convolution @@ -69,32 +69,40 @@ library's `preprocess()` driven by extracted values rather than anything hand-wr ## Looking at the output -`run_mmdet` writes detections as raw `float32` — six numbers per box, `x1 y1 x2 y2 score label` -— because that is what comparing against a reference implementation needs. It also prints the -highest-scoring ones: +The extension of the output path decides what `run_mmdet` writes. +```sh +run_mmdet model.gguf image.jpg detected.png 512 # the image, boxes drawn on it +run_mmdet model.gguf image.jpg boxes.bin 512 # raw float32, six numbers per box ``` -- detect(anchor): 100 boxes → boxes.bin (x1,y1,x2,y2,score,label f32*6) +An image is the default because that is what the rest of `vision-cli` produces and what a person +looking at a result wants. Raw `float32` — `x1 y1 x2 y2 score label` — is what comparing against +a reference implementation needs, so it stays one extension away. + +Either way the highest-scoring detections are printed: + +``` # x1 y1 x2 y2 score label 0 459.3 241.1 512.0 263.3 0.837 63 1 306.4 69.4 361.2 84.5 0.835 63 ... (98 more) ``` -`VISP_PRINT_DETS` changes how many are printed, or turns the table off with `0`. +The image carries where, the table carries what. No text is drawn into the image; class is +encoded as colour, which keeps a font out of the runner. -To see the boxes on the image: - -```sh -python tools/verify/draw_boxes.py image.jpg boxes.bin -o annotated.png -t 0.3 -``` +| Variable | Effect | +| :--- | :--- | +| `VISP_DRAW_THRESHOLD` | Minimum score to draw. Default `0.3` | +| `VISP_PRINT_DETS` | How many rows to print. `0` turns the table off | -`--size` if the detector ran at something other than 512, `--labels` for a class list other than -COCO 80. +`tools/verify/draw_boxes.py` draws a `.bin` that was written earlier, which is the way to look +at a file kept for comparison. It adds class names and scores as text, which the C++ path does +not. -`run_dump` does the same for any generated graph — it prints each output tensor's shape and -writes it as raw `float32`, which is the way to inspect a backbone that has no head attached. +`run_dump` covers any generated graph — it prints each output tensor's shape and writes it as +raw `float32`, which is how to inspect a backbone with no head attached. ## Heads diff --git a/tools/detect/draw.h b/tools/detect/draw.h new file mode 100644 index 0000000..83cdb53 --- /dev/null +++ b/tools/detect/draw.h @@ -0,0 +1,77 @@ +// 검출 결과를 이미지 위에 그린다. 러너의 기본 출력이 이미지이기 위한 최소한의 부품. +// +// 글자는 그리지 않는다 — 폰트를 들여오면 러너가 무거워지고, 클래스·점수는 러너가 표로 +// 찍는다. 이미지는 "어디"를, 터미널 표는 "무엇"을 담당한다. +#pragma once + +#include "visp/image.h" +#include "visp/postproc.h" + +#include +#include +#include + +namespace visp { + +// 클래스끼리 구분되게. 범례 없이도 같은 클래스가 같은 색으로 보이면 충분하다. +inline std::array detection_colour(int label) { + static constexpr uint8_t table[][3] = { + {230, 60, 60}, {60, 160, 230}, {70, 190, 110}, {240, 160, 40}, {170, 100, 220}, + {40, 200, 200}, {230, 110, 170}, {150, 160, 60}, {110, 130, 240}, {200, 90, 60}, + }; + int n = int(sizeof(table) / sizeof(table[0])); + int i = ((label % n) + n) % n; + return {table[i][0], table[i][1], table[i][2]}; +} + +namespace detail { + +inline void put_pixel(image_span const& img, int x, int y, std::array c) { + if (x < 0 || y < 0 || x >= img.extent[0] || y >= img.extent[1]) { + return; + } + int nc = n_channels(img.format); + auto* p = static_cast(img.data) + size_t(y) * img.stride + size_t(x) * nc; + p[0] = c[0]; + if (nc > 1) p[1] = c[1]; + if (nc > 2) p[2] = c[2]; +} + +} // namespace detail + +// 박스 외곽선 하나. thickness 픽셀 두께로 안쪽에 그린다. +inline void draw_box(image_span const& img, float x1, float y1, float x2, float y2, + std::array colour, int thickness = 2) { + int ix1 = int(std::min(x1, x2)), ix2 = int(std::max(x1, x2)); + int iy1 = int(std::min(y1, y2)), iy2 = int(std::max(y1, y2)); + for (int t = 0; t < thickness; ++t) { + for (int x = ix1; x <= ix2; ++x) { + detail::put_pixel(img, x, iy1 + t, colour); + detail::put_pixel(img, x, iy2 - t, colour); + } + for (int y = iy1; y <= iy2; ++y) { + detail::put_pixel(img, ix1 + t, y, colour); + detail::put_pixel(img, ix2 - t, y, colour); + } + } +} + +// 검출 목록을 그린다. +// 좌표는 검출기가 돈 정사각 입력 기준이므로 원본 크기로 되돌린다(scale_x/scale_y). +// 반환: 실제로 그린 개수. +inline int draw_detections(image_span const& img, std::vector const& dets, + float scale_x, float scale_y, float threshold = 0.3f) { + int thickness = std::max(2, img.extent[1] / 300); + int drawn = 0; + for (detection const& d : dets) { + if (d.score < threshold) { + continue; + } + draw_box(img, d.x1 * scale_x, d.y1 * scale_y, d.x2 * scale_x, d.y2 * scale_y, + detection_colour(d.label), thickness); + ++drawn; + } + return drawn; +} + +} // namespace visp diff --git a/tools/verify/backbone/run_mmdet.cpp b/tools/verify/backbone/run_mmdet.cpp index f141c1e..837eeee 100755 --- a/tools/verify/backbone/run_mmdet.cpp +++ b/tools/verify/backbone/run_mmdet.cpp @@ -9,10 +9,12 @@ // libvisioncpp 와 링크(build_mmdet_cpp.sh). run_yolo_cpp 와 동일한 -DARCH 매크로 방식. // // 컴파일: -DARCH=<클래스명> -DVISP_ARCH_HEADER='"/.h"' -// 실행: run_mmdet [size=512] +// 실행: run_mmdet [size=512] +// 출력 확장자가 .bin 이면 원시 f32(레퍼런스 대조용), 그 외에는 박스를 그린 이미지. #include VISP_ARCH_HEADER // 백본: _forward / _params / _detect_params #include "head.h" // head 부품: anchor_head_forward (같은 폴더) +#include "draw.h" // 검출 결과를 이미지에 그린다 (같은 폴더) #include MMDET_PARAMS_HEADER // 생성된 mmdet_params() — 값이 실행 파일 안에 있다 #include "visp/image.h" // image_load (이미지 입력 pre) #include "visp/ml.h" @@ -55,13 +57,20 @@ static std::vector to_vec(tensor t) { // 입력이 이미지(.jpg/.png…)면 preprocess()(resize+normalize+to_rgb, 생성된 상수)로 텐서 생성. // .bin 이면 이미 전처리된 CWHN f32 텐서로 간주. → yolo run_yolo_cpp 는 .bin(외부 전처리)만, 여기선 둘 다. -static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c) { +static bool has_ext(std::string const& s, const char* e) { + size_t n = std::strlen(e); + return s.size() >= n && s.compare(s.size() - n, n, e) == 0; +} + +static bool is_image_path(std::string const& s) { + return has_ext(s, ".jpg") || has_ext(s, ".jpeg") || has_ext(s, ".png") || has_ext(s, ".bmp"); +} + +// `source` 가 비어 있지 않게 채워지면 입력이 이미지였다는 뜻 — 결과를 그 위에 그릴 수 있다. +static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c, + image_data* source) { std::string s(path); - auto ext = [&](const char* e) { - size_t n = std::strlen(e); - return s.size() >= n && s.compare(s.size() - n, n, e) == 0; - }; - if (ext(".jpg") || ext(".jpeg") || ext(".png") || ext(".bmp")) { + if (is_image_path(s)) { image_data img = image_load(path); int iw = img.extent[0], ih = img.extent[1]; int ic = n_channels(img.format); // stbi_load(...,0)=네이티브 채널수 (JPEG=3, PNG+α=4) @@ -70,14 +79,21 @@ static std::vector load_input(const char* path, int SZ, mmdet_cfg const& bool to_rgb = c.to_rgb; printf("- preprocess: image %dx%dx%d → %dx%d (mean %.1f,%.1f,%.1f std %.1f,%.1f,%.1f to_rgb=%d)\n", iw, ih, ic, SZ, SZ, mean[0], mean[1], mean[2], sd[0], sd[1], sd[2], (int)to_rgb); - return preprocess(img.data.get(), ih, iw, ic, SZ, mean, sd, to_rgb); + auto tensor_data = preprocess(img.data.get(), ih, iw, ic, SZ, mean, sd, to_rgb); + if (source) { + *source = std::move(img); + } + return tensor_data; } return load_bin(path, (size_t)3 * SZ * SZ); } int main(int argc, char** argv) { if (argc < 4) { - fprintf(stderr, "usage: %s [size=512]\n", argv[0]); + fprintf(stderr, + "usage: %s [size=512]\n" + " output ending in .bin holds raw float32 detections;\n" + " any other extension is an image with the boxes drawn on it\n", argv[0]); return 1; } const char* gguf = argv[1]; @@ -129,7 +145,8 @@ int main(int argc, char** argv) { // 5) 계산 (입력: 이미지면 preprocess, .bin 이면 전처리된 텐서) compute_graph_allocate(graph, backend); - auto in_data = load_input(inp, SZ, cfg); + image_data source; + auto in_data = load_input(inp, SZ, cfg, &source); if (const char* dp = std::getenv("MMDET_DUMP_PRE")) { // 디버그: 전처리 텐서 덤프 FILE* f = fopen(dp, "wb"); if (f) { fwrite(in_data.data(), sizeof(float), in_data.size(), f); fclose(f); } } @@ -150,13 +167,37 @@ int main(int argc, char** argv) { } std::vector dets = detect_anchor(cls_v, box_v, feat_hw, dp); - FILE* f = fopen(outp, "wb"); - for (detection const& d : dets) { - float rec[6] = { d.x1, d.y1, d.x2, d.y2, d.score, (float)d.label }; - fwrite(rec, sizeof(float), 6, f); + // 기본은 이미지 — vision-cli 의 다른 명령들과 같다. 원시 수치는 .bin 으로 요청한다. + std::string out_s(outp); + bool want_raw = has_ext(out_s, ".bin"); + if (!want_raw && source.extent[0] == 0) { + fprintf(stderr, "- input was a tensor, so there is no image to draw on; writing raw\n"); + want_raw = true; + } + + if (want_raw) { + FILE* f = fopen(outp, "wb"); + if (!f) { fprintf(stderr, "cannot write %s\n", outp); return 1; } + for (detection const& d : dets) { + float rec[6] = { d.x1, d.y1, d.x2, d.y2, d.score, (float)d.label }; + fwrite(rec, sizeof(float), 6, f); + } + fclose(f); + printf("- detect(anchor): %zu boxes → %s (x1,y1,x2,y2,score,label f32*6)\n", + dets.size(), outp); + } else { + float thr = 0.3f; + if (const char* e = std::getenv("VISP_DRAW_THRESHOLD")) { + thr = (float)atof(e); + } + // 좌표는 정사각 입력 기준이므로 원본 해상도로 되돌린다. + float sx = float(source.extent[0]) / float(SZ); + float sy = float(source.extent[1]) / float(SZ); + int drawn = draw_detections(source, dets, sx, sy, thr); + image_save(source, outp); + printf("- detect(anchor): %zu boxes, %d drawn at score >= %.2f → %s\n", + dets.size(), drawn, thr, outp); } - fclose(f); - printf("- detect(anchor): %zu boxes → %s (x1,y1,x2,y2,score,label f32*6)\n", dets.size(), outp); // 파일은 대조용 raw f32 다. 사람이 결과를 눈으로 확인할 수 있게 상위 몇 개는 표로도 찍는다 // (개수는 VISP_PRINT_DETS 로 조절, 0 이면 끄기). From 97a631ec3f0a36723cdba0960509ccee377958c1 Mon Sep 17 00:00:00 2001 From: eunchae Date: Thu, 6 Aug 2026 18:00:13 +0900 Subject: [PATCH 5/8] docs(mmdet): state the resolution requirement and name the parameters header Running the documented steps from an empty directory found two gaps. The build step was shown without the parameters header, but export writes it next to backbone.pt rather than into the generated directory, so the command as printed could not find it. The header is now named in the command, and the script says what to pass when it is missing instead of reporting an empty path. The compile step did not say which resolution to use. Tracing records the operations for one input shape, so a graph built at a different size aborts in ggml_can_repeat once a tensor of the wrong extent reaches a residual addition -- a failure a long way from its cause. Both documents now say to compile at the --size given to export. Co-Authored-By: Claude Opus 5 --- docs/mmdet-detectors.md | 7 ++++++- tools/README.md | 5 ++++- tools/build/build_mmdet_cpp.sh | 14 +++++++++----- tools/detect/draw.h | 15 +++++++-------- tools/detect/head.h | 7 ++++--- tools/frontend/mmdet/mmdet_to_pt.py | 13 +++++++------ tools/verify/backbone/run_mmdet.cpp | 26 +++++++++++++------------- 7 files changed, 50 insertions(+), 37 deletions(-) diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md index 173570e..c92bae4 100644 --- a/docs/mmdet-detectors.md +++ b/docs/mmdet-detectors.md @@ -95,6 +95,11 @@ The `.pt` file pickles by module name `mmdet_wrap`, so the export directory must PyTorch-to-ggml model compiler. That compiler is outside the scope of this document; what matters is the interface the generated code must satisfy. +**Compile at the resolution `--size` used in step 1.** Tracing records the operations for one +input shape, so the graph runs at that shape and no other. A graph built for a different size +aborts at run time in `ggml_can_repeat` once a tensor of the wrong extent reaches a residual +addition. + **Generated files** — for an architecture named `MMDetBackbone`: | File | Contents | @@ -134,7 +139,7 @@ generated graph did not name its outputs as expected. ### Step 3 — Build the runner ```sh -bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone +bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h ``` The script compiles three translation units together, with the generated parameters diff --git a/tools/README.md b/tools/README.md index eeec448..77354ef 100755 --- a/tools/README.md +++ b/tools/README.md @@ -49,10 +49,13 @@ python tools/frontend/mmdet/mmdet_to_pt.py \ --out backbone.pt --size 512 # 2. Compile backbone.pt to a vision.cpp arch module: .cpp, .h, .gguf. +# Compile it at the same resolution --size used above. Tracing records the operations for +# one input shape, and a graph built for another size aborts in ggml_can_repeat at run time. # The interface the generated code must satisfy is in docs/mmdet-detectors.md. # 3. Build the runner: the generated graph, head.cpp and run_mmdet.cpp together. -bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone +# The parameters header is the one export wrote next to backbone.pt. +bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h # 4. Run. output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg detected.png 512 diff --git a/tools/build/build_mmdet_cpp.sh b/tools/build/build_mmdet_cpp.sh index 3b0e1f2..19a242e 100755 --- a/tools/build/build_mmdet_cpp.sh +++ b/tools/build/build_mmdet_cpp.sh @@ -5,10 +5,10 @@ # head.cpp 는 라이브러리가 아니라 여기서 러너와 함께 컴파일된다 → g2c output/.cpp 를 직접 # 컴파일(arch/ 복사·cli REG 없음). # -# 사용: build_mmdet_cpp.sh [arch_name] +# usage: build_mmdet_cpp.sh [arch_name] # gen_dir = g2c --output 디렉토리 (예: output/MMDetBackbone) — .cpp/.h/.gguf 있음 -# params.h = mmdet_to_pt.py 가 생성한 .postproc.h (mmdet_params()). 생략하면 -# gen_dir 안의 *.postproc.h 를 찾는다. +# params.h = .postproc.h written by mmdet_to_pt.py next to its --out. When omitted, +# a *.postproc.h inside gen_dir is used. # arch_name= 클래스명(생략 시 gen_dir 의 *.cpp 에서 자동) # env: VISP_BUILD = libvisioncpp 빌드 디렉토리 (기본: /build) # @@ -19,7 +19,7 @@ DETECT="$V/tools/detect" # 공용 head/decode 부품 (head.cpp/ RUN="$V/tools/verify" # E2E 검증 러너 GEN="${1:?usage: build_mmdet_cpp.sh [arch_name]}" GEN="$(cd "$GEN" && pwd)" -# 2번째 인자는 params 헤더거나(.h) 아키텍처 이름이다 — 확장자로 가른다. +# The second argument is either the parameters header (.h) or an architecture name. PARAMS="" ARCH="" case "${2:-}" in @@ -30,7 +30,11 @@ esac if [ -z "$PARAMS" ]; then PARAMS="$(ls "$GEN"/*.postproc.h 2>/dev/null | head -1)" fi -[ -f "$PARAMS" ] || { echo "params 헤더를 못 찾음 (mmdet_to_pt.py 가 만든 .postproc.h): ${PARAMS:-없음}"; exit 1; } +if [ ! -f "$PARAMS" ]; then + echo "no parameters header. mmdet_to_pt.py writes .postproc.h next to its --out." + echo " usage: $(basename "$0") $GEN .postproc.h" + exit 1 +fi if [ -z "$ARCH" ]; then ARCH="$(basename "$(ls "$GEN"/*.cpp | grep -v run_ | head -1)" .cpp)" fi diff --git a/tools/detect/draw.h b/tools/detect/draw.h index 83cdb53..0b5bf46 100644 --- a/tools/detect/draw.h +++ b/tools/detect/draw.h @@ -1,7 +1,7 @@ -// 검출 결과를 이미지 위에 그린다. 러너의 기본 출력이 이미지이기 위한 최소한의 부품. +// Draw detections onto an image -- the minimum needed for the runner's default output. // -// 글자는 그리지 않는다 — 폰트를 들여오면 러너가 무거워지고, 클래스·점수는 러너가 표로 -// 찍는다. 이미지는 "어디"를, 터미널 표는 "무엇"을 담당한다. +// No text is drawn. A font would grow the runner for nothing, because the runner also prints +// the detections as a table: the image carries where, the table carries what. #pragma once #include "visp/image.h" @@ -13,7 +13,7 @@ namespace visp { -// 클래스끼리 구분되게. 범례 없이도 같은 클래스가 같은 색으로 보이면 충분하다. +// Enough separation to tell classes apart. Same class, same colour, no legend needed. inline std::array detection_colour(int label) { static constexpr uint8_t table[][3] = { {230, 60, 60}, {60, 160, 230}, {70, 190, 110}, {240, 160, 40}, {170, 100, 220}, @@ -39,7 +39,7 @@ inline void put_pixel(image_span const& img, int x, int y, std::array colour, int thickness = 2) { int ix1 = int(std::min(x1, x2)), ix2 = int(std::max(x1, x2)); @@ -56,9 +56,8 @@ inline void draw_box(image_span const& img, float x1, float y1, float x2, float } } -// 검출 목록을 그린다. -// 좌표는 검출기가 돈 정사각 입력 기준이므로 원본 크기로 되돌린다(scale_x/scale_y). -// 반환: 실제로 그린 개수. +// Draw a list of detections. Coordinates are in the square input the detector ran on, so +// scale_x / scale_y put them back on the original image. Returns how many were drawn. inline int draw_detections(image_span const& img, std::vector const& dets, float scale_x, float scale_y, float threshold = 0.3f) { int thickness = std::max(2, img.extent[1] / 300); diff --git a/tools/detect/head.h b/tools/detect/head.h index 871c9fe..d1c6832 100755 --- a/tools/detect/head.h +++ b/tools/detect/head.h @@ -25,9 +25,10 @@ struct anchor_head_cfg { bool head_has_norm = false; // 타워에 norm(GN 등) — 이번 PoC(RetinaNet)=false }; -// 러너가 검출기 하나를 돌리는 데 필요한 설정 전부. -// 모델이 정해지면 이 값들은 **상수**다 → mmdet_to_pt.py 가 `.postproc.h` 로 -// `mmdet_params()` 를 생성하고, 러너와 함께 컴파일된다. 런타임에 읽는 파일이 없다. +// Everything the runner needs to run one detector. +// Once an architecture is fixed these are constants, so mmdet_to_pt.py emits them as +// mmdet_params() in .postproc.h and they are compiled into the runner. Nothing is read +// at run time. struct mmdet_cfg { anchor_head_cfg head; det_params det; diff --git a/tools/frontend/mmdet/mmdet_to_pt.py b/tools/frontend/mmdet/mmdet_to_pt.py index d509d44..9e0ad60 100755 --- a/tools/frontend/mmdet/mmdet_to_pt.py +++ b/tools/frontend/mmdet/mmdet_to_pt.py @@ -23,7 +23,7 @@ def _f(v): - """C++ float 리터럴. repr 로 유효숫자를 보존한다 (0.1 이 0.1f 로 반올림되지 않게).""" + """A C++ float literal. repr keeps the significant digits.""" return f"{float(v)!r}f" @@ -32,10 +32,11 @@ def _arr(name, values): def emit_params(cfg, config_name): - """decode/anchor config → `mmdet_params()` 를 담은 C++ 헤더. + """Decoding and anchor configuration as a C++ header holding mmdet_params(). - JSON 사이드카를 대신한다. 값이 실행 파일 안으로 들어가므로 배포물이 하나 줄고, - gguf 와 설정이 서로 다른 export 에서 나와 짝이 어긋나는 사고가 원천적으로 사라진다. + Replaces the JSON sidecar. The values end up inside the executable, which removes a + deployed file and makes it impossible to pair a .gguf with a configuration from a + different export. """ h = cfg.get("head_type", "raw") out = [ @@ -100,8 +101,8 @@ def main(argv=None): n_head = sum(1 for k in m.state_dict() if k.startswith("bbox_head")) print(f" → saved: {a.out} (state_dict {len(m.state_dict())} tensors, head {n_head} 포함)") - # decode/anchor + head-conv config → **C++ 함수**로 생성. - # 모델이 정해지면 이 값들은 상수라 런타임에 읽을 이유가 없다 → 러너와 함께 컴파일한다. + # Decoding, anchor and head-conv configuration, emitted as a C++ function. Once the + # architecture is fixed these are constants, so they are compiled into the runner. header = os.path.splitext(a.out)[0] + ".postproc.h" with open(header, "w") as f: f.write(emit_params(cfg, os.path.basename(a.config))) diff --git a/tools/verify/backbone/run_mmdet.cpp b/tools/verify/backbone/run_mmdet.cpp index 837eeee..d0c3a71 100755 --- a/tools/verify/backbone/run_mmdet.cpp +++ b/tools/verify/backbone/run_mmdet.cpp @@ -3,19 +3,19 @@ // 백본 = g2c 가 생성한 output/.cpp (그대로 컴파일) → _forward // head = tools/detect/head.cpp 부품 (러너와 함께 컴파일, 라이브러리 아님) // decode+NMS = src/visp/postproc.cpp detect_anchor (라이브러리) -// cfg = .postproc.h (mmdet_to_pt.py 가 생성한 mmdet_params(), 함께 컴파일) +// cfg = .postproc.h (mmdet_params() from mmdet_to_pt.py, compiled in) // // 백본을 arch/ 로 복사하거나 cli REG 에 등록하지 않는다 — output/.cpp 를 직접 컴파일해 // libvisioncpp 와 링크(build_mmdet_cpp.sh). run_yolo_cpp 와 동일한 -DARCH 매크로 방식. // // 컴파일: -DARCH=<클래스명> -DVISP_ARCH_HEADER='"/.h"' -// 실행: run_mmdet [size=512] -// 출력 확장자가 .bin 이면 원시 f32(레퍼런스 대조용), 그 외에는 박스를 그린 이미지. +// run: run_mmdet [size=512] +// an output ending in .bin holds raw f32 for comparison; anything else is an image #include VISP_ARCH_HEADER // 백본: _forward / _params / _detect_params #include "head.h" // head 부품: anchor_head_forward (같은 폴더) -#include "draw.h" // 검출 결과를 이미지에 그린다 (같은 폴더) -#include MMDET_PARAMS_HEADER // 생성된 mmdet_params() — 값이 실행 파일 안에 있다 +#include "draw.h" // draws detections onto the image (same folder) +#include MMDET_PARAMS_HEADER // generated mmdet_params(); values live in the binary #include "visp/image.h" // image_load (이미지 입력 pre) #include "visp/ml.h" #include "visp/postproc.h" // detect_anchor, preprocess, det_params, detection @@ -55,8 +55,8 @@ static std::vector to_vec(tensor t) { return d; } -// 입력이 이미지(.jpg/.png…)면 preprocess()(resize+normalize+to_rgb, 생성된 상수)로 텐서 생성. -// .bin 이면 이미 전처리된 CWHN f32 텐서로 간주. → yolo run_yolo_cpp 는 .bin(외부 전처리)만, 여기선 둘 다. +// An image input goes through preprocess() with the generated constants; +// a .bin is taken as an already pre-processed CWHN f32 tensor. static bool has_ext(std::string const& s, const char* e) { size_t n = std::strlen(e); return s.size() >= n && s.compare(s.size() - n, n, e) == 0; @@ -66,7 +66,7 @@ static bool is_image_path(std::string const& s) { return has_ext(s, ".jpg") || has_ext(s, ".jpeg") || has_ext(s, ".png") || has_ext(s, ".bmp"); } -// `source` 가 비어 있지 않게 채워지면 입력이 이미지였다는 뜻 — 결과를 그 위에 그릴 수 있다. +// A non-empty `source` means the input was an image, so the result can be drawn on it. static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c, image_data* source) { std::string s(path); @@ -117,7 +117,7 @@ int main(int argc, char** argv) { tensor bb = FWD(m, input, p); ggml_build_forward_expand(graph, bb); - // 3) 설정 — 컴파일 시점에 박힌 상수. 런타임에 읽는 파일이 없다. + // 3) Configuration -- constants fixed at compile time. No file is read. mmdet_cfg cfg = mmdet_params(); anchor_head_cfg& hc = cfg.head; det_params& dp = cfg.det; @@ -167,7 +167,7 @@ int main(int argc, char** argv) { } std::vector dets = detect_anchor(cls_v, box_v, feat_hw, dp); - // 기본은 이미지 — vision-cli 의 다른 명령들과 같다. 원시 수치는 .bin 으로 요청한다. + // An image by default, as with every other entry point here. Raw numbers on request. std::string out_s(outp); bool want_raw = has_ext(out_s, ".bin"); if (!want_raw && source.extent[0] == 0) { @@ -190,7 +190,7 @@ int main(int argc, char** argv) { if (const char* e = std::getenv("VISP_DRAW_THRESHOLD")) { thr = (float)atof(e); } - // 좌표는 정사각 입력 기준이므로 원본 해상도로 되돌린다. + // Coordinates are in the square input, so scale them back to the original resolution. float sx = float(source.extent[0]) / float(SZ); float sy = float(source.extent[1]) / float(SZ); int drawn = draw_detections(source, dets, sx, sy, thr); @@ -199,8 +199,8 @@ int main(int argc, char** argv) { dets.size(), drawn, thr, outp); } - // 파일은 대조용 raw f32 다. 사람이 결과를 눈으로 확인할 수 있게 상위 몇 개는 표로도 찍는다 - // (개수는 VISP_PRINT_DETS 로 조절, 0 이면 끄기). + // The highest-scoring detections, so a run says something without opening the output. + // VISP_PRINT_DETS sets how many; 0 turns it off. int n_print = 10; if (const char* e = std::getenv("VISP_PRINT_DETS")) { n_print = atoi(e); From 09fc5f290a77653ade72c70f583a4c36e8a7c108 Mon Sep 17 00:00:00 2001 From: eunchae Date: Fri, 7 Aug 2026 11:01:53 +0900 Subject: [PATCH 6/8] docs(mmdet): align the commands in the guide and the tools README Extracting every shell command from both documents and comparing them found one that would fail. The VISP_BUILD example in the guide -- the form to use when the library was configured somewhere other than build/ -- omitted the parameters header, so copying that line hit the same "no parameters header" stop that the plain form had already been fixed for. The README gained the same note in the same shape, since a reader who built elsewhere had nothing to go on there at all. Nine of the ten commands are now identical between the two. The remaining differences are deliberate: the guide shows an argument-form line for run_mmdet that the README does not need, and spells the config path as /path/to/... where the README uses a bare file name. Co-Authored-By: Claude Opus 5 --- docs/mmdet-detectors.md | 3 ++- tools/README.md | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md index c92bae4..d46114f 100644 --- a/docs/mmdet-detectors.md +++ b/docs/mmdet-detectors.md @@ -159,7 +159,8 @@ The library is looked up in `build/`, which is where [Building](../README.md#bui If you configured elsewhere, point `VISP_BUILD` at that directory: ```sh -VISP_BUILD=/path/to/that/directory bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone +VISP_BUILD=/path/to/that/directory \ + bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h ``` The result is `/run_mmdet`. diff --git a/tools/README.md b/tools/README.md index 77354ef..1defa6b 100755 --- a/tools/README.md +++ b/tools/README.md @@ -61,6 +61,13 @@ bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg detected.png 512 ``` +Step 3 looks for the library in `build/`. If you configured elsewhere, name it: + +```sh +VISP_BUILD=/path/to/that/directory \ + bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h +``` + `backbone.postproc.h` holds a generated `mmdet_params()` — anchor scales, head convolution layout, normalisation values. Once an architecture is fixed those are constants, so they are compiled into the runner rather than read at run time, and the deployed set is the executable From 57df89127e0011d79b7ced8eddc6664a8e5984aa Mon Sep 17 00:00:00 2001 From: eunchae Date: Fri, 7 Aug 2026 11:04:00 +0900 Subject: [PATCH 7/8] docs(mmdet): drop inline emphasis, and stand alone Removes bold and italic markers from the prose. Structure -- headings, tables, code, links, lists -- is untouched; only the markers a reader sees as noise when looking at the source are gone. Also removes the link to overview.md, which lives in a different pull request. This one has to make sense on its own. Co-Authored-By: Claude Opus 5 --- docs/mmdet-detectors.md | 27 +++++++++++++-------------- tools/README.md | 6 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md index d46114f..b93efa0 100644 --- a/docs/mmdet-detectors.md +++ b/docs/mmdet-detectors.md @@ -1,13 +1,12 @@ # Object Detection with MMDetection Models This guide describes how to run detectors from -[MMDetection](https://github.com/open-mmlab/mmdetection) with _vision_.cpp. -If you are new to the library, read the [Overview](overview.md) first. +[MMDetection](https://github.com/open-mmlab/mmdetection) with vision.cpp. MMDetection defines hundreds of detectors as compositions of a backbone, a neck and a head. The backbone and neck are plain feed-forward networks and translate directly into a ggml graph. The head does not: it contains data-dependent control flow (NMS, dynamic offsets, variable -proposal counts) that cannot be captured by tracing. _vision_.cpp therefore uses a **hybrid** +proposal counts) that cannot be captured by tracing. vision.cpp therefore uses a hybrid structure — the feature extractor runs as a compiled ggml graph, and the head, decoding and post-processing are hand-written C++ built from library primitives. @@ -42,7 +41,7 @@ objects. ## Prerequisites -- _vision_.cpp built from source — see [Building](../README.md#building). +- vision.cpp built from source — see [Building](../README.md#building). The runners link against `libvisioncpp` plus the ggml libraries. - A Python environment with MMDetection installed, for the export step only. Nothing in the runtime path depends on Python. @@ -91,24 +90,24 @@ The `.pt` file pickles by module name `mmdet_wrap`, so the export directory must ### Step 2 — Compile the backbone -`backbone.pt` is a plain PyTorch module and is compiled to a _vision_.cpp arch module by a +`backbone.pt` is a plain PyTorch module and is compiled to a vision.cpp arch module by a PyTorch-to-ggml model compiler. That compiler is outside the scope of this document; what matters is the interface the generated code must satisfy. -**Compile at the resolution `--size` used in step 1.** Tracing records the operations for one +Compile at the resolution `--size` used in step 1. Tracing records the operations for one input shape, so the graph runs at that shape and no other. A graph built for a different size aborts at run time in `ggml_can_repeat` once a tensor of the wrong extent reaches a residual addition. -**Generated files** — for an architecture named `MMDetBackbone`: +Generated files — for an architecture named `MMDetBackbone`: | File | Contents | | :--- | :--- | | `MMDetBackbone.h` | Declarations (see below). | | `MMDetBackbone.cpp` | Builds the ggml graph for backbone and neck. | -| `MMDetBackbone.gguf` | Weights for **both** the backbone and the head, under their original `state_dict` names. | +| `MMDetBackbone.gguf` | Weights for both the backbone and the head, under their original `state_dict` names. | -**Header contract** — the runner includes the header and reaches the graph through three +Header contract — the runner includes the header and reaches the graph through three macro-expanded names: ```c++ @@ -122,7 +121,7 @@ MMDetBackbone_params MMDetBackbone_detect_params(model_file const& f); } // namespace visp ``` -**Graph contract** +Graph contract - The input tensor is named `x`, type `f32`, shape `{3, size, size, 1}` in ggml `ne` order (CWHN — channels vary fastest). @@ -439,7 +438,7 @@ a mask sub-graph, and host-side mask pasting. run_maskrcnn [size=800] ``` -> **Note** +> Note > ggml's `conv_transpose_2d_p0` does not support batching. `run_maskrcnn` therefore evaluates > the mask sub-graph one RoI at a time. Running it batched leaves only the first RoI correct. @@ -468,7 +467,7 @@ match to drop it. Passing `frame_id == 0` resets the tracker. `mmdet_params()` in `.postproc.h` is generated by the export step and compiled into the runner. Its fields are grouped here by consumer. -**Pre-processing** (`c.img_mean`, `c.img_std`, `c.to_rgb`) — used only when the runner is +Pre-processing (`c.img_mean`, `c.img_std`, `c.to_rgb`) — used only when the runner is given an image rather than a `.bin`. | Field | Description | @@ -476,7 +475,7 @@ given an image rather than a `.bin`. | `img_mean`, `img_std` | Per-channel normalisation, taken from the config's `data_preprocessor`. | | `to_rgb` | Whether to swap channel order before normalising. | -**Head reconstruction** (`c.head`) — maps onto `anchor_head_cfg`. +Head reconstruction (`c.head`) — maps onto `anchor_head_cfg`. | Field | Description | | :--- | :--- | @@ -485,7 +484,7 @@ given an image rather than a `.bin`. | `cls_head`, `reg_head` | Names of the final convolutions. | | `head_has_norm` | Whether the tower contains normalisation layers. | -**Decoding** (`c.det`) — maps onto `det_params`. +Decoding (`c.det`) — maps onto `det_params`. | Field | Description | | :--- | :--- | diff --git a/tools/README.md b/tools/README.md index 1defa6b..ae93fba 100755 --- a/tools/README.md +++ b/tools/README.md @@ -1,6 +1,6 @@ # tools — detection frontends, components and runners -Everything needed to run a detector with _vision_.cpp that does not belong in the library +Everything needed to run a detector with vision.cpp that does not belong in the library itself. [docs/mmdet-detectors.md](../docs/mmdet-detectors.md) explains the pipeline end to end; this file describes how the directory is arranged. @@ -34,9 +34,9 @@ tracing records only the path one input happened to take. Two arrangements keep framework knowledge from spreading: -- **`detect/head.cpp` is not part of `libvisioncpp`.** It is compiled together with the runner, +- `detect/head.cpp` is not part of `libvisioncpp`. It is compiled together with the runner, so detector-specific structure never enters the core library. -- **Decoding lives in the library, not in the frontend.** `detect_anchor`, `roi_align` and +- Decoding lives in the library, not in the frontend. `detect_anchor`, `roi_align` and `rpn_proposals` in `src/visp/postproc.h` take numbers, not configuration objects, which is why they are reusable for detectors that never went through MMDetection. From bb86244c85bca7c0f3589a8591ad741867735a5f Mon Sep 17 00:00:00 2001 From: eunchae Date: Fri, 7 Aug 2026 11:34:59 +0900 Subject: [PATCH 8/8] fix(mmdet): emit infinities as INFINITY, not inff repr(float('inf')) is "inf", so the float-literal helper produced "inff", which C++ does not accept -- the generated header would fail to compile. Configs reach this. FCOS bounds its last regression range with INF (regress_ranges=((-1, 64), ..., (512, INF))), so any head whose parameters include such a range hits it as soon as those values are emitted. Non-finite values now become INFINITY / -INFINITY / NAN, and the generated header includes for them. Co-Authored-By: Claude Opus 5 --- tools/frontend/mmdet/mmdet_to_pt.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/frontend/mmdet/mmdet_to_pt.py b/tools/frontend/mmdet/mmdet_to_pt.py index 9e0ad60..dd8b88c 100755 --- a/tools/frontend/mmdet/mmdet_to_pt.py +++ b/tools/frontend/mmdet/mmdet_to_pt.py @@ -12,6 +12,7 @@ PYTHONPATH=:<이 폴더> g2c --model retinanet_bb.pt --name Retina --output output/retina """ import argparse +import math import os import sys import torch @@ -23,8 +24,17 @@ def _f(v): - """A C++ float literal. repr keeps the significant digits.""" - return f"{float(v)!r}f" + """A C++ float literal. repr keeps the significant digits. + + Infinities reach here from configs that bound a regression range with one -- FCOS writes + regress_ranges=((-1, 64), ..., (512, INF)) -- and `inff` is not something C++ accepts. + """ + v = float(v) + if math.isinf(v): + return "-INFINITY" if v < 0 else "INFINITY" + if math.isnan(v): + return "NAN" + return f"{v!r}f" def _arr(name, values): @@ -44,6 +54,7 @@ def emit_params(cfg, config_name): f"// source: {config_name}\n", f"// head_type: {h}\n", "#pragma once\n\n", + "#include \n", '#include "head.h"\n\n', "namespace visp {\n\n", "inline mmdet_cfg mmdet_params() {\n",