[RNE Rewrite] feat: add instance segmentation pipeline

packages/react-native-executorch/cpp/core/tensor.cpp

@@ -39,8 +39,8 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name)
     if (nameStr == "copyTo") {
         auto self = shared_from_this();
         auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
-            if (count != 1) {
-                throw jsi::JSError(rt, "copyTo: Usage: copyTo(dst)");
+            if (count != 1 && count != 2) {
+                throw jsi::JSError(rt, "copyTo: Usage: copyTo(dst, options?)");
             }
 
             if (!args[0].isObject() || !args[0].asObject(rt).isHostObject<TensorHostObject>(rt)) {
@@ -71,11 +71,34 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name)
                 throw jsi::JSError(rt, "copyTo: dst tensor has been disposed");
             }
 
-            if (self->size_ != dst->size_) {
-                throw jsi::JSError(rt, "copyTo: size mismatch between src and dst tensors");
+            size_t srcOffset = 0;
+
+            if (count == 2 && args[1].isObject()) {
+                auto optsObj = args[1].asObject(rt);
+                if (optsObj.hasProperty(rt, "offset")) {
+                    srcOffset = static_cast<size_t>(optsObj.getProperty(rt, "offset").asNumber());
+                }
+            }
+
+            size_t copyLen = self->numel_ - srcOffset;
+
+            if (count == 2 && args[1].isObject()) {
+                auto optsObj = args[1].asObject(rt);
+                if (optsObj.hasProperty(rt, "length")) {
+                    copyLen = static_cast<size_t>(optsObj.getProperty(rt, "length").asNumber());
+                }
+            }
+
+            if (srcOffset + copyLen > self->numel_) {
+                throw jsi::JSError(rt, "copyTo: out of bounds offset and length for src tensor");
+            }
+
+            const auto elemSize = rnexecutorch::core::types::elementSize(self->dtype_);
+            if (copyLen * elemSize != dst->size_) {
+                throw jsi::JSError(rt, "copyTo: size mismatch between copy byte size and dst tensor size");
             }
 
-            std::memcpy(dst->data_.get(), self->data_.get(), self->size_);
+            std::memcpy(dst->data_.get(), self->data_.get() + (srcOffset * elemSize), copyLen * elemSize);
 
             return jsi::Value(rt, args[0].asObject(rt));
         };

packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp

@@ -9,6 +9,8 @@
 #include <utility>
 #include <vector>
 
+#include <opencv2/core.hpp>
+
 #include "core/dtype.h"
 #include "core/tensor.h"
 
@@ -243,4 +245,136 @@ void install_nms(jsi::Runtime &rt, jsi::Object &module) {
 
     module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody));
 }
+
+namespace {
+int dtypeToCvDepth(rnexecutorch::core::types::DType dtype) {
+    switch (dtype) {
+    case rnexecutorch::core::types::DType::uint8:
+        return CV_8U;
+    case rnexecutorch::core::types::DType::int32:
+        return CV_32S;
+    case rnexecutorch::core::types::DType::float32:
+        return CV_32F;
+    }
+    throw std::invalid_argument("unsupported dtype");
+}
+} // namespace
+
+void install_restrictToBox(jsi::Runtime &rt, jsi::Object &module) {
+    auto name = "restrictToBox";
+    auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
+        if (count != 4) {
+            throw jsi::JSError(rt, "Usage: restrictToBox(src, dst, boxTuple, format)");
+        }
+
+        if (!args[0].isObject() || !args[0].asObject(rt).isHostObject<TensorHostObject>(rt)) {
+            throw jsi::JSError(rt, "restrictToBox: src must be a Tensor");
+        }
+
+        if (!args[1].isObject() || !args[1].asObject(rt).isHostObject<TensorHostObject>(rt)) {
+            throw jsi::JSError(rt, "restrictToBox: dst must be a Tensor");
+        }
+
+        if (!args[2].isObject() || !args[2].asObject(rt).isArray(rt)) {
+            throw jsi::JSError(rt, "restrictToBox: boxTuple must be an Array");
+        }
+
+        if (!args[3].isString()) {
+            throw jsi::JSError(rt, "restrictToBox: format must be a String");
+        }
+
+        auto src = args[0].asObject(rt).getHostObject<TensorHostObject>(rt);
+        auto dst = args[1].asObject(rt).getHostObject<TensorHostObject>(rt);
+        auto boxTuple = args[2].asObject(rt).asArray(rt);
+        std::string boxFormatStr = args[3].asString(rt).utf8(rt);
+
+        if (boxTuple.size(rt) != 4) {
+            throw jsi::JSError(rt, "restrictToBox: boxTuple must contain exactly 4 coordinates");
+        }
+
+        BoxFormat boxFormat;
+        try {
+            boxFormat = parseBoxFormat(boxFormatStr);
+        } catch (const std::invalid_argument &e) {
+            throw jsi::JSError(rt, "restrictToBox: " + std::string(e.what()));
+        }
+
+        float a = static_cast<float>(boxTuple.getValueAtIndex(rt, 0).asNumber());
+        float b = static_cast<float>(boxTuple.getValueAtIndex(rt, 1).asNumber());
+        float c = static_cast<float>(boxTuple.getValueAtIndex(rt, 2).asNumber());
+        float d = static_cast<float>(boxTuple.getValueAtIndex(rt, 3).asNumber());
+
+        auto [xmin, ymin, xmax, ymax] = decodeToXyxy(a, b, c, d, boxFormat);
+
+        if (src.get() == dst.get()) {
+            throw jsi::JSError(rt, "restrictToBox: In-place operations (src == dst) are not supported.");
+        }
+
+        if (src->shape_ != dst->shape_) {
+            throw jsi::JSError(rt, "restrictToBox: src and dst must have the same shape");
+        }
+
+        if (src->shape_.size() < 2) {
+            throw jsi::JSError(rt, "restrictToBox: src must have at least 2 dimensions [H, W, ...]");
+        }
+
+        int32_t H = src->shape_[0];
+        int32_t W = src->shape_[1];
+        int32_t C = 1;
+        for (size_t i = 2; i < src->shape_.size(); ++i) {
+            C *= src->shape_[i];
+        }
+
+        int32_t x1 = static_cast<int32_t>(std::ceil(xmin));
+        int32_t y1 = static_cast<int32_t>(std::ceil(ymin));
+        int32_t x2 = static_cast<int32_t>(std::floor(xmax));
+        int32_t y2 = static_cast<int32_t>(std::floor(ymax));
+
+        x1 = std::max(0, x1);
+        y1 = std::max(0, y1);
+        x2 = std::min(W - 1, x2);
+        y2 = std::min(H - 1, y2);
+
+        bool isEmpty = (x2 < x1) || (y2 < y1);
+
+        if (src->dtype_ != dst->dtype_) {
+            throw jsi::JSError(rt, "restrictToBox: src and dst must have the same dtype");
+        }
+
+        std::shared_lock<std::shared_mutex> srcLock(src->mutex_, std::try_to_lock);
+        std::unique_lock<std::shared_mutex> dstLock(dst->mutex_, std::try_to_lock);
+        if (!srcLock.owns_lock() || !dstLock.owns_lock()) {
+            throw jsi::JSError(rt, "restrictToBox: tensors in use");
+        }
+
+        if (!src->data_ || !dst->data_) {
+            throw jsi::JSError(rt, "restrictToBox: tensors must not be disposed");
+        }
+
+        int32_t cvType;
+        try {
+            cvType = CV_MAKETYPE(dtypeToCvDepth(src->dtype_), C);
+        } catch (const std::invalid_argument &e) {
+            throw jsi::JSError(rt, "restrictToBox: " + std::string(e.what()));
+        }
+
+        ::cv::Mat srcMat(H, W, cvType, src->data_.get());
+        ::cv::Mat dstMat(H, W, cvType, dst->data_.get());
+
+        if (isEmpty) {
+            dstMat.setTo(::cv::Scalar::all(0));
+        } else {
+            dstMat.setTo(::cv::Scalar::all(0));
+            int32_t boxW = x2 - x1 + 1;
+            int32_t boxH = y2 - y1 + 1;
+            ::cv::Rect roi(x1, y1, boxW, boxH);
+            srcMat(roi).copyTo(dstMat(roi));
+        }
+
+        return jsi::Value(rt, args[1]);
+    };
+
+    module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 6, fnBody));
+}
+
 } // namespace rnexecutorch::extensions::cv::box_ops

packages/react-native-executorch/cpp/extensions/cv/box_ops.h

@@ -4,4 +4,5 @@
 
 namespace rnexecutorch::extensions::cv::box_ops {
 void install_nms(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+void install_restrictToBox(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
 } // namespace rnexecutorch::extensions::cv::box_ops

packages/react-native-executorch/cpp/extensions/cv/install.cpp

@@ -16,6 +16,7 @@ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) {
     image_ops::install_applyColormap(rt, cvModule);
 
     box_ops::install_nms(rt, cvModule);
+    box_ops::install_restrictToBox(rt, cvModule);
 
     module.setProperty(rt, "cv", cvModule);
 }

packages/react-native-executorch/cpp/extensions/math/install.cpp

@@ -10,6 +10,7 @@ void install(jsi::Runtime &rt, jsi::Object &module) {
     install_sigmoid(rt, mathModule);
     install_softmax(rt, mathModule);
     install_argmax(rt, mathModule);
+    install_threshold(rt, mathModule);
 
     module.setProperty(rt, "math", mathModule);
 }

packages/react-native-executorch/cpp/extensions/math/operations.cpp

@@ -301,4 +301,71 @@ void install_argmax(jsi::Runtime &rt, jsi::Object &module) {
     };
     module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody));
 }
+
+void install_threshold(jsi::Runtime &rt, jsi::Object &module) {
+    auto name = "threshold";
+    auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
+        if (count != 3) {
+            throw jsi::JSError(rt, "Usage: threshold(src, dst, threshold)");
+        }
+
+        if (!args[0].isObject() || !args[0].asObject(rt).isHostObject<TensorHostObject>(rt)) {
+            throw jsi::JSError(rt, "threshold: src must be a Tensor");
+        }
+
+        if (!args[1].isObject() || !args[1].asObject(rt).isHostObject<TensorHostObject>(rt)) {
+            throw jsi::JSError(rt, "threshold: dst must be a Tensor");
+        }
+
+        if (!args[2].isNumber()) {
+            throw jsi::JSError(rt, "threshold: threshold must be a number");
+        }
+
+        auto src = args[0].asObject(rt).getHostObject<TensorHostObject>(rt);
+        auto dst = args[1].asObject(rt).getHostObject<TensorHostObject>(rt);
+        float thresholdVal = static_cast<float>(args[2].asNumber());
+
+        if (src.get() == dst.get()) {
+            throw jsi::JSError(rt, "threshold: In-place operations (src == dst) are not supported.");
+        }
+
+        if (src->shape_ != dst->shape_) {
+            throw jsi::JSError(rt, "threshold: src and dst must have the same shape");
+        }
+
+        if (src->dtype_ != rnexecutorch::core::types::DType::float32) {
+            throw jsi::JSError(rt, "threshold: src must be a float32 tensor");
+        }
+
+        if (dst->dtype_ != rnexecutorch::core::types::DType::float32) {
+            throw jsi::JSError(rt, "threshold: dst must be a float32 tensor");
+        }
+
+        std::shared_lock<std::shared_mutex> srcLock(src->mutex_, std::try_to_lock);
+        std::unique_lock<std::shared_mutex> dstLock(dst->mutex_, std::try_to_lock);
+        if (!srcLock.owns_lock() || !dstLock.owns_lock()) {
+            throw jsi::JSError(rt, "threshold: tensors in use");
+        }
+
+        if (!src->data_) {
+            throw jsi::JSError(rt, "threshold: src tensor has been disposed");
+        }
+
+        if (!dst->data_) {
+            throw jsi::JSError(rt, "threshold: dst tensor has been disposed");
+        }
+
+        const auto *srcData = reinterpret_cast<const float *>(src->data_.get());
+        auto *dstData = reinterpret_cast<float *>(dst->data_.get());
+
+        for (size_t i = 0; i < src->numel_; ++i) {
+            dstData[i] = (srcData[i] >= thresholdVal) ? 1.0f : 0.0f;
+        }
+
+        return jsi::Value(rt, args[1]);
+    };
+
+    module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody));
+}
+
 } // namespace rnexecutorch::extensions::math

packages/react-native-executorch/cpp/extensions/math/operations.h

@@ -6,4 +6,5 @@ namespace rnexecutorch::extensions::math {
 void install_sigmoid(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
 void install_softmax(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
 void install_argmax(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+void install_threshold(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
 } // namespace rnexecutorch::extensions::math

packages/react-native-executorch/src/core/tensor.ts

@@ -28,11 +28,17 @@ export type Tensor = {
   readonly numel: number;
 
   /**
-   * Copies this tensor's data into another tensor with equal number of bytes.
+   * Copies this tensor's data into another tensor.
    * @param dst The destination tensor to copy data into.
+   * @param options Optional configuration for the copy operation.
+   * @param options.offset The start offset in elements in the source tensor.
+   * Defaults to `0`.
+   * @param options.length The number of elements to copy. Defaults to
+   * `numel - offset`, i.e. copies from `offset` to the end of the source
+   * tensor.
    * @returns The destination tensor `dst`.
    */
-  copyTo(dst: Tensor): Tensor;
+  copyTo(dst: Tensor, options?: { offset?: number; length?: number }): Tensor;
 
   /**
    * Releases the underlying native C++ memory held by this tensor.

packages/react-native-executorch/src/extensions/cv/image.ts

@@ -2,7 +2,7 @@
  * Supported pixel format layouts for image buffers.
  * @category Types
  */
-export type ImageFormat = 'rgb' | 'rgba' | 'bgr' | 'bgra';
+export type ImageFormat = 'rgb' | 'rgba' | 'bgr' | 'bgra' | 'gray';
 
 /**
  * Represents a raw CPU image buffer in HWC (Height, Width, Channel) layout.

packages/react-native-executorch/src/extensions/cv/ops/boxes.ts

@@ -165,3 +165,34 @@ export function nms(boxes: Tensor, scores: Tensor, opts: NmsOptions): number[] |
   'worklet';
   return rnexecutorchJsi.cv.nms(boxes, scores, opts);
 }
+
+/**
+ * Masks the source tensor by keeping only the elements inside the specified
+ * bounding box, writing the result to a pre-allocated destination tensor.
+ *
+ * Note: This operation does not change the tensor dimensions (it does not crop
+ * the shape). Instead, it copies the elements within the box coordinates from
+ * `src` to `dst`, and sets all elements outside the box to `0`.
+ * @category Typescript API
+ * @param src The source tensor of shape [H, W, ...] with at least 2 dimensions.
+ * @param dst The pre-allocated destination tensor of the same shape and data
+ * type as `src`.
+ * @param box The bounding box defining the region of interest to copy.
+ * @returns The destination tensor containing the masked output.
+ */
+export function restrictToBox(src: Tensor, dst: Tensor, box: BoundingBox<BoxFormat>): Tensor {
+  'worklet';
+  let [a, b, c, d] = [0, 0, 0, 0];
+  switch (box.format) {
+    case 'xyxy':
+      [a, b, c, d] = [box.xmin, box.ymin, box.xmax, box.ymax];
+      break;
+    case 'xywh':
+      [a, b, c, d] = [box.xmin, box.ymin, box.w, box.h];
+      break;
+    case 'cxcywh':
+      [a, b, c, d] = [box.cx, box.cy, box.w, box.h];
+      break;
+  }
+  return rnexecutorchJsi.cv.restrictToBox(src, dst, [a, b, c, d], box.format);
+}

packages/react-native-executorch/src/extensions/cv/ops/image.ts

@@ -31,10 +31,11 @@ export const FORMAT_CONVERSION: Record<
   ImageFormat,
   Record<ImageFormat, ColorConversionCode | null>
 > = {
-  rgb: { rgb: null, rgba: 'RGB2RGBA', bgr: 'RGB2BGR', bgra: null },
-  bgr: { rgb: 'BGR2RGB', rgba: 'BGR2RGBA', bgr: null, bgra: null },
-  rgba: { rgb: 'RGBA2RGB', rgba: null, bgr: 'RGBA2BGR', bgra: null },
-  bgra: { rgb: 'BGRA2RGB', rgba: 'BGRA2RGBA', bgr: 'BGRA2BGR', bgra: null },
+  rgb: { rgb: null, rgba: 'RGB2RGBA', bgr: 'RGB2BGR', bgra: null, gray: 'RGB2GRAY' },
+  bgr: { rgb: 'BGR2RGB', rgba: 'BGR2RGBA', bgr: null, bgra: null, gray: 'BGR2GRAY' },
+  rgba: { rgb: 'RGBA2RGB', rgba: null, bgr: 'RGBA2BGR', bgra: null, gray: 'RGBA2GRAY' },
+  bgra: { rgb: 'BGRA2RGB', rgba: 'BGRA2RGBA', bgr: 'BGRA2BGR', bgra: null, gray: 'BGRA2GRAY' },
+  gray: { rgb: 'GRAY2RGBA', rgba: 'GRAY2RGBA', bgr: null, bgra: null, gray: null },
 };
 
 /**
@@ -47,6 +48,7 @@ export const FORMAT_CHANNELS: Record<ImageFormat, number> = {
   bgr: 3,
   rgba: 4,
   bgra: 4,
+  gray: 1,
 };
 
 /**