From 6bf7db1f4ccfbe94c6a3e598341620428cd0b311 Mon Sep 17 00:00:00 2001 From: yizhi <946185759@qq.com> Date: Mon, 17 Mar 2025 12:24:30 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4cv=EF=BC=8C=E8=AF=B7=E4=BD=BF?= =?UTF-8?q?=E7=94=A8@yizhi/cv=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 23 ----- README.md | 3 +- cxx/cv/node.cc | 145 --------------------------- cxx/cv/node.h | 8 -- cxx/node.cc | 6 -- package.json | 8 +- src/config.ts | 1 - src/cv/index.ts | 1 - src/cv/main.ts | 1 - src/cv/mat.ts | 80 --------------- src/deploy/common/model.ts | 6 +- src/deploy/facealign/landmark1000.ts | 15 ++- src/deploy/facealign/pfld.ts | 15 ++- src/deploy/faceattr/gender-age.ts | 6 +- src/deploy/facedet/common.ts | 2 +- src/deploy/facedet/yolov5.ts | 10 +- src/deploy/faceid/adaface.ts | 8 +- src/deploy/faceid/common.ts | 2 +- src/deploy/faceid/insightface.ts | 8 +- src/main.ts | 1 - src/test.ts | 49 +++++---- thirdpart/install.js | 29 ------ 22 files changed, 72 insertions(+), 355 deletions(-) delete mode 100644 cxx/cv/node.cc delete mode 100644 cxx/cv/node.h delete mode 100644 src/cv/index.ts delete mode 100644 src/cv/main.ts delete mode 100644 src/cv/mat.ts diff --git a/CMakeLists.txt b/CMakeLists.txt index a89b960..cc4324b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,24 +85,6 @@ if(EXISTS ${MNN_CMAKE_FILE}) endif() endif() -# OpenCV -set(OpenCV_CMAKE_FILE ${CMAKE_SOURCE_DIR}/thirdpart/OpenCV/${CMAKE_BUILD_TYPE}/config.cmake) -if(EXISTS ${OpenCV_CMAKE_FILE}) - include(${OpenCV_CMAKE_FILE}) - message(STATUS "OpenCV_LIB_DIR: ${OpenCV_LIB_DIR}") - message(STATUS "OpenCV_INCLUDE_DIR: ${OpenCV_INCLUDE_DIR}") - message(STATUS "OpenCV_LIBS: ${OpenCV_LIBS}") - include_directories(${OpenCV_INCLUDE_DIRS}) - link_directories(${OpenCV_LIB_DIR}) - - if(NODE_ADDON_FOUND) - add_node_targert(cv cxx/cv/node.cc) - target_link_libraries(cv ${OpenCV_LIBS}) - target_compile_definitions(cv PUBLIC USE_OPENCV) - list(APPEND NODE_COMMON_SOURCES cxx/cv/node.cc) - endif() -endif() - # OnnxRuntime set(ONNXRuntime_CMAKE_FILE ${CMAKE_SOURCE_DIR}/thirdpart/ONNXRuntime/config.cmake) if(EXISTS ${ONNXRuntime_CMAKE_FILE}) @@ -137,11 +119,6 @@ if(NODE_ADDON_FOUND) target_link_libraries(addon ${ONNXRuntime_LIBS}) target_compile_definitions(addon PUBLIC USE_ONNXRUNTIME) endif() - # OpenCV - if(EXISTS ${OpenCV_CMAKE_FILE}) - target_link_libraries(addon ${OpenCV_LIBS}) - target_compile_definitions(addon PUBLIC USE_OPENCV) - endif() endif() if(MSVC AND NODE_ADDON_FOUND) diff --git a/README.md b/README.md index 10af61f..04dea96 100644 --- a/README.md +++ b/README.md @@ -29,14 +29,13 @@ const outputs = session.run(inputs); ## 插件编译 1. 依赖 - 1. python3 1. cmake 1. ninja 1. c++编译器(gcc,clang,Visual Studio ...) 1. 编译第三方库 ``` -node thirdpart/install.js --with-mnn --with-onnx --with-opencv +node thirdpart/install.js --with-mnn --with-onnx ``` 1. 编译插件 ``` diff --git a/cxx/cv/node.cc b/cxx/cv/node.cc deleted file mode 100644 index b8b744d..0000000 --- a/cxx/cv/node.cc +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include -#include -#include "node.h" - -using namespace Napi; - -#define MAT_INSTANCE_METHOD(method) InstanceMethod<&CVMat::method>(#method, static_cast(napi_writable | napi_configurable)) - -static FunctionReference *constructor = nullptr; - -class CVMat : public ObjectWrap { - public: - static Napi::Object Init(Napi::Env env, Napi::Object exports) - { - Function func = DefineClass(env, "Mat", { - MAT_INSTANCE_METHOD(IsEmpty), - MAT_INSTANCE_METHOD(GetCols), - MAT_INSTANCE_METHOD(GetRows), - MAT_INSTANCE_METHOD(GetChannels), - MAT_INSTANCE_METHOD(Resize), - MAT_INSTANCE_METHOD(Crop), - MAT_INSTANCE_METHOD(Rotate), - MAT_INSTANCE_METHOD(Clone), - - MAT_INSTANCE_METHOD(DrawCircle), - - MAT_INSTANCE_METHOD(Data), - MAT_INSTANCE_METHOD(Encode), - }); - constructor = new FunctionReference(); - *constructor = Napi::Persistent(func); - exports.Set("Mat", func); - env.SetInstanceData(constructor); - return exports; - } - - CVMat(const CallbackInfo &info) - : ObjectWrap(info) - { - int mode = cv::IMREAD_COLOR_BGR; - if (info.Length() > 1 && info[1].IsObject()) { - Object options = info[1].As(); - if (options.Has("mode") && options.Get("mode").IsNumber()) mode = options.Get("mode").As().Int32Value(); - } - - if (info[0].IsString()) im_ = cv::imread(info[0].As().Utf8Value(), mode); - else if (info[0].IsTypedArray()) { - auto buffer = info[0].As().ArrayBuffer(); - uint8_t *bufferPtr = static_cast(buffer.Data()); - std::vector data(bufferPtr, bufferPtr + buffer.ByteLength()); - im_ = cv::imdecode(data, mode); - } - } - - ~CVMat() { im_.release(); } - - Napi::Value IsEmpty(const Napi::CallbackInfo &info) { return Boolean::New(info.Env(), im_.empty()); } - Napi::Value GetCols(const Napi::CallbackInfo &info) { return Number::New(info.Env(), im_.cols); } - Napi::Value GetRows(const Napi::CallbackInfo &info) { return Number::New(info.Env(), im_.rows); } - Napi::Value GetChannels(const Napi::CallbackInfo &info) { return Number::New(info.Env(), im_.channels()); } - Napi::Value Resize(const Napi::CallbackInfo &info) - { - return CreateMat(info.Env(), [this, &info](auto &mat) { cv::resize(im_, mat.im_, cv::Size(info[0].As().Int32Value(), info[1].As().Int32Value())); }); - } - Napi::Value Crop(const Napi::CallbackInfo &info) - { - return CreateMat(info.Env(), [this, &info](auto &mat) { - mat.im_ = im_(cv::Rect( - info[0].As().Int32Value(), info[1].As().Int32Value(), - info[2].As().Int32Value(), info[3].As().Int32Value())); - }); - } - Napi::Value Rotate(const Napi::CallbackInfo &info) - { - return CreateMat(info.Env(), [this, &info](auto &mat) { - auto x = info[0].As().DoubleValue(); - auto y = info[1].As().DoubleValue(); - auto angle = info[2].As().DoubleValue(); - cv::Mat rotation_matix = cv::getRotationMatrix2D(cv::Point2f(x, y), angle, 1.0); - cv::warpAffine(im_, mat.im_, rotation_matix, im_.size()); - }); - } - - Napi::Value Clone(const Napi::CallbackInfo &info) - { - return CreateMat(info.Env(), [this, &info](auto &mat) { mat.im_ = im_.clone(); }); - } - - Napi::Value DrawCircle(const Napi::CallbackInfo &info) - { - int x = info[0].As().Int32Value(); - int y = info[1].As().Int32Value(); - int radius = info[2].As().Int32Value(); - int b = info[3].As().Int32Value(); - int g = info[4].As().Int32Value(); - int r = info[5].As().Int32Value(); - int thickness = info[6].As().Int32Value(); - int lineType = info[7].As().Int32Value(); - int shift = info[8].As().Int32Value(); - - cv::circle(im_, cv::Point(x, y), radius, cv::Scalar(b, g, r), thickness, lineType, shift); - return info.Env().Undefined(); - } - - Napi::Value Data(const Napi::CallbackInfo &info) { return ArrayBuffer::New(info.Env(), im_.ptr(), im_.elemSize() * im_.total()); } - Napi::Value Encode(const Napi::CallbackInfo &info) - { - auto options = info[0].As(); - auto extname = options.Get("extname").As().Utf8Value(); - cv::imencode(extname, im_, encoded_); - return ArrayBuffer::New(info.Env(), encoded_.data(), encoded_.size()); - } - - - private: - inline Napi::Object EmptyMat(Napi::Env env) { return constructor->New({}).As(); } - inline CVMat &GetMat(Napi::Object obj) { return *ObjectWrap::Unwrap(obj); } - inline Napi::Object CreateMat(Napi::Env env, std::function callback) - { - auto obj = EmptyMat(env); - callback(GetMat(obj)); - return obj; - } - - private: - cv::Mat im_; - std::vector encoded_; -}; - - -void InstallOpenCVAPI(Env env, Object exports) -{ - CVMat::Init(env, exports); -} - -#if defined(USE_OPENCV) && !defined(BUILD_MAIN_WORD) -static Object Init(Env env, Object exports) -{ - InstallOpenCVAPI(env, exports); - return exports; -} -NODE_API_MODULE(addon, Init) - -#endif diff --git a/cxx/cv/node.h b/cxx/cv/node.h deleted file mode 100644 index 498a7d2..0000000 --- a/cxx/cv/node.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __CV_NODE_H__ -#define __CV_NODE_H__ - -#include "common/node.h" - -void InstallOpenCVAPI(Napi::Env env, Napi::Object exports); - -#endif diff --git a/cxx/node.cc b/cxx/node.cc index b502d65..31cf39c 100644 --- a/cxx/node.cc +++ b/cxx/node.cc @@ -1,5 +1,4 @@ #include "common/node.h" -#include "cv/node.h" #include "mnn/node.h" #include "ort/node.h" @@ -8,11 +7,6 @@ using namespace Napi; #if defined(BUILD_MAIN_WORD) Object Init(Env env, Object exports) { -// OpenCV -#ifdef USE_OPENCV - printf("use opencv\n"); - InstallOpenCVAPI(env, exports); -#endif // OnnxRuntime #ifdef USE_ONNXRUNTIME InstallOrtAPI(env, exports); diff --git a/package.json b/package.json index f23683e..d46defc 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,11 @@ { "name": "@yizhi/ai", - "version": "1.0.5", + "version": "1.0.6", + "releaseVersion": "1.0.6", "main": "dist/index.js", "types": "typing/index.d.ts", "scripts": { - "build": "tsc", + "build": "rm -rf dist typing && tsc", "watch": "tsc -w --inlineSourceMap" }, "keywords": [], @@ -17,5 +18,8 @@ "compressing": "^1.10.1", "node-addon-api": "^8.3.1", "unbzip2-stream": "^1.4.3" + }, + "dependencies": { + "@yizhi/cv": "^1.0.2" } } \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 9e5655a..f5934cf 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,6 @@ import path from "path"; const defaultAddonDir = path.join(__dirname, "../build") const aiConfig = { - "CV_ADDON_FILE": path.join(defaultAddonDir, "cv.node"), "MNN_ADDON_FILE": path.join(defaultAddonDir, "mnn.node"), "ORT_ADDON_FILE": path.join(defaultAddonDir, "ort.node"), }; diff --git a/src/cv/index.ts b/src/cv/index.ts deleted file mode 100644 index 69b587f..0000000 --- a/src/cv/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * as cv from "./main"; diff --git a/src/cv/main.ts b/src/cv/main.ts deleted file mode 100644 index ca1ede5..0000000 --- a/src/cv/main.ts +++ /dev/null @@ -1 +0,0 @@ -export { Mat, ImreadModes } from "./mat"; diff --git a/src/cv/mat.ts b/src/cv/mat.ts deleted file mode 100644 index c597dd1..0000000 --- a/src/cv/mat.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { getConfig } from "../config"; - -export enum ImreadModes { - IMREAD_UNCHANGED = -1, - IMREAD_GRAYSCALE = 0, - IMREAD_COLOR_BGR = 1, - IMREAD_COLOR = 1, - IMREAD_ANYDEPTH = 2, - IMREAD_ANYCOLOR = 4, - IMREAD_LOAD_GDAL = 8, - IMREAD_REDUCED_GRAYSCALE_2 = 16, - IMREAD_REDUCED_COLOR_2 = 17, - IMREAD_REDUCED_GRAYSCALE_4 = 32, - IMREAD_REDUCED_COLOR_4 = 33, - IMREAD_REDUCED_GRAYSCALE_8 = 64, - IMREAD_REDUCED_COLOR_8 = 65, - IMREAD_IGNORE_ORIENTATION = 128, - IMREAD_COLOR_RGB = 256, -}; - -interface MatConstructorOption { - mode?: ImreadModes; -} - -export class Mat { - #mat: any - - public static async load(image: string, option?: MatConstructorOption) { - let buffer: Uint8Array - if (/^https?:\/\//.test(image)) buffer = await fetch(image).then(res => res.arrayBuffer()).then(res => new Uint8Array(res)); - else buffer = await import("fs").then(fs => fs.promises.readFile(image)); - return new Mat(buffer, option); - } - - public constructor(imageData: Uint8Array, option?: MatConstructorOption) { - const addon = require(getConfig("CV_ADDON_FILE")); - if ((imageData as any) instanceof addon.Mat) this.#mat = imageData; - else this.#mat = new addon.Mat(imageData, option); - } - - public get empty(): boolean { return this.#mat.IsEmpty() } - - public get cols(): number { return this.#mat.GetCols(); } - - public get rows(): number { return this.#mat.GetRows(); } - - public get width() { return this.cols; } - - public get height() { return this.rows; } - - public get channels() { return this.#mat.GetChannels(); } - - public resize(width: number, height: number) { return new Mat(this.#mat.Resize.bind(this.#mat)(width, height)); } - - public crop(sx: number, sy: number, sw: number, sh: number) { return new Mat(this.#mat.Crop(sx, sy, sw, sh)); } - - public rotate(sx: number, sy: number, angleDeg: number) { return new Mat(this.#mat.Rotate(sx, sy, angleDeg)); } - - public get data() { return new Uint8Array(this.#mat.Data()); } - - public encode(extname: string) { return new Uint8Array(this.#mat.Encode({ extname })); } - - public clone() { return new Mat(this.#mat.Clone()); } - - public circle(x: number, y: number, radius: number, options?: { - color?: { r: number, g: number, b: number }, - thickness?: number - lineType?: number - }) { - this.#mat.DrawCircle( - x, y, radius, - options?.color?.b ?? 0, - options?.color?.g ?? 0, - options?.color?.r ?? 0, - options?.thickness ?? 1, - options?.lineType ?? 8, - 0, - ); - } -} diff --git a/src/deploy/common/model.ts b/src/deploy/common/model.ts index cd4c735..fb15e65 100644 --- a/src/deploy/common/model.ts +++ b/src/deploy/common/model.ts @@ -1,5 +1,5 @@ +import { cv } from "@yizhi/cv"; import { backend } from "../../backend"; -import { cv } from "../../cv"; export type ModelConstructor = new (session: backend.CommonSession) => T; @@ -7,7 +7,7 @@ export type ImageSource = cv.Mat | Uint8Array | string; export interface ImageCropOption { /** 图片裁剪区域 */ - crop?: { sx: number, sy: number, sw: number, sh: number } + crop?: { x: number, y: number, width: number, height: number } } export type ModelType = "onnx" | "mnn" @@ -34,7 +34,7 @@ export abstract class Model { if (/^https?:\/\//.test(image)) image = await fetch(image).then(res => res.arrayBuffer()).then(buffer => new Uint8Array(buffer)); else image = await import("fs").then(fs => fs.promises.readFile(image as string)); } - if (image instanceof Uint8Array) image = new cv.Mat(image, { mode: cv.ImreadModes.IMREAD_COLOR_BGR }) + if (image instanceof Uint8Array) image = cv.imdecode(image, cv.IMREAD_COLOR_BGR); if (image instanceof cv.Mat) return await resolver(image); else throw new Error("Invalid image"); } diff --git a/src/deploy/facealign/landmark1000.ts b/src/deploy/facealign/landmark1000.ts index 9226653..5941bca 100644 --- a/src/deploy/facealign/landmark1000.ts +++ b/src/deploy/facealign/landmark1000.ts @@ -1,5 +1,4 @@ -import { writeFileSync } from "fs"; -import { cv } from "../../cv"; +import cv from "@yizhi/cv"; import { ImageCropOption, ImageSource, Model } from "../common/model"; import { convertImage } from "../common/processors"; import { FaceAlignmentResult, FacePoint, indexFromTo } from "./common"; @@ -38,10 +37,10 @@ export class FaceLandmark1000 extends Model { public async doPredict(image: cv.Mat, option?: FaceLandmark1000PredictOption) { const input = this.input; - if (option?.crop) image = image.crop(option.crop.sx, option.crop.sy, option.crop.sw, option.crop.sh); - const ratioWidth = image.width / input.shape[3]; - const ratioHeight = image.height / input.shape[2]; - image = image.resize(input.shape[3], input.shape[2]); + if (option?.crop) image = cv.crop(image, option?.crop); + const ratioWidth = image.cols / input.shape[3]; + const ratioHeight = image.rows / input.shape[2]; + image = cv.resize(image, input.shape[3], input.shape[2]); const nchwImageData = convertImage(image.data, { sourceImageFormat: "bgr", targetColorFormat: "gray", targetShapeFormat: "nchw", targetNormalize: { mean: [0], std: [1] } }); @@ -55,8 +54,8 @@ export class FaceLandmark1000 extends Model { const points: FacePoint[] = []; for (let i = 0; i < res.length; i += 2) { - const x = res[i] * image.width * ratioWidth; - const y = res[i + 1] * image.height * ratioHeight; + const x = res[i] * image.cols * ratioWidth; + const y = res[i + 1] * image.rows * ratioHeight; points.push({ x, y }); } diff --git a/src/deploy/facealign/pfld.ts b/src/deploy/facealign/pfld.ts index 2166087..71354eb 100644 --- a/src/deploy/facealign/pfld.ts +++ b/src/deploy/facealign/pfld.ts @@ -1,5 +1,4 @@ -import { writeFileSync } from "fs"; -import { cv } from "../../cv"; +import cv from "@yizhi/cv"; import { ImageCropOption, ImageSource, Model } from "../common/model"; import { convertImage } from "../common/processors"; import { FaceAlignmentResult, FacePoint } from "./common"; @@ -39,10 +38,10 @@ export class PFLD extends Model { private async doPredict(image: cv.Mat, option?: PFLDPredictOption) { const input = this.input; - if (option?.crop) image = image.crop(option.crop.sx, option.crop.sy, option.crop.sw, option.crop.sh); - const ratioWidth = image.width / input.shape[3]; - const ratioHeight = image.height / input.shape[2]; - image = image.resize(input.shape[3], input.shape[2]); + if (option?.crop) image = cv.crop(image, option.crop); + const ratioWidth = image.cols / input.shape[3]; + const ratioHeight = image.rows / input.shape[2]; + image = cv.resize(image, input.shape[3], input.shape[2]); const nchwImageData = convertImage(image.data, { sourceImageFormat: "bgr", targetColorFormat: "bgr", targetShapeFormat: "nchw", targetNormalize: { mean: [0], std: [255] } }) @@ -59,8 +58,8 @@ export class PFLD extends Model { const points: FacePoint[] = []; for (let i = 0; i < pointsBuffer.length; i += 2) { - const x = pointsBuffer[i] * image.width * ratioWidth; - const y = pointsBuffer[i + 1] * image.height * ratioHeight; + const x = pointsBuffer[i] * image.cols * ratioWidth; + const y = pointsBuffer[i + 1] * image.rows * ratioHeight; points.push({ x, y }); } diff --git a/src/deploy/faceattr/gender-age.ts b/src/deploy/faceattr/gender-age.ts index e5249e9..4c8b311 100644 --- a/src/deploy/faceattr/gender-age.ts +++ b/src/deploy/faceattr/gender-age.ts @@ -1,4 +1,4 @@ -import { cv } from "../../cv"; +import cv from "@yizhi/cv"; import { ImageCropOption, ImageSource, Model } from "../common/model"; import { convertImage } from "../common/processors"; @@ -25,8 +25,8 @@ export class GenderAge extends Model { private async doPredict(image: cv.Mat, option?: GenderAgePredictOption): Promise { const input = this.input; const output = this.output; - if (option?.crop) image = image.crop(option.crop.sx, option.crop.sy, option.crop.sw, option.crop.sh); - image = image.resize(input.shape[3], input.shape[2]); + if (option?.crop) image = cv.crop(image, option.crop); + image = cv.resize(image, input.shape[3], input.shape[2]); const nchwImage = convertImage(image.data, { sourceImageFormat: "bgr", targetColorFormat: "rgb", targetShapeFormat: "nchw", targetNormalize: { mean: [0], std: [1] } }); diff --git a/src/deploy/facedet/common.ts b/src/deploy/facedet/common.ts index 94a1eb2..c88f5a3 100644 --- a/src/deploy/facedet/common.ts +++ b/src/deploy/facedet/common.ts @@ -1,4 +1,4 @@ -import { cv } from "../../cv" +import cv from "@yizhi/cv" import { ImageSource, Model } from "../common/model" interface IFaceBoxConstructorOption { diff --git a/src/deploy/facedet/yolov5.ts b/src/deploy/facedet/yolov5.ts index 0a39327..7264362 100644 --- a/src/deploy/facedet/yolov5.ts +++ b/src/deploy/facedet/yolov5.ts @@ -1,4 +1,4 @@ -import { cv } from "../../cv"; +import cv from "@yizhi/cv"; import { convertImage } from "../common/processors"; import { FaceBox, FaceDetectOption, FaceDetector, nms } from "./common"; @@ -15,9 +15,9 @@ export class Yolov5Face extends FaceDetector { public async doPredict(image: cv.Mat, option?: FaceDetectOption): Promise { const input = this.input; - const resizedImage = image.resize(input.shape[2], input.shape[3]); - const ratioWidth = image.width / resizedImage.width; - const ratioHeight = image.height / resizedImage.height; + const resizedImage = cv.resize(image, input.shape[2], input.shape[3]); + const ratioWidth = image.cols / resizedImage.cols; + const ratioHeight = image.rows / resizedImage.rows; const nchwImageData = convertImage(resizedImage.data, { sourceImageFormat: "bgr", targetColorFormat: "bgr", targetShapeFormat: "nchw", targetNormalize: { mean: [127.5], std: [127.5] } }); const outputData = await this.session.run({ input: nchwImageData }).then(r => r.output); @@ -37,7 +37,7 @@ export class Yolov5Face extends FaceDetector { faces.push(new FaceBox({ x1: x - w / 2, y1: y - h / 2, x2: x + w / 2, y2: y + h / 2, - score, imw: image.width, imh: image.height, + score, imw: image.cols, imh: image.rows, })) } return nms(faces, option?.mnsThreshold ?? 0.3).map(box => box.toInt()); diff --git a/src/deploy/faceid/adaface.ts b/src/deploy/faceid/adaface.ts index 6d771a1..7ebc631 100644 --- a/src/deploy/faceid/adaface.ts +++ b/src/deploy/faceid/adaface.ts @@ -1,4 +1,4 @@ -import { Mat } from "../../cv/mat"; +import cv from "@yizhi/cv"; import { convertImage } from "../common/processors"; import { FaceRecognition, FaceRecognitionPredictOption } from "./common"; @@ -13,12 +13,12 @@ export class AdaFace extends FaceRecognition { return this.cacheModel(MODEL_URL_CONFIG[type ?? "MOBILEFACENET_ADAFACE_ONNX"], { createModel: true }).then(r => r.model); } - public async doPredict(image: Mat, option?: FaceRecognitionPredictOption): Promise { + public async doPredict(image: cv.Mat, option?: FaceRecognitionPredictOption): Promise { const input = this.input; const output = this.output; - if (option?.crop) image = image.crop(option.crop.sx, option.crop.sy, option.crop.sw, option.crop.sh); - image = image.resize(input.shape[3], input.shape[2]); + if (option?.crop) image = cv.crop(image, option.crop); + image = cv.resize(image, input.shape[3], input.shape[2]); const nchwImageData = convertImage(image.data, { sourceImageFormat: "bgr", targetColorFormat: "rgb", targetShapeFormat: "nchw", targetNormalize: { mean: [127.5], std: [127.5] } }); diff --git a/src/deploy/faceid/common.ts b/src/deploy/faceid/common.ts index 0320046..8a1ffa4 100644 --- a/src/deploy/faceid/common.ts +++ b/src/deploy/faceid/common.ts @@ -1,4 +1,4 @@ -import { cv } from "../../cv"; +import cv from "@yizhi/cv"; import { ImageCropOption, ImageSource, Model } from "../common/model"; export interface FaceRecognitionPredictOption extends ImageCropOption { } diff --git a/src/deploy/faceid/insightface.ts b/src/deploy/faceid/insightface.ts index 606ef2b..16438db 100644 --- a/src/deploy/faceid/insightface.ts +++ b/src/deploy/faceid/insightface.ts @@ -1,4 +1,4 @@ -import { Mat } from "../../cv/mat"; +import cv from "@yizhi/cv"; import { convertImage } from "../common/processors"; import { FaceRecognition, FaceRecognitionPredictOption } from "./common"; @@ -29,11 +29,11 @@ const MODEL_URL_CONFIG_PARTIAL_FC = { export class Insightface extends FaceRecognition { - public async doPredict(image: Mat, option?: FaceRecognitionPredictOption): Promise { + public async doPredict(image: cv.Mat, option?: FaceRecognitionPredictOption): Promise { const input = this.input; const output = this.output; - if (option?.crop) image = image.crop(option.crop.sx, option.crop.sy, option.crop.sw, option.crop.sh); - image = image.resize(input.shape[3], input.shape[2]); + if (option?.crop) image = cv.crop(image, option.crop); + image = cv.resize(image, input.shape[3], input.shape[2]); const nchwImageData = convertImage(image.data, { sourceImageFormat: "bgr", targetColorFormat: "bgr", targetShapeFormat: "nchw", targetNormalize: { mean: [127.5], std: [127.5] } }); const embedding = await this.session.run({ diff --git a/src/main.ts b/src/main.ts index b9010e1..b12ad4a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,4 +1,3 @@ export { deploy } from "./deploy"; -export { cv } from "./cv"; export { backend } from "./backend"; export { setConfig as config } from "./config"; diff --git a/src/test.ts b/src/test.ts index 9d3f3e9..06ef8f5 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,10 +1,20 @@ import fs from "fs"; +import cv from "@yizhi/cv"; import { deploy } from "./deploy"; -import { cv } from "./cv"; import { faceidTestData } from "./test_data/faceid"; import path from "path"; import crypto from "crypto"; -import ai from "."; + +cv.config("ADDON_PATH", path.join(__dirname, "../build/cv.node")); + +function loadImage(url: string) { + if (/https?:\/\//.test(url)) return fetch(url).then(res => res.arrayBuffer()).then(data => cv.imdecode(new Uint8Array(data))); + else return import("fs").then(fs => fs.promises.readFile(url)).then(buffer => cv.imdecode(buffer)); +} + +function rotate(im: cv.Mat, x: number, y: number, angle: number) { + return cv.warpAffine(im, cv.getRotationMatrix2D(x, y, angle, 1), im.cols, im.rows); +} async function cacheImage(group: string, url: string) { const _url = new URL(url); @@ -34,11 +44,11 @@ async function testGenderTest() { const facedet = await deploy.facedet.Yolov5Face.load("YOLOV5S_MNN"); const detector = await deploy.faceattr.GenderAgeDetector.load("INSIGHT_GENDER_AGE_MNN"); - const image = await cv.Mat.load("https://b0.bdstatic.com/ugc/iHBWUj0XqytakT1ogBfBJwc7c305331d2cf904b9fb3d8dd3ed84f5.jpg"); + const image = await loadImage("https://b0.bdstatic.com/ugc/iHBWUj0XqytakT1ogBfBJwc7c305331d2cf904b9fb3d8dd3ed84f5.jpg"); const boxes = await facedet.predict(image); if (!boxes.length) return console.error("未检测到人脸"); for (const [idx, box] of boxes.entries()) { - const res = await detector.predict(image, { crop: { sx: box.left, sy: box.top, sw: box.width, sh: box.height } }); + const res = await detector.predict(image, { crop: { x: box.left, y: box.top, width: box.width, height: box.height } }); console.log(`[${idx + 1}]`, res); } @@ -54,10 +64,10 @@ async function testFaceID() { const { basic, tests } = faceidTestData.stars; console.log("正在加载图片资源"); - const basicImage = await cv.Mat.load(await cacheImage("faceid", basic.image)); + const basicImage = await loadImage(await cacheImage("faceid", basic.image)); const testsImages: Record = {}; for (const [name, imgs] of Object.entries(tests)) { - testsImages[name] = await Promise.all(imgs.map(img => cacheImage("faceid", img).then(img => cv.Mat.load(img)))); + testsImages[name] = await Promise.all(imgs.map(img => cacheImage("faceid", img).then(img => loadImage(img)))); } console.log("正在检测基本数据"); @@ -69,9 +79,9 @@ async function testFaceID() { async function getEmbd(image: cv.Mat, box: deploy.facedet.FaceBox) { box = box.toSquare(); - const alignResult = await facealign.predict(image, { crop: { sx: box.left, sy: box.top, sw: box.width, sh: box.height } }); - let faceImage = image.rotate(box.centerX, box.centerY, -alignResult.direction * 180 / Math.PI); - return faceid.predict(faceImage, { crop: { sx: box.left, sy: box.top, sw: box.width, sh: box.height } }); + const alignResult = await facealign.predict(image, { crop: { x: box.left, y: box.top, width: box.width, height: box.height } }); + let faceImage = rotate(image, box.centerX, box.centerY, -alignResult.direction * 180 / Math.PI); + return faceid.predict(faceImage, { crop: { x: box.left, y: box.top, width: box.width, height: box.height } }); } const basicEmbds: number[][] = []; @@ -116,33 +126,34 @@ async function testFaceAlign() { const fd = await deploy.facedet.Yolov5Face.load("YOLOV5S_ONNX"); const fa = await deploy.facealign.PFLD.load("PFLD_106_LITE_MNN"); // const fa = await deploy.facealign.FaceLandmark1000.load("FACELANDMARK1000_ONNX"); - let image = await cv.Mat.load("https://i0.hdslb.com/bfs/archive/64e47ec9fdac9e24bc2b49b5aaad5560da1bfe3e.jpg"); - image = image.rotate(image.width / 2, image.height / 2, 0); + let image = await loadImage("https://i0.hdslb.com/bfs/archive/64e47ec9fdac9e24bc2b49b5aaad5560da1bfe3e.jpg"); + image = rotate(image, image.cols / 2, image.rows / 2, 0); const face = await fd.predict(image).then(res => { console.log(res); return res[0].toSquare() }); - const points = await fa.predict(image, { crop: { sx: face.left, sy: face.top, sw: face.width, sh: face.height } }); + const points = await fa.predict(image, { crop: { x: face.left, y: face.top, width: face.width, height: face.height } }); points.points.forEach((point, idx) => { - image.circle(face.left + point.x, face.top + point.y, 2); + cv.circle(image, face.left + point.x, face.top + point.y, 2, 0, 0, 0); }) // const point = points.getPointsOf("rightEye")[1]; // image.circle(face.left + point.x, face.top + point.y, 2); - fs.writeFileSync("testdata/xx.jpg", image.encode(".jpg")); + // fs.writeFileSync("testdata/xx.jpg", image.encode(".jpg")); + cv.imwrite("testdata/xx.jpg", image); - let faceImage = image.rotate(face.centerX, face.centerY, -points.direction * 180 / Math.PI); - faceImage = faceImage.crop(face.left, face.top, face.width, face.height); - fs.writeFileSync("testdata/face.jpg", faceImage.encode(".jpg")); + let faceImage = rotate(image, face.centerX, face.centerY, -points.direction * 180 / Math.PI); + faceImage = cv.crop(faceImage, { x: face.left, y: face.top, width: face.width, height: face.height }); + fs.writeFileSync("testdata/face.jpg", cv.imencode(".jpg", faceImage)!); console.log(points); console.log(points.direction * 180 / Math.PI); } async function test() { - // await testGenderTest(); - // await testFaceID(); + await testGenderTest(); + await testFaceID(); await testFaceAlign(); } diff --git a/thirdpart/install.js b/thirdpart/install.js index 5262dce..0d9aab5 100644 --- a/thirdpart/install.js +++ b/thirdpart/install.js @@ -28,7 +28,6 @@ function assert(cond, message) { const buildOptions = { withMNN: findArg("with-mnn", true) ?? false, - withOpenCV: findArg("with-opencv", true) ?? false, withONNX: findArg("with-onnx", true) ?? false, buildType: findArg("build-type", false) ?? "Release", proxy: findArg("proxy"), @@ -107,9 +106,6 @@ async function downloadFromURL(name, url, resolver) { if (!checkFile(saveName)) { console.log(`开始下载${name}, 地址:${url}`); - await fetch(url).then(res => { - console.log(res.status) - }) const result = spawnSync("curl", ["-o", saveName + ".cache", "-L", url, "-s", "-w", "%{http_code}"], { ...spawnOption, stdio: "pipe" }); assert(result.status == 0 && result.stdout.toString() == "200", `下载${name}失败`); fs.renameSync(saveName + ".cache", saveName); @@ -163,31 +159,6 @@ async function main() { ` set(MNN_LIBS \${MNN_LIB_DIR}/libMNN.a)`, `endif()`, ].join("\n")); - //OpenCV - if (buildOptions.withOpenCV) cmakeBuildFromSource("OpenCV", "https://github.com/opencv/opencv.git", "4.11.0", null, [ - "-DBUILD_SHARED_LIBS=OFF", - "-DBUILD_opencv_apps=OFF", - "-DBUILD_opencv_js=OFF", - "-DBUILD_opencv_python2=OFF", - "-DBUILD_opencv_python3=OFF", - "-DBUILD_ANDROID_PROJECTS=OFF", - "-DBUILD_ANDROID_EXAMPLES=OFF", - "-DBUILD_TESTS=OFF", - "-DBUILD_FAT_JAVA_LIB=OFF", - "-DBUILD_ANDROID_SERVICE=OFF", - "-DBUILD_JAVA=OFF", - "-DBUILD_PERF_TESTS=OFF" - ], (root) => [ - `set(OpenCV_STATIC ON)`, - os.platform() == "win32" ? - `include(${JSON.stringify(P(path.join(root, "OpenCVConfig.cmake")))})` : - `include(${JSON.stringify(P(path.join(root, "lib/cmake/opencv4/OpenCVConfig.cmake")))})`, - `set(OpenCV_INCLUDE_DIR \${OpenCV_INCLUDE_DIRS})`, - os.platform() == "win32" ? - "set(OpenCV_LIB_DIR ${OpenCV_LIB_PATH})" : - `set(OpenCV_LIB_DIR ${JSON.stringify(P(path.join(root, "lib")))})`, - // `set(OpenCV_LIBS OpenCV_LIBS)`, - ].join("\n")) //ONNXRuntime if (buildOptions.withONNX && !checkFile(THIRDPARTY_DIR, "ONNXRuntime/config.cmake")) { let url = "";