From c1e666ce5b269db22fc4871e72aa002a1a8cff34 Mon Sep 17 00:00:00 2001 From: Yizhi <946185759@qq.com> Date: Thu, 6 Mar 2025 14:45:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=89=E6=96=B9=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=BC=96=E8=AF=91=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- README.md | 1 + thirdpart/cmake-js-util.js | 18 +++ thirdpart/install.js | 249 +++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 thirdpart/cmake-js-util.js create mode 100644 thirdpart/install.js diff --git a/.gitignore b/.gitignore index 14fe965..c689dba 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ /data /dist /testdata -/thirdpart +/thirdpart/**/* /typing /package-lock.json /models diff --git a/README.md b/README.md index e69de29..f92c19a 100644 --- a/README.md +++ b/README.md @@ -0,0 +1 @@ +## AI工具箱 diff --git a/thirdpart/cmake-js-util.js b/thirdpart/cmake-js-util.js new file mode 100644 index 0000000..b8cfb92 --- /dev/null +++ b/thirdpart/cmake-js-util.js @@ -0,0 +1,18 @@ +const os = require("os"); +const path = require("path"); +const { execSync, spawnSync, exec } = require("child_process"); + +const args = process.argv.slice(2); + +const cmakeJS = path.join(__dirname, "../node_modules/.bin/cmake-js") + + +function runCmakeJS(args) { + const child = exec(`${cmakeJS} ${args.join(" ")}`, (err, stdout, stderr) => console.log(stdout)); + child.once("close", code => process.exit(code)); +} + +if (args.includes("--include")) runCmakeJS(["print-cmakejs-include"]); +else if (args.includes("--src")) runCmakeJS(["print-cmakejs-src"]); +else if (args.includes("--lib")) runCmakeJS(["print-cmakejs-lib"]); +else if (args.includes("--napi")) console.log(require("node-addon-api").include.replace(/^"/, "").replace(/"$/, "")); \ No newline at end of file diff --git a/thirdpart/install.js b/thirdpart/install.js new file mode 100644 index 0000000..c4bcc1d --- /dev/null +++ b/thirdpart/install.js @@ -0,0 +1,249 @@ + + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { spawnSync } = require("child_process"); +const args = process.argv.slice(2); +const THIRDPARTY_DIR = __dirname; + +function findArg(argname, isBoolean) { + let prefix = `--${argname}`; + let idx = args.findIndex(arg => arg == prefix); + if (idx >= 0) { + if (isBoolean) return true; + else return args[idx + 1]; + } + idx = args.findIndex(arg => arg.startsWith(prefix)) + if (idx >= 0) return args[idx].substring(prefix.length + 1); + return null; +} + +function assert(cond, message) { + if (!cond) { + console.error(message); + process.exit(1); + } +} + +const buildOptions = { + withMNN: findArg("with-mnn", true) ?? false, + withOpenCV: findArg("with-opencv", true) ?? false, + withONNX: findArg("with-onnx", true) ?? false, + buildType: findArg("build-type", false) ?? "Release", + proxy: findArg("proxy"), + generator: findArg("generator"), +} + +const spawnOption = { + stdio: "inherit", env: { + ...process.env, ...buildOptions.proxy ? { + "http_proxy": buildOptions.proxy, + "https_proxy": buildOptions.proxy, + } : {} + } +}; + +function checkFile(...items) { + return fs.existsSync(path.resolve(...items)); +} + +function downloadSource(name, repo, branch, recursive, checkedFile) { + const sourceRootDir = path.join(THIRDPARTY_DIR, "_source"); + fs.mkdirSync(sourceRootDir, { recursive: true }); + const sourceDir = path.join(sourceRootDir, name); + if (checkFile(sourceDir, checkedFile)) return sourceDir; + + console.log(`${name}源代码不存在,正在下载...`); + assert(spawnSync("git", [ + "clone", + ...recursive ? ["--recursive"] : [], + repo, "-b", branch, + sourceDir + ], { ...spawnOption }).status == 0, `下载${name}源代码失败`); + + return sourceDir; +} + +function cmakeBuildFromSource(name, repo, branch, downloader, buildArgs, cmakeCreator) { + const sourceDir = downloader ? downloader(name, repo, branch) : downloadSource(name, repo, branch, false, "CMakeLists.txt"); + const buildDir = path.join(sourceDir, "build", buildOptions.buildType); + const installDir = path.join(THIRDPARTY_DIR, name, buildOptions.buildType); + const configFile = path.join(installDir, "config.cmake"); + + if (typeof buildArgs == "function") buildArgs = buildArgs(sourceDir, buildDir, installDir); + + //编译源代码 + if (checkFile(configFile)) return + console.log(`${name}目标不存在,正在编译...`); + fs.rmSync(buildDir, { recursive: true, force: true }); + fs.mkdirSync(buildDir, { recursive: true }); + + assert(spawnSync("cmake", [ + "-G", buildOptions.generator ?? "Ninja", sourceDir, + "-DCMAKE_BUILD_TYPE=" + buildOptions.buildType, + "-DCMAKE_INSTALL_PREFIX=" + installDir, + ...(buildArgs ?? []).map(arg => arg.startsWith("-D") ? arg : `-D${arg}`), + ], { ...spawnOption, cwd: buildDir }).status == 0, `配置${name}失败`); + + assert(spawnSync("cmake", [ + "--build", ".", + "--parallel", os.cpus().length.toString(), + "--config", buildOptions.buildType, + "--target", "install", + ], { ...spawnOption, cwd: buildDir }).status == 0, `编译${name}失败`); + + fs.writeFileSync(configFile, cmakeCreator(installDir)); +} + +async function downloadFromURL(name, url, resolver) { + const cacheDir = path.join(THIRDPARTY_DIR, "_cache"); + fs.mkdirSync(cacheDir, { recursive: true }); + const saveName = path.join(cacheDir, path.basename(url)); + const outputDir = path.join(cacheDir, name); + fs.rmSync(outputDir, { force: true, recursive: true }); + + if (!checkFile(saveName)) { + const result = spawnSync("curl", ["-o", saveName + ".cache", "-L", url, "-s", "-w", "%{http_code}"], { ...spawnOption, stdio: "pipe" }); + assert(result.status == 0 && result.stdout.toString() == "200", `下载${name}失败`); + fs.renameSync(saveName + ".cache", saveName); + } + let currentName = saveName; + + if (saveName.endsWith(".bz2")) { + currentName = saveName.substring(0, saveName.length - 4); + const input = fs.createReadStream(saveName); + const out = fs.createWriteStream(currentName); + const bz2 = require("unbzip2-stream")(); + await new Promise((resolve, reject) => { + input.pipe(bz2).pipe(out); + out.on("close", resolve); + input.on("error", reject); + bz2.on("error", reject); + out.on("error", reject); + }); + } + + const format = path.extname(currentName); + assert([".tar", ".gz", ".tgz", ".zip"].includes(format), "仅支持.tar, .gz, .tgz, .zip格式的压缩文件"); + const method = { + ".tar": require("compressing").tar, + ".gz": require("compressing").gzip, + ".tgz": require("compressing").tgz, + ".zip": require("compressing").zip, + }[format]; + await method.uncompress(currentName, outputDir); + + await resolver(outputDir); + + fs.rmSync(outputDir, { recursive: true, force: true }); +} + + +async function main() { + //MNN + if (buildOptions.withMNN) cmakeBuildFromSource("MNN", "https://github.com/alibaba/MNN.git", "3.0.4", null, [ + "-DMNN_BUILD_SHARED_LIBS=OFF", + "-DMNN_AVX512=ON", + "-DMNN_BUILD_TOOLS=ON", + "-DMNN_BUILD_CONVERTER=OFF", + "-DMNN_WIN_RUNTIME_MT=OFF", + ], (root) => [ + `set(MNN_INCLUDE_DIR ${JSON.stringify(path.join(root, "include"))})`, + `set(MNN_LIB_DIR ${JSON.stringify(path.join(root, "lib"))})`, + `set(MNN_LIBS MNN)`, + ].join("\n")); + //OpenCV + if (buildOptions.withOpenCV) cmakeBuildFromSource("OpenCV", "https://github.com/opencv/opencv.git", null, "4.11.0", [ + "-DBUILD_SHARED_LIBS=OFF", + "-DBUILD_opencv_apps=OFF", + "-DBUILD_opencv_js=OFF", + "-DBUILD_opencv_python2=OFF", + "-DBUILD_opencv_python3=OFF", + "-DBUILD_ANDROID_PROJECTS=OFF", + "-DBUILD_ANDROID_EXAMPLES=OFF", + "-DBUILD_TESTS=OFF", + "-DBUILD_FAT_JAVA_LIB=OFF", + "-DBUILD_ANDROID_SERVICE=OFF", + "-DBUILD_JAVA=OFF", + "-DBUILD_PERF_TESTS=OFF" + ], (root) => [ + `set(OpenCV_STATIC ON)`, + os.platform() == "win32" ? + `include(${JSON.stringify(path.join(root, "OpenCVConfig.cmake"))})` : + `include(${JSON.stringify(path.join(root, "lib/cmake/opencv4/OpenCVConfig.cmake"))})`, + `set(OpenCV_INCLUDE_DIR \${OpenCV_INCLUDE_DIRS})`, + os.platform() == "win32" ? + "set(OpenCV_LIB_DIR ${OpenCV_LIB_PATH})" : + `set(OpenCV_LIB_DIR ${JSON.stringify(path.join(root, "lib"))})`, + // `set(OpenCV_LIBS OpenCV_LIBS)`, + ].join("\n")) + //ONNXRuntime + if (buildOptions.withONNX && !checkFile(THIRDPARTY_DIR, "ONNXRuntime/config.cmake")) { + let url = ""; + switch (os.platform()) { + case "win32": { + switch (os.arch()) { + case "x64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x64-static_lib-MinSizeRel-1.20.1.tar.bz2"; + break; + case "arm64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-arm64-static_lib-1.20.1.tar.bz2"; + break; + default: assert(false, `不支持的架构${os.arch()}`); + } + break; + } + case "linux": { + switch (os.arch()) { + case "x64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-x64-static_lib-1.20.1-glibc2_17.zip"; + break; + case "arm64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-static_lib-1.20.1-glibc2_17.zip"; + break; + default: assert(false, `不支持的架构${os.arch()}`); + } + break; + } + case "darwin": { + switch (os.arch()) { + case "x64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-x86_64-static_lib-1.20.1.zip"; + break; + case "arm64": + url = "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-arm64-static_lib-1.20.1.zip"; + break; + default: assert(false, `不支持的架构${os.arch()}`); + } + break; + } + default: assert(false, `不支持的平台${os.platform()}`); + } + await downloadFromURL("ONNXRuntime", url, (savedir) => { + const dirname = fs.readdirSync(savedir)[0]; + fs.cpSync(path.join(savedir, dirname), path.join(THIRDPARTY_DIR, "ONNXRuntime"), { recursive: true }); + }); + fs.writeFileSync(path.join(THIRDPARTY_DIR, "ONNXRuntime/config.cmake"), [ + `set(ONNXRuntime_INCLUDE_DIR ${JSON.stringify(path.join(THIRDPARTY_DIR, "ONNXRuntime/include"))})`, + `set(ONNXRuntime_LIB_DIR ${JSON.stringify(path.join(THIRDPARTY_DIR, "ONNXRuntime/lib"))})`, + `set(ONNXRuntime_LIBS onnxruntime)`, + ].join("\n")); + } + // if (buildOptions.withONNX) cmakeBuildFromSource("ONNXRuntime", "https://github.com/csukuangfj/onnxruntime-build.git", "main", (name, repo, branch) => { + // const sourceDir = downloadSource(name, repo, branch, false, "README.md") + // if (!checkFile(sourceDir, "onnxruntime/README.md")) { + // fs.rmSync(path.join(sourceDir, "onnxruntime"), { recursive: true, force: true }); + // downloadSource("ONNXRuntime/onnxruntime", "https://github.com/microsoft/onnxruntime.git", "v1.20.2", true, "README.md") + // } + // return sourceDir; + // }, (sourceDir, buildDir, installDir) => [ + // `-DONNXRUNTIME_SOURCE_DIR=${path.join(sourceDir, "onnxruntime")}` + // ], (root) => [ + // `set(OnnxRuntime_INCLUDE_DIR ${JSON.stringify(path.join(root, "include"))})`, + // `set(OnnxRuntime_LIB_DIR ${JSON.stringify(path.join(root, "lib"))})`, + // `set(OnnxRuntime_LIBS onnxruntime)`, + // ].join("\n")) +} + +main(); \ No newline at end of file