Unit_AI/node_modules/@dqbd/tiktoken/lite/tiktoken.cjs
2024-06-01 16:24:36 -04:00

39 lines
No EOL
1 KiB
JavaScript

const wasm = require("./tiktoken_bg.cjs");
let imports = {};
imports["./tiktoken_bg.js"] = wasm;
const path = require("path");
const fs = require("fs");
const candidates = __dirname
.split(path.sep)
.reduce((memo, _, index, array) => {
const prefix = array.slice(0, index + 1).join(path.sep) + path.sep;
if (!prefix.includes("node_modules" + path.sep)) {
memo.unshift(
path.join(
prefix,
"node_modules",
"@dqbd",
"tiktoken",
"lite",
"./tiktoken_bg.wasm"
)
);
}
return memo;
}, [])
candidates.unshift(path.join(__dirname, "./tiktoken_bg.wasm"));
let bytes = null;
for (const candidate of candidates) {
try {
bytes = fs.readFileSync(candidate);
break;
} catch {}
}
if (bytes == null) throw new Error("Missing tiktoken_bg.wasm");
const wasmModule = new WebAssembly.Module(bytes);
const wasmInstance = new WebAssembly.Instance(wasmModule, imports);
wasm.__wbg_set_wasm(wasmInstance.exports);
exports["Tiktoken"] = wasm["Tiktoken"];