let wasm; export function __wbg_set_wasm(val) { wasm = val; } const heap = new Array(128).fill(undefined); heap.push(undefined, null, true, false); function getObject(idx) { return heap[idx]; } let heap_next = heap.length; function dropObject(idx) { if (idx < 132) return; heap[idx] = heap_next; heap_next = idx; } function takeObject(idx) { const ret = getObject(idx); dropObject(idx); return ret; } let WASM_VECTOR_LEN = 0; let cachedUint8Memory0 = null; function getUint8Memory0() { if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) { cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer); } return cachedUint8Memory0; } const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder; let cachedTextEncoder = new lTextEncoder('utf-8'); const encodeString = (typeof cachedTextEncoder.encodeInto === 'function' ? function (arg, view) { return cachedTextEncoder.encodeInto(arg, view); } : function (arg, view) { const buf = cachedTextEncoder.encode(arg); view.set(buf); return { read: arg.length, written: buf.length }; }); function passStringToWasm0(arg, malloc, realloc) { if (realloc === undefined) { const buf = cachedTextEncoder.encode(arg); const ptr = malloc(buf.length); getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf); WASM_VECTOR_LEN = buf.length; return ptr; } let len = arg.length; let ptr = malloc(len); const mem = getUint8Memory0(); let offset = 0; for (; offset < len; offset++) { const code = arg.charCodeAt(offset); if (code > 0x7F) break; mem[ptr + offset] = code; } if (offset !== len) { if (offset !== 0) { arg = arg.slice(offset); } ptr = realloc(ptr, len, len = offset + arg.length * 3); const view = getUint8Memory0().subarray(ptr + offset, ptr + len); const ret = encodeString(arg, view); offset += ret.written; } WASM_VECTOR_LEN = offset; return ptr; } function isLikeNone(x) { return x === undefined || x === null; } let cachedInt32Memory0 = null; function getInt32Memory0() { if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) { cachedInt32Memory0 = new Int32Array(wasm.memory.buffer); } return cachedInt32Memory0; } const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder; let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true }); cachedTextDecoder.decode(); function getStringFromWasm0(ptr, len) { return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)); } function addHeapObject(obj) { if (heap_next === heap.length) heap.push(heap.length + 1); const idx = heap_next; heap_next = heap[idx]; heap[idx] = obj; return idx; } let cachedUint32Memory0 = null; function getUint32Memory0() { if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) { cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer); } return cachedUint32Memory0; } function getArrayU32FromWasm0(ptr, len) { return getUint32Memory0().subarray(ptr / 4, ptr / 4 + len); } function passArray8ToWasm0(arg, malloc) { const ptr = malloc(arg.length * 1); getUint8Memory0().set(arg, ptr / 1); WASM_VECTOR_LEN = arg.length; return ptr; } function passArray32ToWasm0(arg, malloc) { const ptr = malloc(arg.length * 4); getUint32Memory0().set(arg, ptr / 4); WASM_VECTOR_LEN = arg.length; return ptr; } function getArrayU8FromWasm0(ptr, len) { return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len); } function handleError(f, args) { try { return f.apply(this, args); } catch (e) { wasm.__wbindgen_export_3(addHeapObject(e)); } } /** */ export class Tiktoken { static __wrap(ptr) { const obj = Object.create(Tiktoken.prototype); obj.ptr = ptr; return obj; } __destroy_into_raw() { const ptr = this.ptr; this.ptr = 0; return ptr; } free() { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); const ptr = this.__destroy_into_raw(); wasm.__wbg_tiktoken_free(ptr); } /** * @param {string} tiktoken_bfe * @param {any} special_tokens * @param {string} pat_str */ constructor(tiktoken_bfe, special_tokens, pat_str) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); const ptr0 = passStringToWasm0(tiktoken_bfe, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); const len0 = WASM_VECTOR_LEN; const ptr1 = passStringToWasm0(pat_str, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); const len1 = WASM_VECTOR_LEN; const ret = wasm.tiktoken_new(ptr0, len0, addHeapObject(special_tokens), ptr1, len1); return Tiktoken.__wrap(ret); } /** * @returns {string | undefined} */ get name() { try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); wasm.tiktoken_name(retptr, this.ptr); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; let v0; if (r0 !== 0) { v0 = getStringFromWasm0(r0, r1).slice(); wasm.__wbindgen_export_2(r0, r1 * 1); } return v0; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @param {string} text * @param {any} allowed_special * @param {any} disallowed_special * @returns {Uint32Array} */ encode(text, allowed_special, disallowed_special) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); const len0 = WASM_VECTOR_LEN; wasm.tiktoken_encode(retptr, this.ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special)); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; var r2 = getInt32Memory0()[retptr / 4 + 2]; var r3 = getInt32Memory0()[retptr / 4 + 3]; if (r3) { throw takeObject(r2); } var v1 = getArrayU32FromWasm0(r0, r1).slice(); wasm.__wbindgen_export_2(r0, r1 * 4); return v1; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @param {string} text * @returns {Uint32Array} */ encode_ordinary(text) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); const len0 = WASM_VECTOR_LEN; wasm.tiktoken_encode_ordinary(retptr, this.ptr, ptr0, len0); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; var v1 = getArrayU32FromWasm0(r0, r1).slice(); wasm.__wbindgen_export_2(r0, r1 * 4); return v1; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @param {string} text * @param {any} allowed_special * @param {any} disallowed_special * @returns {any} */ encode_with_unstable(text, allowed_special, disallowed_special) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); const len0 = WASM_VECTOR_LEN; wasm.tiktoken_encode_with_unstable(retptr, this.ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special)); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; var r2 = getInt32Memory0()[retptr / 4 + 2]; if (r2) { throw takeObject(r1); } return takeObject(r0); } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @param {Uint8Array} bytes * @returns {number} */ encode_single_token(bytes) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export_0); const len0 = WASM_VECTOR_LEN; const ret = wasm.tiktoken_encode_single_token(this.ptr, ptr0, len0); return ret >>> 0; } /** * @param {Uint32Array} tokens * @returns {Uint8Array} */ decode(tokens) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passArray32ToWasm0(tokens, wasm.__wbindgen_export_0); const len0 = WASM_VECTOR_LEN; wasm.tiktoken_decode(retptr, this.ptr, ptr0, len0); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; var v1 = getArrayU8FromWasm0(r0, r1).slice(); wasm.__wbindgen_export_2(r0, r1 * 1); return v1; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @param {number} token * @returns {Uint8Array} */ decode_single_token_bytes(token) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); wasm.tiktoken_decode_single_token_bytes(retptr, this.ptr, token); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; var v0 = getArrayU8FromWasm0(r0, r1).slice(); wasm.__wbindgen_export_2(r0, r1 * 1); return v0; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } } /** * @returns {any} */ token_byte_values() { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); const ret = wasm.tiktoken_token_byte_values(this.ptr); return takeObject(ret); } } export function __wbindgen_is_undefined(arg0) { const ret = getObject(arg0) === undefined; return ret; }; export function __wbg_stringify_029a979dfb73aa17() { return handleError(function (arg0) { const ret = JSON.stringify(getObject(arg0)); return addHeapObject(ret); }, arguments) }; export function __wbindgen_object_drop_ref(arg0) { takeObject(arg0); }; export function __wbindgen_string_get(arg0, arg1) { if (wasm == null) throw new Error("@dqbd/tiktoken: WASM binary has not been propery initialized."); const obj = getObject(arg1); const ret = typeof(obj) === 'string' ? obj : undefined; var ptr0 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); var len0 = WASM_VECTOR_LEN; getInt32Memory0()[arg0 / 4 + 1] = len0; getInt32Memory0()[arg0 / 4 + 0] = ptr0; }; export function __wbindgen_error_new(arg0, arg1) { const ret = new Error(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }; export function __wbg_parse_3ac95b51fc312db8() { return handleError(function (arg0, arg1) { const ret = JSON.parse(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }, arguments) }; export function __wbindgen_throw(arg0, arg1) { throw new Error(getStringFromWasm0(arg0, arg1)); };