{"versions":{"0.1.2":{"name":"@agnai/web-tokenizers","version":"0.1.2","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"module","publishConfig":{"access":"public"},"scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"gitHead":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_id":"@agnai/web-tokenizers@0.1.2","_nodeVersion":"16.17.0","_npmVersion":"8.15.0","dist":{"integrity":"sha512-nmn9Oiv1gvUKGi+J53M2Zvm0iVnMrSveemWUD2H9uPY1fqd5K1EmcZPo7aXZ90Rrv7Z36DFyCrfmQF9huB6SkA==","shasum":"0c135aa6e1e586e782c8fed6981faccd6e88c070","tarball":"http://123.232.10.234:8212/nexus/content/groups/npm-public/@agnai/web-tokenizers/-/web-tokenizers-0.1.2.tgz","fileCount":7,"unpackedSize":4158298,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDAF/MQDLTVZgDuMtSGI2Uu0ctEdnPrtyWZmTh7avWQIAiB1KPZl1z2LZDxfRTUBS6Kd9FyZMxO+flMFkdw5wQx5yg=="}],"size":1472517},"_npmUser":{"name":"anonymous","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"anonymous","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/web-tokenizers_0.1.2_1691904583673_0.4845431704275971"},"_hasShrinkwrap":false,"_cnpmcore_publish_time":"2023-08-13T05:29:44.021Z","publish_time":1691904584021,"_source_registry_name":"default","contributors":[]},"0.1.3":{"name":"@agnai/web-tokenizers","version":"0.1.3","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"commonjs","publishConfig":{"access":"public"},"scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"gitHead":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_id":"@agnai/web-tokenizers@0.1.3","_nodeVersion":"16.17.0","_npmVersion":"8.15.0","dist":{"integrity":"sha512-KlmTftToTtmb6aLVdne4NluS+POWputPF5J8v25UN/EQS+K9vahWEIe1NPRSFqBQclObkqHaj7JOnFrmnSm5MA==","shasum":"b1b9b43d53a0bfeeee363d52d309a8e43fccacd6","tarball":"http://123.232.10.234:8212/nexus/content/groups/npm-public/@agnai/web-tokenizers/-/web-tokenizers-0.1.3.tgz","fileCount":7,"unpackedSize":4158300,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIElWx+LM22Yxk9rKA0WdWNI7XFZNuBvsaHv9VCMDJXGGAiEAgQn759lhR9O+ZA4KiI4X4CQQ1wSf1lGnfgfHj9UuUAQ="}],"size":1472518},"_npmUser":{"name":"anonymous","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"anonymous","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/web-tokenizers_0.1.3_1691904868649_0.08884971140313547"},"_hasShrinkwrap":false,"_cnpmcore_publish_time":"2023-08-13T05:34:28.994Z","publish_time":1691904868994,"_source_registry_name":"default","contributors":[]},"0.1.6-pre":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"module","scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"_id":"@agnai/web-tokenizers@0.1.6-pre","gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_nodeVersion":"20.18.1","_npmVersion":"10.8.2","dist":{"integrity":"sha512-Bd3vBRxSp8NHo1PCVqRFdDetBR76tC3yk7nRyNLFtcjeRTMxKSr3Ka0dGJ6HFjNCvFNEZPisdipY5vaDyImtzw==","shasum":"d96eaf8e7634fe51b12125c22c0796eca3634fe3","tarball":"http://123.232.10.234:8212/nexus/content/groups/npm-public/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre.tgz","fileCount":8,"unpackedSize":17041326,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEQCIH7oTkrK8MWCoMYtMLFAXePszqjZUTIIoSE2ljgptAdYAiA6zFHBnHxww+FpvuLj/eoUS85nqfrlZ8DPv4F23vlXXw=="}],"size":2897600},"_npmUser":{"name":"anonymous","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"anonymous","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/web-tokenizers_0.1.6-pre_1765681518545_0.060549449463055405"},"_hasShrinkwrap":false,"_cnpmcore_publish_time":"2025-12-14T03:05:18.837Z","publish_time":1765681518837,"_source_registry_name":"default","contributors":[]},"0.1.6-pre2":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre2","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"module","scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"exports":{".":{"common":"./lib/index.cjs","import":"./lib/index.js"}},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"_id":"@agnai/web-tokenizers@0.1.6-pre2","gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_nodeVersion":"20.18.1","_npmVersion":"10.8.2","dist":{"integrity":"sha512-gRi45yQKkWhZcAgXnrqSWfPnC88OlMy/PTfk217t5K2teNVNjXv1rEsXORgDK4JywrMud8lT3jn6iNgJxrjvBQ==","shasum":"181225293a63bf6275b80c6bb111355bcf47754b","tarball":"http://123.232.10.234:8212/nexus/content/groups/npm-public/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre2.tgz","fileCount":9,"unpackedSize":21280154,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIQC9Zl7FDQ5ygbnFvC9CaJEhGK+oiiNBfhiO4PWPJHuVUgIgOPICCeL7sO5tjkeKC63qi4P2LJDdfMtQZVtII3QOC9k="}],"size":4171275},"_npmUser":{"name":"anonymous","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"anonymous","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/web-tokenizers_0.1.6-pre2_1765715520912_0.7650883845230878"},"_hasShrinkwrap":false,"_cnpmcore_publish_time":"2025-12-14T12:32:01.191Z","publish_time":1765715521191,"_source_registry_name":"default","contributors":[]},"0.1.6-pre3":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre3","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"module","scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"exports":{".":{"require":"./lib/index.cjs","import":"./lib/index.js"}},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"_id":"@agnai/web-tokenizers@0.1.6-pre3","gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_nodeVersion":"20.18.1","_npmVersion":"10.8.2","dist":{"integrity":"sha512-oJ2JqB+YB76ksZxFjtmZyWxrKxuKF50faSFIfrxMppijOH5EZwb7znwg/w4/EDFF0TE3s5jfOSUyqCSqAKQS+Q==","shasum":"f42eb9eaa95efe8ae49b9da39e0e956e3369b484","tarball":"http://123.232.10.234:8212/nexus/content/groups/npm-public/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre3.tgz","fileCount":9,"unpackedSize":21280155,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIQD9UdgIvVR9pfMSUSnIs6n71PZMTjdMlbE5mhI26U5ZegIgCCIxuJrq/MtUOwTANDWiLXsTRSYfxMnU1wdoogbQAmQ="}],"size":4171260},"_npmUser":{"name":"anonymous","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"anonymous","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/web-tokenizers_0.1.6-pre3_1765716255459_0.817846659388404"},"_hasShrinkwrap":false,"_cnpmcore_publish_time":"2025-12-14T12:44:15.743Z","publish_time":1765716255743,"_source_registry_name":"default","contributors":[]}},"dist-tags":{"latest":"0.1.6-pre3"},"name":"@agnai/web-tokenizers","time":{"created":"2023-08-13T05:31:19.973Z","modified":"2025-12-14T12:44:37.552Z","0.1.2":"2023-08-13T05:29:44.021Z","0.1.3":"2023-08-13T05:34:28.994Z","0.1.6-pre":"2025-12-14T03:05:18.837Z","0.1.6-pre2":"2025-12-14T12:32:01.191Z","0.1.6-pre3":"2025-12-14T12:44:15.743Z"},"readme":"# WebTokenizers\n\n| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |\n\nWebTokenizers is a javascript binding library that can be universally deployed.\nIt wraps and binds the [HuggingFace tokenizers library](https://github.com/huggingface/tokenizers)\nand [sentencepiece](https://github.com/google/sentencepiece) and provides a minimum common interface\nin havascript.\n\n## Build from Srouce\n\n```bash\nsource /path/to/emsdk_env.sh\nnpm install\nnpm run build\n```\n\nTo try out the test webpage\n```bash\ncd tests\nnpm start\n```","users":{}}