fix: importing/requiring under certain setups was not working

fixes #4

.config/beemo/typescript.ts

@@ -0,0 +1,10 @@
+import { TypeScriptConfig } from '@beemo/driver-typescript'
+
+const config: TypeScriptConfig = {
+  compilerOptions: {
+    allowJs: true,
+  },
+  include: ['src'],
+}
+
+export default config

.gitignore

@@ -50,3 +50,5 @@ node_modules/
 
 .DS_Store
 *.local
+
+tsconfig.json

package.json

@@ -58,6 +58,10 @@
     "./esm/*": {
       "import": "./esm/*.js"
     },
+    "./data/*": {
+      "import": "./data/*",
+      "require": "./data/*"
+    },
     "./package.json": "./package.json"
   },
   "main": "esm/main.js",
@@ -73,7 +77,7 @@
   "scripts": {
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022",
-    "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022",
+    "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
     "build:umd": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=gpt3encoder.js'",
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
@@ -108,7 +112,7 @@
   },
   "devDependencies": {
     "@niieani/scaffold": "^1.6.7",
-    "tsx": "^3.12.6"
+    "tsx": "^3.12.7"
   },
   "packageManager": "[email protected]",
   "publishConfig": {

src/getBpe.ts

@@ -16,8 +16,19 @@ const bpeMerges = lines.slice(1, -1).map((x) =>
 )
 
 const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length))
+const encoder = require('../data/encoder.json')
 
+fs.mkdirSync(path.join(__dirname, 'data'), { recursive: true })
 fs.writeFileSync(
-  path.join(__dirname, '../data/bpeRanks.json'),
-  JSON.stringify(bpeRanks),
+  path.join(__dirname, 'data/bpeRanks.js'),
+  `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(
+    bpeRanks,
+  )}`,
 )
+fs.writeFileSync(
+  path.join(__dirname, 'data/encoder.js'),
+  `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(
+    encoder,
+  )}`,
+)
+

src/encoder.test.ts → src/main.test.ts

@@ -5,7 +5,7 @@ import {
   decodeGenerator,
   encode,
   isWithinTokenLimit,
-} from './encoder'
+} from './main'
 
 describe('basic functionality', () => {
   it('empty string', () => {