From 0f3b89c1ea611e8a0fee1b27c88a1da248c89925 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 12 Jul 2024 14:57:16 -0700 Subject: [PATCH 01/22] Import as single file initial --- packages/compiler/src/core/formatter.ts | 21 ++++++ packages/compiler/src/formatter/parser.ts | 2 +- packages/importer/cmd/cli.js | 5 ++ packages/importer/package.json | 56 ++++++++++++++ packages/importer/src/cli.ts | 73 +++++++++++++++++++ packages/importer/src/importer-host.ts | 6 ++ packages/importer/src/index.ts | 0 packages/importer/test/scenarios/simple/a.tsp | 7 ++ packages/importer/test/scenarios/simple/b.tsp | 13 ++++ .../importer/test/scenarios/simple/main.tsp | 5 ++ packages/importer/test/test.test.ts | 8 ++ packages/importer/tsconfig.config.json | 4 + packages/importer/tsconfig.json | 11 +++ packages/importer/vitest.config.ts | 4 + pnpm-lock.yaml | 34 +++++++++ tsconfig.ws.json | 1 + 16 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 packages/importer/cmd/cli.js create mode 100644 packages/importer/package.json create mode 100644 packages/importer/src/cli.ts create mode 100644 packages/importer/src/importer-host.ts create mode 100644 packages/importer/src/index.ts create mode 100644 packages/importer/test/scenarios/simple/a.tsp create mode 100644 packages/importer/test/scenarios/simple/b.tsp create mode 100644 packages/importer/test/scenarios/simple/main.tsp create mode 100644 packages/importer/test/test.test.ts create mode 100644 packages/importer/tsconfig.config.json create mode 100644 packages/importer/tsconfig.json create mode 100644 packages/importer/vitest.config.ts diff --git a/packages/compiler/src/core/formatter.ts b/packages/compiler/src/core/formatter.ts index 1df67125f0..251c73f8a2 100644 --- a/packages/compiler/src/core/formatter.ts +++ b/packages/compiler/src/core/formatter.ts @@ -1,6 +1,27 @@ import type { Options } from "prettier"; import { check, format } from "prettier/standalone"; import * as typespecPrettierPlugin from "../formatter/index.js"; +import { flattenNamespaces } from "../formatter/parser.js"; +import type { Node } from "./types.js"; + +export function printTypeSpecNode(node: Node): Promise { + flattenNamespaces(node); + + return format(".", { + parser: "typespec", + plugins: [ + { + ...typespecPrettierPlugin, + parsers: { + typespec: { + ...typespecPrettierPlugin.parsers.typespec, + parse: () => node, + }, + }, + }, + ], + }); +} export async function formatTypeSpec(code: string, prettierConfig?: Options): Promise { const output = await format(code, { diff --git a/packages/compiler/src/formatter/parser.ts b/packages/compiler/src/formatter/parser.ts index 2e5d45d614..1930c9ae70 100644 --- a/packages/compiler/src/formatter/parser.ts +++ b/packages/compiler/src/formatter/parser.ts @@ -25,7 +25,7 @@ export function parse(text: string, options: ParserOptions): TypeSpecScript * This causes prettier to not know where comments belong. * https://github.com/microsoft/typespec/pull/2061 */ -function flattenNamespaces(base: Node) { +export function flattenNamespaces(base: Node) { visitChildren(base, (node) => { if (node.kind === SyntaxKind.NamespaceStatement) { let current = node; diff --git a/packages/importer/cmd/cli.js b/packages/importer/cmd/cli.js new file mode 100644 index 0000000000..44e278c31b --- /dev/null +++ b/packages/importer/cmd/cli.js @@ -0,0 +1,5 @@ +/** + * File serving as an entrypoint to resolve a local tsp install from a global install. + * DO NOT MOVE or this will create a breaking change for user of global cli. + */ +import "../dist/cli.js"; diff --git a/packages/importer/package.json b/packages/importer/package.json new file mode 100644 index 0000000000..47c9e492fa --- /dev/null +++ b/packages/importer/package.json @@ -0,0 +1,56 @@ +{ + "name": "@typespec/importer", + "private": true, + "version": "0.0.1", + "author": "Microsoft Corporation", + "description": "Package to import TypeSpec files into a single one", + "homepage": "https://typespec.io", + "readme": "https://github.com/microsoft/typespec/blob/main/README.md", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/typespec.git" + }, + "bugs": { + "url": "https://github.com/microsoft/typespec/issues" + }, + "keywords": [ + "typespec" + ], + "type": "module", + "main": "dist/src/index.js", + "bin": "dist/src/cli.js", + "engines": { + "node": ">=18.0.0" + }, + "scripts": { + "bundle": "node ./dist/src/cli.js", + "clean": "rimraf ./dist ./temp", + "build": "tsc -p .", + "watch": "tsc -p . --watch", + "test": "vitest run", + "test:ui": "vitest --ui", + "test:ci": "vitest run --coverage --reporter=junit --reporter=default", + "lint": "eslint . --max-warnings=0", + "lint:fix": "eslint . --fix" + }, + "files": [ + "lib/*.tsp", + "dist/**", + "!dist/test/**" + ], + "dependencies": { + "@typespec/compiler": "workspace:~", + "picocolors": "~1.0.1" + }, + "devDependencies": { + "@types/node": "~18.11.19", + "@vitest/coverage-v8": "^1.6.0", + "@vitest/ui": "^1.6.0", + "c8": "^10.1.2", + "rimraf": "~5.0.7", + "typescript": "~5.5.3", + "vite": "^5.3.2", + "vitest": "^1.6.0" + } +} diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts new file mode 100644 index 0000000000..7f605e0514 --- /dev/null +++ b/packages/importer/src/cli.ts @@ -0,0 +1,73 @@ +import { + compile, + getLocationContext, + normalizePath, + printTypeSpecNode, + type TypeSpecScriptNode, +} from "@typespec/compiler"; +import { resolve } from "path"; +import pc from "picocolors"; +import { parseArgs } from "util"; +import { ImporterHost } from "./importer-host.js"; + +function log(...args: any[]) { + // eslint-disable-next-line no-console + console.log(...args); +} +const result = parseArgs({ + options: {}, + args: process.argv.slice(2), + allowPositionals: true, +}); + +const entrypoint = normalizePath(resolve(result.positionals[0])); + +const program = await compile(ImporterHost, entrypoint); + +const errors = []; +const libraries = new Set(); + +for (const [name, file] of program.jsSourceFiles) { + const locContext = getLocationContext(program, file); + switch (locContext.type) { + case "project": + errors.push(`Importer doesn't support JS files in project: ${name}`); + break; + case "library": + libraries.add(locContext.metadata.name); + break; + case "compiler": + // do nothing + } +} + +const sourceFiles: TypeSpecScriptNode[] = []; +for (const file of program.sourceFiles.values()) { + const locContext = getLocationContext(program, file); + switch (locContext.type) { + case "project": + sourceFiles.push(file); + break; + case "library": + libraries.add(locContext.metadata.name); + break; + case "compiler": + // do nothing + } +} + +// console.log( +// "Source files:", +// sourceFiles.map((x) => x.file.path) + +for (const file of sourceFiles) { + const result = await printTypeSpecNode(file); + console.log("Result:----\n", result); +} + +if (errors.length > 0) { + for (const error of errors) { + log(pc.red(error)); + } + process.exit(1); +} diff --git a/packages/importer/src/importer-host.ts b/packages/importer/src/importer-host.ts new file mode 100644 index 0000000000..cd7a959f98 --- /dev/null +++ b/packages/importer/src/importer-host.ts @@ -0,0 +1,6 @@ +import { NodeHost } from "@typespec/compiler"; + +/** + * Special host that tries to load data from additional locations + */ +export const ImporterHost = NodeHost; diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/importer/test/scenarios/simple/a.tsp b/packages/importer/test/scenarios/simple/a.tsp new file mode 100644 index 0000000000..8869479b0a --- /dev/null +++ b/packages/importer/test/scenarios/simple/a.tsp @@ -0,0 +1,7 @@ +import "@typespec/compiler"; + +namespace MyService; + +model Foo { + a: string; +} diff --git a/packages/importer/test/scenarios/simple/b.tsp b/packages/importer/test/scenarios/simple/b.tsp new file mode 100644 index 0000000000..89e4e67dd2 --- /dev/null +++ b/packages/importer/test/scenarios/simple/b.tsp @@ -0,0 +1,13 @@ +import "@typespec/compiler"; + +namespace MyService.Other; + +model Bar { + b: string; +} + +namespace Clean { + model Baz { + c: string; + } +} diff --git a/packages/importer/test/scenarios/simple/main.tsp b/packages/importer/test/scenarios/simple/main.tsp new file mode 100644 index 0000000000..3f7c13f29d --- /dev/null +++ b/packages/importer/test/scenarios/simple/main.tsp @@ -0,0 +1,5 @@ +import "./a.tsp"; +import "./b.tsp"; + +@service +namespace MyService; diff --git a/packages/importer/test/test.test.ts b/packages/importer/test/test.test.ts new file mode 100644 index 0000000000..6924495d38 --- /dev/null +++ b/packages/importer/test/test.test.ts @@ -0,0 +1,8 @@ +import { ok } from "assert"; +import { describe, it } from "vitest"; + +describe("bundler", () => { + it("works", () => { + ok(true); + }); +}); diff --git a/packages/importer/tsconfig.config.json b/packages/importer/tsconfig.config.json new file mode 100644 index 0000000000..79fb341f39 --- /dev/null +++ b/packages/importer/tsconfig.config.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": {} +} diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.json new file mode 100644 index 0000000000..176cbd4e2f --- /dev/null +++ b/packages/importer/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "references": [{ "path": "../compiler/tsconfig.json" }], + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "verbatimModuleSyntax": true, + "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.ts new file mode 100644 index 0000000000..15eeaceb85 --- /dev/null +++ b/packages/importer/vitest.config.ts @@ -0,0 +1,4 @@ +import { defineConfig, mergeConfig } from "vitest/config"; +import { defaultTypeSpecVitestConfig } from "../../vitest.workspace.js"; + +export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91b2587dd3..ced6a71192 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -512,6 +512,40 @@ importers: specifier: ~5.5.3 version: 5.5.3 + packages/importer: + dependencies: + '@typespec/compiler': + specifier: workspace:~ + version: link:../compiler + picocolors: + specifier: ~1.0.1 + version: 1.0.1 + devDependencies: + '@types/node': + specifier: ~18.11.19 + version: 18.11.19 + '@vitest/coverage-v8': + specifier: ^1.6.0 + version: 1.6.0(vitest@1.6.0(@types/node@18.11.19)(@vitest/ui@1.6.0)(happy-dom@14.12.3)(jsdom@19.0.0)(terser@5.30.0)) + '@vitest/ui': + specifier: ^1.6.0 + version: 1.6.0(vitest@1.6.0) + c8: + specifier: ^10.1.2 + version: 10.1.2 + rimraf: + specifier: ~5.0.7 + version: 5.0.7 + typescript: + specifier: ~5.5.3 + version: 5.5.3 + vite: + specifier: ^5.3.2 + version: 5.3.3(@types/node@18.11.19)(terser@5.30.0) + vitest: + specifier: ^1.6.0 + version: 1.6.0(@types/node@18.11.19)(@vitest/ui@1.6.0(vitest@1.6.0))(happy-dom@14.12.3)(jsdom@19.0.0)(terser@5.30.0) + packages/internal-build-utils: dependencies: '@pnpm/find-workspace-packages': diff --git a/tsconfig.ws.json b/tsconfig.ws.json index f20e06a030..8aa8f8404e 100644 --- a/tsconfig.ws.json +++ b/tsconfig.ws.json @@ -15,6 +15,7 @@ { "path": "packages/openapi3/tsconfig.json" }, { "path": "packages/monarch/tsconfig.json" }, { "path": "packages/bundler/tsconfig.json" }, + { "path": "packages/importer/tsconfig.json" }, { "path": "packages/tspd/tsconfig.json" }, { "path": "packages/samples/tsconfig.json" }, { "path": "packages/json-schema/tsconfig.json" }, From c22b89ded78f75c5ae1480e67e876adeda95c1c4 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 12 Jul 2024 15:18:41 -0700 Subject: [PATCH 02/22] Fix --- packages/importer/src/cli.ts | 63 ++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 7f605e0514..82005f5ead 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -3,6 +3,9 @@ import { getLocationContext, normalizePath, printTypeSpecNode, + SyntaxKind, + type ImportStatementNode, + type Statement, type TypeSpecScriptNode, } from "@typespec/compiler"; import { resolve } from "path"; @@ -14,13 +17,13 @@ function log(...args: any[]) { // eslint-disable-next-line no-console console.log(...args); } -const result = parseArgs({ +const args = parseArgs({ options: {}, args: process.argv.slice(2), allowPositionals: true, }); -const entrypoint = normalizePath(resolve(result.positionals[0])); +const entrypoint = normalizePath(resolve(args.positionals[0])); const program = await compile(ImporterHost, entrypoint); @@ -56,15 +59,61 @@ for (const file of program.sourceFiles.values()) { } } -// console.log( -// "Source files:", -// sourceFiles.map((x) => x.file.path) +const imports: Record = {}; +const statements: Statement[] = []; for (const file of sourceFiles) { - const result = await printTypeSpecNode(file); - console.log("Result:----\n", result); + let currentStatements = statements; + for (const statement of file.statements) { + switch (statement.kind) { + case SyntaxKind.ImportStatement: + if (!statement.path.value.startsWith(".")) { + imports[statement.path.value] = statement; + } + break; + case SyntaxKind.NamespaceStatement: + let current = statement; + const ids = [statement.id]; + while (current.statements && "kind" in current.statements) { + current = current.statements; + ids.push(current.id); + } + if (current.statements === undefined) { + currentStatements = []; + statements.push({ ...current, statements: currentStatements, ...({ ids } as any) }); + } else { + currentStatements.push({ ...current, ...({ ids } as any) }); + } + break; + default: + currentStatements.push(statement); + } + } } +const newSourceFile: TypeSpecScriptNode = { + kind: SyntaxKind.TypeSpecScript, + statements: [...Object.values(imports), ...statements], + comments: [], + file: undefined as any, + pos: 0, + end: 0, + parseOptions: sourceFiles[0].parseOptions, + // Binder items + usings: [], + inScopeNamespaces: [], + namespaces: [], + parseDiagnostics: [], + printable: true, + id: undefined as any, + flags: 0, + symbol: undefined as any, + locals: undefined as any, +}; + +const result = await printTypeSpecNode(newSourceFile); +console.log("Result:----\n", result); + if (errors.length > 0) { for (const error of errors) { log(pc.red(error)); From 9ae1564e8427421852c08ecb65071e3f0736c611 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 12 Jul 2024 15:35:18 -0700 Subject: [PATCH 03/22] Try stuff --- packages/importer/src/cli.ts | 7 +++++++ packages/importer/src/importer-host.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 82005f5ead..c3b1194ba1 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -1,6 +1,8 @@ import { compile, getLocationContext, + logDiagnostics, + NodeHost, normalizePath, printTypeSpecNode, SyntaxKind, @@ -27,6 +29,11 @@ const entrypoint = normalizePath(resolve(args.positionals[0])); const program = await compile(ImporterHost, entrypoint); +if (program.hasError()) { + logDiagnostics(program.diagnostics, NodeHost.logSink); + process.exit(1); +} + const errors = []; const libraries = new Set(); diff --git a/packages/importer/src/importer-host.ts b/packages/importer/src/importer-host.ts index cd7a959f98..b2676cf0ff 100644 --- a/packages/importer/src/importer-host.ts +++ b/packages/importer/src/importer-host.ts @@ -1,6 +1,28 @@ -import { NodeHost } from "@typespec/compiler"; +import { createSourceFile, NodeHost, type CompilerHost } from "@typespec/compiler"; /** * Special host that tries to load data from additional locations */ -export const ImporterHost = NodeHost; +export const ImporterHost: CompilerHost = { + ...NodeHost, + stat: async (pathOrUrl) => { + console.log("State", pathOrUrl); + if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { + const res = await fetch(pathOrUrl); + console.log("Res", res); + + return { + isFile: () => res.status === 200, + isDirectory: () => false, + }; + } + return NodeHost.stat(pathOrUrl); + }, + readFile: async (pathOrUrl) => { + if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { + const res = await fetch(pathOrUrl); + return createSourceFile(await res.text(), pathOrUrl); + } + return NodeHost.readFile(pathOrUrl); + }, +}; From 5c95afef7def5009553ae920f8ac06d7cdfef848 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 12 Jul 2024 15:37:52 -0700 Subject: [PATCH 04/22] tweaks --- packages/importer/src/cli.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index c3b1194ba1..f295a5f8ad 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -25,7 +25,11 @@ const args = parseArgs({ allowPositionals: true, }); -const entrypoint = normalizePath(resolve(args.positionals[0])); +const rawEntrypoint = args.positionals[0]; +const entrypoint = + rawEntrypoint.startsWith("http://") || rawEntrypoint.startsWith("https://") + ? rawEntrypoint + : normalizePath(resolve(rawEntrypoint)); const program = await compile(ImporterHost, entrypoint); From 2b8e2aaea6932b4896d2eaa5fd803dd3dacc71d8 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 9 Sep 2024 14:46:06 -0400 Subject: [PATCH 05/22] Migrate to source loader --- packages/compiler/src/core/index.ts | 1 + packages/compiler/src/index.ts | 1 - packages/importer/src/cli.ts | 18 +++++++++--------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/core/index.ts b/packages/compiler/src/core/index.ts index 1166028cef..d64741e7b6 100644 --- a/packages/compiler/src/core/index.ts +++ b/packages/compiler/src/core/index.ts @@ -53,5 +53,6 @@ export { isProjectedProgram } from "./projected-program.js"; export * from "./scanner.js"; export * from "./semantic-walker.js"; export { createSourceFile, getSourceFileKindFromExt } from "./source-file.js"; +export { createSourceLoader } from "./source-loader.js"; export * from "./type-utils.js"; export * from "./types.js"; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 7d7736d040..ef5fb3167b 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -4,7 +4,6 @@ export * from "./lib/decorators.js"; export * from "./server/index.js"; import * as formatter from "./formatter/index.js"; export const TypeSpecPrettierPlugin = formatter; - // DO NOT ADD ANYMORE EXPORTS HERE, this is for backcompat. Utils should be exported from the utils folder. export { /** @deprecated use import from @typespec/compiler/utils */ diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index f295a5f8ad..a476af86a7 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -1,6 +1,5 @@ import { - compile, - getLocationContext, + createSourceLoader, logDiagnostics, NodeHost, normalizePath, @@ -31,18 +30,19 @@ const entrypoint = ? rawEntrypoint : normalizePath(resolve(rawEntrypoint)); -const program = await compile(ImporterHost, entrypoint); +const loader = await createSourceLoader(ImporterHost); +await loader.importFile(entrypoint); -if (program.hasError()) { - logDiagnostics(program.diagnostics, NodeHost.logSink); +if (loader.resolution.diagnostics.length > 0) { + logDiagnostics(loader.resolution.diagnostics, NodeHost.logSink); process.exit(1); } const errors = []; const libraries = new Set(); -for (const [name, file] of program.jsSourceFiles) { - const locContext = getLocationContext(program, file); +for (const [name, file] of loader.resolution.jsSourceFiles) { + const locContext = loader.resolution.locationContexts.get(file.file)!; switch (locContext.type) { case "project": errors.push(`Importer doesn't support JS files in project: ${name}`); @@ -56,8 +56,8 @@ for (const [name, file] of program.jsSourceFiles) { } const sourceFiles: TypeSpecScriptNode[] = []; -for (const file of program.sourceFiles.values()) { - const locContext = getLocationContext(program, file); +for (const file of loader.resolution.sourceFiles.values()) { + const locContext = loader.resolution.locationContexts.get(file.file)!; switch (locContext.type) { case "project": sourceFiles.push(file); From 1a93dd43f31f4136c0576bbedc1bf78a38558b77 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 9 Sep 2024 15:06:51 -0400 Subject: [PATCH 06/22] Setup debugger --- .vscode/launch.json | 20 ++++++++++++++++++++ packages/compiler/src/core/source-loader.ts | 6 +++++- packages/importer/package.json | 1 + packages/importer/src/cli.ts | 8 ++++++++ pnpm-lock.yaml | 8 ++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 802349582d..4517f62001 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -93,6 +93,26 @@ "order": 2 } }, + { + "type": "node", + "request": "launch", + "name": "Debug importer", + "program": "${workspaceFolder}/packages/importer/dist/cli.js", + "args": [ + "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/batch/Azure.Batch/main.tsp" + ], + "smartStep": true, + "sourceMaps": true, + "skipFiles": ["/**/*.js"], + "outFiles": [ + "${workspaceFolder}/packages/*/dist/**/*.js", + "${workspaceFolder}/packages/*/dist-dev/**/*.js" + ], + "cwd": "${workspaceFolder}/packages/importer", + "presentation": { + "order": 2 + } + }, { "name": "Regenerate .tmlanguage", "type": "node", diff --git a/packages/compiler/src/core/source-loader.ts b/packages/compiler/src/core/source-loader.ts index 0438c68b23..94db8b2fb6 100644 --- a/packages/compiler/src/core/source-loader.ts +++ b/packages/compiler/src/core/source-loader.ts @@ -181,7 +181,11 @@ export async function createSourceLoader( function getSourceFileLocationContext(sourcefile: SourceFile): LocationContext { const locationContext = sourceFileLocationContexts.get(sourcefile); - compilerAssert(locationContext, "SourceFile should have a declaration locationContext."); + compilerAssert( + locationContext, + `SourceFile ${sourcefile.path} should have a declaration locationContext.`, + { file: sourcefile, pos: 0, end: 0 } + ); return locationContext; } diff --git a/packages/importer/package.json b/packages/importer/package.json index 47c9e492fa..340b75e051 100644 --- a/packages/importer/package.json +++ b/packages/importer/package.json @@ -49,6 +49,7 @@ "@vitest/ui": "^1.6.0", "c8": "^10.1.2", "rimraf": "~5.0.7", + "source-map-support": "~0.5.21", "typescript": "~5.5.3", "vite": "^5.3.2", "vitest": "^1.6.0" diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index a476af86a7..591bac96b6 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -1,3 +1,11 @@ +try { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + await import("source-map-support/register.js"); +} catch { + // package only present in dev. +} + import { createSourceLoader, logDiagnostics, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f0191ab10..f8ce2d442e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -597,6 +597,9 @@ importers: rimraf: specifier: ~5.0.7 version: 5.0.7 + source-map-support: + specifier: ~0.5.21 + version: 0.5.21 typescript: specifier: ~5.5.3 version: 5.5.3 @@ -3758,6 +3761,7 @@ packages: '@humanwhocodes/config-array@0.11.14': resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} @@ -3769,6 +3773,7 @@ packages: '@humanwhocodes/object-schema@2.0.3': resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} + deprecated: Use @eslint/object-schema instead '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} @@ -7571,6 +7576,7 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -7922,6 +7928,7 @@ packages: inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. inherits@2.0.3: resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} @@ -10355,6 +10362,7 @@ packages: right-pad@1.0.1: resolution: {integrity: sha512-bYBjgxmkvTAfgIYy328fmkwhp39v8lwVgWhhrzxPV3yHtcSqyYKe9/XOhvW48UFjATg3VuJbpsp5822ACNvkmw==} engines: {node: '>= 0.10'} + deprecated: Please use String.prototype.padEnd() over this package. rimraf@2.6.3: resolution: {integrity: sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==} From 3044f7d48224d4b04e8cd30b76dce0bc23e73652 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 9 Sep 2024 15:08:53 -0400 Subject: [PATCH 07/22] fix source loader --- packages/compiler/src/core/source-loader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compiler/src/core/source-loader.ts b/packages/compiler/src/core/source-loader.ts index 94db8b2fb6..48894fd79c 100644 --- a/packages/compiler/src/core/source-loader.ts +++ b/packages/compiler/src/core/source-loader.ts @@ -86,7 +86,7 @@ export async function createSourceLoader( async function importFile( path: string, - locationContext: LocationContext, + locationContext: LocationContext = { type: "project" }, kind: "import" | "entrypoint" = "import" ) { const sourceFileKind = host.getSourceFileKind(path); From 2a6af84425fc369244124d3b1fb3c2cbcae7ddca Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 9 Sep 2024 15:32:43 -0400 Subject: [PATCH 08/22] Progress --- packages/compiler/src/core/module-resolver.ts | 6 +++++ packages/compiler/src/core/source-loader.ts | 23 ++++++++++++++++++- packages/importer/src/cli.ts | 6 ++++- packages/importer/src/importer-host.ts | 16 ++++++++++--- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/core/module-resolver.ts b/packages/compiler/src/core/module-resolver.ts index f1673003fd..d737225310 100644 --- a/packages/compiler/src/core/module-resolver.ts +++ b/packages/compiler/src/core/module-resolver.ts @@ -108,10 +108,12 @@ export async function resolveModule( const { baseDir } = options; const absoluteStart = baseDir === "" ? "." : await realpath(resolvePath(baseDir)); + console.log("Resolve3", name); if (!(await isDirectory(host, absoluteStart))) { throw new TypeError(`Provided basedir '${baseDir}'is not a directory.`); } + console.log("Resolve4", name); // Check if the module name is referencing a path(./foo, /foo, file:/foo) if (/^(?:\.\.?(?:\/|$)|\/|([A-Za-z]:)?[/\\])/.test(name)) { @@ -122,7 +124,11 @@ export async function resolveModule( } } + console.log("Resolve5", name); + const module = await findAsNodeModule(name, absoluteStart); + console.log("Resolve6", name); + if (module) return module; throw new ResolveModuleError( diff --git a/packages/compiler/src/core/source-loader.ts b/packages/compiler/src/core/source-loader.ts index 48894fd79c..1ce6d2b492 100644 --- a/packages/compiler/src/core/source-loader.ts +++ b/packages/compiler/src/core/source-loader.ts @@ -38,6 +38,9 @@ export interface SourceResolution { readonly locationContexts: WeakMap; readonly loadedLibraries: Map; + /** List of imports that were marked as external and not loaded. */ + readonly externals: string[]; + readonly diagnostics: readonly Diagnostic[]; } @@ -50,6 +53,10 @@ export interface LoadSourceOptions { readonly parseOptions?: ParseOptions; readonly tracer?: Tracer; getCachedScript?: (file: SourceFile) => TypeSpecScriptNode | undefined; + /** + * List or callback to determine if a module is external and should not be loaded. + */ + externals?: string[] | ((path: string) => boolean); } export interface SourceLoader { @@ -83,6 +90,14 @@ export async function createSourceLoader( const sourceFiles = new Map(); const jsSourceFiles = new Map(); const loadedLibraries = new Map(); + const externals: string[] = []; + + const externalsOpts = options?.externals; + const isExternal = externalsOpts + ? typeof externalsOpts === "function" + ? externalsOpts + : (x: string) => externalsOpts.includes(x) + : () => false; async function importFile( path: string, @@ -117,6 +132,7 @@ export async function createSourceLoader( locationContexts: sourceFileLocationContexts, loadedLibraries: loadedLibraries, diagnostics: diagnostics.diagnostics, + externals, }, }; @@ -206,10 +222,15 @@ export async function createSourceLoader( relativeTo: string, locationContext: LocationContext = { type: "project" } ) { + if (isExternal(path)) { + externals.push(path); + return; + } const library = await resolveTypeSpecLibrary(path, relativeTo, target); if (library === undefined) { return; } + if (library.type === "module") { loadedLibraries.set(library.manifest.name, { path: library.path, @@ -226,8 +247,8 @@ export async function createSourceLoader( metadata, }; } - const importFilePath = library.type === "module" ? library.mainFile : library.path; + const importFilePath = library.type === "module" ? library.mainFile : library.path; const isDirectory = (await host.stat(importFilePath)).isDirectory(); if (isDirectory) { await loadDirectory(importFilePath, locationContext, target); diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 591bac96b6..345402c3c8 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -38,7 +38,11 @@ const entrypoint = ? rawEntrypoint : normalizePath(resolve(rawEntrypoint)); -const loader = await createSourceLoader(ImporterHost); +const loader = await createSourceLoader(ImporterHost, { + externals: (path: string) => { + return !(path === entrypoint || path.startsWith(".")); + }, +}); await loader.importFile(entrypoint); if (loader.resolution.diagnostics.length > 0) { diff --git a/packages/importer/src/importer-host.ts b/packages/importer/src/importer-host.ts index b2676cf0ff..deb34cde47 100644 --- a/packages/importer/src/importer-host.ts +++ b/packages/importer/src/importer-host.ts @@ -1,4 +1,9 @@ -import { createSourceFile, NodeHost, type CompilerHost } from "@typespec/compiler"; +import { + createSourceFile, + getAnyExtensionFromPath, + NodeHost, + type CompilerHost, +} from "@typespec/compiler"; /** * Special host that tries to load data from additional locations @@ -9,11 +14,10 @@ export const ImporterHost: CompilerHost = { console.log("State", pathOrUrl); if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { const res = await fetch(pathOrUrl); - console.log("Res", res); return { isFile: () => res.status === 200, - isDirectory: () => false, + isDirectory: () => getAnyExtensionFromPath(pathOrUrl) === "", }; } return NodeHost.stat(pathOrUrl); @@ -25,4 +29,10 @@ export const ImporterHost: CompilerHost = { } return NodeHost.readFile(pathOrUrl); }, + realpath: async (path) => { + if (path.startsWith("http://") || path.startsWith("https://")) { + return path; + } + return NodeHost.realpath(path); + }, }; From cae4de4dc6fb47a660f4d0fd788ccbdedcb5b369 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 07:37:22 -0400 Subject: [PATCH 09/22] Generate --- packages/compiler/src/core/module-resolver.ts | 12 ++++-------- packages/importer/src/cli.ts | 1 + packages/importer/src/importer-host.ts | 1 - 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/core/module-resolver.ts b/packages/compiler/src/core/module-resolver.ts index d737225310..fdbea28b04 100644 --- a/packages/compiler/src/core/module-resolver.ts +++ b/packages/compiler/src/core/module-resolver.ts @@ -108,12 +108,11 @@ export async function resolveModule( const { baseDir } = options; const absoluteStart = baseDir === "" ? "." : await realpath(resolvePath(baseDir)); - console.log("Resolve3", name); - if (!(await isDirectory(host, absoluteStart))) { - throw new TypeError(`Provided basedir '${baseDir}'is not a directory.`); - } - console.log("Resolve4", name); + // TODO: can this check be disabled? + // if (!(await isDirectory(host, absoluteStart))) { + // throw new TypeError(`Provided basedir '${baseDir}'is not a directory.`); + // } // Check if the module name is referencing a path(./foo, /foo, file:/foo) if (/^(?:\.\.?(?:\/|$)|\/|([A-Za-z]:)?[/\\])/.test(name)) { @@ -124,10 +123,7 @@ export async function resolveModule( } } - console.log("Resolve5", name); - const module = await findAsNodeModule(name, absoluteStart); - console.log("Resolve6", name); if (module) return module; diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 345402c3c8..27dc209168 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -143,3 +143,4 @@ if (errors.length > 0) { } process.exit(1); } +process.exit(0); diff --git a/packages/importer/src/importer-host.ts b/packages/importer/src/importer-host.ts index deb34cde47..9da83e574e 100644 --- a/packages/importer/src/importer-host.ts +++ b/packages/importer/src/importer-host.ts @@ -11,7 +11,6 @@ import { export const ImporterHost: CompilerHost = { ...NodeHost, stat: async (pathOrUrl) => { - console.log("State", pathOrUrl); if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { const res = await fetch(pathOrUrl); From f38faa8a58aaae5e49ee64402a559b0641db121d Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 07:49:38 -0400 Subject: [PATCH 10/22] Tweaks --- packages/importer/src/cli.ts | 132 +++------------------------- packages/importer/src/importer.ts | 139 ++++++++++++++++++++++++++++++ packages/importer/src/index.ts | 1 + 3 files changed, 153 insertions(+), 119 deletions(-) create mode 100644 packages/importer/src/importer.ts diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 27dc209168..013743b4b7 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -6,21 +6,10 @@ try { // package only present in dev. } -import { - createSourceLoader, - logDiagnostics, - NodeHost, - normalizePath, - printTypeSpecNode, - SyntaxKind, - type ImportStatementNode, - type Statement, - type TypeSpecScriptNode, -} from "@typespec/compiler"; -import { resolve } from "path"; -import pc from "picocolors"; +import { getDirectoryPath, logDiagnostics, NodeHost } from "@typespec/compiler"; +import { mkdir, writeFile } from "fs/promises"; import { parseArgs } from "util"; -import { ImporterHost } from "./importer-host.js"; +import { combineProjectIntoFile } from "./importer.js"; function log(...args: any[]) { // eslint-disable-next-line no-console @@ -33,114 +22,19 @@ const args = parseArgs({ }); const rawEntrypoint = args.positionals[0]; -const entrypoint = - rawEntrypoint.startsWith("http://") || rawEntrypoint.startsWith("https://") - ? rawEntrypoint - : normalizePath(resolve(rawEntrypoint)); -const loader = await createSourceLoader(ImporterHost, { - externals: (path: string) => { - return !(path === entrypoint || path.startsWith(".")); - }, -}); -await loader.importFile(entrypoint); +const { content, diagnostics } = await combineProjectIntoFile(rawEntrypoint); -if (loader.resolution.diagnostics.length > 0) { - logDiagnostics(loader.resolution.diagnostics, NodeHost.logSink); +if (diagnostics.length > 0) { + logDiagnostics(diagnostics, NodeHost.logSink); process.exit(1); } - -const errors = []; -const libraries = new Set(); - -for (const [name, file] of loader.resolution.jsSourceFiles) { - const locContext = loader.resolution.locationContexts.get(file.file)!; - switch (locContext.type) { - case "project": - errors.push(`Importer doesn't support JS files in project: ${name}`); - break; - case "library": - libraries.add(locContext.metadata.name); - break; - case "compiler": - // do nothing - } -} - -const sourceFiles: TypeSpecScriptNode[] = []; -for (const file of loader.resolution.sourceFiles.values()) { - const locContext = loader.resolution.locationContexts.get(file.file)!; - switch (locContext.type) { - case "project": - sourceFiles.push(file); - break; - case "library": - libraries.add(locContext.metadata.name); - break; - case "compiler": - // do nothing - } -} - -const imports: Record = {}; -const statements: Statement[] = []; - -for (const file of sourceFiles) { - let currentStatements = statements; - for (const statement of file.statements) { - switch (statement.kind) { - case SyntaxKind.ImportStatement: - if (!statement.path.value.startsWith(".")) { - imports[statement.path.value] = statement; - } - break; - case SyntaxKind.NamespaceStatement: - let current = statement; - const ids = [statement.id]; - while (current.statements && "kind" in current.statements) { - current = current.statements; - ids.push(current.id); - } - if (current.statements === undefined) { - currentStatements = []; - statements.push({ ...current, statements: currentStatements, ...({ ids } as any) }); - } else { - currentStatements.push({ ...current, ...({ ids } as any) }); - } - break; - default: - currentStatements.push(statement); - } - } -} - -const newSourceFile: TypeSpecScriptNode = { - kind: SyntaxKind.TypeSpecScript, - statements: [...Object.values(imports), ...statements], - comments: [], - file: undefined as any, - pos: 0, - end: 0, - parseOptions: sourceFiles[0].parseOptions, - // Binder items - usings: [], - inScopeNamespaces: [], - namespaces: [], - parseDiagnostics: [], - printable: true, - id: undefined as any, - flags: 0, - symbol: undefined as any, - locals: undefined as any, -}; - -const result = await printTypeSpecNode(newSourceFile); -console.log("Result:----\n", result); - -if (errors.length > 0) { - for (const error of errors) { - log(pc.red(error)); - } +if (content) { + const outputFile = "tsp-output/main.tsp"; + await mkdir(getDirectoryPath(outputFile), { recursive: true }); + log(`Writing output to ${outputFile}`); + await writeFile(outputFile, content); + process.exit(0); +} else { process.exit(1); } -process.exit(0); diff --git a/packages/importer/src/importer.ts b/packages/importer/src/importer.ts new file mode 100644 index 0000000000..b8903b3089 --- /dev/null +++ b/packages/importer/src/importer.ts @@ -0,0 +1,139 @@ +import { + createSourceLoader, + normalizePath, + printTypeSpecNode, + SyntaxKind, + type Diagnostic, + type ImportStatementNode, + type Statement, + type TypeSpecScriptNode, +} from "@typespec/compiler"; +import { resolve } from "path"; +import { ImporterHost } from "./importer-host.js"; + +export interface ImportResult { + /** TypeSpec Content */ + readonly content?: string; + /** Diagnostics */ + readonly diagnostics: readonly Diagnostic[]; +} + +/** + * Combine a TypeSpec project into a single file. + * Supports importing files from http/https with limitations: + * - directory import are not supported + * @param rawEntrypoint TypeSpec project entrypoint + */ +export async function combineProjectIntoFile(rawEntrypoint: string): Promise { + const entrypoint = + rawEntrypoint.startsWith("http://") || rawEntrypoint.startsWith("https://") + ? rawEntrypoint + : normalizePath(resolve(rawEntrypoint)); + + const loader = await createSourceLoader(ImporterHost, { + externals: (path: string) => { + return !(path === entrypoint || path.startsWith(".")); + }, + }); + await loader.importFile(entrypoint); + + if (loader.resolution.diagnostics.length > 0) { + return { diagnostics: loader.resolution.diagnostics }; + } + + const diagnostics: Diagnostic[] = []; + const libraries = new Set(); + + for (const [name, file] of loader.resolution.jsSourceFiles) { + const locContext = loader.resolution.locationContexts.get(file.file)!; + switch (locContext.type) { + case "project": + diagnostics.push({ + severity: "error", + code: "no-js", + message: `Importer doesn't support JS files in project: ${name}`, + target: { file: file.file, pos: 0, end: 0 }, + }); + break; + case "library": + libraries.add(locContext.metadata.name); + break; + case "compiler": + // do nothing + } + } + + const sourceFiles: TypeSpecScriptNode[] = []; + for (const file of loader.resolution.sourceFiles.values()) { + const locContext = loader.resolution.locationContexts.get(file.file)!; + switch (locContext.type) { + case "project": + sourceFiles.push(file); + break; + case "library": + libraries.add(locContext.metadata.name); + break; + case "compiler": + // do nothing + } + } + + const imports: Record = {}; + const statements: Statement[] = []; + + for (const file of sourceFiles) { + let currentStatements = statements; + for (const statement of file.statements) { + switch (statement.kind) { + case SyntaxKind.ImportStatement: + if (!statement.path.value.startsWith(".")) { + imports[statement.path.value] = statement; + } + break; + case SyntaxKind.NamespaceStatement: + let current = statement; + const ids = [statement.id]; + while (current.statements && "kind" in current.statements) { + current = current.statements; + ids.push(current.id); + } + if (current.statements === undefined) { + currentStatements = []; + statements.push({ ...current, statements: currentStatements, ...({ ids } as any) }); + } else { + currentStatements.push({ ...current, ...({ ids } as any) }); + } + break; + default: + currentStatements.push(statement); + } + } + } + + const newSourceFile: TypeSpecScriptNode = { + kind: SyntaxKind.TypeSpecScript, + statements: [...Object.values(imports), ...statements], + comments: [], + file: undefined as any, + pos: 0, + end: 0, + parseOptions: sourceFiles[0].parseOptions, + // Binder items + usings: [], + inScopeNamespaces: [], + namespaces: [], + parseDiagnostics: [], + printable: true, + id: undefined as any, + flags: 0, + symbol: undefined as any, + locals: undefined as any, + }; + + const content = await printTypeSpecNode(newSourceFile); + + return { + content, + diagnostics, + }; +} diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts index e69de29bb2..a2873a32f7 100644 --- a/packages/importer/src/index.ts +++ b/packages/importer/src/index.ts @@ -0,0 +1 @@ +export { combineProjectIntoFile } from "./importer.js"; From 7865753bf29cb18a9a5007c5d588f5e499d97aab Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 08:08:50 -0400 Subject: [PATCH 11/22] tweaks --- .vscode/launch.json | 5 +++-- packages/importer/test.txt | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 packages/importer/test.txt diff --git a/.vscode/launch.json b/.vscode/launch.json index 4517f62001..cbe204fe59 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -99,7 +99,8 @@ "name": "Debug importer", "program": "${workspaceFolder}/packages/importer/dist/cli.js", "args": [ - "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/batch/Azure.Batch/main.tsp" + // "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/batch/Azure.Batch/main.tsp" + "main.tsp" ], "smartStep": true, "sourceMaps": true, @@ -108,7 +109,7 @@ "${workspaceFolder}/packages/*/dist/**/*.js", "${workspaceFolder}/packages/*/dist-dev/**/*.js" ], - "cwd": "${workspaceFolder}/packages/importer", + "cwd": "${workspaceFolder}/packages/samples/scratch", "presentation": { "order": 2 } diff --git a/packages/importer/test.txt b/packages/importer/test.txt new file mode 100644 index 0000000000..83f1116f90 --- /dev/null +++ b/packages/importer/test.txt @@ -0,0 +1,3 @@ +TODO: delete + +node ./dist/cli.js https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/batch/Azure.Batch/main.tsp From 3eba9796e55595f661d975c0b0bd0d8802c43d62 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 08:18:06 -0400 Subject: [PATCH 12/22] Fix printing of raw text --- packages/compiler/src/formatter/print/printer.ts | 9 ++++++--- packages/importer/src/importer.ts | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/compiler/src/formatter/print/printer.ts b/packages/compiler/src/formatter/print/printer.ts index ef37a59c5a..ab9d4a13f0 100644 --- a/packages/compiler/src/formatter/print/printer.ts +++ b/packages/compiler/src/formatter/print/printer.ts @@ -520,7 +520,7 @@ export function printComment( case SyntaxKind.BlockComment: return printBlockComment(commentPath as AstPath, options); case SyntaxKind.LineComment: - return `${options.originalText.slice(comment.pos, comment.end).trimEnd()}`; + return `${getRawText(comment, options).trimEnd()}`; default: throw new Error(`Not a comment: ${JSON.stringify(comment)}`); } @@ -1660,7 +1660,7 @@ function printNumberLiteral( options: TypeSpecPrettierOptions ): Doc { const node = path.node; - return getRawText(node, options); + return node.valueAsString; } function printBooleanLiteral( @@ -1975,7 +1975,10 @@ function printItemList( * @param options Prettier options * @returns Raw text in the file for the given node. */ -function getRawText(node: TextRange, options: TypeSpecPrettierOptions) { +function getRawText(node: TextRange, options: TypeSpecPrettierOptions): string { + if ("rawText" in node) { + return node.rawText as string; + } return options.originalText.slice(node.pos, node.end); } diff --git a/packages/importer/src/importer.ts b/packages/importer/src/importer.ts index b8903b3089..40c6d28807 100644 --- a/packages/importer/src/importer.ts +++ b/packages/importer/src/importer.ts @@ -3,6 +3,7 @@ import { normalizePath, printTypeSpecNode, SyntaxKind, + visitChildren, type Diagnostic, type ImportStatementNode, type Statement, @@ -108,6 +109,13 @@ export async function combineProjectIntoFile(rawEntrypoint: string): Promise Date: Tue, 10 Sep 2024 08:34:22 -0400 Subject: [PATCH 13/22] Fix --- packages/compiler/src/core/formatter.ts | 3 --- packages/importer/src/importer.ts | 15 ++++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/core/formatter.ts b/packages/compiler/src/core/formatter.ts index 251c73f8a2..091eb7742d 100644 --- a/packages/compiler/src/core/formatter.ts +++ b/packages/compiler/src/core/formatter.ts @@ -1,12 +1,9 @@ import type { Options } from "prettier"; import { check, format } from "prettier/standalone"; import * as typespecPrettierPlugin from "../formatter/index.js"; -import { flattenNamespaces } from "../formatter/parser.js"; import type { Node } from "./types.js"; export function printTypeSpecNode(node: Node): Promise { - flattenNamespaces(node); - return format(".", { parser: "typespec", plugins: [ diff --git a/packages/importer/src/importer.ts b/packages/importer/src/importer.ts index 40c6d28807..20b5fd00fb 100644 --- a/packages/importer/src/importer.ts +++ b/packages/importer/src/importer.ts @@ -83,6 +83,13 @@ export async function combineProjectIntoFile(rawEntrypoint: string): Promise Date: Tue, 10 Sep 2024 08:40:28 -0400 Subject: [PATCH 14/22] fix usings --- packages/importer/src/importer.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/importer/src/importer.ts b/packages/importer/src/importer.ts index 20b5fd00fb..96296debbd 100644 --- a/packages/importer/src/importer.ts +++ b/packages/importer/src/importer.ts @@ -5,9 +5,12 @@ import { SyntaxKind, visitChildren, type Diagnostic, + type IdentifierNode, type ImportStatementNode, + type MemberExpressionNode, type Statement, type TypeSpecScriptNode, + type UsingStatementNode, } from "@typespec/compiler"; import { resolve } from "path"; import { ImporterHost } from "./importer-host.js"; @@ -23,6 +26,7 @@ export interface ImportResult { * Combine a TypeSpec project into a single file. * Supports importing files from http/https with limitations: * - directory import are not supported + * - different files that would result in merging ambiguous using statements will have conflict * @param rawEntrypoint TypeSpec project entrypoint */ export async function combineProjectIntoFile(rawEntrypoint: string): Promise { @@ -80,6 +84,7 @@ export async function combineProjectIntoFile(rawEntrypoint: string): Promise = {}; + const usings: Record = {}; const statements: Statement[] = []; for (const file of sourceFiles) { @@ -98,6 +103,12 @@ export async function combineProjectIntoFile(rawEntrypoint: string): Promise Date: Tue, 10 Sep 2024 08:57:56 -0400 Subject: [PATCH 15/22] Importer in playgroud --- packages/importer/package.json | 3 ++ packages/playground-website/package.json | 1 + .../src/import-openapi3.tsx | 46 +++++++++++++++++-- pnpm-lock.yaml | 3 ++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/importer/package.json b/packages/importer/package.json index 340b75e051..d4be8d471f 100644 --- a/packages/importer/package.json +++ b/packages/importer/package.json @@ -19,6 +19,9 @@ ], "type": "module", "main": "dist/src/index.js", + "exports": { + ".": "./dist/index.js" + }, "bin": "dist/src/cli.js", "engines": { "node": ">=18.0.0" diff --git a/packages/playground-website/package.json b/packages/playground-website/package.json index 7a4e51eec5..d3043a1fea 100644 --- a/packages/playground-website/package.json +++ b/packages/playground-website/package.json @@ -65,6 +65,7 @@ "@typespec/protobuf": "workspace:~", "@typespec/rest": "workspace:~", "@typespec/versioning": "workspace:~", + "@typespec/importer": "workspace:~", "es-module-shims": "~1.10.0", "react": "~18.3.1", "react-dom": "~18.3.1" diff --git a/packages/playground-website/src/import-openapi3.tsx b/packages/playground-website/src/import-openapi3.tsx index e45fa0e1cc..ef054569b5 100644 --- a/packages/playground-website/src/import-openapi3.tsx +++ b/packages/playground-website/src/import-openapi3.tsx @@ -5,6 +5,8 @@ import { DialogContent, DialogSurface, DialogTitle, + Input, + Label, Menu, MenuItem, MenuList, @@ -14,13 +16,15 @@ import { Tooltip, } from "@fluentui/react-components"; import { ArrowUploadFilled } from "@fluentui/react-icons"; +import { formatDiagnostic } from "@typespec/compiler"; +import { combineProjectIntoFile } from "@typespec/importer"; import { Editor, useMonacoModel, usePlaygroundContext } from "@typespec/playground/react"; import { useState } from "react"; import { parse } from "yaml"; import style from "./import-openapi3.module.css"; export const ImportToolbarButton = () => { - const [open, setOpen] = useState(false); + const [open, setOpen] = useState<"openapi3" | "tsp" | undefined>(); return ( <> @@ -36,17 +40,19 @@ export const ImportToolbarButton = () => { - setOpen(true)}>From OpenAPI 3 spec + setOpen("tsp")}>From OpenAPI 3 spec + setOpen("openapi3")}>From OpenAPI 3 spec - setOpen(data.open)}> + setOpen(undefined)}> Settings - setOpen(false)} /> + {open === "openapi3" && setOpen(undefined)} />} + {open === "tsp" && setOpen(undefined)} />} @@ -88,3 +94,35 @@ const ImportOpenAPI3 = ({ onImport }: { onImport: () => void }) => { ); }; + +const ImportTsp = ({ onImport }: { onImport: () => void }) => { + const [error, setError] = useState(null); + const [value, setValue] = useState(""); + const context = usePlaygroundContext(); + + const importSpec = async () => { + const content = value; + const result = await combineProjectIntoFile(content); + if (result.diagnostics.length > 0) { + setError(result.diagnostics.map(formatDiagnostic).join("\n")); + return; + } else if (result.content) { + context.setContent(result.content); + onImport(); + } + }; + return ( +
+

Import Remote Tsp Document

+ +
+ + setValue(data.value)} /> +
+ {error &&
{error}
} + +
+ ); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8ce2d442e..22c1b35e35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1013,6 +1013,9 @@ importers: '@typespec/http': specifier: workspace:~ version: link:../http + '@typespec/importer': + specifier: workspace:~ + version: link:../importer '@typespec/json-schema': specifier: workspace:~ version: link:../json-schema From cc0213882385abc146e64994846fa46db2d9a2fa Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 09:29:06 -0400 Subject: [PATCH 16/22] Add to the playground --- packages/importer/src/cli.ts | 8 ++-- packages/importer/src/importer-host.ts | 33 ++----------- packages/importer/src/importer.ts | 12 +++-- packages/importer/src/index.ts | 1 + packages/importer/src/remote-host.ts | 47 +++++++++++++++++++ .../src/import-openapi3.tsx | 18 +++---- 6 files changed, 73 insertions(+), 46 deletions(-) create mode 100644 packages/importer/src/remote-host.ts diff --git a/packages/importer/src/cli.ts b/packages/importer/src/cli.ts index 013743b4b7..8d5c0b93cc 100644 --- a/packages/importer/src/cli.ts +++ b/packages/importer/src/cli.ts @@ -6,9 +6,11 @@ try { // package only present in dev. } -import { getDirectoryPath, logDiagnostics, NodeHost } from "@typespec/compiler"; +import { getDirectoryPath, logDiagnostics, NodeHost, normalizePath } from "@typespec/compiler"; import { mkdir, writeFile } from "fs/promises"; +import { resolve } from "path"; import { parseArgs } from "util"; +import { ImporterHost } from "./importer-host.js"; import { combineProjectIntoFile } from "./importer.js"; function log(...args: any[]) { @@ -21,9 +23,9 @@ const args = parseArgs({ allowPositionals: true, }); -const rawEntrypoint = args.positionals[0]; +const rawEntrypoint = normalizePath(resolve(args.positionals[0])); -const { content, diagnostics } = await combineProjectIntoFile(rawEntrypoint); +const { content, diagnostics } = await combineProjectIntoFile(ImporterHost, rawEntrypoint); if (diagnostics.length > 0) { logDiagnostics(diagnostics, NodeHost.logSink); diff --git a/packages/importer/src/importer-host.ts b/packages/importer/src/importer-host.ts index 9da83e574e..bc5991401f 100644 --- a/packages/importer/src/importer-host.ts +++ b/packages/importer/src/importer-host.ts @@ -1,37 +1,10 @@ -import { - createSourceFile, - getAnyExtensionFromPath, - NodeHost, - type CompilerHost, -} from "@typespec/compiler"; +import { NodeHost, type CompilerHost } from "@typespec/compiler"; +import { createRemoteHost } from "./remote-host.js"; /** * Special host that tries to load data from additional locations */ export const ImporterHost: CompilerHost = { ...NodeHost, - stat: async (pathOrUrl) => { - if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { - const res = await fetch(pathOrUrl); - - return { - isFile: () => res.status === 200, - isDirectory: () => getAnyExtensionFromPath(pathOrUrl) === "", - }; - } - return NodeHost.stat(pathOrUrl); - }, - readFile: async (pathOrUrl) => { - if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { - const res = await fetch(pathOrUrl); - return createSourceFile(await res.text(), pathOrUrl); - } - return NodeHost.readFile(pathOrUrl); - }, - realpath: async (path) => { - if (path.startsWith("http://") || path.startsWith("https://")) { - return path; - } - return NodeHost.realpath(path); - }, + ...createRemoteHost(NodeHost), }; diff --git a/packages/importer/src/importer.ts b/packages/importer/src/importer.ts index 96296debbd..823e435468 100644 --- a/packages/importer/src/importer.ts +++ b/packages/importer/src/importer.ts @@ -4,6 +4,7 @@ import { printTypeSpecNode, SyntaxKind, visitChildren, + type CompilerHost, type Diagnostic, type IdentifierNode, type ImportStatementNode, @@ -12,8 +13,6 @@ import { type TypeSpecScriptNode, type UsingStatementNode, } from "@typespec/compiler"; -import { resolve } from "path"; -import { ImporterHost } from "./importer-host.js"; export interface ImportResult { /** TypeSpec Content */ @@ -29,13 +28,16 @@ export interface ImportResult { * - different files that would result in merging ambiguous using statements will have conflict * @param rawEntrypoint TypeSpec project entrypoint */ -export async function combineProjectIntoFile(rawEntrypoint: string): Promise { +export async function combineProjectIntoFile( + host: CompilerHost, + rawEntrypoint: string +): Promise { const entrypoint = rawEntrypoint.startsWith("http://") || rawEntrypoint.startsWith("https://") ? rawEntrypoint - : normalizePath(resolve(rawEntrypoint)); + : normalizePath(rawEntrypoint); - const loader = await createSourceLoader(ImporterHost, { + const loader = await createSourceLoader(host, { externals: (path: string) => { return !(path === entrypoint || path.startsWith(".")); }, diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts index a2873a32f7..7083b88258 100644 --- a/packages/importer/src/index.ts +++ b/packages/importer/src/index.ts @@ -1 +1,2 @@ export { combineProjectIntoFile } from "./importer.js"; +export { createRemoteHost } from "./remote-host.js"; diff --git a/packages/importer/src/remote-host.ts b/packages/importer/src/remote-host.ts new file mode 100644 index 0000000000..ee7c791685 --- /dev/null +++ b/packages/importer/src/remote-host.ts @@ -0,0 +1,47 @@ +import { + createSourceFile, + getAnyExtensionFromPath, + getSourceFileKindFromExt, + type CompilerHost, +} from "@typespec/compiler"; + +export class NotUrlError extends Error {} +export function createRemoteHost(base?: CompilerHost): CompilerHost { + return { + ...base, + getSourceFileKind: getSourceFileKindFromExt, + stat: async (pathOrUrl: string) => { + if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { + const res = await fetch(pathOrUrl); + + return { + isFile: () => res.status === 200, + isDirectory: () => getAnyExtensionFromPath(pathOrUrl) === "", + }; + } else if (base) { + return base.stat(pathOrUrl); + } else { + throw new NotUrlError(pathOrUrl); + } + }, + readFile: async (pathOrUrl: string) => { + if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { + const res = await fetch(pathOrUrl); + return createSourceFile(await res.text(), pathOrUrl); + } else if (base) { + return base.stat(pathOrUrl); + } else { + throw new NotUrlError(pathOrUrl); + } + }, + realpath: async (pathOrUrl: string) => { + if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { + return pathOrUrl; + } else if (base) { + return base.stat(pathOrUrl); + } else { + throw new NotUrlError(pathOrUrl); + } + }, + } as any; +} diff --git a/packages/playground-website/src/import-openapi3.tsx b/packages/playground-website/src/import-openapi3.tsx index ef054569b5..404def6edf 100644 --- a/packages/playground-website/src/import-openapi3.tsx +++ b/packages/playground-website/src/import-openapi3.tsx @@ -17,7 +17,7 @@ import { } from "@fluentui/react-components"; import { ArrowUploadFilled } from "@fluentui/react-icons"; import { formatDiagnostic } from "@typespec/compiler"; -import { combineProjectIntoFile } from "@typespec/importer"; +import { combineProjectIntoFile, createRemoteHost } from "@typespec/importer"; import { Editor, useMonacoModel, usePlaygroundContext } from "@typespec/playground/react"; import { useState } from "react"; import { parse } from "yaml"; @@ -40,7 +40,7 @@ export const ImportToolbarButton = () => { - setOpen("tsp")}>From OpenAPI 3 spec + setOpen("tsp")}>Remote TypeSpec setOpen("openapi3")}>From OpenAPI 3 spec @@ -102,7 +102,7 @@ const ImportTsp = ({ onImport }: { onImport: () => void }) => { const importSpec = async () => { const content = value; - const result = await combineProjectIntoFile(content); + const result = await combineProjectIntoFile(createRemoteHost(), content); if (result.diagnostics.length > 0) { setError(result.diagnostics.map(formatDiagnostic).join("\n")); return; @@ -115,14 +115,16 @@ const ImportTsp = ({ onImport }: { onImport: () => void }) => {

Import Remote Tsp Document

-
+
- setValue(data.value)} />
+ setValue(data.value)} /> {error &&
{error}
} - +
+ +
); }; From a50b53556a825a8e5e3dadf960d7761168f53b00 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 09:29:18 -0400 Subject: [PATCH 17/22] tweaks --- packages/importer/src/remote-host.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/importer/src/remote-host.ts b/packages/importer/src/remote-host.ts index ee7c791685..cc36442cca 100644 --- a/packages/importer/src/remote-host.ts +++ b/packages/importer/src/remote-host.ts @@ -8,8 +8,8 @@ import { export class NotUrlError extends Error {} export function createRemoteHost(base?: CompilerHost): CompilerHost { return { - ...base, getSourceFileKind: getSourceFileKindFromExt, + ...base, stat: async (pathOrUrl: string) => { if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) { const res = await fetch(pathOrUrl); From d243bf1b2bfb22eb19db4fe6af206a5f8a9485fe Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 10 Sep 2024 09:37:26 -0400 Subject: [PATCH 18/22] cleanup --- ...-openapi3.module.css => import.module.css} | 5 +++++ .../src/{import-openapi3.tsx => import.tsx} | 22 +++++++++++++------ packages/playground-website/src/index.ts | 2 +- packages/playground-website/src/main.tsx | 2 +- packages/playground/src/react/index.ts | 2 ++ 5 files changed, 24 insertions(+), 9 deletions(-) rename packages/playground-website/src/{import-openapi3.module.css => import.module.css} (68%) rename packages/playground-website/src/{import-openapi3.tsx => import.tsx} (87%) diff --git a/packages/playground-website/src/import-openapi3.module.css b/packages/playground-website/src/import.module.css similarity index 68% rename from packages/playground-website/src/import-openapi3.module.css rename to packages/playground-website/src/import.module.css index 0a56eb3562..85aec9384e 100644 --- a/packages/playground-website/src/import-openapi3.module.css +++ b/packages/playground-website/src/import.module.css @@ -3,3 +3,8 @@ border: 1px solid var(--colorNeutralStroke1); margin-bottom: 20px; } + +.url-input { + width: 100%; + margin-bottom: 20px; +} diff --git a/packages/playground-website/src/import-openapi3.tsx b/packages/playground-website/src/import.tsx similarity index 87% rename from packages/playground-website/src/import-openapi3.tsx rename to packages/playground-website/src/import.tsx index 404def6edf..b8fea66d25 100644 --- a/packages/playground-website/src/import-openapi3.tsx +++ b/packages/playground-website/src/import.tsx @@ -16,12 +16,16 @@ import { Tooltip, } from "@fluentui/react-components"; import { ArrowUploadFilled } from "@fluentui/react-icons"; -import { formatDiagnostic } from "@typespec/compiler"; import { combineProjectIntoFile, createRemoteHost } from "@typespec/importer"; -import { Editor, useMonacoModel, usePlaygroundContext } from "@typespec/playground/react"; -import { useState } from "react"; +import { + DiagnosticList, + Editor, + useMonacoModel, + usePlaygroundContext, +} from "@typespec/playground/react"; +import { ReactNode, useState } from "react"; import { parse } from "yaml"; -import style from "./import-openapi3.module.css"; +import style from "./import.module.css"; export const ImportToolbarButton = () => { const [open, setOpen] = useState<"openapi3" | "tsp" | undefined>(); @@ -96,7 +100,7 @@ const ImportOpenAPI3 = ({ onImport }: { onImport: () => void }) => { }; const ImportTsp = ({ onImport }: { onImport: () => void }) => { - const [error, setError] = useState(null); + const [error, setError] = useState(null); const [value, setValue] = useState(""); const context = usePlaygroundContext(); @@ -104,7 +108,7 @@ const ImportTsp = ({ onImport }: { onImport: () => void }) => { const content = value; const result = await combineProjectIntoFile(createRemoteHost(), content); if (result.diagnostics.length > 0) { - setError(result.diagnostics.map(formatDiagnostic).join("\n")); + setError(); return; } else if (result.content) { context.setContent(result.content); @@ -118,7 +122,11 @@ const ImportTsp = ({ onImport }: { onImport: () => void }) => {
- setValue(data.value)} /> + setValue(data.value)} + className={style["url-input"]} + /> {error &&
{error}
}