From e41b8b5ef861d779f043a13d66227ba946527224 Mon Sep 17 00:00:00 2001 From: "FAREAST\\chunyu" Date: Mon, 13 Mar 2023 16:52:17 +0800 Subject: [PATCH 1/5] typespec renaming --- ...Scripts.md => Typespec-Project-Scripts.md} | 58 +++++++++---------- ...rate.ps1 => Typespec-Project-Generate.ps1} | 26 ++++----- ...ect-Sync.ps1 => Typespec-Project-Sync.ps1} | 16 ++--- 3 files changed, 50 insertions(+), 50 deletions(-) rename doc/common/{Cadl-Project-Scripts.md => Typespec-Project-Scripts.md} (57%) rename eng/common/scripts/{Cadl-Project-Generate.ps1 => Typespec-Project-Generate.ps1} (72%) rename eng/common/scripts/{Cadl-Project-Sync.ps1 => Typespec-Project-Sync.ps1} (88%) diff --git a/doc/common/Cadl-Project-Scripts.md b/doc/common/Typespec-Project-Scripts.md similarity index 57% rename from doc/common/Cadl-Project-Scripts.md rename to doc/common/Typespec-Project-Scripts.md index 054e09e0f22..179cd9a54c5 100644 --- a/doc/common/Cadl-Project-Scripts.md +++ b/doc/common/Typespec-Project-Scripts.md @@ -1,22 +1,22 @@ -# Integrating with cadl sync and generate scripts +# Integrating with typespec sync and generate scripts There are 2 common scripts provided for each language to be able to generate from within the language -repo and use the remote cadl definition in the spec repo. +repo and use the remote typespec definition in the spec repo. ## One time language repo setup There are 3 things that these two scripts expect are set up in your language repo before they will run correctly. -1. Make sure your .gitignore is ignoring the TempCadlFiles +1. Make sure your .gitignore is ignoring the TempTypespecFiles 2. Create a common emitter-package.json for your language 3. Write the language specific hooks in Language-Settings.ps1 -### TempCadlFiles +### TempTypespecFiles You should add a new entry in your .gitignore for your repo so that none of these files are accidentally checked in if [cleanup](#cleanup-anchor) is turned off. ``` # .gitignore file -TempCadlFiles/ +TempTypespecFiles/ ``` ### emitter-package.json @@ -30,7 +30,7 @@ Example { "main": "dist/src/index.js", "dependencies": { - "@azure-tools/cadl-csharp": "0.1.11-beta.20230123.1" + "@azure-tools/typespec-csharp": "0.1.11-beta.20230123.1" } } ``` @@ -50,7 +50,7 @@ Example ```powershell function Get-dotnet-EmitterName() { - return "@azure-tools/cadl-csharp" + return "@azure-tools/typespec-csharp" } ``` @@ -68,31 +68,31 @@ function Get-dotnet-EmitterPackageJsonPath() { #### Get-${Language}-EmitterAdditionalOptions (Optional) -This function allows you to append additional `--option` arguments that will be passed into cadl compile. One example of this is the `emitter-output-dir`. For dotnet we want the location of the generated files to be `{projectDir}/src` however in other languages they will have other conventions. This method will take in a fully qualified path to the project directory so you can construct your relative path to that as the output. +This function allows you to append additional `--option` arguments that will be passed into typespec compile. One example of this is the `emitter-output-dir`. For dotnet we want the location of the generated files to be `{projectDir}/src` however in other languages they will have other conventions. This method will take in a fully qualified path to the project directory so you can construct your relative path to that as the output. Example ```powershell function Get-dotnet-EmitterAdditionalOptions([string]$projectDirectory) { - return "--option @azure-tools/cadl-csharp.emitter-output-dir=$projectDirectory/src" + return "--option @azure-tools/typespec-csharp.emitter-output-dir=$projectDirectory/src" } ``` ## Per project setup -Each project will need to have a configuration file that will tell the scripts where to find the cadl spec. +Each project will need to have a configuration file that will tell the scripts where to find the typespec spec. -### cadl-location.yaml +### typespec-location.yaml This file should live under the project directory for each service and has the following properties | Property | Description | IsRequired | | --- | --- | --- | -| directory | The top level directory where the main.cadl for the service lives. This should be relative to the spec repo root such as `specification/cognitiveservices/OpenAI.Inference` | true | -| additionalDirectories | Sometimes a cadl file will use a relative import that might not be under the main directory. In this case a single `directory` will not be enough to pull down all necessary files. To support this you can specify additional directories as a list to sync so that all needed files are synced. | false: default = null | -| commit | The commit sha for the version of the cadl files you want to generate off of. This allows us to have idempotence on generation until we opt into pointing at a later version. | true | +| directory | The top level directory where the main.tso for the service lives. This should be relative to the spec repo root such as `specification/cognitiveservices/OpenAI.Inference` | true | +| additionalDirectories | Sometimes a typespec file will use a relative import that might not be under the main directory. In this case a single `directory` will not be enough to pull down all necessary files. To support this you can specify additional directories as a list to sync so that all needed files are synced. | false: default = null | +| commit | The commit sha for the version of the typespec files you want to generate off of. This allows us to have idempotence on generation until we opt into pointing at a later version. | true | | repo | The repo this spec lives in. This should be either `Azure/azure-rest-api-specs` or `Azure/azure-rest-api-specs-pr`. Note that pr will work locally but not in CI until we add another change to handle token based auth. | true | -| cleanup | This will remove the TempCadlFiles directory after generation is complete if true otherwise this directory will be left to support local changes to the files to see how different changes would affect the generation. | false: default = true | +| cleanup | This will remove the TempTypespecFiles directory after generation is complete if true otherwise this directory will be left to support local changes to the files to see how different changes would affect the generation. | false: default = true | Example @@ -105,42 +105,42 @@ repo: Azure/azure-rest-api-specs cleanup: false ``` -## Cadl-Project-Sync.ps1 +## Typespec-Project-Sync.ps1 -This is the first script that should be called and can be found at `./eng/common/scripts/Cadl-Project-Sync.ps1`. It takes in one parameter which is the root directory of the project which is typically one layer lower than the service directory. As an example for dotnet this is `./sdk/openai/Azure.AI.OpenAI` where `openai` is the service directory and `Azure.AI.OpenAI` is the project directory. +This is the first script that should be called and can be found at `./eng/common/scripts/Typespec-Project-Sync.ps1`. It takes in one parameter which is the root directory of the project which is typically one layer lower than the service directory. As an example for dotnet this is `./sdk/openai/Azure.AI.OpenAI` where `openai` is the service directory and `Azure.AI.OpenAI` is the project directory. ```powershell -./eng/common/scripts/Cadl-Project-Sync.ps1 ./sdk/openai/Azure.AI.OpenAI +./eng/common/scripts/Typespec-Project-Sync.ps1 ./sdk/openai/Azure.AI.OpenAI ``` -This script will create a `sparse-spec` folder as a sibling to the root of your current git clone. Each project that is generated will get a sub directory inside of this folder named after the project you are generating. It will then automatically filter to only the files in the [directory](#directory-anchor) defined in cadl-location.yaml, and sync to the [commit sha](#commit-anchor) defined in cadl-location.yaml. +This script will create a `sparse-spec` folder as a sibling to the root of your current git clone. Each project that is generated will get a sub directory inside of this folder named after the project you are generating. It will then automatically filter to only the files in the [directory](#directory-anchor) defined in typespec-location.yaml, and sync to the [commit sha](#commit-anchor) defined in typespec-location.yaml. As an example if you have your language repo at `D:\git\azure-sdk-for-net` there will be a new directory `D:\git\sparse-spec\Azure.AI.OpenAI` where the sparse spec will live. -This is then copied over to your project directory so that you can make temporary changes if needed. The location will be `./{projectDir}/TempCadlFiles`. This temporary directory will be [cleaned up](#cleanup-anchor) at the end of the generate script if set in the cadl-location.yaml. +This is then copied over to your project directory so that you can make temporary changes if needed. The location will be `./{projectDir}/TempTypespecFiles`. This temporary directory will be [cleaned up](#cleanup-anchor) at the end of the generate script if set in the typespec-location.yaml. -## Cadl-Project-Generate.ps1 +## Typespec-Project-Generate.ps1 -This is the second script that should be called and can be found at `./eng/common/scripts/Cadl-Project-Generate.ps1`. It takes the exact same parameter as the sync script. +This is the second script that should be called and can be found at `./eng/common/scripts/Typespec-Project-Generate.ps1`. It takes the exact same parameter as the sync script. ```powershell -./eng/common/scripts/Cadl-Project-Generate.ps1 ./sdk/openai/Azure.AI.OpenAI +./eng/common/scripts/Typespec-Project-Generate.ps1 ./sdk/openai/Azure.AI.OpenAI ``` -The first thing this does is clean up the npm install that might exist in `./{projectDir}/TempCadlFiles`, followed by replacing the package.json with the language static one. +The first thing this does is clean up the npm install that might exist in `./{projectDir}/TempTypespecFiles`, followed by replacing the package.json with the language static one. -Once this is done it will run `npm install` followed by `cadl compile` which is the standard way to generate a cadl project. +Once this is done it will run `npm install` followed by `tsp compile` which is the standard way to generate a typespec project. The exact command that gets run is output stdout to enable debugging if needed. -We currently don't do anything to the cadl-project.yaml that gets pulled in from the spec repo to limit to just your language emitter instead we use the filter option on the command line `--emit $emitterName`. This allows you to isolate the generation to only things owned by your language so you can safely add generation dependencies in CI without needing to worry about noisy neighbors. +We currently don't do anything to the typespec-project.yaml that gets pulled in from the spec repo to limit to just your language emitter instead we use the filter option on the command line `--emit $emitterName`. This allows you to isolate the generation to only things owned by your language so you can safely add generation dependencies in CI without needing to worry about noisy neighbors. ## Build tool integration -One use case that some languages have is to have their CI regenerate the project and then do a `git diff` to validate that there are no differences. This helps detect if people modify the generated files manually. To support this its valuable to have the exact same command to generate a project regardless of whether the individual library is autorest or cadl. +One use case that some languages have is to have their CI regenerate the project and then do a `git diff` to validate that there are no differences. This helps detect if people modify the generated files manually. To support this its valuable to have the exact same command to generate a project regardless of whether the individual library is autorest or typespec. -To achieve this each language will have their own idiomatic tool set but whatever that toolset is can check to see if a cadl-location.yaml file exists, and if it does they can call the Cadl-Project-Sync.ps1 and Cadl-Project-Generate.ps1 scripts, otherwise they can call the autorest command they call today for all other libraries. +To achieve this each language will have their own idiomatic tool set but whatever that toolset is can check to see if a typespec-location.yaml file exists, and if it does they can call the Typespec-Project-Sync.ps1 and Typespec-Project-Generate.ps1 scripts, otherwise they can call the autorest command they call today for all other libraries. In dotnet this is achieved by running `dotnet build /t:GenerateCode` regardless of which type of project it is the correct commands get called and it remains consistent and idiomatic to the language. In other languages this could be `npm generate` or `python generate.py` to do the same. -Since the generate script simply is a wrapper for `npm install` and `cadl compile` you can still run those commands directly manually after the sync if you want to instead. +Since the generate script simply is a wrapper for `npm install` and `tsp compile` you can still run those commands directly manually after the sync if you want to instead. diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Typespec-Project-Generate.ps1 similarity index 72% rename from eng/common/scripts/Cadl-Project-Generate.ps1 rename to eng/common/scripts/Typespec-Project-Generate.ps1 index 3e7ee781b05..43b6beab649 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Typespec-Project-Generate.ps1 @@ -1,4 +1,4 @@ -# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Cadl-Project-Scripts.md +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Typespec-Project-Scripts.md [CmdletBinding()] param ( @@ -6,7 +6,7 @@ param ( [ValidateNotNullOrEmpty()] [string] $ProjectDirectory, [Parameter(Position=1)] - [string] $CadlAdditionalOptions ## addtional cadl emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 + [string] $TypespecAdditionalOptions ## addtional typespec emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 ) $ErrorActionPreference = "Stop" @@ -55,17 +55,17 @@ function NpmInstallForProject([string]$workingDirectory) { $resolvedProjectDirectory = Resolve-Path $ProjectDirectory $emitterName = &$GetEmitterNameFn -$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" +$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/typespec-location.yaml" -Write-Host "Reading configuration from $cadlConfigurationFile" -$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml +Write-Host "Reading configuration from $typespecConfigurationFile" +$configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml $specSubDirectory = $configuration["directory"] $innerFolder = Split-Path $specSubDirectory -Leaf -$tempFolder = "$ProjectDirectory/TempCadlFiles" +$tempFolder = "$ProjectDirectory/TempTypespecFiles" $npmWorkingDir = Resolve-Path $tempFolder/$innerFolder -$mainCadlFile = If (Test-Path "$npmWorkingDir/client.cadl") { Resolve-Path "$npmWorkingDir/client.cadl" } Else { Resolve-Path "$npmWorkingDir/main.cadl"} +$mainTypespecFile = If (Test-Path "$npmWorkingDir/client.tsp") { Resolve-Path "$npmWorkingDir/client.tsp" } Else { Resolve-Path "$npmWorkingDir/main.tsp"} try { Push-Location $npmWorkingDir @@ -79,15 +79,15 @@ try { $emitterAdditionalOptions = " $emitterAdditionalOptions" } } - $cadlCompileCommand = "npx cadl compile $mainCadlFile --emit $emitterName$emitterAdditionalOptions" - if ($CadlAdditionalOptions) { - $options = $CadlAdditionalOptions.Split(";"); + $typespecCompileCommand = "npx tsp compile $mainTypespecFile --emit $emitterName$emitterAdditionalOptions" + if ($TypespecAdditionalOptions) { + $options = $TypespecAdditionalOptions.Split(";"); foreach ($option in $options) { - $cadlCompileCommand += " --option $emitterName.$option" + $typespecCompileCommand += " --option $emitterName.$option" } } - Write-Host($cadlCompileCommand) - Invoke-Expression $cadlCompileCommand + Write-Host($typespecCompileCommand) + Invoke-Expression $typespecCompileCommand if ($LASTEXITCODE) { exit $LASTEXITCODE } } diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Typespec-Project-Sync.ps1 similarity index 88% rename from eng/common/scripts/Cadl-Project-Sync.ps1 rename to eng/common/scripts/Typespec-Project-Sync.ps1 index c6e8cf5ce53..24f68a8c54f 100644 --- a/eng/common/scripts/Cadl-Project-Sync.ps1 +++ b/eng/common/scripts/Typespec-Project-Sync.ps1 @@ -1,4 +1,4 @@ -# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Cadl-Project-Scripts.md +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Typespec-Project-Scripts.md [CmdletBinding()] param ( @@ -86,11 +86,11 @@ function GetSpecCloneDir([string]$projectName) { return $createResult } -$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" -Write-Host "Reading configuration from $cadlConfigurationFile" -$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml +$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/typespec-location.yaml" +Write-Host "Reading configuration from $typespecConfigurationFile" +$configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml -$pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/") +$pieces = $typespecConfigurationFile.Path.Replace("\","/").Split("/") $projectName = $pieces[$pieces.Count - 2] $specSubDirectory = $configuration["directory"] @@ -118,10 +118,10 @@ if ( $configuration["repo"] -and $configuration["commit"]) { } -$tempCadlDir = "$ProjectDirectory/TempCadlFiles" -New-Item $tempCadlDir -Type Directory -Force | Out-Null +$tempTypespecDir = "$ProjectDirectory/TempTypespecFiles" +New-Item $tempTypespecDir -Type Directory -Force | Out-Null CopySpecToProjectIfNeeded ` -specCloneRoot $specCloneDir ` -mainSpecDir $specSubDirectory ` - -dest $tempCadlDir ` + -dest $tempTypespecDir ` -specAdditionalSubDirectories $configuration["additionalDirectories"] From fcadc763a936e181030d94c13f8e71fc66a28524 Mon Sep 17 00:00:00 2001 From: "FAREAST\\chunyu" Date: Tue, 14 Mar 2023 09:25:21 +0800 Subject: [PATCH 2/5] add back scripts for cadl --- eng/common/scripts/Cadl-Project-Generate.ps1 | 101 +++++++++++++++ eng/common/scripts/Cadl-Project-Sync.ps1 | 127 +++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 eng/common/scripts/Cadl-Project-Generate.ps1 create mode 100644 eng/common/scripts/Cadl-Project-Sync.ps1 diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 new file mode 100644 index 00000000000..3e7ee781b05 --- /dev/null +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -0,0 +1,101 @@ +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Cadl-Project-Scripts.md + +[CmdletBinding()] +param ( + [Parameter(Position=0)] + [ValidateNotNullOrEmpty()] + [string] $ProjectDirectory, + [Parameter(Position=1)] + [string] $CadlAdditionalOptions ## addtional cadl emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +. $PSScriptRoot/common.ps1 +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + +function NpmInstallForProject([string]$workingDirectory) { + Push-Location $workingDirectory + try { + $currentDur = Resolve-Path "." + Write-Host "Generating from $currentDur" + + if (Test-Path "package.json") { + Remove-Item -Path "package.json" -Force + } + + if (Test-Path ".npmrc") { + Remove-Item -Path ".npmrc" -Force + } + + if (Test-Path "node_modules") { + Remove-Item -Path "node_modules" -Force -Recurse + } + + if (Test-Path "package-lock.json") { + Remove-Item -Path "package-lock.json" -Force + } + + #default to root/eng/emitter-package.json but you can override by writing + #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1 + $replacementPackageJson = "$PSScriptRoot/../../emitter-package.json" + if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { + $replacementPackageJson = &$GetEmitterPackageJsonPathFn + } + + Write-Host("Copying package.json from $replacementPackageJson") + Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force + npm install --no-lock-file + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } + finally { + Pop-Location + } +} + +$resolvedProjectDirectory = Resolve-Path $ProjectDirectory +$emitterName = &$GetEmitterNameFn +$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" + +Write-Host "Reading configuration from $cadlConfigurationFile" +$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml + +$specSubDirectory = $configuration["directory"] +$innerFolder = Split-Path $specSubDirectory -Leaf + +$tempFolder = "$ProjectDirectory/TempCadlFiles" +$npmWorkingDir = Resolve-Path $tempFolder/$innerFolder +$mainCadlFile = If (Test-Path "$npmWorkingDir/client.cadl") { Resolve-Path "$npmWorkingDir/client.cadl" } Else { Resolve-Path "$npmWorkingDir/main.cadl"} + +try { + Push-Location $npmWorkingDir + NpmInstallForProject $npmWorkingDir + + if ($LASTEXITCODE) { exit $LASTEXITCODE } + + if (Test-Path "Function:$GetEmitterAdditionalOptionsFn") { + $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn $resolvedProjectDirectory + if ($emitterAdditionalOptions.Length -gt 0) { + $emitterAdditionalOptions = " $emitterAdditionalOptions" + } + } + $cadlCompileCommand = "npx cadl compile $mainCadlFile --emit $emitterName$emitterAdditionalOptions" + if ($CadlAdditionalOptions) { + $options = $CadlAdditionalOptions.Split(";"); + foreach ($option in $options) { + $cadlCompileCommand += " --option $emitterName.$option" + } + } + Write-Host($cadlCompileCommand) + Invoke-Expression $cadlCompileCommand + + if ($LASTEXITCODE) { exit $LASTEXITCODE } +} +finally { + Pop-Location +} + +$shouldCleanUp = $configuration["cleanup"] ?? $true +if ($shouldCleanUp) { + Remove-Item $tempFolder -Recurse -Force +} \ No newline at end of file diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Cadl-Project-Sync.ps1 new file mode 100644 index 00000000000..c6e8cf5ce53 --- /dev/null +++ b/eng/common/scripts/Cadl-Project-Sync.ps1 @@ -0,0 +1,127 @@ +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Cadl-Project-Scripts.md + +[CmdletBinding()] +param ( + [Parameter(Position=0)] + [ValidateNotNullOrEmpty()] + [string] $ProjectDirectory +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module +$sparseCheckoutFile = ".git/info/sparse-checkout" + +function AddSparseCheckoutPath([string]$subDirectory) { + if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) { + Write-Output $subDirectory >> .git/info/sparse-checkout + } +} + +function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) { + $source = "$specCloneRoot/$mainSpecDir" + Copy-Item -Path $source -Destination $dest -Recurse -Force + Write-Host "Copying spec from $source to $dest" + + foreach ($additionalDir in $specAdditionalSubDirectories) { + $source = "$specCloneRoot/$additionalDir" + Write-Host "Copying spec from $source to $dest" + Copy-Item -Path $source -Destination $dest -Recurse -Force + } +} + +function UpdateSparseCheckoutFile([string]$mainSpecDir, [string[]]$specAdditionalSubDirectories) { + AddSparseCheckoutPath $mainSpecDir + foreach ($subDir in $specAdditionalSubDirectories) { + AddSparseCheckoutPath $subDir + } +} + +function GetGitRemoteValue([string]$repo) { + Push-Location $ProjectDirectory + $result = "" + try { + $gitRemotes = (git remote -v) + foreach ($remote in $gitRemotes) { + if ($remote.StartsWith("origin")) { + if ($remote -match 'https://github.com/\S+') { + $result = "https://github.com/$repo.git" + break + } elseif ($remote -match "git@github.com:\S+"){ + $result = "git@github.com:$repo.git" + break + } else { + throw "Unknown git remote format found: $remote" + } + } + } + } + finally { + Pop-Location + } + + return $result +} + +function InitializeSparseGitClone([string]$repo) { + git clone --no-checkout --filter=tree:0 $repo . + if ($LASTEXITCODE) { exit $LASTEXITCODE } + git sparse-checkout init + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Remove-Item $sparseCheckoutFile -Force +} + +function GetSpecCloneDir([string]$projectName) { + Push-Location $ProjectDirectory + try { + $root = git rev-parse --show-toplevel + } + finally { + Pop-Location + } + + $sparseSpecCloneDir = "$root/../sparse-spec/$projectName" + New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null + $createResult = Resolve-Path $sparseSpecCloneDir + return $createResult +} + +$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" +Write-Host "Reading configuration from $cadlConfigurationFile" +$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml + +$pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/") +$projectName = $pieces[$pieces.Count - 2] + +$specSubDirectory = $configuration["directory"] + +if ( $configuration["repo"] -and $configuration["commit"]) { + $specCloneDir = GetSpecCloneDir $projectName + $gitRemoteValue = GetGitRemoteValue $configuration["repo"] + + Write-Host "Setting up sparse clone for $projectName at $specCloneDir" + + Push-Location $specCloneDir.Path + try { + if (!(Test-Path ".git")) { + InitializeSparseGitClone $gitRemoteValue + UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"] + } + git checkout $configuration["commit"] + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } + finally { + Pop-Location + } +} elseif ( $configuration["spec-root-dir"] ) { + $specCloneDir = $configuration["spec-root-dir"] +} + + +$tempCadlDir = "$ProjectDirectory/TempCadlFiles" +New-Item $tempCadlDir -Type Directory -Force | Out-Null +CopySpecToProjectIfNeeded ` + -specCloneRoot $specCloneDir ` + -mainSpecDir $specSubDirectory ` + -dest $tempCadlDir ` + -specAdditionalSubDirectories $configuration["additionalDirectories"] From aa20a57446619e8d7fbedcb046d7ca8958ef96e8 Mon Sep 17 00:00:00 2001 From: "FAREAST\\chunyu" Date: Tue, 14 Mar 2023 11:15:41 +0800 Subject: [PATCH 3/5] support .cadl and .tsp --- eng/common/scripts/Typespec-Project-Generate.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Typespec-Project-Generate.ps1 b/eng/common/scripts/Typespec-Project-Generate.ps1 index 43b6beab649..3dc9a6ec1b0 100644 --- a/eng/common/scripts/Typespec-Project-Generate.ps1 +++ b/eng/common/scripts/Typespec-Project-Generate.ps1 @@ -65,7 +65,7 @@ $innerFolder = Split-Path $specSubDirectory -Leaf $tempFolder = "$ProjectDirectory/TempTypespecFiles" $npmWorkingDir = Resolve-Path $tempFolder/$innerFolder -$mainTypespecFile = If (Test-Path "$npmWorkingDir/client.tsp") { Resolve-Path "$npmWorkingDir/client.tsp" } Else { Resolve-Path "$npmWorkingDir/main.tsp"} +$mainTypespecFile = If (Test-Path "$npmWorkingDir/client.*") { Resolve-Path "$npmWorkingDir/client.*" } Else { Resolve-Path "$npmWorkingDir/main.*"} try { Push-Location $npmWorkingDir From 0dfa000a0480d022c618e648c58741058e6aefe6 Mon Sep 17 00:00:00 2001 From: "FAREAST\\chunyu" Date: Wed, 15 Mar 2023 10:48:20 +0800 Subject: [PATCH 4/5] rename --- doc/common/Typespec-Project-Scripts.md | 32 +++++++++---------- ...rate.ps1 => TypeSpec-Project-Generate.ps1} | 16 +++++----- ...ect-Sync.ps1 => TypeSpec-Project-Sync.ps1} | 10 +++--- 3 files changed, 29 insertions(+), 29 deletions(-) rename eng/common/scripts/{Typespec-Project-Generate.ps1 => TypeSpec-Project-Generate.ps1} (86%) rename eng/common/scripts/{Typespec-Project-Sync.ps1 => TypeSpec-Project-Sync.ps1} (93%) diff --git a/doc/common/Typespec-Project-Scripts.md b/doc/common/Typespec-Project-Scripts.md index 179cd9a54c5..1e7343fc543 100644 --- a/doc/common/Typespec-Project-Scripts.md +++ b/doc/common/Typespec-Project-Scripts.md @@ -6,17 +6,17 @@ repo and use the remote typespec definition in the spec repo. ## One time language repo setup There are 3 things that these two scripts expect are set up in your language repo before they will run correctly. -1. Make sure your .gitignore is ignoring the TempTypespecFiles +1. Make sure your .gitignore is ignoring the TempTypeSpecFiles 2. Create a common emitter-package.json for your language 3. Write the language specific hooks in Language-Settings.ps1 -### TempTypespecFiles +### TempTypeSpecFiles You should add a new entry in your .gitignore for your repo so that none of these files are accidentally checked in if [cleanup](#cleanup-anchor) is turned off. ``` # .gitignore file -TempTypespecFiles/ +TempTypeSpecFiles/ ``` ### emitter-package.json @@ -82,7 +82,7 @@ function Get-dotnet-EmitterAdditionalOptions([string]$projectDirectory) { Each project will need to have a configuration file that will tell the scripts where to find the typespec spec. -### typespec-location.yaml +### tsp-location.yaml This file should live under the project directory for each service and has the following properties @@ -92,7 +92,7 @@ This file should live under the project directory for each service and has the f | additionalDirectories | Sometimes a typespec file will use a relative import that might not be under the main directory. In this case a single `directory` will not be enough to pull down all necessary files. To support this you can specify additional directories as a list to sync so that all needed files are synced. | false: default = null | | commit | The commit sha for the version of the typespec files you want to generate off of. This allows us to have idempotence on generation until we opt into pointing at a later version. | true | | repo | The repo this spec lives in. This should be either `Azure/azure-rest-api-specs` or `Azure/azure-rest-api-specs-pr`. Note that pr will work locally but not in CI until we add another change to handle token based auth. | true | -| cleanup | This will remove the TempTypespecFiles directory after generation is complete if true otherwise this directory will be left to support local changes to the files to see how different changes would affect the generation. | false: default = true | +| cleanup | This will remove the TempTypeSpecFiles directory after generation is complete if true otherwise this directory will be left to support local changes to the files to see how different changes would affect the generation. | false: default = true | Example @@ -105,41 +105,41 @@ repo: Azure/azure-rest-api-specs cleanup: false ``` -## Typespec-Project-Sync.ps1 +## TypeSpec-Project-Sync.ps1 -This is the first script that should be called and can be found at `./eng/common/scripts/Typespec-Project-Sync.ps1`. It takes in one parameter which is the root directory of the project which is typically one layer lower than the service directory. As an example for dotnet this is `./sdk/openai/Azure.AI.OpenAI` where `openai` is the service directory and `Azure.AI.OpenAI` is the project directory. +This is the first script that should be called and can be found at `./eng/common/scripts/TypeSpec-Project-Sync.ps1`. It takes in one parameter which is the root directory of the project which is typically one layer lower than the service directory. As an example for dotnet this is `./sdk/openai/Azure.AI.OpenAI` where `openai` is the service directory and `Azure.AI.OpenAI` is the project directory. ```powershell -./eng/common/scripts/Typespec-Project-Sync.ps1 ./sdk/openai/Azure.AI.OpenAI +./eng/common/scripts/TypeSpec-Project-Sync.ps1 ./sdk/openai/Azure.AI.OpenAI ``` -This script will create a `sparse-spec` folder as a sibling to the root of your current git clone. Each project that is generated will get a sub directory inside of this folder named after the project you are generating. It will then automatically filter to only the files in the [directory](#directory-anchor) defined in typespec-location.yaml, and sync to the [commit sha](#commit-anchor) defined in typespec-location.yaml. +This script will create a `sparse-spec` folder as a sibling to the root of your current git clone. Each project that is generated will get a sub directory inside of this folder named after the project you are generating. It will then automatically filter to only the files in the [directory](#directory-anchor) defined in tsp-location.yaml, and sync to the [commit sha](#commit-anchor) defined in tsp-location.yaml. As an example if you have your language repo at `D:\git\azure-sdk-for-net` there will be a new directory `D:\git\sparse-spec\Azure.AI.OpenAI` where the sparse spec will live. -This is then copied over to your project directory so that you can make temporary changes if needed. The location will be `./{projectDir}/TempTypespecFiles`. This temporary directory will be [cleaned up](#cleanup-anchor) at the end of the generate script if set in the typespec-location.yaml. +This is then copied over to your project directory so that you can make temporary changes if needed. The location will be `./{projectDir}/TempTypeSpecFiles`. This temporary directory will be [cleaned up](#cleanup-anchor) at the end of the generate script if set in the tsp-location.yaml. -## Typespec-Project-Generate.ps1 +## TypeSpec-Project-Generate.ps1 -This is the second script that should be called and can be found at `./eng/common/scripts/Typespec-Project-Generate.ps1`. It takes the exact same parameter as the sync script. +This is the second script that should be called and can be found at `./eng/common/scripts/TypeSpec-Project-Generate.ps1`. It takes the exact same parameter as the sync script. ```powershell -./eng/common/scripts/Typespec-Project-Generate.ps1 ./sdk/openai/Azure.AI.OpenAI +./eng/common/scripts/TypeSpec-Project-Generate.ps1 ./sdk/openai/Azure.AI.OpenAI ``` -The first thing this does is clean up the npm install that might exist in `./{projectDir}/TempTypespecFiles`, followed by replacing the package.json with the language static one. +The first thing this does is clean up the npm install that might exist in `./{projectDir}/TempTypeSpecFiles`, followed by replacing the package.json with the language static one. Once this is done it will run `npm install` followed by `tsp compile` which is the standard way to generate a typespec project. The exact command that gets run is output stdout to enable debugging if needed. -We currently don't do anything to the typespec-project.yaml that gets pulled in from the spec repo to limit to just your language emitter instead we use the filter option on the command line `--emit $emitterName`. This allows you to isolate the generation to only things owned by your language so you can safely add generation dependencies in CI without needing to worry about noisy neighbors. +We currently don't do anything to the tspconfig.yaml that gets pulled in from the spec repo to limit to just your language emitter instead we use the filter option on the command line `--emit $emitterName`. This allows you to isolate the generation to only things owned by your language so you can safely add generation dependencies in CI without needing to worry about noisy neighbors. ## Build tool integration One use case that some languages have is to have their CI regenerate the project and then do a `git diff` to validate that there are no differences. This helps detect if people modify the generated files manually. To support this its valuable to have the exact same command to generate a project regardless of whether the individual library is autorest or typespec. -To achieve this each language will have their own idiomatic tool set but whatever that toolset is can check to see if a typespec-location.yaml file exists, and if it does they can call the Typespec-Project-Sync.ps1 and Typespec-Project-Generate.ps1 scripts, otherwise they can call the autorest command they call today for all other libraries. +To achieve this each language will have their own idiomatic tool set but whatever that toolset is can check to see if a tsp-location.yaml file exists, and if it does they can call the TypeSpec-Project-Sync.ps1 and TypeSpec-Project-Generate.ps1 scripts, otherwise they can call the autorest command they call today for all other libraries. In dotnet this is achieved by running `dotnet build /t:GenerateCode` regardless of which type of project it is the correct commands get called and it remains consistent and idiomatic to the language. In other languages this could be `npm generate` or `python generate.py` to do the same. diff --git a/eng/common/scripts/Typespec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 similarity index 86% rename from eng/common/scripts/Typespec-Project-Generate.ps1 rename to eng/common/scripts/TypeSpec-Project-Generate.ps1 index 3dc9a6ec1b0..d1e728895ab 100644 --- a/eng/common/scripts/Typespec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -1,4 +1,4 @@ -# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Typespec-Project-Scripts.md +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/TypeSpec-Project-Scripts.md [CmdletBinding()] param ( @@ -6,7 +6,7 @@ param ( [ValidateNotNullOrEmpty()] [string] $ProjectDirectory, [Parameter(Position=1)] - [string] $TypespecAdditionalOptions ## addtional typespec emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 + [string] $typespecAdditionalOptions ## addtional typespec emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 ) $ErrorActionPreference = "Stop" @@ -55,7 +55,7 @@ function NpmInstallForProject([string]$workingDirectory) { $resolvedProjectDirectory = Resolve-Path $ProjectDirectory $emitterName = &$GetEmitterNameFn -$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/typespec-location.yaml" +$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/tsp-location.yaml" Write-Host "Reading configuration from $typespecConfigurationFile" $configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml @@ -63,9 +63,9 @@ $configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom $specSubDirectory = $configuration["directory"] $innerFolder = Split-Path $specSubDirectory -Leaf -$tempFolder = "$ProjectDirectory/TempTypespecFiles" +$tempFolder = "$ProjectDirectory/TempTypeSpecFiles" $npmWorkingDir = Resolve-Path $tempFolder/$innerFolder -$mainTypespecFile = If (Test-Path "$npmWorkingDir/client.*") { Resolve-Path "$npmWorkingDir/client.*" } Else { Resolve-Path "$npmWorkingDir/main.*"} +$mainTypeSpecFile = If (Test-Path "$npmWorkingDir/client.*") { Resolve-Path "$npmWorkingDir/client.*" } Else { Resolve-Path "$npmWorkingDir/main.*"} try { Push-Location $npmWorkingDir @@ -79,9 +79,9 @@ try { $emitterAdditionalOptions = " $emitterAdditionalOptions" } } - $typespecCompileCommand = "npx tsp compile $mainTypespecFile --emit $emitterName$emitterAdditionalOptions" - if ($TypespecAdditionalOptions) { - $options = $TypespecAdditionalOptions.Split(";"); + $typespecCompileCommand = "npx tsp compile $mainTypeSpecFile --emit $emitterName$emitterAdditionalOptions" + if ($typespecAdditionalOptions) { + $options = $typespecAdditionalOptions.Split(";"); foreach ($option in $options) { $typespecCompileCommand += " --option $emitterName.$option" } diff --git a/eng/common/scripts/Typespec-Project-Sync.ps1 b/eng/common/scripts/TypeSpec-Project-Sync.ps1 similarity index 93% rename from eng/common/scripts/Typespec-Project-Sync.ps1 rename to eng/common/scripts/TypeSpec-Project-Sync.ps1 index 24f68a8c54f..f8656bc1d60 100644 --- a/eng/common/scripts/Typespec-Project-Sync.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Sync.ps1 @@ -1,4 +1,4 @@ -# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/Typespec-Project-Scripts.md +# For details see https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/TypeSpec-Project-Scripts.md [CmdletBinding()] param ( @@ -86,7 +86,7 @@ function GetSpecCloneDir([string]$projectName) { return $createResult } -$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/typespec-location.yaml" +$typespecConfigurationFile = Resolve-Path "$ProjectDirectory/tsp-location.yaml" Write-Host "Reading configuration from $typespecConfigurationFile" $configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml @@ -118,10 +118,10 @@ if ( $configuration["repo"] -and $configuration["commit"]) { } -$tempTypespecDir = "$ProjectDirectory/TempTypespecFiles" -New-Item $tempTypespecDir -Type Directory -Force | Out-Null +$tempTypeSpecDir = "$ProjectDirectory/TempTypeSpecFiles" +New-Item $tempTypeSpecDir -Type Directory -Force | Out-Null CopySpecToProjectIfNeeded ` -specCloneRoot $specCloneDir ` -mainSpecDir $specSubDirectory ` - -dest $tempTypespecDir ` + -dest $tempTypeSpecDir ` -specAdditionalSubDirectories $configuration["additionalDirectories"] From 0e1be3f58f0a855e498ce156e0e03dce89d322a5 Mon Sep 17 00:00:00 2001 From: "FAREAST\\chunyu" Date: Thu, 16 Mar 2023 10:54:01 +0800 Subject: [PATCH 5/5] add newline at the end of file --- eng/common/scripts/TypeSpec-Project-Generate.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/TypeSpec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 index d1e728895ab..feba00d37ed 100644 --- a/eng/common/scripts/TypeSpec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -98,4 +98,4 @@ finally { $shouldCleanUp = $configuration["cleanup"] ?? $true if ($shouldCleanUp) { Remove-Item $tempFolder -Recurse -Force -} \ No newline at end of file +}