[sdlf-*] restore legacy seedfarmer deployspec

awslabs · Dec 2, 2024 · 28ca94c · 28ca94c
1 parent 0c815a6
commit 28ca94c
Show file tree

Hide file tree

Showing 6 changed files with 510 additions and 61 deletions.
diff --git a/sdlf-dataset/deployspec.yaml b/sdlf-dataset/deployspec.yaml
@@ -1,29 +1,142 @@
-publishGenericEnvVariables: True
+publishGenericEnvVariables: true
 deploy:
   phases:
     install:
       commands:
-        - npm install -g aws-cdk
         - |-
-          pip install --upgrade pip setuptools
-          pip install poetry
-          poetry config virtualenvs.create false --local
-          poetry install -v
+            # force Python3.12
+            rm -Rf ~/.venv/
+            pyenv global 3.12
+            python3 -m venv ~/.venv
+            . ~/.venv/bin/activate
+            pip install --upgrade pip setuptools
+            pip install aws-codeseeder~=1.1.0 seed-farmer==5.0.0
+        - |-
+            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
+            ARTIFACTS_BUCKET=$(aws cloudformation --endpoint-url "$CFN_ENDPOINT" describe-stacks --query "Stacks[?StackName=='aws-codeseeder-$SEEDFARMER_PROJECT_NAME'][].Outputs[?OutputKey=='Bucket'].OutputValue" --output text)
+
+            deps=(
+              "https://raw.githubusercontent.com/aws/serverless-application-model/develop/bin/sam-translate.py"
+              "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip"
+              "https://raw.githubusercontent.com/awslabs/aws-serverless-data-lake-framework/main/sdlf-cicd/template-generic-cfn-module.yaml"
+            )
+            for u in "${deps[@]}"; do
+              aws s3api get-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" "${u##*/}" || {
+                curl -L -O "$u"
+                aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" --body "${u##*/}"
+              }
+            done
+        - |-
+            pip uninstall -y aws-sam-cli && unzip -q aws-sam-cli-linux-x86_64.zip -d sam-installation
+            ./sam-installation/install && sam --version
+            pip install "cfn-lint<1" cloudformation-cli poetry
+            npm install -g [email protected]
+            poetry config virtualenvs.create false --local
     build:
       commands:
         - |-
-          cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json
-        - seedfarmer metadata convert -f cdk-exports.json || true
+          # deployment-type possible values: cfn-module, cdk-construct
+          # cfn-module creates a CloudFormation Registry module out of template.yaml
+          # cdk-construct publishes a pip library out of template.py on CodeArtifact
+          echo "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
+        - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
+            # SEEDFARMER_PARAMETER_LIBRARY_MODULE can be used to pass the module name. If not, the module name is inferred from SEEDFARMER_MODULE_NAME
+            # by removing everything up to the first hyphen, then anything that isn't a letter/number, and lower-casing everything.
+            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
+            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
+            MODULE="${sf_module_name_alnum,,}"
+
+            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
+            cd src || exit
+            sam package --template-file ./dataset.yaml --s3-bucket "$ARTIFACTS_BUCKET" --s3-prefix sdlf --output-template-file template.yaml
+            python3 ../sam-translate.py --template-file=template.yaml --output-template=translated-template.json
+
+            SSM_ENDPOINT="https://ssm.$AWS_REGION.amazonaws.com"
+            TEMPLATE_BASE_FILE_PATH="modules/$SEEDFARMER_PARAMETER_LIBRARY_ORG/$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK/$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH/translated-template.json" --body translated-template.json
+            TEMPLATE_URL="https://$ARTIFACTS_BUCKET.s3.$AWS_REGION.amazonaws.com/$TEMPLATE_BASE_FILE_PATH/translated-template.json"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" validate-template --template-url "$TEMPLATE_URL"
+
+            mkdir module
+            cd module || exit
+            cfn init --artifact-type MODULE --type-name "$SEEDFARMER_PARAMETER_LIBRARY_ORG::$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK::$SEEDFARMER_PARAMETER_LIBRARY_MODULE::MODULE" && rm fragments/sample.json
+            cp -i -a ../translated-template.json fragments/
+            cfn generate
+            zip -q -r "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" .rpdk-config fragments/ schema.json
+
+            NEW_MODULE="$(sha256sum "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" | cut -c1-12)"
+            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH-$NEW_MODULE.zip" --body "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip"
+
+            if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT" get-parameter --name "/SDLF/CFN/$SEEDFARMER_PARAMETER_LIBRARY_ORG-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE-MODULE" --query "Parameter.Value" --output text); then
+              echo "Current module hash: $CURRENT_MODULE / New module hash: $NEW_MODULE"
+              if [ "$NEW_MODULE" == "$CURRENT_MODULE" ]; then
+                echo "No change since last build, exiting module creation."
+                exit 0
+              fi
+            fi
+
+            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" deploy \
+              --stack-name "$STACK_NAME" \
+              --template-file ../../template-generic-cfn-module.yaml \
+              --parameter-overrides \
+                  pArtifactsBucket="$ARTIFACTS_BUCKET" \
+                  pLibraryOrg="$SEEDFARMER_PARAMETER_LIBRARY_ORG" \
+                  pLibraryFramework="$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK" \
+                  pLibraryModule="$SEEDFARMER_PARAMETER_LIBRARY_MODULE" \
+                  pModuleGitRef="$NEW_MODULE" \
+              --tags Framework=sdlf || exit 1
+            echo "done"
+            cd .. && rm -Rf module
+          fi
+        - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
+            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
+            CA_REPOSITORY_ENDPOINT=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-repository-endpoint --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --query repositoryEndpoint --output text)
+            CA_TOKEN=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-authorization-token --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --query authorizationToken --output text)
+            poetry source add private "https://aws:${CA_TOKEN}@${CA_REPOSITORY_ENDPOINT#https://}simple/"
+            poetry install -v
+            poetry config repositories.private "$CA_REPOSITORY_ENDPOINT"
+            poetry config http-basic.private aws "$CA_TOKEN"
+            poetry publish --skip-existing --build -r private || exit 1
+          fi
+    post_build:
+      commands:
+      - echo "Deploy successful"
 destroy:
   phases:
     install:
       commands:
-        - npm install -g aws-cdk
         - |-
-          pip install --upgrade pip setuptools
-          pip install poetry
-          poetry config virtualenvs.create false --local
-          poetry install -v
+            pip install --upgrade pip setuptools
+            pip install poetry
     build:
       commands:
-        - cdk destroy --force --all --app "python src/app.py"
+      - |-
+          echo "DESTROY! $SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
+      - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
+            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
+            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
+            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
+            MODULE="${sf_module_name_alnum,,}"
+            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
+            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" delete-stack --stack-name "$STACK_NAME" || exit 1
+          fi
+      - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
+            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
+            package_version_poetry=$(poetry version --short)
+            package_prerelease_id=$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f1)
+            pypi_prerelease_id=$(test "$package_prerelease_id" = "rc" && echo "rc" || echo "${package_prerelease_id:0:1}")
+            pypi_package_version="$(echo "$package_version_poetry" | cut -d'-' -f1)$pypi_prerelease_id$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f2)"
+            aws codeartifact --endpoint-url "$CA_ENDPOINT" delete-package-versions \
+              --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" \
+              --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --package "$(poetry version | cut -d' ' -f1)" --versions "$pypi_package_version"
+          fi
+    post_build:
+      commands:
+      - echo "Destroy successful"
+build_type: BUILD_GENERAL1_SMALL
diff --git a/sdlf-foundations/deployspec.yaml b/sdlf-foundations/deployspec.yaml
@@ -1,30 +1,142 @@
-publishGenericEnvVariables: True
+publishGenericEnvVariables: true
 deploy:
   phases:
     install:
       commands:
-        - npm install -g aws-cdk
         - |-
-          pip install --upgrade pip setuptools
-          pip install poetry
-          poetry config virtualenvs.create false --local
-          poetry install -v
+            # force Python3.12
+            rm -Rf ~/.venv/
+            pyenv global 3.12
+            python3 -m venv ~/.venv
+            . ~/.venv/bin/activate
+            pip install --upgrade pip setuptools
+            pip install aws-codeseeder~=1.1.0 seed-farmer==5.0.0
+        - |-
+            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
+            ARTIFACTS_BUCKET=$(aws cloudformation --endpoint-url "$CFN_ENDPOINT" describe-stacks --query "Stacks[?StackName=='aws-codeseeder-$SEEDFARMER_PROJECT_NAME'][].Outputs[?OutputKey=='Bucket'].OutputValue" --output text)
+
+            deps=(
+              "https://raw.githubusercontent.com/aws/serverless-application-model/develop/bin/sam-translate.py"
+              "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip"
+              "https://raw.githubusercontent.com/awslabs/aws-serverless-data-lake-framework/main/sdlf-cicd/template-generic-cfn-module.yaml"
+            )
+            for u in "${deps[@]}"; do
+              aws s3api get-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" "${u##*/}" || {
+                curl -L -O "$u"
+                aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" --body "${u##*/}"
+              }
+            done
+        - |-
+            pip uninstall -y aws-sam-cli && unzip -q aws-sam-cli-linux-x86_64.zip -d sam-installation
+            ./sam-installation/install && sam --version
+            pip install "cfn-lint<1" cloudformation-cli poetry
+            npm install -g [email protected]
+            poetry config virtualenvs.create false --local
     build:
       commands:
         - |-
-          env | grep "SEEDFARMER"
-          cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json
-        - seedfarmer metadata convert -f cdk-exports.json || true
+          # deployment-type possible values: cfn-module, cdk-construct
+          # cfn-module creates a CloudFormation Registry module out of template.yaml
+          # cdk-construct publishes a pip library out of template.py on CodeArtifact
+          echo "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
+        - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
+            # SEEDFARMER_PARAMETER_LIBRARY_MODULE can be used to pass the module name. If not, the module name is inferred from SEEDFARMER_MODULE_NAME
+            # by removing everything up to the first hyphen, then anything that isn't a letter/number, and lower-casing everything.
+            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
+            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
+            MODULE="${sf_module_name_alnum,,}"
+
+            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
+            cd src || exit
+            sam package --template-file ./foundations.yaml --s3-bucket "$ARTIFACTS_BUCKET" --s3-prefix sdlf --output-template-file template.yaml
+            python3 ../sam-translate.py --template-file=template.yaml --output-template=translated-template.json
+
+            SSM_ENDPOINT="https://ssm.$AWS_REGION.amazonaws.com"
+            TEMPLATE_BASE_FILE_PATH="modules/$SEEDFARMER_PARAMETER_LIBRARY_ORG/$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK/$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH/translated-template.json" --body translated-template.json
+            TEMPLATE_URL="https://$ARTIFACTS_BUCKET.s3.$AWS_REGION.amazonaws.com/$TEMPLATE_BASE_FILE_PATH/translated-template.json"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" validate-template --template-url "$TEMPLATE_URL"
+
+            mkdir module
+            cd module || exit
+            cfn init --artifact-type MODULE --type-name "$SEEDFARMER_PARAMETER_LIBRARY_ORG::$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK::$SEEDFARMER_PARAMETER_LIBRARY_MODULE::MODULE" && rm fragments/sample.json
+            cp -i -a ../translated-template.json fragments/
+            cfn generate
+            zip -q -r "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" .rpdk-config fragments/ schema.json
+
+            NEW_MODULE="$(sha256sum "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" | cut -c1-12)"
+            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH-$NEW_MODULE.zip" --body "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip"
+
+            if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT" get-parameter --name "/SDLF/CFN/$SEEDFARMER_PARAMETER_LIBRARY_ORG-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE-MODULE" --query "Parameter.Value" --output text); then
+              echo "Current module hash: $CURRENT_MODULE / New module hash: $NEW_MODULE"
+              if [ "$NEW_MODULE" == "$CURRENT_MODULE" ]; then
+                echo "No change since last build, exiting module creation."
+                exit 0
+              fi
+            fi
+
+            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" deploy \
+              --stack-name "$STACK_NAME" \
+              --template-file ../../template-generic-cfn-module.yaml \
+              --parameter-overrides \
+                  pArtifactsBucket="$ARTIFACTS_BUCKET" \
+                  pLibraryOrg="$SEEDFARMER_PARAMETER_LIBRARY_ORG" \
+                  pLibraryFramework="$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK" \
+                  pLibraryModule="$SEEDFARMER_PARAMETER_LIBRARY_MODULE" \
+                  pModuleGitRef="$NEW_MODULE" \
+              --tags Framework=sdlf || exit 1
+            echo "done"
+            cd .. && rm -Rf module
+          fi
+        - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
+            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
+            CA_REPOSITORY_ENDPOINT=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-repository-endpoint --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --query repositoryEndpoint --output text)
+            CA_TOKEN=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-authorization-token --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --query authorizationToken --output text)
+            poetry source add private "https://aws:${CA_TOKEN}@${CA_REPOSITORY_ENDPOINT#https://}simple/"
+            poetry install -v
+            poetry config repositories.private "$CA_REPOSITORY_ENDPOINT"
+            poetry config http-basic.private aws "$CA_TOKEN"
+            poetry publish --skip-existing --build -r private || exit 1
+          fi
+    post_build:
+      commands:
+      - echo "Deploy successful"
 destroy:
   phases:
     install:
       commands:
-        - npm install -g aws-cdk
         - |-
-          pip install --upgrade pip setuptools
-          pip install poetry
-          poetry config virtualenvs.create false --local
-          poetry install -v
+            pip install --upgrade pip setuptools
+            pip install poetry
     build:
       commands:
-        - cdk destroy --force --all --app "python src/app.py"
+      - |-
+          echo "DESTROY! $SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
+      - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
+            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
+            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
+            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
+            MODULE="${sf_module_name_alnum,,}"
+            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
+            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
+            aws cloudformation --endpoint-url "$CFN_ENDPOINT" delete-stack --stack-name "$STACK_NAME" || exit 1
+          fi
+      - |-
+          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
+            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
+            package_version_poetry=$(poetry version --short)
+            package_prerelease_id=$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f1)
+            pypi_prerelease_id=$(test "$package_prerelease_id" = "rc" && echo "rc" || echo "${package_prerelease_id:0:1}")
+            pypi_package_version="$(echo "$package_version_poetry" | cut -d'-' -f1)$pypi_prerelease_id$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f2)"
+            aws codeartifact --endpoint-url "$CA_ENDPOINT" delete-package-versions \
+              --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" \
+              --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --package "$(poetry version | cut -d' ' -f1)" --versions "$pypi_package_version"
+          fi
+    post_build:
+      commands:
+      - echo "Destroy successful"
+build_type: BUILD_GENERAL1_SMALL