Adding infra generation pipeline template with keyvault enabled (#400)

* adding key vault guidance

* updates to infra kv guidance

* syntax update

* adding permit access note

* Pushing Keyvault generation pipeline

* updating docs var names

* update kv doc on versioning

* updating docs

* removing duplicate var set

Author: NathanielRose

azure-pipelines/templates/infra-generation-kv-pipeline.yml

@@ -0,0 +1,145 @@
+trigger:
+  branches:
+    include:
+    - master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+variables:
+  - group: 'spk-infra-hld-vg-kv'
+
+steps:
+- checkout: self
+  persistCredentials: true
+  clean: true
+
+- bash: |
+    curl $BEDROCK_BUILD_SCRIPT > build.sh
+    chmod +x ./build.sh
+  displayName: Download Bedrock orchestration script
+  env:
+    BEDROCK_BUILD_SCRIPT: https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh
+
+- script: |
+    dpkg --configure -a
+    curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+    az extension add --name azure-devops
+    az --version
+    wget https://releases.hashicorp.com/terraform/$(tf_version)/terraform_$(tf_version)_linux_amd64.zip -q
+    unzip -q terraform_$(tf_version)_linux_amd64.zip
+    sudo mv terraform /usr/local/bin
+    terraform -version
+  displayName: 'Install az-cli, az devops extension, and Terraform'
+
+- script: |
+    . build.sh --source-only
+    init
+    get_os_spk
+    get_spk_version
+    download_spk
+    echo 'SPK Version: '
+    spk --version
+    set -e
+    echo "GENERATING $(PROJECTDIRECTORY)"
+    cd $(PROJECTDIRECTORY)
+    spk infra generate -p $(CLUSTER)
+  env:
+    ARM_CLIENT_ID: $(ARMCLIENTID)
+    ARM_CLIENT_SECRET: $(ARMCLIENTSECRET)
+    ARM_TENANT_ID: $(ARMTENANTID)
+    ARM_SUBSCRIPTION_ID: $(ARMSUBSCRIPTIONID)
+  displayName: 'SPK'
+
+- script: |
+    . build.sh --source-only
+    init
+    cd $(PROJECTDIRECTORY)-generated/$(CLUSTER)
+    terraform init -backend-config=./backend.tfvars
+  displayName: 'Terraform init'
+
+- script: |
+    . build.sh --source-only
+    init
+    cd $(PROJECTDIRECTORY)-generated/$(CLUSTER)
+    terraform plan -var-file=./spk.tfvars
+  env:
+    ARM_CLIENT_ID: $(ARMCLIENTID)
+    ARM_CLIENT_SECRET: $(ARMCLIENTSECRET)
+    ARM_TENANT_ID: $(ARMTENANTID)
+    ARM_SUBSCRIPTION_ID: $(ARMSUBSCRIPTIONID)
+  displayName: 'Terraform plan'
+
+- script: |
+    # Exit on error
+    set -e
+
+    # Retrieve most recent commit hash from HLD repo
+    commit_hash=$(git rev-parse HEAD)
+    echo "Commit Hash: $commit_hash"
+
+    # Clone Generated Repo and copy generated components over
+    echo "Cloning Generated Repo: $(GENERATEDREPO)"
+    git clone $(GENERATEDREPO)
+
+    # Extract repo name from url
+    repo_url=$(GENERATEDREPO)
+    repo=${repo_url##*/}
+    repo_name=${repo%.*}
+
+    cd "$repo_name"
+    rsync -rv --exclude=.terraform $HOME/$(PROJECTDIRECTORY)-generated .
+
+    # Set git identity
+    git config user.email "[email protected]"
+    git config user.name "Automated Account"
+
+    # Following variables have to be set for TeamCity
+    export GIT_AUTHOR_NAME="Automated Account"
+    export GIT_COMMITTER_NAME="Automated Account"
+    export EMAIL="[email protected]"
+
+    # Format Terraform files
+    terraform fmt
+    
+    # Add generated files to repository
+    git status
+    git add .
+    git commit -m "Adding generated components for $(PROJECTDIRECTORY)/$(CLUSTER)"
+
+    # Create a new branch
+    PR_BRANCH_NAME=pr_$commit_hash
+    git checkout -b $PR_BRANCH_NAME
+
+    # Git Push
+    repo_url=$(GENERATEDREPO)
+    repo_url="${repo_url#http://}"
+    repo_url="${repo_url#https://}"
+    echo "$repo_url"
+    echo "GIT PUSH: https://<ACCESS_TOKEN_SECRET>@$repo_url"
+    git push "https://$ACCESS_TOKEN_SECRET@$repo_url" $PR_BRANCH_NAME
+
+    # Create PR
+    # If using GitHub repos:
+    echo "CREATE PULL REQUEST"
+    if [[ $(GENERATEDREPO) == *"github"* ]]; then
+      echo "Installing Hub"
+      sudo add-apt-repository ppa:cpick/hub
+      sudo apt-get update
+      sudo apt-get install hub
+      export GITHUB_TOKEN=$ACCESS_TOKEN_SECRET
+      hub pull-request -m "Adding Generated Components for $PROJECT_DIRECTORY/$CLUSTER"
+    elif [[ $(GENERATEDREPO) == *"azure"* ]] || [[ $(GENERATEDREPO) == *"visualstudio"* ]]; then
+    # If using AzDo repos:
+    # You will need to specify variables $AZDO-ORG-NAME and $AZDO-PROJECT-NAME
+      export AZURE_DEVOPS_EXT_PAT=$ACCESS_TOKEN_SECRET
+      az devops configure --defaults organization=$(AZDOORGNAME) project=$(AZDOPROJECTNAME)
+      echo "Making pull request for $PR_BRANCH_NAME against master"
+      az repos pr create --description "Automated PR for $PR_BRANCH_NAME against master"
+    else
+      echo "Unable to create pull request."
+      exit 1
+    fi
+  env:
+    ACCESS_TOKEN_SECRET: $(ACCESSTOKENSECRET)
+  displayName: 'Commit and Push to Generated Repository' 

azure-pipelines/templates/infra-generation-pipeline.yml

@@ -81,7 +81,6 @@ steps:
     # Clone Generated Repo and copy generated components over
     echo "Cloning Generated Repo: $GENERATED_REPO"
     git clone $GENERATED_REPO
-    repo_url=$GENERATED_REPO
 
     # Extract repo name from url
     repo_url=$GENERATED_REPO

guides/infra/spk-infra-generation-pipeline.md

@@ -120,25 +120,25 @@ name: "spk-infra-hld-vg-kv"
 description: "key vault variable group for infra hld"
 type: "AzureKeyVault"
 variables:
-  ACCESS-TOKEN-SECRET:
+  ACCESSTOKENSECRET:
     enabled: true
-  ARM-CLIENT-ID:
+  ARMCLIENTID:
     enabled: true
-  ARM-CLIENT-SECRET:
+  ARMCLIENTSECRET:
     enabled: true
-  ARM-SUBSCRIPTION-ID:
+  ARMSUBSCRIPTIONID:
     enabled: true
-  ARM-TENANT-ID:
+  ARMTENANTID:
     enabled: true
   CLUSTER:
     enabled: true
-  GENERATED-REPO:
+  GENERATEDREPO:
     enabled: true
-  PROJECT-DIRECTORY:
+  PROJECTDIRECTORY:
     enabled: true
-  AZDO-ORG-NAME: (optional)
+  AZDOORGNAME: (optional)
     enabled: true
-  AZDO-PROJECT-NAME: (optional)
+  AZDOPROJECTNAME: (optional)
     enabled: true
 key_vault_provider:
   name: "myvault"                                                # name of the Azure Key Vaukt with Secrets
@@ -167,16 +167,23 @@ previously provisioned.
 
 ![](../images/kvsetupvg.png)
 
-Additionally, be sure to select the respective Key secrets you wish to map to
-your variable group.
+Additionally, be sure to select the respective key secrets you wish to map to
+your variable group. Only the secret _names_ are mapped to the variable group,
+not the secret values. The latest version of the value of each secret is fetched
+from the vault and used in the pipeline linked to the variable group during the
+run.
 
 ![](../images/secrets-kv-vg.png)
 
-> When using a variable group, you may be prompted to grant access permission to
-> all pipelines in order for your newly created pipeline to have access to an
-> agent pool and specific service connections. Be sure to navigate to the
-> Pipeline UI to permit permission to use the agent pool and the service
-> connection to authenticate against your key vault.
+A template generation pipeline that uses a variable group backed by key vault is
+provided in the
+[infra-generation-pipeline.yml](../../azure-pipelines/templates/infra-generation-kv-pipeline.yml)
+
+> When using a variable group with key vault, you may be prompted to grant
+> access permission to all pipelines in order for your newly created pipeline to
+> have access to an agent pool and specific service connections in AzDO. Be sure
+> to navigate to the Pipeline UI to permit permissions to use the agent pool and
+> the service connection to authenticate against your key vault.
 
 ![](../images/permit_access.jpg)