From 481a552486a61518a3c1e9f2baddda8082164d13 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 10 May 2019 19:44:08 -0400 Subject: [PATCH] Boilerplate commit --- .gitignore | 45 +++ .ruby-version | 1 + CHANGELOG.md | 19 ++ CONTRIBUTING.md | 110 +++++++ Gemfile | 19 ++ LICENSE | 1 + Makefile | 148 +++++++++ README.md | 71 ++++- examples/simple_example/README.md | 13 + examples/simple_example/main.tf | 26 ++ examples/simple_example/outputs.tf | 20 ++ examples/simple_example/variables.tf | 25 ++ helpers/combine_docfiles.py | 67 +++++ kitchen.yml | 43 +++ main.tf | 24 ++ outputs.tf | 19 ++ test/boilerplate/boilerplate.Dockerfile.txt | 13 + test/boilerplate/boilerplate.Makefile.txt | 13 + test/boilerplate/boilerplate.go.txt | 15 + test/boilerplate/boilerplate.py.txt | 13 + test/boilerplate/boilerplate.sh.txt | 13 + test/boilerplate/boilerplate.tf.txt | 15 + test/boilerplate/boilerplate.xml.txt | 15 + test/boilerplate/boilerplate.yaml.txt | 13 + test/ci_integration.sh | 64 ++++ test/fixtures/simple_example/main.tf | 32 ++ test/fixtures/simple_example/outputs.tf | 25 ++ test/fixtures/simple_example/variables.tf | 20 ++ .../simple_example/controls/gcloud.rb | 23 ++ .../simple_example/controls/gcp.rb | 21 ++ .../simple_example/controls/gsutil.rb | 23 ++ test/integration/simple_example/inspec.yml | 12 + test/make.sh | 162 ++++++++++ test/test_verify_boilerplate.py | 140 +++++++++ test/verify_boilerplate.py | 283 ++++++++++++++++++ variables.tf | 23 ++ 36 files changed, 1588 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .ruby-version create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 Gemfile create mode 100644 Makefile create mode 100644 examples/simple_example/README.md create mode 100644 examples/simple_example/main.tf create mode 100644 examples/simple_example/outputs.tf create mode 100644 examples/simple_example/variables.tf create mode 100755 helpers/combine_docfiles.py create mode 100644 kitchen.yml create mode 100644 main.tf create mode 100644 outputs.tf create mode 100644 test/boilerplate/boilerplate.Dockerfile.txt create mode 100644 test/boilerplate/boilerplate.Makefile.txt create mode 100644 test/boilerplate/boilerplate.go.txt create mode 100644 test/boilerplate/boilerplate.py.txt create mode 100644 test/boilerplate/boilerplate.sh.txt create mode 100644 test/boilerplate/boilerplate.tf.txt create mode 100644 test/boilerplate/boilerplate.xml.txt create mode 100644 test/boilerplate/boilerplate.yaml.txt create mode 100755 test/ci_integration.sh create mode 100644 test/fixtures/simple_example/main.tf create mode 100644 test/fixtures/simple_example/outputs.tf create mode 100644 test/fixtures/simple_example/variables.tf create mode 100644 test/integration/simple_example/controls/gcloud.rb create mode 100644 test/integration/simple_example/controls/gcp.rb create mode 100644 test/integration/simple_example/controls/gsutil.rb create mode 100644 test/integration/simple_example/inspec.yml create mode 100755 test/make.sh create mode 100755 test/test_verify_boilerplate.py create mode 100644 test/verify_boilerplate.py create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9ce300b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# OSX leaves these everywhere on SMB shares +._* + +# OSX trash +.DS_Store + +# Python +*.pyc + +# Emacs save files +*~ +\#*\# +.\#* + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +### https://raw.github.com/github/gitignore/90f149de451a5433aebd94d02d11b0e28843a1af/Terraform.gitignore + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Kitchen files +**/inspec.lock +**/.kitchen +**/kitchen.local.yml +**/Gemfile.lock + +# Ignore any .tfvars files that are generated automatically for each Terraform run. Most +# .tfvars files are managed as part of configuration and so should be included in +# version control. +**/*.tfvars + +credentials.json diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..aedc15bb --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.5.3 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..625d8501 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 20XX-YY-ZZ + +### Added + +- Initial release + +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-cloud-storage/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/terraform-google-modules/terraform-google-cloud-storage/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..43e610e6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing + +This document provides guidelines for contributing to the module. + +## Generating Documentation for Inputs and Outputs + +The Inputs and Outputs tables in the READMEs of the root module, +submodules, and example modules are automatically generated based on +the `variables` and `outputs` of the respective modules. These tables +must be refreshed if the module interfaces are changed. + +### Dependencies + +The following dependencies must be installed on the development system: + +- [make] +- [terraform-docs] v0.6.0 + +### Execution + +Run `make generate_docs` to generate new Inputs and Outputs tables. + +## Integration Testing + +Integration tests are used to verify the behaviour of the root module, +submodules, and example modules. Additions, changes, and fixes should +be accompanied with tests. + +The integration tests are run using [Kitchen][kitchen], +[Kitchen-Terraform][kitchen-terraform], and [InSpec][inspec]. These +tools are packaged within a Docker image for convenience. + +The general strategy for these tests is to verify the behaviour of the +[example modules](./examples), thus ensuring that the root module, +submodules, and example modules are all functionally correct. + +### Dependencies + +The following dependencies must be installed on the development system: + +- [Docker Engine][docker-engine] +- [Google Cloud SDK][google-cloud-sdk] +- [make] + +### Inputs + +Test instances are defined in the +[Kitchen configuration file](./kitchen.yml). The inputs of each Kitchen +instance may be configured with the `driver.variables` key in a +local Kitchen configuration file located at `./kitchen.local.yml` or in +a Terraform variables file located at +`./test/fixtures//variables.tfvars`. + +### Credentials + +Download the key of a Service Account with the +[required roles][required-roles] to `./credentials.json`. + +### Interactive Execution + +1. Run `make docker_run` to start the testing Docker container in + interactive mode. + +1. Run `kitchen create ` to initialize the working + directory for an example module. + +1. Run `kitchen converge ` to apply the example module. + +1. Run `kitchen verify ` to test the example module. + +1. Run `kitchen destroy ` to destroy the example module + state. + +### Noninteractive Execution + +Run `make test_integration_docker` to test all of the example modules +noninteractively. + +## Linting and Formatting + +Many of the files in the repository can be linted or formatted to +maintain a standard of quality. + +### Dependencies + +The following dependencies must be installed on the development system: + +- [flake8] +- [gofmt] +- [hadolint] +- [make] +- [shellcheck] +- [Terraform][terraform] v0.11 + +### Execution + +Run `make check`. + +[docker-engine]: https://www.docker.com/products/docker-engine +[flake8]: http://flake8.pycqa.org/en/latest/ +[gofmt]: https://golang.org/cmd/gofmt/ +[google-cloud-sdk]: https://cloud.google.com/sdk/install +[hadolint]: https://github.com/hadolint/hadolint +[inspec]: https://inspec.io/ +[kitchen-terraform]: https://github.com/newcontext-oss/kitchen-terraform +[kitchen]: https://kitchen.ci/ +[make]: https://en.wikipedia.org/wiki/Make_(software) +[shellcheck]: https://www.shellcheck.net/ +[terraform-docs]: https://github.com/segmentio/terraform-docs +[terraform]: https://terraform.io/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..4dbdea25 --- /dev/null +++ b/Gemfile @@ -0,0 +1,19 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ruby '2.5.3' + +source 'https://rubygems.org/' do + gem 'kitchen-terraform', '~> 4.8' +end diff --git a/LICENSE b/LICENSE index 261eeb9e..d6456956 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..730d0aeb --- /dev/null +++ b/Makefile @@ -0,0 +1,148 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! + +# Make will use bash instead of sh +SHELL := /usr/bin/env bash + +# Docker build config variables +CREDENTIALS_PATH ?= /cft/workdir/credentials.json +DOCKER_ORG := gcr.io/cloud-foundation-cicd +DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 1.0.1 +DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} + +# All is the first target in the file so it will get picked up when you just run 'make' on its own +.PHONY: all +all: check generate_docs + +# Run all available linters +.PHONY: check +check: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace + +# The .PHONY directive tells make that this isn't a real target and so +# the presence of a file named 'check_shell' won't cause this target to stop +# working +.PHONY: check_shell +check_shell: + @source test/make.sh && check_shell + +.PHONY: check_python +check_python: + @source test/make.sh && check_python + +.PHONY: check_golang +check_golang: + @source test/make.sh && golang + +.PHONY: check_terraform +check_terraform: + @source test/make.sh && check_terraform + +.PHONY: check_docker +check_docker: + @source test/make.sh && docker + +.PHONY: check_base_files +check_base_files: + @source test/make.sh && basefiles + +.PHONY: check_trailing_whitespace +check_trailing_whitespace: + @source test/make.sh && check_trailing_whitespace + +.PHONY: test_check_headers +test_check_headers: + @echo "Testing the validity of the header check" + @python test/test_verify_boilerplate.py + +.PHONY: check_headers +check_headers: + @source test/make.sh && check_headers + +# Integration tests +.PHONY: test_integration +test_integration: + test/ci_integration.sh + +.PHONY: generate_docs +generate_docs: + @source test/make.sh && generate_docs + +# Versioning +.PHONY: version +version: + @source helpers/version-repo.sh + +# Run docker +.PHONY: docker_run +docker_run: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + /bin/bash -c "source test/ci_integration.sh && setup_environment && exec /bin/bash" + +.PHONY: docker_create +docker_create: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen create" + +.PHONY: docker_converge +docker_converge: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen converge" + +.PHONY: docker_verify +docker_verify: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen verify" + +.PHONY: docker_destroy +docker_destroy: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen destroy" + +.PHONY: test_integration_docker +test_integration_docker: + docker run --rm -it \ + -e PROJECT_ID \ + -e SERVICE_ACCOUNT_JSON \ + -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ + -v $(CURDIR):/cft/workdir \ + ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + make test_integration diff --git a/README.md b/README.md index 90cabaf1..c9bcd019 100644 --- a/README.md +++ b/README.md @@ -1 +1,70 @@ -# terraform-google-cloud-storage \ No newline at end of file +# terraform-google-cloud-storage + +This module was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template/), which by default generates a module that simply creates a GCS bucket. As the module develops, this README should be updated. + +The resources/services/activations/deletions that this module will create/trigger are: + +- Create a GCS bucket with the provided name + +## Usage + +Basic usage of this module is as follows: + +```hcl +module "cloud_storage" { + source = "terraform-google-modules/cloud-storage/google" + version = "~> 0.1" + + project_id = "" + bucket_name = "gcs-test-bucket" +} +``` + +Functional examples are included in the +[examples](./examples/) directory. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +## Requirements + +These sections describe requirements for using this module. + +### Software + +The following dependencies must be available: + +- [Terraform][terraform] v0.11 +- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0 + +### Service Account + +A service account with the following roles must be used to provision +the resources of this module: + +- Storage Admin: `roles/storage.admin` + +The [Project Factory module][project-factory-module] and the +[IAM module][iam-module] may be used in combination to provision a +service account with the necessary roles applied. + +### APIs + +A project with the following APIs enabled must be used to host the +resources of this module: + +- Google Cloud Storage JSON API: `storage-api.googleapis.com` + +The [Project Factory module][project-factory-module] can be used to +provision a project with the necessary APIs enabled. + +## Contributing + +Refer to the [contribution guidelines](./CONTRIBUTING.md) for +information on contributing to this module. + +[iam-module]: https://registry.terraform.io/modules/terraform-google-modules/iam/google +[project-factory-module]: https://registry.terraform.io/modules/terraform-google-modules/project-factory/google +[terraform-provider-gcp]: https://www.terraform.io/docs/providers/google/index.html +[terraform]: https://www.terraform.io/downloads.html diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md new file mode 100644 index 00000000..e7cb1094 --- /dev/null +++ b/examples/simple_example/README.md @@ -0,0 +1,13 @@ +# Simple Example + +This example illustrates how to use the `cloud-storage` module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf new file mode 100644 index 00000000..81edc1e9 --- /dev/null +++ b/examples/simple_example/main.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + version = "~> 2.0" +} + +module "cloud_storage" { + source = "../.." + + project_id = "${var.project_id}" + bucket_name = "${var.bucket_name}" +} diff --git a/examples/simple_example/outputs.tf b/examples/simple_example/outputs.tf new file mode 100644 index 00000000..ecc8657c --- /dev/null +++ b/examples/simple_example/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket_name" { + description = "The name of the bucket." + value = "${module.cloud_storage.bucket_name}" +} diff --git a/examples/simple_example/variables.tf b/examples/simple_example/variables.tf new file mode 100644 index 00000000..5835ff11 --- /dev/null +++ b/examples/simple_example/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = "string" +} + +variable "bucket_name" { + description = "The name of the bucket to create." + type = "string" +} diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py new file mode 100755 index 00000000..5da02e94 --- /dev/null +++ b/helpers/combine_docfiles.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please note that this file was generated from +# [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! + +''' Combine file from: + * script argument 1 + with content of file from: + * script argument 2 + using the beginning of line separators + hardcoded using regexes in this file: + + We exclude any text using the separate + regex specified here +''' + +import os +import re +import sys + +insert_separator_regex = r'(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' # noqa: E501 +exclude_separator_regex = r'(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' # noqa: E501 + +if len(sys.argv) != 3: + sys.exit(1) + +if not os.path.isfile(sys.argv[1]): + sys.exit(0) + +input = open(sys.argv[1], "r").read() +replace_content = open(sys.argv[2], "r").read() + +# Exclude the specified content from the replacement content +groups = re.match( + exclude_separator_regex, + replace_content, + re.DOTALL +).groups(0) +replace_content = groups[0] + groups[1] + +# Find where to put the replacement content, overwrite the input file +match = re.match(insert_separator_regex, input, re.DOTALL) +if match is None: + print("ERROR: Could not find autogen docs anchors in", sys.argv[1]) + print("To fix this, insert the following anchors in your README where " + "module inputs and outputs should be documented.") + print("[^]: (autogen_docs_start)") + print("[^]: (autogen_docs_end)") + sys.exit(1) +groups = match.groups(0) +output = groups[0] + replace_content + groups[2] + "\n" +open(sys.argv[1], "w").write(output) diff --git a/kitchen.yml b/kitchen.yml new file mode 100644 index 00000000..f3c9c354 --- /dev/null +++ b/kitchen.yml @@ -0,0 +1,43 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +driver: + name: terraform + +provisioner: + name: terraform + +verifier: + name: terraform + +platforms: + - name: default + +suites: + - name: simple_example + driver: + root_module_directory: test/fixtures/simple_example/ + verifier: + color: false + systems: + - name: simple_example local + backend: local + controls: + - gcloud + - gsutil + - name: simple_example gcp + backend: gcp + controls: + - gcp diff --git a/main.tf b/main.tf new file mode 100644 index 00000000..d8f51a8e --- /dev/null +++ b/main.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.11.0" +} + +resource "google_storage_bucket" "main" { + project = "${var.project_id}" + name = "${var.bucket_name}" +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 00000000..8305339b --- /dev/null +++ b/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket_name" { + value = "${google_storage_bucket.main.name}" +} diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 00000000..b0c7da3d --- /dev/null +++ b/test/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 00000000..b0c7da3d --- /dev/null +++ b/test/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt new file mode 100644 index 00000000..557e16f0 --- /dev/null +++ b/test/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt new file mode 100644 index 00000000..b0c7da3d --- /dev/null +++ b/test/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt new file mode 100644 index 00000000..2e94f3e5 --- /dev/null +++ b/test/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.tf.txt b/test/boilerplate/boilerplate.tf.txt new file mode 100644 index 00000000..cfccff84 --- /dev/null +++ b/test/boilerplate/boilerplate.tf.txt @@ -0,0 +1,15 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt new file mode 100644 index 00000000..3d98cdc6 --- /dev/null +++ b/test/boilerplate/boilerplate.xml.txt @@ -0,0 +1,15 @@ + diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt new file mode 100644 index 00000000..b0c7da3d --- /dev/null +++ b/test/boilerplate/boilerplate.yaml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/ci_integration.sh b/test/ci_integration.sh new file mode 100755 index 00000000..6faf2f5f --- /dev/null +++ b/test/ci_integration.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Always clean up. +DELETE_AT_EXIT="$(mktemp -d)" +finish() { + echo 'BEGIN: finish() trap handler' >&2 + kitchen destroy "$SUITE" + [[ -d "${DELETE_AT_EXIT}" ]] && rm -rf "${DELETE_AT_EXIT}" + echo 'END: finish() trap handler' >&2 +} + +# Map the input parameters provided by Concourse CI, or whatever mechanism is +# running the tests to Terraform input variables. Also setup credentials for +# use with kitchen-terraform, inspec, and gcloud. +setup_environment() { + local tmpfile + tmpfile="$(mktemp)" + echo "${SERVICE_ACCOUNT_JSON}" > "${tmpfile}" + + # gcloud variables + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${tmpfile}" + # Application default credentials (Terraform google provider and inspec-gcp) + export GOOGLE_APPLICATION_CREDENTIALS="${tmpfile}" + + # Terraform variables + export TF_VAR_project_id="$PROJECT_ID" +} + +main() { + export SUITE="${SUITE:-}" + + set -eu + # Setup trap handler to auto-cleanup + export TMPDIR="${DELETE_AT_EXIT}" + trap finish EXIT + + # Setup environment variables + setup_environment + set -x + + # Execute the test lifecycle + kitchen create "$SUITE" + kitchen converge "$SUITE" + kitchen verify "$SUITE" +} + +# if script is being executed and not sourced. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/test/fixtures/simple_example/main.tf b/test/fixtures/simple_example/main.tf new file mode 100644 index 00000000..747394e8 --- /dev/null +++ b/test/fixtures/simple_example/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "random" { + version = "~> 2.0" +} + +resource "random_pet" "main" { + length = 1 + prefix = "simple-example" + separator = "-" +} + +module "example" { + source = "../../../examples/simple_example" + + project_id = "${var.project_id}" + bucket_name = "${random_pet.main.id}" +} diff --git a/test/fixtures/simple_example/outputs.tf b/test/fixtures/simple_example/outputs.tf new file mode 100644 index 00000000..e252498f --- /dev/null +++ b/test/fixtures/simple_example/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket_name" { + description = "The name of the bucket." + value = "${module.example.bucket_name}" +} + +output "project_id" { + description = "The ID of the project in which resources are provisioned." + value = "${var.project_id}" +} diff --git a/test/fixtures/simple_example/variables.tf b/test/fixtures/simple_example/variables.tf new file mode 100644 index 00000000..093f6b11 --- /dev/null +++ b/test/fixtures/simple_example/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = "string" +} diff --git a/test/integration/simple_example/controls/gcloud.rb b/test/integration/simple_example/controls/gcloud.rb new file mode 100644 index 00000000..8b7da404 --- /dev/null +++ b/test/integration/simple_example/controls/gcloud.rb @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gcloud" do + title "gcloud" + + describe command("gcloud --project=#{attribute("project_id")} services list --enabled") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq "" } + its(:stdout) { should match "storage-api.googleapis.com" } + end +end diff --git a/test/integration/simple_example/controls/gcp.rb b/test/integration/simple_example/controls/gcp.rb new file mode 100644 index 00000000..0ece33ab --- /dev/null +++ b/test/integration/simple_example/controls/gcp.rb @@ -0,0 +1,21 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gcp" do + title "GCP Resources" + + describe google_storage_bucket(name: attribute("bucket_name")) do + it { should exist } + end +end diff --git a/test/integration/simple_example/controls/gsutil.rb b/test/integration/simple_example/controls/gsutil.rb new file mode 100644 index 00000000..52c179e0 --- /dev/null +++ b/test/integration/simple_example/controls/gsutil.rb @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gsutil" do + title "gsutil" + + describe command("gsutil ls -p #{attribute("project_id")}") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq "" } + its(:stdout) { should match "gs://#{attribute("bucket_name")}" } + end +end diff --git a/test/integration/simple_example/inspec.yml b/test/integration/simple_example/inspec.yml new file mode 100644 index 00000000..2fd718c0 --- /dev/null +++ b/test/integration/simple_example/inspec.yml @@ -0,0 +1,12 @@ +name: simple_example +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: bucket_name + required: true + type: string diff --git a/test/make.sh b/test/make.sh new file mode 100755 index 00000000..a5b3aeba --- /dev/null +++ b/test/make.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! + +# Create a temporary directory that's auto-cleaned, even if the process aborts. +DELETE_AT_EXIT="$(mktemp -d)" +finish() { + [[ -d "${DELETE_AT_EXIT}" ]] && rm -rf "${DELETE_AT_EXIT}" +} +trap finish EXIT +# Create a temporary file in the auto-cleaned up directory while avoiding +# overwriting TMPDIR for other processes. +# shellcheck disable=SC2120 # (Arguments may be passed, e.g. maketemp -d) +maketemp() { + TMPDIR="${DELETE_AT_EXIT}" mktemp "$@" +} + +# find_files is a helper to exclude .git directories and match only regular +# files to avoid double-processing symlinks. +find_files() { + local pth="$1" + shift + find "${pth}" '(' -path '*/.git' -o -path '*/.terraform' ')' \ + -prune -o -type f "$@" +} + +# Compatibility with both GNU and BSD style xargs. +compat_xargs() { + local compat=() + # Test if xargs is GNU or BSD style. GNU xargs will succeed with status 0 + # when given --no-run-if-empty and no input on STDIN. BSD xargs will fail and + # exit status non-zero If xargs fails, assume it is BSD style and proceed. + # stderr is silently redirected to avoid console log spam. + if xargs --no-run-if-empty /dev/null; then + compat=("--no-run-if-empty") + fi + xargs "${compat[@]}" "$@" +} + +# This function makes sure that the required files for +# releasing to OSS are present +function basefiles() { + local fn required_files="LICENSE README.md" + echo "Checking for required files ${required_files}" + for fn in ${required_files}; do + test -f "${fn}" || echo "Missing required file ${fn}" + done +} + +# This function runs the hadolint linter on +# every file named 'Dockerfile' +function docker() { + echo "Running hadolint on Dockerfiles" + find_files . -name "Dockerfile" -print0 \ + | compat_xargs -0 hadolint +} + +# This function runs 'terraform validate' and 'terraform fmt' +# against all directory paths which contain *.tf files. +function check_terraform() { + set -e + echo "Running terraform validate" + find_files . -name "*.tf" -print0 \ + | compat_xargs -0 -n1 dirname \ + | sort -u \ + | compat_xargs -t -n1 terraform validate --check-variables=false + echo "Running terraform fmt" + find_files . -name "*.tf" -print0 \ + | compat_xargs -0 -n1 dirname \ + | sort -u \ + | compat_xargs -t -n1 terraform fmt -check=true -write=false +} + +# This function runs 'go fmt' and 'go vet' on every file +# that ends in '.go' +function golang() { + echo "Running go fmt and go vet" + find_files . -name "*.go" -print0 | compat_xargs -0 -n1 go fmt + find_files . -name "*.go" -print0 | compat_xargs -0 -n1 go vet +} + +# This function runs the flake8 linter on every file +# ending in '.py' +function check_python() { + echo "Running flake8" + find_files . -name "*.py" -print0 | compat_xargs -0 flake8 + return 0 +} + +# This function runs the shellcheck linter on every +# file ending in '.sh' +function check_shell() { + echo "Running shellcheck" + find_files . -name "*.sh" -print0 | compat_xargs -0 shellcheck -x +} + +# This function makes sure that there is no trailing whitespace +# in any files in the project. +# There are some exclusions +function check_trailing_whitespace() { + local rc + echo "Checking for trailing whitespace" + find_files . -print \ + | grep -v -E '\.(pyc|png)$' \ + | compat_xargs grep -H -n '[[:blank:]]$' + rc=$? + if [[ ${rc} -eq 0 ]]; then + return 1 + fi +} + +function generate_docs() { + echo "Generating markdown docs with terraform-docs" + local path tmpfile + while read -r path; do + if [[ -e "${path}/README.md" ]]; then + # shellcheck disable=SC2119 + tmpfile="$(maketemp)" + echo "terraform-docs markdown ${path}" + terraform-docs markdown "${path}" > "${tmpfile}" + helpers/combine_docfiles.py "${path}"/README.md "${tmpfile}" + else + echo "Skipping ${path} because README.md does not exist." + fi + done < <(find_files . -name '*.tf' -print0 \ + | compat_xargs -0 -n1 dirname \ + | sort -u) +} + +function prepare_test_variables() { + echo "Preparing terraform.tfvars files for integration tests" + #shellcheck disable=2044 + for i in $(find ./test/fixtures -type f -name terraform.tfvars.sample); do + destination=${i/%.sample/} + if [ ! -f "${destination}" ]; then + cp "${i}" "${destination}" + echo "${destination} has been created. Please edit it to reflect your GCP configuration." + fi + done +} + +function check_headers() { + echo "Checking file headers" + # Use the exclusion behavior of find_files + find_files . -type f -print0 \ + | compat_xargs -0 python test/verify_boilerplate.py +} diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py new file mode 100755 index 00000000..dd870ba5 --- /dev/null +++ b/test/test_verify_boilerplate.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please note that this file was generated from +# [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! + +''' A simple test for the verify_boilerplate python script. +This will create a set of test files, both valid and invalid, +and confirm that the has_valid_header call returns the correct +value. + +It also checks the number of files that are found by the +get_files call. +''' +from copy import deepcopy +from tempfile import mkdtemp +from shutil import rmtree +import unittest +from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ + get_args, get_files + + +class AllTestCase(unittest.TestCase): + """ + All of the setup, teardown, and tests are contained in this + class. + """ + + def write_file(self, filename, content, expected): + """ + A utility method that creates test files, and adds them to + the cases that will be tested. + + Args: + filename: (string) the file name (path) to be created. + content: (list of strings) the contents of the file. + expected: (boolean) True if the header is expected to be valid, + false if not. + """ + + file = open(filename, 'w+') + for line in content: + file.write(line + "\n") + file.close() + self.cases[filename] = expected + + def create_test_files(self, tmp_path, extension, header): + """ + Creates 2 test files for .tf, .xml, .go, etc and one for + Dockerfile, and Makefile. + + The reason for the difference is that Makefile and Dockerfile + don't have an extension. These would be substantially more + difficult to create negative test cases, unless the files + were written, deleted, and re-written. + + Args: + tmp_path: (string) the path in which to create the files + extension: (string) the file extension + header: (list of strings) the header/boilerplate content + """ + + content = "\n...blah \ncould be code or could be garbage\n" + special_cases = ["Dockerfile", "Makefile"] + header_template = deepcopy(header) + valid_filename = tmp_path + extension + valid_content = header_template.append(content) + if extension not in special_cases: + # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) + invalid_header = [] + for line in header_template: + if "2018" in line: + invalid_header.append(line.replace('2018', 'YEAR')) + else: + invalid_header.append(line) + invalid_header.append(content) + invalid_content = invalid_header + invalid_filename = tmp_path + "invalid." + extension + self.write_file(invalid_filename, invalid_content, False) + valid_filename = tmp_path + "testfile." + extension + + valid_content = header_template + self.write_file(valid_filename, valid_content, True) + + def setUp(self): + """ + Set initial counts and values, and initializes the setup of the + test files. + """ + self.cases = {} + self.tmp_path = mkdtemp() + "/" + self.my_args = get_args() + self.my_refs = get_refs(self.my_args) + self.my_regex = get_regexs() + self.prexisting_file_count = len( + get_files(self.my_refs.keys(), self.my_args)) + for key in self.my_refs: + self.create_test_files(self.tmp_path, key, + self.my_refs.get(key)) + + def tearDown(self): + """ Delete the test directory. """ + rmtree(self.tmp_path) + + def test_files_headers(self): + """ + Confirms that the expected output of has_valid_header is correct. + """ + for case in self.cases: + if self.cases[case]: + self.assertTrue(has_valid_header(case, self.my_refs, + self.my_regex)) + else: + self.assertFalse(has_valid_header(case, self.my_refs, + self.my_regex)) + + def test_invalid_count(self): + """ + Test that the initial files found isn't zero, indicating + a problem with the code. + """ + self.assertFalse(self.prexisting_file_count == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py new file mode 100644 index 00000000..21bc83f3 --- /dev/null +++ b/test/verify_boilerplate.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Verifies that all source files contain the necessary copyright boilerplate +# snippet. +# This is based on existing work +# https://github.com/kubernetes/test-infra/blob/master/hack +# /verify_boilerplate.py + +# Please note that this file was generated from +# [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! +from __future__ import print_function +import argparse +import glob +import os +import re +import sys + + +def get_args(): + """Parses command line arguments. + + Configures and runs argparse.ArgumentParser to extract command line + arguments. + + Returns: + An argparse.Namespace containing the arguments parsed from the + command line + """ + parser = argparse.ArgumentParser() + parser.add_argument("filenames", + help="list of files to check, " + "all files if unspecified", + nargs='*') + rootdir = os.path.dirname(__file__) + "/../" + rootdir = os.path.abspath(rootdir) + parser.add_argument( + "--rootdir", + default=rootdir, + help="root directory to examine") + + default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") + parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) + return parser.parse_args() + + +def get_refs(ARGS): + """Converts the directory of boilerplate files into a map keyed by file + extension. + + Reads each boilerplate file's contents into an array, then adds that array + to a map keyed by the file extension. + + Returns: + A map of boilerplate lines, keyed by file extension. For example, + boilerplate.py.txt would result in the k,v pair {".py": py_lines} where + py_lines is an array containing each line of the file. + """ + refs = {} + + # Find and iterate over the absolute path for each boilerplate template + for path in glob.glob(os.path.join( + ARGS.boilerplate_dir, + "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + return refs + + +# pylint: disable=too-many-locals +def has_valid_header(filename, refs, regexs): + """Test whether a file has the correct boilerplate header. + + Tests each file against the boilerplate stored in refs for that file type + (based on extension), or by the entire filename (eg Dockerfile, Makefile). + Some heuristics are applied to remove build tags and shebangs, but little + variance in header formatting is tolerated. + + Args: + filename: A string containing the name of the file to test + refs: A map of boilerplate headers, keyed by file extension + regexs: a map of compiled regex objects used in verifying boilerplate + + Returns: + True if the file has the correct boilerplate header, otherwise returns + False. + """ + try: + with open(filename, 'r') as fp: # pylint: disable=invalid-name + data = fp.read() + except IOError: + return False + basename = os.path.basename(filename) + extension = get_file_extension(filename) + if extension: + ref = refs[extension] + else: + ref = refs[basename] + # remove build tags from the top of Go files + if extension == "go": + con = regexs["go_build_constraints"] + (data, found) = con.subn("", data, 1) + # remove shebang + elif extension == "sh" or extension == "py": + she = regexs["shebang"] + (data, found) = she.subn("", data, 1) + data = data.splitlines() + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + return False + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + year = regexs["year"] + for datum in data: + if year.search(datum): + return False + + # if we don't match the reference at this point, fail + if ref != data: + return False + return True + + +def get_file_extension(filename): + """Extracts the extension part of a filename. + + Identifies the extension as everything after the last period in filename. + + Args: + filename: string containing the filename + + Returns: + A string containing the extension in lowercase + """ + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# These directories will be omitted from header checks +SKIPPED_DIRS = [ + 'Godeps', 'third_party', '_gopath', '_output', + '.git', 'vendor', '__init__.py', 'node_modules' +] + + +def normalize_files(files): + """Extracts the files that require boilerplate checking from the files + argument. + + A new list will be built. Each path from the original files argument will + be added unless it is within one of SKIPPED_DIRS. All relative paths will + be converted to absolute paths by prepending the root_dir path parsed from + the command line, or its default value. + + Args: + files: a list of file path strings + + Returns: + A modified copy of the files list where any any path in a skipped + directory is removed, and all paths have been made absolute. + """ + newfiles = [] + for pathname in files: + if any(x in pathname for x in SKIPPED_DIRS): + continue + newfiles.append(pathname) + for idx, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[idx] = os.path.join(ARGS.rootdir, pathname) + return newfiles + + +def get_files(extensions, ARGS): + """Generates a list of paths whose boilerplate should be verified. + + If a list of file names has been provided on the command line, it will be + treated as the initial set to search. Otherwise, all paths within rootdir + will be discovered and used as the initial set. + + Once the initial set of files is identified, it is normalized via + normalize_files() and further stripped of any file name whose extension is + not in extensions. + + Args: + extensions: a list of file extensions indicating which file types + should have their boilerplate verified + + Returns: + A list of absolute file paths + """ + files = [] + if ARGS.filenames: + files = ARGS.filenames + else: + for root, dirs, walkfiles in os.walk(ARGS.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for dpath in SKIPPED_DIRS: + if dpath in dirs: + dirs.remove(dpath) + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = get_file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + """Builds a map of regular expressions used in boilerplate validation. + + There are two scenarios where these regexes are used. The first is in + validating the date referenced is the boilerplate, by ensuring it is an + acceptable year. The second is in identifying non-boilerplate elements, + like shebangs and compiler hints that should be ignored when validating + headers. + + Returns: + A map of compiled regular expression objects, keyed by mnemonic. + """ + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the + # real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016 or 2017, company holder names can be + # anything + regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", + re.MULTILINE) + # strip #!.* from shell/python scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(args): + """Identifies and verifies files that should have the desired boilerplate. + + Retrieves the lists of files to be validated and tests each one in turn. + If all files contain correct boilerplate, this function terminates + normally. Otherwise it prints the name of each non-conforming file and + exists with a non-zero status code. + """ + regexs = get_regexs() + refs = get_refs(args) + filenames = get_files(refs.keys(), args) + nonconforming_files = [] + for filename in filenames: + if not has_valid_header(filename, refs, regexs): + nonconforming_files.append(filename) + if nonconforming_files: + print('%d files have incorrect boilerplate headers:' % len( + nonconforming_files)) + for filename in sorted(nonconforming_files): + print(os.path.relpath(filename, args.rootdir)) + sys.exit(1) + + +if __name__ == "__main__": + ARGS = get_args() + main(ARGS) diff --git a/variables.tf b/variables.tf new file mode 100644 index 00000000..65ec922c --- /dev/null +++ b/variables.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to deploy to" +} + +variable "bucket_name" { + description = "The name of the bucket to create" +}