From 2ee00e0f88054b93da26037e0f85a457ab28619c Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 20:36:54 -0600 Subject: [PATCH 1/8] ci: add a validation step for starters --- .circleci/config.yml | 45 ++++++------------- ...{clone-folder.sh => clone-and-validate.sh} | 18 ++++++-- 2 files changed, 27 insertions(+), 36 deletions(-) rename scripts/{clone-folder.sh => clone-and-validate.sh} (69%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59d0a86634dd2..c816e6e497d3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,6 +167,16 @@ jobs: - e2e-test: test_path: e2e-tests/production-runtime + starters_validate: + executor: node + steps: + - checkout + - run: ./scripts/assert-changed-files.sh "starters/*|.circleci/*" + - <<: *restore_cache + - <<: *install_node_modules + - <<: *persist_cache + - run: sh ./scripts/clone-and-validate.sh starters false + starters_publish: executor: node steps: @@ -178,40 +188,11 @@ jobs: - run: sudo apt-get update && sudo apt-get install jq # jq is helpful for parsing json - run: git config --global user.name "GatsbyJS Bot" - run: git config --global user.email "admin@gatsbyjs.com" - - run: sh ./scripts/clone-folder.sh starters + - run: sh ./scripts/clone-and-validate.sh starters workflows: version: 2 build-test: jobs: - - bootstrap - - lint - - unit_tests_node6: - <<: *ignore_docs - requires: - - bootstrap - - unit_tests_node8: - <<: *ignore_docs - requires: - - bootstrap - - unit_tests_node10: - <<: *ignore_docs - requires: - - bootstrap - - integration_tests: - <<: *ignore_docs - - e2e_tests_gatsbygram: - <<: *e2e-test-workflow - - e2e_tests_path-prefix: - <<: *e2e-test-workflow - - e2e_tests_gatsby-image: - <<: *e2e-test-workflow - - e2e_tests_development_runtime: - <<: *e2e-test-workflow - - e2e_tests_production_runtime: - <<: *e2e-test-workflow - - starters_publish: - filters: - branches: - only: - - master + - ? starters:validate + <<: *ignore_master diff --git a/scripts/clone-folder.sh b/scripts/clone-and-validate.sh similarity index 69% rename from scripts/clone-folder.sh rename to scripts/clone-and-validate.sh index 21e8fa52a7a30..99303e9be42aa 100755 --- a/scripts/clone-folder.sh +++ b/scripts/clone-and-validate.sh @@ -1,5 +1,6 @@ #!/bin/bash FOLDER=$1 +CLONE=$2 BASE=$(pwd) COMMIT_MESSAGE=$(git log -1 --pretty=%B) @@ -15,10 +16,19 @@ for folder in $FOLDER/*; do git clone --depth 1 https://$GITHUB_API_TOKEN@github.com/gatsbyjs/$NAME.git $CLONE_DIR cd $CLONE_DIR find . | grep -v ".git" | grep -v "^\.*$" | xargs rm -rf # delete all files (to handle deletions in monorepo) - cp -r $BASE/$folder/. . # copy all content - git add . - git commit --message "$COMMIT_MESSAGE" - git push origin master + cp -r $BASE/$folder/. . + + # validate + yarn import + yarn build + + # sync to read-only clones + if [ "$CLONE" != false ]; then + echo "syncing to read-only clones" + git add . + git commit --message "$COMMIT_MESSAGE" + git push origin master + fi cd $BASE done From 582071cdf2eb5adfa842c0c70052b7fe5213b6f7 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 20:40:49 -0600 Subject: [PATCH 2/8] ci: fix circleci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c816e6e497d3c..50100c1c4bc34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -194,5 +194,5 @@ workflows: version: 2 build-test: jobs: - - ? starters:validate + - ? starters_validate <<: *ignore_master From d3f7e36c9676dcdd0967c57e09d52b3ed834bcf5 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 20:45:48 -0600 Subject: [PATCH 3/8] chore: run yarn to install deps --- .circleci/config.yml | 4 ++-- scripts/clone-and-validate.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50100c1c4bc34..b5503e83b5365 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -194,5 +194,5 @@ workflows: version: 2 build-test: jobs: - - ? starters_validate - <<: *ignore_master + - starters_validate: + <<: *ignore_master diff --git a/scripts/clone-and-validate.sh b/scripts/clone-and-validate.sh index 99303e9be42aa..fb6b9d280c792 100755 --- a/scripts/clone-and-validate.sh +++ b/scripts/clone-and-validate.sh @@ -20,6 +20,7 @@ for folder in $FOLDER/*; do # validate yarn import + yarn yarn build # sync to read-only clones From 1c39ddf881a26eeb5d27ad110ed3a7acb740215e Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 21:18:04 -0600 Subject: [PATCH 4/8] chore: tweak yarn import statement --- scripts/clone-and-validate.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/clone-and-validate.sh b/scripts/clone-and-validate.sh index fb6b9d280c792..8fa2fe2a93cea 100755 --- a/scripts/clone-and-validate.sh +++ b/scripts/clone-and-validate.sh @@ -19,13 +19,14 @@ for folder in $FOLDER/*; do cp -r $BASE/$folder/. . # validate - yarn import - yarn - yarn build + npm install + npm run build # sync to read-only clones if [ "$CLONE" != false ]; then - echo "syncing to read-only clones" + rm -rf yarn.lock + yarn import + git add . git commit --message "$COMMIT_MESSAGE" git push origin master From 3692e5d6b86591e2e7a9e8bd8bf7662809897d15 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 22:25:25 -0600 Subject: [PATCH 5/8] chore: add a comment --- scripts/clone-and-validate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/clone-and-validate.sh b/scripts/clone-and-validate.sh index 8fa2fe2a93cea..e9a8540c3c755 100755 --- a/scripts/clone-and-validate.sh +++ b/scripts/clone-and-validate.sh @@ -25,7 +25,7 @@ for folder in $FOLDER/*; do # sync to read-only clones if [ "$CLONE" != false ]; then rm -rf yarn.lock - yarn import + yarn import # generate a new yarn.lock file based on package-lock.json git add . git commit --message "$COMMIT_MESSAGE" From c6d18f00d6bfa66f4297adde402b662ba924e2a9 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 22:27:44 -0600 Subject: [PATCH 6/8] chore: tweak how jq is installed; minor tweaks --- .circleci/config.yml | 1 - scripts/clone-and-validate.sh | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5503e83b5365..68fc91355215e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,7 +185,6 @@ jobs: - <<: *install_node_modules - <<: *persist_cache - run: yarn markdown - - run: sudo apt-get update && sudo apt-get install jq # jq is helpful for parsing json - run: git config --global user.name "GatsbyJS Bot" - run: git config --global user.email "admin@gatsbyjs.com" - run: sh ./scripts/clone-and-validate.sh starters diff --git a/scripts/clone-and-validate.sh b/scripts/clone-and-validate.sh index e9a8540c3c755..6ea57b3974e32 100755 --- a/scripts/clone-and-validate.sh +++ b/scripts/clone-and-validate.sh @@ -1,9 +1,14 @@ #!/bin/bash FOLDER=$1 CLONE=$2 +IS_CI="${CI:-false}" BASE=$(pwd) COMMIT_MESSAGE=$(git log -1 --pretty=%B) +if [ "$IS_CI" = true ]; then + sudo apt-get update && sudo apt-get install jq +fi + for folder in $FOLDER/*; do [ -d "$folder" ] || continue # only directories cd $BASE From 77e21185d1f04e602abd78eebba8dbc7941a738c Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 22:28:37 -0600 Subject: [PATCH 7/8] chore: add back circleci config --- .circleci/config.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 68fc91355215e..0e0d47e205f44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,13 +185,45 @@ jobs: - <<: *install_node_modules - <<: *persist_cache - run: yarn markdown + - run: sudo apt-get update && sudo apt-get install jq # jq is helpful for parsing json - run: git config --global user.name "GatsbyJS Bot" - run: git config --global user.email "admin@gatsbyjs.com" - - run: sh ./scripts/clone-and-validate.sh starters + - run: sh ./scripts/clone-folder.sh starters workflows: version: 2 build-test: jobs: + - bootstrap + - lint + - unit_tests_node6: + <<: *ignore_docs + requires: + - bootstrap + - unit_tests_node8: + <<: *ignore_docs + requires: + - bootstrap + - unit_tests_node10: + <<: *ignore_docs + requires: + - bootstrap + - integration_tests: + <<: *ignore_docs + - e2e_tests_gatsbygram: + <<: *e2e-test-workflow + - e2e_tests_path-prefix: + <<: *e2e-test-workflow + - e2e_tests_gatsby-image: + <<: *e2e-test-workflow + - e2e_tests_development_runtime: + <<: *e2e-test-workflow + - e2e_tests_production_runtime: + <<: *e2e-test-workflow - starters_validate: <<: *ignore_master + - starters_publish: + filters: + branches: + only: + - master From a8df49f5d1d255dce8490438ae4d50179cc0c918 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Thu, 27 Dec 2018 22:34:40 -0600 Subject: [PATCH 8/8] chore: add an audit step --- scripts/clone-and-validate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/clone-and-validate.sh b/scripts/clone-and-validate.sh index 6ea57b3974e32..7280ddc731ccd 100755 --- a/scripts/clone-and-validate.sh +++ b/scripts/clone-and-validate.sh @@ -24,6 +24,7 @@ for folder in $FOLDER/*; do cp -r $BASE/$folder/. . # validate + npm audit npm install npm run build