From ebe495802b8a6e889192c063df718c5bd7242fc9 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sat, 13 May 2017 12:06:35 +0530 Subject: [PATCH 01/10] ccache template changes --- conda_smithy/templates/circle.yml.tmpl | 17 +++++++++++++++++ conda_smithy/templates/travis.yml.tmpl | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index bd32de35c..4bfce72fd 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -24,14 +24,31 @@ dependencies: override: - docker pull condaforge/linux-anvil +{%- if matrix %} +{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} + # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. + - ./ci_support/run_docker_build.sh || touch ./build_failed + cache_directories: + - "./build_artefacts/.ccache" +{% else %} +{% endif -%} +{% endif -%} + {% endif -%} {% endblock -%} + test: override: {%- block env_test %} {%- if matrix %} + +{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} + - if test -f ./build_failed; then exit 1; fi +{% else %} # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. - ./ci_support/run_docker_build.sh +{% endif -%} + {% else %} # The Circle-CI build should not be active, but if this is not true for some reason, do a fast finish. - exit 0 diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index 2a3dcf6c9..ea5fdfa7e 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -11,6 +11,13 @@ language: generic os: osx osx_image: xcode6.4 +{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} +cache: + branch: no-md5deep + directories: + - $HOME/.ccache +{%- endif %} + {% block env -%} {% if matrix[0] or travis.secure -%} env: From 58b1aec54452e017f25453ab47ca768af9d05d6a Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 16 Jun 2017 16:23:06 +0530 Subject: [PATCH 02/10] Use --no-build-id --- conda_smithy/templates/run_docker_build.tmpl | 4 ++++ conda_smithy/templates/travis.yml.tmpl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index 90d57163f..83347fd64 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -60,7 +60,11 @@ conda install --yes --quiet conda-forge-build-setup {{ build_setup }}{% endif -%} {%- block build %} +{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} +conda build --no-build-id /recipe_root --quiet || exit 1 +{%- else %} conda build /recipe_root --quiet || exit 1 +{%- endif %} {%- for owner, channel in channels['targets'] %} {{ upload_script }} /recipe_root {{ owner }} --channel={{ channel }} || exit 1 {%- endfor -%} diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index ea5fdfa7e..1188574a6 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -77,7 +77,11 @@ install: {{ build_setup }}{% endif %} script: +{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} + - conda build --no-build-id ./{{ recipe_dir }} +{%- else %} - conda build ./{{ recipe_dir }} +{%- endif %} {% for owner, channel in channels['targets'] %} - {{ upload_script }} ./{{ recipe_dir }} {{ owner }} --channel={{ channel }} {% endfor %} From 33950e67c7c74748ac8f0a0186107a1164fc35be Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 16 Jun 2017 21:02:44 +0530 Subject: [PATCH 03/10] Add comments --- conda_smithy/templates/circle.yml.tmpl | 2 +- conda_smithy/templates/run_docker_build.tmpl | 3 +++ conda_smithy/templates/travis.yml.tmpl | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index 4bfce72fd..167c955a1 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -43,7 +43,7 @@ test: {%- if matrix %} {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} - - if test -f ./build_failed; then exit 1; fi + - if test -f ./build_failed; echo "Build step failed. Please look at the exact error message above"; then exit 1; fi {% else %} # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. - ./ci_support/run_docker_build.sh diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index 83347fd64..8977859d4 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -61,6 +61,9 @@ conda install --yes --quiet conda-forge-build-setup {%- block build %} {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} +# ccache uses the compiler command line to generate the cache keys, so we can't have a +# different path for every build. ccache provides CCACHE_BASEDIR variable to solve this +# but it's better to avoid setting it as it has some side effects. conda build --no-build-id /recipe_root --quiet || exit 1 {%- else %} conda build /recipe_root --quiet || exit 1 diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index 1188574a6..ccc54853c 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -78,6 +78,9 @@ install: script: {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} + # ccache uses the compiler command line to generate the cache keys, so we can't have a + # different path for every build. ccache provides CCACHE_BASEDIR variable to solve this + # but it's better to avoid setting it as it has some side effects. - conda build --no-build-id ./{{ recipe_dir }} {%- else %} - conda build ./{{ recipe_dir }} From 024a2f6b379b35d8bc17de6a8c291233d5a84be2 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 16 Jun 2017 21:05:38 +0530 Subject: [PATCH 04/10] no-md5deep branch is no longer needed --- conda_smithy/templates/travis.yml.tmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index ccc54853c..90b4a330c 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -13,7 +13,6 @@ osx_image: xcode6.4 {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} cache: - branch: no-md5deep directories: - $HOME/.ccache {%- endif %} From ddef6a8b1c97ed9dd771e7f313d7f445f029d229 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 16 Jun 2017 21:31:09 +0530 Subject: [PATCH 05/10] More comments --- conda_smithy/templates/circle.yml.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index 167c955a1..3e6ef77b7 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -27,6 +27,8 @@ dependencies: {%- if matrix %} {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. + # If any command present on dependencies section fails, CircleCI doesn't + # cache anything. That's why we need to check for failure later - ./ci_support/run_docker_build.sh || touch ./build_failed cache_directories: - "./build_artefacts/.ccache" From 27922ad9f453218733aea5443a9500afcc5d4f31 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 25 Jul 2017 10:24:09 +0530 Subject: [PATCH 06/10] Escape jinja templating inside jinja --- conda_smithy/templates/circle.yml.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index fcb01c4dd..8a02b38b0 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -51,7 +51,7 @@ jobs: {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} - restore_cache: keys: - - ccache-{{ .Branch }} + - ccache-{{ '{{' }} .Branch {{ '}}' }} - ccache {%- endif %} - run: @@ -59,7 +59,7 @@ jobs: command: ./ci_support/run_docker_build.sh {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} - save_cache: - key: ccache-{{ .Branch }} + key: ccache-{{ '{{' }} .Branch {{ '}}' }} paths: - ./build_artefacts/.ccache when: always From f8596195f1dee4165e57e9f3a1f5000620442f9a Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 25 Jul 2017 10:30:55 +0530 Subject: [PATCH 07/10] Always cache --- conda_smithy/templates/circle.yml.tmpl | 4 ---- conda_smithy/templates/run_docker_build.tmpl | 4 ---- conda_smithy/templates/travis.yml.tmpl | 6 ------ 3 files changed, 14 deletions(-) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index 8a02b38b0..aa2478dc3 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -48,16 +48,13 @@ jobs: echo {{ '"' }}{{ dep }}{{ '=${' }}{{ dep }}{{ '}"' }} {%- endfor -%} {%- endif %} - {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} - restore_cache: keys: - ccache-{{ '{{' }} .Branch {{ '}}' }} - ccache - {%- endif %} - run: # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. command: ./ci_support/run_docker_build.sh - {%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} - save_cache: key: ccache-{{ '{{' }} .Branch {{ '}}' }} paths: @@ -68,7 +65,6 @@ jobs: paths: - ./build_artefacts/.ccache when: always - {%- endif %} {%- endfor -%} {%- endif -%} {%- endblock %} diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index b8528f00e..8435df669 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -72,14 +72,10 @@ conda install --yes --quiet conda-forge-build-setup {% if build_setup -%} {{ build_setup }}{% endif -%} -{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} # ccache uses the compiler command line to generate the cache keys, so we can't have a # different path for every build. ccache provides CCACHE_BASEDIR variable to solve this # but it's better to avoid setting it as it has some side effects. conda build --no-build-id /recipe_root --quiet || exit 1 -{%- else %} -conda build /recipe_root --quiet || exit 1 -{%- endif %} {%- for owner, channel in channels['targets'] %} {{ upload_script }} /recipe_root {{ owner }} --channel={{ channel }} || exit 1 {%- endfor %} diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index 90b4a330c..397aa54b1 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -11,11 +11,9 @@ language: generic os: osx osx_image: xcode6.4 -{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} cache: directories: - $HOME/.ccache -{%- endif %} {% block env -%} {% if matrix[0] or travis.secure -%} @@ -76,14 +74,10 @@ install: {{ build_setup }}{% endif %} script: -{%- if 'ccache-toolchain' in ' '.join(package['meta']['requirements']['build']) %} # ccache uses the compiler command line to generate the cache keys, so we can't have a # different path for every build. ccache provides CCACHE_BASEDIR variable to solve this # but it's better to avoid setting it as it has some side effects. - conda build --no-build-id ./{{ recipe_dir }} -{%- else %} - - conda build ./{{ recipe_dir }} -{%- endif %} {% for owner, channel in channels['targets'] %} - {{ upload_script }} ./{{ recipe_dir }} {{ owner }} --channel={{ channel }} {% endfor %} From 62171b9a0ebf26b7ed6ba41300d950ba2d403611 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 25 Jul 2017 10:46:22 +0530 Subject: [PATCH 08/10] write to main cache only on master --- conda_smithy/templates/circle.yml.tmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index aa2478dc3..853bb6bd6 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -61,6 +61,10 @@ jobs: - ./build_artefacts/.ccache when: always - save_cache: + filters: + branches: + only: + - master key: ccache paths: - ./build_artefacts/.ccache From 937f02508e28144120e8dc840c8d1e2cffd7ac0e Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 25 Jul 2017 10:49:10 +0530 Subject: [PATCH 09/10] use cache from master --- conda_smithy/templates/circle.yml.tmpl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index 853bb6bd6..e96f62950 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -51,7 +51,7 @@ jobs: - restore_cache: keys: - ccache-{{ '{{' }} .Branch {{ '}}' }} - - ccache + - ccache-master - run: # Run, test and (if we have a BINSTAR_TOKEN) upload the distributions. command: ./ci_support/run_docker_build.sh @@ -60,15 +60,6 @@ jobs: paths: - ./build_artefacts/.ccache when: always - - save_cache: - filters: - branches: - only: - - master - key: ccache - paths: - - ./build_artefacts/.ccache - when: always {%- endfor -%} {%- endif -%} {%- endblock %} From 95dd2a9805628a1ef21a5677284c112a2dd01952 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 28 Feb 2018 14:04:39 -0600 Subject: [PATCH 10/10] Update circle.yml.tmpl --- conda_smithy/templates/circle.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl index 58b405a01..4c20b7daf 100644 --- a/conda_smithy/templates/circle.yml.tmpl +++ b/conda_smithy/templates/circle.yml.tmpl @@ -39,7 +39,7 @@ jobs: - save_cache: key: ccache-{{ '{{' }} .Branch {{ '}}' }} paths: - - ./build_artefacts/.ccache + - ./build_artifacts/.ccache when: always {%- endfor -%} {%- endif -%}