From f8b4ffcc7e303b3f8f7107ff4a94f80c6c1436d2 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:24:31 -0500 Subject: [PATCH 01/14] Create fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pipelines/main/misc/fullsrcdist.yml diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml new file mode 100644 index 00000000..30da48ea --- /dev/null +++ b/pipelines/main/misc/fullsrcdist.yml @@ -0,0 +1,31 @@ +steps: + - group: "Check" + steps: + - label: "fullsrcdist" + key: "fullsrcdist" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + workspaces: + - "/cache/repos:/cache/repos" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + timeout_in_minutes: 10 + commands: | + make full-source-dist USE_BINARYBUILDER=0 + agents: + queue: "julia" + # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" + arch: "x86_64" From 976b93874384dca3f0d0870a800c2fe5c09c1d26 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:30:31 -0500 Subject: [PATCH 02/14] Update launch_unsigned_jobs.yml --- pipelines/main/launch_unsigned_jobs.yml | 49 +++++++++++++------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index a27ed1fa..9f197824 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -13,6 +13,32 @@ # something about the privileged steps. steps: + - group: "Check" + notify: + - github_commit_status: + context: "Check" + steps: + - label: "Launch check jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # We launch the whitespace check first, because we want it to finish quickly. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + agents: + queue: "julia" + os: "linux" - group: "Build" notify: - github_commit_status: @@ -44,29 +70,6 @@ steps: agents: queue: "julia" os: "linux" - - group: "Check" - notify: - - github_commit_status: - context: "Check" - steps: - - label: "Launch check jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml - - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml - agents: - queue: "julia" - os: "linux" - group: "Test" notify: - github_commit_status: From 8a195b7411bc79134c188fca7d39f33aa9b55d3f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:34:41 -0500 Subject: [PATCH 03/14] Update fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 30da48ea..0aaf1831 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -1,8 +1,10 @@ steps: - - group: "Check" + - group: "Full Source Dist" # TODO: delete this line + # - group: "Check" # TODO: uncomment this line steps: - label: "fullsrcdist" key: "fullsrcdist" + soft_fail: true # TODO: delete this line plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" From 442fa80e54d7339847db6e3a798d4eb37440f428 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:41:43 -0500 Subject: [PATCH 04/14] Update launch_unsigned_jobs.yml --- pipelines/main/launch_unsigned_jobs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 9f197824..850a389e 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -30,10 +30,11 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml agents: From 85fe41e82ba7f981bde703e4fa8cc33f07b9eca9 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:46:39 -0500 Subject: [PATCH 05/14] Update launch_unsigned_jobs.yml --- pipelines/main/launch_unsigned_jobs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 850a389e..bb5fc6c8 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -29,7 +29,7 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml + # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml # TODO: uncomment this line buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml @@ -37,6 +37,7 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml # TODO: delete this line agents: queue: "julia" os: "linux" From 82b1ecf103e12a32079e1f1163760322e8e9cc6d Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 17:58:17 -0500 Subject: [PATCH 06/14] Update launch_unsigned_jobs.yml --- pipelines/main/launch_unsigned_jobs.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index bb5fc6c8..d2054d5b 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -37,10 +37,18 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml # TODO: delete this line agents: queue: "julia" os: "linux" + - group: "Full Source Dist" # TODO: delete this line + steps: # TODO: delete this line + - label: "Launch fullsrcdist jobs" # TODO: delete this line + plugins: # TODO: delete this line + - JuliaCI/external-buildkite#v1: # TODO: delete this line + version: "./.buildkite-external-version" # TODO: delete this line + repo_url: "https://github.com/JuliaCI/julia-buildkite" # TODO: delete this line + commands: | # TODO: delete this line + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml # TODO: delete this line - group: "Build" notify: - github_commit_status: From cacef30fb2dcbd449c804a0c7bdba70fe8b16db5 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 18:21:25 -0500 Subject: [PATCH 07/14] Update launch_unsigned_jobs.yml --- pipelines/main/launch_unsigned_jobs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index d2054d5b..1352e25f 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -49,6 +49,9 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" # TODO: delete this line commands: | # TODO: delete this line buildkite-agent pipeline upload .buildkite/pipelines/main/misc/fullsrcdist.yml # TODO: delete this line + agents: # TODO: delete this line + queue: "julia" # TODO: delete this line + os: "linux" # TODO: delete this line - group: "Build" notify: - github_commit_status: From 1b1a9bf1ecd090ab111a9b19390dcb0b3f394698 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 15 Nov 2022 18:47:54 -0500 Subject: [PATCH 08/14] Apply suggestions from code review Co-authored-by: Alex Arslan --- pipelines/main/misc/fullsrcdist.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 0aaf1831..6022d4c2 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -25,6 +25,7 @@ steps: timeout_in_minutes: 10 commands: | make full-source-dist USE_BINARYBUILDER=0 + make light-source-dist USE_BINARYBUILDER=0 agents: queue: "julia" # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing From 2f096b945271588c3324d7666ded151169958ef2 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 16 Nov 2022 23:10:11 -0500 Subject: [PATCH 09/14] Update fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 6022d4c2..2f1033b2 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -14,8 +14,10 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.26/package_linux.x86_64.tar.gz + rootfs_treehash: "5b0b851aca3c941b900a1301c13922c6cfc7f211" + uid: 1000 + gid: 1000 workspaces: - "/cache/repos:/cache/repos" - JuliaCI/julia#v1: From f3335e4b972bdac54bd7dc09148df8461c8ffab4 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 17 Nov 2022 00:35:08 -0500 Subject: [PATCH 10/14] Download pre-built Julia --- pipelines/main/misc/fullsrcdist.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 2f1033b2..bfa7ddbd 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -4,6 +4,8 @@ steps: steps: - label: "fullsrcdist" key: "fullsrcdist" + depends_on: + - "build_x86_64-linux-gnu" soft_fail: true # TODO: delete this line plugins: - JuliaCI/external-buildkite#v1: @@ -24,9 +26,20 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.6' - timeout_in_minutes: 10 + # timeout_in_minutes: 10 # TODO: uncomment this line + timeout_in_minutes: 90 # TODO: delete this line commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz + ln -s ./usr/bin/julia ./julia + echo "--- Print Julia version info" + export JULIA_NUM_THREADS=1 + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + echo "--- Full-source-dist" make full-source-dist USE_BINARYBUILDER=0 + echo "--- Light-source-dist" make light-source-dist USE_BINARYBUILDER=0 agents: queue: "julia" From 21bba88dbf8f869a8d66cbc133f85c68c1614a64 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 17 Nov 2022 02:36:26 -0500 Subject: [PATCH 11/14] Update fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index bfa7ddbd..8b92995f 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -16,8 +16,8 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.26/package_linux.x86_64.tar.gz - rootfs_treehash: "5b0b851aca3c941b900a1301c13922c6cfc7f211" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.26/llvm_passes.x86_64.tar.gz + rootfs_treehash: "5b0b851aca3c941b900a1301c139000000000000" uid: 1000 gid: 1000 workspaces: From e0101962a910996b7ae376e1dcbad4f8452a21a4 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 17 Nov 2022 11:23:26 -0500 Subject: [PATCH 12/14] Update fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 8b92995f..98093798 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -17,7 +17,7 @@ steps: version: '1.6' - staticfloat/sandbox#v1: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.26/llvm_passes.x86_64.tar.gz - rootfs_treehash: "5b0b851aca3c941b900a1301c139000000000000" + rootfs_treehash: "ab2b59a86d9bec3a4bc701127037637bedf9fe51" uid: 1000 gid: 1000 workspaces: From 69aed0b32cfad7f80d8d94f61789e3416ea31625 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 24 Nov 2022 20:52:28 -0500 Subject: [PATCH 13/14] Update fullsrcdist.yml --- pipelines/main/misc/fullsrcdist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 98093798..1f0c6818 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -38,9 +38,9 @@ steps: export JULIA_NUM_THREADS=1 ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' echo "--- Full-source-dist" - make full-source-dist USE_BINARYBUILDER=0 + make full-source-dist echo "--- Light-source-dist" - make light-source-dist USE_BINARYBUILDER=0 + make light-source-dist agents: queue: "julia" # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing From ec9235679dca493c95c50a3726b0e832d4f043ea Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 24 Nov 2022 20:54:12 -0500 Subject: [PATCH 14/14] I'm --- pipelines/main/misc/fullsrcdist.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pipelines/main/misc/fullsrcdist.yml b/pipelines/main/misc/fullsrcdist.yml index 1f0c6818..8ab07a8b 100644 --- a/pipelines/main/misc/fullsrcdist.yml +++ b/pipelines/main/misc/fullsrcdist.yml @@ -30,17 +30,16 @@ steps: timeout_in_minutes: 90 # TODO: delete this line commands: | # Download pre-built julia, extract into `usr/` - buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . - mkdir -p ./usr - tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz - ln -s ./usr/bin/julia ./julia - echo "--- Print Julia version info" - export JULIA_NUM_THREADS=1 - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "--- Full-source-dist" - make full-source-dist + # buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . + # mkdir -p ./usr + # tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz + # ln -s ./usr/bin/julia ./julia + # echo "--- Print Julia version info" + # ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' echo "--- Light-source-dist" make light-source-dist + echo "--- Full-source-dist" + make full-source-dist agents: queue: "julia" # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing