From 9de0ce25b5b7e51e33fafac4e4b6768d9dedd3fc Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 17:34:33 +0800 Subject: [PATCH 01/20] df -h --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d9bcdfcbfa47..c7f05b018b20 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -430,6 +430,7 @@ jobs: env: ${{ fromJSON(inputs.envs) }} shell: 'script -q -e -c "bash {0}"' run: | + df -h if [[ "$MODULES_TO_TEST" == "pyspark-errors" ]]; then export PATH=$PATH:$HOME/miniconda/bin export SKIP_PACKAGING=false From eabfbb171284404510c9fa0a51549466199c3ca9 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 17:55:49 +0800 Subject: [PATCH 02/20] rm in job --- .github/workflows/build_and_test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c7f05b018b20..29421e8de0a9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -410,6 +410,12 @@ jobs: key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | pyspark-coursier- + - name: Free up disk space + run: | + sudo rm -rf /__t/CodeQL + sudo rm -rf /__t/go + du -sh /* + du -sh /*/* - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From 36bcd35cb8e6cad5b6b5bf6f1c26682cf3222a7a Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 19:15:39 +0800 Subject: [PATCH 03/20] rm --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 29421e8de0a9..bc8ccb8a04a0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -412,10 +412,10 @@ jobs: pyspark-coursier- - name: Free up disk space run: | - sudo rm -rf /__t/CodeQL - sudo rm -rf /__t/go du -sh /* du -sh /*/* + rm -rf /__t/CodeQL + rm -rf /__t/go - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From 03a87841f6ed80fc9e68221d14fe0fde8eb56bb5 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 19:26:21 +0800 Subject: [PATCH 04/20] x --- .github/workflows/build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index bc8ccb8a04a0..d6c8e8bab466 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -412,10 +412,10 @@ jobs: pyspark-coursier- - name: Free up disk space run: | - du -sh /* - du -sh /*/* - rm -rf /__t/CodeQL - rm -rf /__t/go + du -sh /* || echo "fail to du" + du -sh /*/* || echo "fail to du" + rm -rf /__t/CodeQL || echo "fail to rm /__t/CodeQL" + rm -rf /__t/go || echo "fail to rm /__t/go" - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From cbffaa460af03dccf6a0ed2a52018fe31289ea0d Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 20:12:14 +0800 Subject: [PATCH 05/20] z --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d6c8e8bab466..810a2f6327af 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -412,6 +412,7 @@ jobs: pyspark-coursier- - name: Free up disk space run: | + dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 du -sh /* || echo "fail to du" du -sh /*/* || echo "fail to du" rm -rf /__t/CodeQL || echo "fail to rm /__t/CodeQL" From 38629ce0999a87274d7eea9a718bee187f8508c5 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 31 Jul 2023 20:33:49 +0800 Subject: [PATCH 06/20] df --- .github/workflows/build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 810a2f6327af..b95e9b89ac56 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -413,10 +413,14 @@ jobs: - name: Free up disk space run: | dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 + df -h du -sh /* || echo "fail to du" du -sh /*/* || echo "fail to du" rm -rf /__t/CodeQL || echo "fail to rm /__t/CodeQL" rm -rf /__t/go || echo "fail to rm /__t/go" + du -sh /* || echo "fail to du" + du -sh /*/* || echo "fail to du" + df -h - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From a48e4aa45355bb69e6377dc3d211f6fcbb2392d6 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 07:50:57 +0800 Subject: [PATCH 07/20] test lib --- .github/workflows/build_and_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b95e9b89ac56..58e85ae817e9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -420,6 +420,8 @@ jobs: rm -rf /__t/go || echo "fail to rm /__t/go" du -sh /* || echo "fail to du" du -sh /*/* || echo "fail to du" + du -sh /usr/share/* || echo "fail to du" + du -sh /usr/local/lib/* || echo "fail to du" df -h - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 From 7dc6ae5d88f679b9bf049031984e41d7e3bb87c7 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 10:02:48 +0800 Subject: [PATCH 08/20] more du --- .github/workflows/build_and_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 58e85ae817e9..5c70e6982266 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -416,6 +416,8 @@ jobs: df -h du -sh /* || echo "fail to du" du -sh /*/* || echo "fail to du" + du -sh /usr/share/* || echo "fail to du" + du -sh /usr/local/lib/* || echo "fail to du" rm -rf /__t/CodeQL || echo "fail to rm /__t/CodeQL" rm -rf /__t/go || echo "fail to rm /__t/go" du -sh /* || echo "fail to du" From a4b2e784bedb0456abdf32dcfd340faacf595d98 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 10:33:45 +0800 Subject: [PATCH 09/20] simplify --- .github/workflows/build_and_test.yml | 13 +++---------- dev/infra/Dockerfile | 4 ++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5c70e6982266..35fc4f1008dd 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -414,16 +414,9 @@ jobs: run: | dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 df -h - du -sh /* || echo "fail to du" - du -sh /*/* || echo "fail to du" - du -sh /usr/share/* || echo "fail to du" - du -sh /usr/local/lib/* || echo "fail to du" - rm -rf /__t/CodeQL || echo "fail to rm /__t/CodeQL" - rm -rf /__t/go || echo "fail to rm /__t/go" - du -sh /* || echo "fail to du" - du -sh /*/* || echo "fail to du" - du -sh /usr/share/* || echo "fail to du" - du -sh /usr/local/lib/* || echo "fail to du" + rm -rf /__t/CodeQL || echo "fail to delete /__t/CodeQL" + rm -rf /__t/go || echo "fail to delete /__t/go" + rm -rf /__t/node || echo "fail to delete /__t/node" df -h - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 9d7b29e25b49..dfe5f87ae0c8 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -25,6 +25,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true ARG APT_INSTALL="apt-get install --no-install-recommends -y" +ARG APT_REMOVE="apt-get remove --purge -y" RUN apt-get clean RUN apt-get update @@ -73,3 +74,6 @@ RUN python3.9 -m pip install grpcio protobuf googleapis-common-protos grpcio-sta # Add torch as a testing dependency for TorchDistributor RUN python3.9 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.9 -m pip install torcheval + +# Uninstall unused packages +RUN $APT_REMOVE libgl1-mesa-dri From be2001fe92a63902b4900c90dbf6ba26dd273348 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 10:38:31 +0800 Subject: [PATCH 10/20] nit --- .github/workflows/build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 35fc4f1008dd..b061f7fd5823 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -438,7 +438,6 @@ jobs: env: ${{ fromJSON(inputs.envs) }} shell: 'script -q -e -c "bash {0}"' run: | - df -h if [[ "$MODULES_TO_TEST" == "pyspark-errors" ]]; then export PATH=$PATH:$HOME/miniconda/bin export SKIP_PACKAGING=false From 105d77c32b7edc206bac5f6c5884674dee31aadb Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 19:03:47 +0800 Subject: [PATCH 11/20] autoremvoe --- dev/infra/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index dfe5f87ae0c8..28093b8d7dd4 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -77,3 +77,5 @@ RUN python3.9 -m pip install torcheval # Uninstall unused packages RUN $APT_REMOVE libgl1-mesa-dri +RUN apt-get autoremove --purge -y +RUN apt-get clean From 5a7bafd472cc63b3a326d192160eb34958cbce60 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 1 Aug 2023 19:20:36 +0800 Subject: [PATCH 12/20] address comments --- dev/infra/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 28093b8d7dd4..7cd84eed0862 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -78,4 +78,4 @@ RUN python3.9 -m pip install torcheval # Uninstall unused packages RUN $APT_REMOVE libgl1-mesa-dri RUN apt-get autoremove --purge -y -RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* From 726e2c4f02a4f6fb36efe3a2d7e1d8df30f2997f Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Aug 2023 11:39:49 +0800 Subject: [PATCH 13/20] update --- dev/infra/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 7cd84eed0862..a00d67b092fe 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -78,4 +78,3 @@ RUN python3.9 -m pip install torcheval # Uninstall unused packages RUN $APT_REMOVE libgl1-mesa-dri RUN apt-get autoremove --purge -y -RUN rm -rf /var/lib/apt/lists/* From a0f8e181c043ffa8b8d57549e7d2393f5a1c9d9d Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Aug 2023 19:13:46 +0800 Subject: [PATCH 14/20] skip docker --- dev/infra/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index a00d67b092fe..9d7b29e25b49 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -25,7 +25,6 @@ ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true ARG APT_INSTALL="apt-get install --no-install-recommends -y" -ARG APT_REMOVE="apt-get remove --purge -y" RUN apt-get clean RUN apt-get update @@ -74,7 +73,3 @@ RUN python3.9 -m pip install grpcio protobuf googleapis-common-protos grpcio-sta # Add torch as a testing dependency for TorchDistributor RUN python3.9 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.9 -m pip install torcheval - -# Uninstall unused packages -RUN $APT_REMOVE libgl1-mesa-dri -RUN apt-get autoremove --purge -y From a778f791a8603fb0199ba4a7b8886bbd37f184bf Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 09:16:43 +0800 Subject: [PATCH 15/20] nit --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b061f7fd5823..1df0f765b19b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -414,6 +414,7 @@ jobs: run: | dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 df -h + echo "Removing large directories" rm -rf /__t/CodeQL || echo "fail to delete /__t/CodeQL" rm -rf /__t/go || echo "fail to delete /__t/go" rm -rf /__t/node || echo "fail to delete /__t/node" From 2a7cfdc826f66860bb3338fb32e7cc75504c635d Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 09:24:24 +0800 Subject: [PATCH 16/20] move to script and try in other container jobs --- .github/workflows/build_and_test.yml | 13 ++++------ dev/free_disk_space_container | 37 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) create mode 100755 dev/free_disk_space_container diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1df0f765b19b..61ce36304907 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -411,14 +411,7 @@ jobs: restore-keys: | pyspark-coursier- - name: Free up disk space - run: | - dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 - df -h - echo "Removing large directories" - rm -rf /__t/CodeQL || echo "fail to delete /__t/CodeQL" - rm -rf /__t/go || echo "fail to delete /__t/go" - rm -rf /__t/node || echo "fail to delete /__t/node" - df -h + run: ./dev/free_disk_space_container - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: @@ -517,6 +510,8 @@ jobs: key: sparkr-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | sparkr-coursier- + - name: Free up disk space + run: ./dev/free_disk_space_container - name: Install Java ${{ inputs.java }} uses: actions/setup-java@v3 with: @@ -630,6 +625,8 @@ jobs: with: distribution: temurin java-version: 8 + - name: Free up disk space + run: ./dev/free_disk_space_container - name: License test run: ./dev/check-license - name: Dependencies test diff --git a/dev/free_disk_space_container b/dev/free_disk_space_container new file mode 100755 index 000000000000..3f18f150f1ec --- /dev/null +++ b/dev/free_disk_space_container @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "==================================" +echo "Free up disk space on CI system" +echo "==================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo rm -rf /__t/CodeQL +sudo rm -rf /__t/go +sudo rm -rf /__t/node + +sudo apt-get remove --purge -y libgl1-mesa-dri +sudo apt-get autoremove --purge -y +sudo apt-get clean + +df -h From 97f1087853335af77e8cc1a833a8768e9007820d Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 09:33:55 +0800 Subject: [PATCH 17/20] rm sudo --- dev/free_disk_space_container | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/free_disk_space_container b/dev/free_disk_space_container index 3f18f150f1ec..13697683251a 100755 --- a/dev/free_disk_space_container +++ b/dev/free_disk_space_container @@ -26,12 +26,12 @@ dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 df -h echo "Removing large packages" -sudo rm -rf /__t/CodeQL -sudo rm -rf /__t/go -sudo rm -rf /__t/node +rm -rf /__t/CodeQL +rm -rf /__t/go +rm -rf /__t/node -sudo apt-get remove --purge -y libgl1-mesa-dri -sudo apt-get autoremove --purge -y -sudo apt-get clean +apt-get remove --purge -y libgl1-mesa-dri +apt-get autoremove --purge -y +apt-get clean df -h From 21d2e85f0274339d2095160eaced99e2bcac97d5 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 09:40:57 +0800 Subject: [PATCH 18/20] add upte --- dev/free_disk_space_container | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/free_disk_space_container b/dev/free_disk_space_container index 13697683251a..a8cf2a14ab7b 100755 --- a/dev/free_disk_space_container +++ b/dev/free_disk_space_container @@ -30,6 +30,8 @@ rm -rf /__t/CodeQL rm -rf /__t/go rm -rf /__t/node +apt-get clean +apt-get update apt-get remove --purge -y libgl1-mesa-dri apt-get autoremove --purge -y apt-get clean From 24a90e0b420f027078ab5c0f2b0cab6012c02e37 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 09:55:36 +0800 Subject: [PATCH 19/20] order --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 61ce36304907..ea0c8e1d7fde 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -620,13 +620,13 @@ jobs: key: docs-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | docs-maven- + - name: Free up disk space + run: ./dev/free_disk_space_container - name: Install Java 8 uses: actions/setup-java@v3 with: distribution: temurin java-version: 8 - - name: Free up disk space - run: ./dev/free_disk_space_container - name: License test run: ./dev/check-license - name: Dependencies test From 01763c2042b55e72d957f7610ef2123285523785 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 3 Aug 2023 14:58:08 +0800 Subject: [PATCH 20/20] fix --- dev/free_disk_space_container | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dev/free_disk_space_container b/dev/free_disk_space_container index a8cf2a14ab7b..cc3b74643e4f 100755 --- a/dev/free_disk_space_container +++ b/dev/free_disk_space_container @@ -30,10 +30,4 @@ rm -rf /__t/CodeQL rm -rf /__t/go rm -rf /__t/node -apt-get clean -apt-get update -apt-get remove --purge -y libgl1-mesa-dri -apt-get autoremove --purge -y -apt-get clean - df -h