Webindex update #2726
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate Download Index in JSON (admin) | |
| on: | |
| repository_dispatch: | |
| types: ["Webindex update"] | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| paths: | |
| - "exposed.map" | |
| - "kernel-description.json" | |
| pull_request: | |
| paths: | |
| - "exposed.map" | |
| concurrency: | |
| group: webindex | |
| cancel-in-progress: false | |
| jobs: | |
| Check: | |
| name: "Check membership" | |
| runs-on: Linux | |
| steps: | |
| - name: "Check membership" | |
| uses: armbian/actions/team-check@main | |
| with: | |
| ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TEAM: "Release manager" | |
| Webindex: | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| name: "Generate JSON Index" | |
| runs-on: ubuntu-latest | |
| needs: Check | |
| steps: | |
| - name: Checkout build framework repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: armbian/build | |
| fetch-depth: 1 | |
| clean: false | |
| path: build | |
| - name: Checkout OS repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: armbian/os | |
| fetch-depth: 1 | |
| clean: false | |
| path: os | |
| - name: Install SSH key | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.KEY_UPLOAD }} | |
| known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} | |
| if_key_exists: replace | |
| - name: "Install dependencies" | |
| run: | | |
| if ! command -v "gh" > /dev/null 2>&1; then sudo apt-get -y -qq install gh; fi | |
| if ! command -v "jc" > /dev/null 2>&1; then sudo apt-get -y -qq install jc; fi | |
| if ! command -v "zip" > /dev/null 2>&1; then sudo apt-get -y -qq install zip; fi | |
| - name: "Generate map file" | |
| run: | | |
| # make sure its cleaned | |
| rm -f exposed.map.autoproposal | |
| # get all stable files we provide | |
| rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -ar [email protected]:/storage/www/dl/ | awk '{ print $5 }' | grep "archive/" | grep -v ".txt\|.xz.\|homeassistant\|openhab\|kali\|omv" | grep ".xz" > /tmp/stable.txt | |
| #rsync -ar rsync://rsync.armbian.com/dl/ | awk '{ print $5 }' | grep "archive/" | grep -v ".txt\|.xz." | grep ".xz" > /tmp/stable.txt | |
| # generate map entries | |
| # $1 = board | |
| # $2 = branch | |
| function check_targets(){ | |
| b=$(echo $1 | rev | cut -d"/" -f1 | rev | cut -d"." -f1) | |
| I=0 | |
| for bs in $(cat /tmp/stable.txt | grep ${b}/); do | |
| GNOME=$(echo $bs | grep "$2" | grep _gnome | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/') | |
| [[ -n $GNOME ]] && echo $GNOME && I=$(( I + 1 )) | |
| XFCE=$(echo $bs | grep $2 | grep _xfce | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/') | |
| [[ -n $XFCE ]] && echo $XFCE && I=$(( I + 1 )) | |
| SERVER=$(echo $bs | grep $2 | grep -v "minimal\|desktop" | grep bookworm | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/' ) | |
| [[ -n ${SERVER} ]] && echo $SERVER && I=$(( I + 1 )) | |
| [[ $I -eq 2 ]] && break | |
| done | |
| JAMMYCLI=$(cat /tmp/stable.txt | grep $b | grep $2 | grep -v "minimal\|desktop" | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/' ) | |
| [[ $I -lt 2 && -n $JAMMYCLI ]] && echo "$JAMMYCLI" | |
| return $I | |
| } | |
| for board in $(ls -1 build/config/boards/*.{conf,csc,wip,tvb}); do | |
| # read kernel target | |
| if [[ $(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g" | wc -w) -gt 2 ]]; then | |
| KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | awk -F, '{ print $1,$3,$2 }') | |
| else | |
| KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g") | |
| fi | |
| VARIANTS=0 | |
| for targets in ${KERNEL_TARGET}; do | |
| if [[ "${targets}" == current && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "current" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| if [[ "${targets}" == legacy && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "legacy" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| if [[ "${targets}" == edge && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "edge" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| done || true | |
| done | tee -a > exposed.map.autoproposal | |
| # | |
| # Lets fix this and go back to manual in case of troubles | |
| # cp exposed.map.autoproposal os/exposed.map | |
| - name: "Build JSON file" # adjust "exposed" property while reading from download | |
| run: | | |
| # prepare feeds | |
| #A=$(rsync -ar rsync://rsync.armbian.com/dl/ | awk '{ print ""$2"|https://dl.armbian.com/"$5"|"$3"T"$4"Z" }' | grep "archive/" | grep -v ".txt") | |
| A=$(rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -ar [email protected]:/storage/www/dl/ | awk '{ print ""$2"|https://dl.armbian.com/"$5"|"$3"T"$4"Z" }' | grep "archive/" | grep -v ".txt\|homeassistant\|openhab\|kali\|omv") | |
| B=$(gh release view --json assets --repo github.com/armbian/community | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| C=$(gh release view --json assets --repo github.com/armbian/os | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| D=$(gh release view --json assets --repo github.com/armbian/distribution | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| # debug | |
| echo "$A" > a.txt | |
| echo "$B" > b.txt | |
| echo "$C" > c.txt | |
| echo "$D" > d.txt | |
| # join all in one file and compress | |
| for line in $A $B $C $D; do | |
| BOARD=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f2) | |
| ARMBIAN_VERSION=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f1) | |
| IMAGE_SIZE=$(echo $line | cut -d"|" -f1 | sed "s/\.//g" | sed "s/\,//g") | |
| KERNEL_BRANCH=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4) | |
| IMAGE_RELEASE=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f3) | |
| IMAGE_URL=$(echo $line | cut -d"|" -f2) | |
| # workarounds | |
| STEP_A=1 | |
| STEP_B=2 | |
| [[ $IMAGE_URL == *i3-wm* || $IMAGE_URL == *kde-* ]] && STEP_A=2 && STEP_B=3 | |
| IMAGE_TARGET=$(echo $IMAGE_URL | cut -d"|" -f2 | grep "minimal\|desktop" | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4- | grep -Po '_[a-z].*' | cut -d"." -f1 | sed "s/_//" | sed "s/_desktop//" | cut -d"-" -f1,${STEP_A}) | |
| IMAGE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/rc[0-9]//g" | cut -d"_" -f4- | cut -d"-" -f${STEP_B}- | cut -d"_" -f1 | cut -d"." -f1) | |
| [[ $IMAGE_EXTENSION == $KERNEL_BRANCH || $IMAGE_EXTENSION == ${IMAGE_TARGET} || $IMAGE_EXTENSION =~ boot|csot|boe|sms ]] && unset IMAGE_EXTENSION | |
| [[ -z ${IMAGE_TARGET} ]] && IMAGE_TARGET="server" | |
| FILE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/-rc[0-9]//g" | rev | cut -d"_" -f1 | rev | sed 's/.*[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)//' | sed "s/desktop.\|minimal.//") | |
| #FILE_EXTENSION=$(echo $IMAGE_URL | grep -o "oowow.*\|img.*") | |
| # Clean out application from extension | |
| FILE_EXTENSION=$(echo $FILE_EXTENSION | sed 's/.*-'$IMAGE_EXTENSION'//g' | sed -e 's/^\.//g') | |
| IMAGE_TYPE=$(echo $IMAGE_URL | cut -d"/" -f5) | |
| IMAGE_CREATED=$(echo $line | cut -d"|" -f3 | sed "s/\//-/g") | |
| EXPOSED=false | |
| # Assemble redirector link | |
| PREFIX="" | |
| [[ "${IMAGE_TYPE}" == "os" ]] && PREFIX="nightly/" | |
| REDI_EXT=$(echo $FILE_EXTENSION | rev | cut -d"." -f1 | rev | sed "s/xz//g") | |
| REDI_URL="https://dl.armbian.com/${PREFIX}${BOARD,,}/${IMAGE_RELEASE^}_${KERNEL_BRANCH}${IMAGE_TARGET:+_$IMAGE_TARGET}${IMAGE_EXTENSION:+-$IMAGE_EXTENSION}${REDI_EXT:+.$REDI_EXT}" | |
| # bypass redirector for weirdos | |
| if [[ "${FILE_EXTENSION}" =~ boot|csot|boe|sms|rootfs|qcow2 ]]; then REDI_URL=${IMAGE_URL}; fi | |
| # check if image is exposed | |
| while read exposed; do | |
| [[ $IMAGE_URL =~ $exposed ]] && EXPOSED=true && echo "Exposed=$IMAGE_URL">> $GITHUB_STEP_SUMMARY | |
| done < os/exposed.map | |
| echo "${BOARD,,}|$ARMBIAN_VERSION|$IMAGE_URL|$REDI_URL|$IMAGE_CREATED|$IMAGE_SIZE|$IMAGE_RELEASE|$KERNEL_BRANCH|$IMAGE_TARGET|$IMAGE_EXTENSION|$EXPOSED|$IMAGE_TYPE|$FILE_EXTENSION" | |
| done | sed '1s/^/"board_slug"|"armbian_version"|"file_url"|"redi_url"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"\n/' | jc --csv | jq '{"assets": .}' > all-images.json | |
| - name: "Download and compress torrent files" | |
| run: | | |
| SOURCE=$(mktemp -d) | |
| DESTINATION=$(mktemp -d) | |
| rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -zqvr --include="*/archive/*.torrent" --exclude="/*/*/*" --exclude="_*/" --exclude="control" --exclude="quotes.txt" --exclude="*/all-torrents.zip" [email protected]:/storage/www/dl/ ${SOURCE} | |
| find ${SOURCE}/. -mindepth 3 -exec mv -i -- {} ${DESTINATION}/ \; | |
| zip -qj all-torrents.zip ${DESTINATION}/*.torrent | |
| - name: "Copy kernel-description JSON" | |
| run: | | |
| cp os/kernel-description.json . | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifacts | |
| path: | | |
| a.txt | |
| b.txt | |
| c.txt | |
| d.txt | |
| all-images.json | |
| kernel-description.json | |
| all-torrents.zip | |
| exposed.map.autoproposal | |
| - name: Upload index | |
| run: | | |
| # torrent index upload to rsync | |
| rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -arvc all-torrents.zip [email protected]:/storage/www/dl/torrent/ | |
| # the rest elsewhere | |
| rsync -e "ssh -p ${{ secrets.HOST_UPLOAD_PORT }}" -arvc --remove-source-files kernel-description.json all-images.json all-torrents.zip ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:/incoming/json | |
| # cleanup | |
| rm -f /tmp/stable.txt |