-
Notifications
You must be signed in to change notification settings - Fork 995
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fdb : suport fdb for metadata engine
- Loading branch information
Showing
38 changed files
with
628 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,226 @@ | ||
| name: 'Version Compatible' | ||
| description: 'Version Compatible action' | ||
| inputs: | ||
| meta_url: | ||
| description: 'meta url' | ||
| required: true | ||
| default: '' | ||
| mount_point: | ||
| description: 'mount point' | ||
| required: true | ||
| default: /tmp/myjfs | ||
| volume_name: | ||
| description: 'volume name' | ||
| required: true | ||
| default: myjfs | ||
| storage: | ||
| description: 'storage' | ||
| required: true | ||
| default: "minio" | ||
| bucket: | ||
| description: 'bucket' | ||
| required: true | ||
| default: "http://127.0.0.1:9000/juicefs-version-compatible" | ||
| access_key: | ||
| description: 'access key' | ||
| required: true | ||
| default: "minioadmin" | ||
| sercret_key: | ||
| description: 'secret key' | ||
| required: true | ||
| default: "minioadmin" | ||
| backward_versions: | ||
| description: 'backward versions' | ||
| required: true | ||
| default: 5 | ||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: '1.18.x' | ||
|
|
||
| - name: Install Fdb C Lib | ||
| shell: bash | ||
| run: | | ||
| wget https://github.com/apple/foundationdb/releases/download/6.3.24/foundationdb-clients_6.3.24-1_amd64.deb | ||
| sudo dpkg -i foundationdb-clients_6.3.24-1_amd64.deb | ||
| - name: Build linux target | ||
| run: | | ||
| make juicefs | ||
| mv juicefs jfs_new | ||
| echo juicefs new version: $(./jfs_new -V) | ||
| shell: bash | ||
|
|
||
| - name: create database | ||
| shell: bash | ||
| run: | | ||
| meta_url=${{inputs.meta_url}} | ||
| db_name=$(basename $meta_url | awk -F? '{print $1}') | ||
| if [[ "$meta_url" == mysql* ]]; then | ||
| user=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $1}') | ||
| password=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $2}') | ||
| test -n "$password" && password="-p$password" || password="" | ||
| host=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $1}') | ||
| port=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $2}') | ||
| test -z "$port" && port="3306" | ||
| mysql -u$user $password -h $host -P $port -e "drop database if exists $db_name; create database $db_name;" | ||
| elif [[ "$meta_url" == postgres* ]]; then | ||
| export PGPASSWORD="postgres" | ||
| printf "\set AUTOCOMMIT on\ndrop database if exists $db_name; create database $db_name; " | psql -U postgres -h localhost | ||
| fi | ||
| - name: Test | ||
| shell: bash | ||
| run: | | ||
| # # Consider command as failed when any component of the pipe fails: | ||
| # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash | ||
| set -o pipefail | ||
| ./mc config host add minio http://127.0.0.1:9000 minioadmin minioadmin | ||
| tests=( | ||
| "format:jfs_new, format:jfs_old, destroy:jfs_new" | ||
| "format:jfs_new, format:jfs_old, destroy:jfs_old" | ||
| "format:jfs_old, format:jfs_new, destroy:jfs_old" | ||
| "format:jfs_old, format:jfs_new, destroy:jfs_new" | ||
| "format:jfs_old, mount:jfs_old, write_file1_100M:jfs_old, umount:jfs_old, dump_backup1:jfs_old, flushmeta:-, load_backup1:jfs_new, config:jfs_new, fsck:jfs_new, mount:jfs_new, read_file1:jfs_new, umount:jfs_new, fsck:jfs_new, destroy:jfs_new " | ||
| "format:jfs_old, mount:jfs_old, write_file1_100M:jfs_old, umount:jfs_old, format:jfs_new, mount:jfs_new, read_file1:jfs_new, umount:jfs_new, fsck:jfs_new, destroy:jfs_new" | ||
| "format:jfs_old, mount:jfs_old, write_file1_10K:jfs_old, umount:jfs_old, format:jfs_new, mount:jfs_new, read_file1:jfs_new, umount:jfs_new, fsck:jfs_new, destroy:jfs_new" | ||
| "format:jfs_old, mount:jfs_old, write_file1_10K:jfs_old, write_file2_10K:jfs_old, umount:jfs_old, dump_backup1:jfs_old, flushmeta:-, load_backup1:jfs_new, config_encrypt:jfs_new, mount:jfs_new, read_file1:jfs_new, read_file2:jfs_new, umount:jfs_new, fsck:jfs_new, destroy:jfs_new " | ||
| ) | ||
| urls=($(curl -s https://api.github.com/repos/juicedata/juicefs/releases | grep browser_download_url | grep linux-amd64.tar.gz | awk -F\" '{print $4}' | head -${{ inputs.backward_versions }})) | ||
| for url in "${urls[@]}"; do | ||
| echo download url is: $url | ||
| wget -q $url | ||
| tar -zxf $(basename $url) | ||
| rm $(basename $url) | ||
| mv juicefs jfs_old | ||
| echo juicefs new version: $(./jfs_new -V) | ||
| echo juicefs old version: $(./jfs_old -V) | ||
| old_rev=$(./jfs_old -V | awk -F- '{print $1}' | awk -F ' ' '{print $3}') | ||
| if [[ "$old_rev" < "1.0.0" ]]; then | ||
| echo "Warning: we don't test compatibily for version under 1.0.0" | ||
| break | ||
| fi | ||
| for test in "${tests[@]}"; do | ||
| echo 'test is ' $test | ||
| echo 'new version is:' $(./jfs_new -V) | ||
| echo 'old version is:' $(./jfs_old -V) | ||
| IFS=',' read -a command_arr <<< "$test" | ||
| for command in "${command_arr[@]}"; do | ||
| jfs_bin=$(echo $command | awk -F: '{print $2}'| xargs) | ||
| jfs_cmd=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $1}') | ||
| echo 'jfs_bin is ' $jfs_bin, "cmd is " $jfs_cmd | ||
| case "$jfs_cmd" in | ||
| "format") | ||
| ./mc admin user remove minio juicedata || true | ||
| ./mc admin user add minio juicedata juicedata | ||
| ./mc admin policy set minio consoleAdmin user=juicedata | ||
| sudo ./$jfs_bin format --trash-days 0 --storage minio --bucket ${{ inputs.bucket }} --access-key juicedata --secret-key juicedata ${{ inputs.meta_url }} juicefs-version-compatible | ||
| echo "format finished" | ||
| ;; | ||
| "mount") | ||
| sudo ./$jfs_bin mount --no-usage-report -d ${{ inputs.meta_url }} ${{ inputs.mount_point }} | ||
| echo "mount finished" | ||
| ;; | ||
| "umount") | ||
| sudo ./$jfs_bin umount ${{ inputs.mount_point }} | ||
| echo "umount finished" | ||
| ;; | ||
| "destroy") | ||
| UUID=$(./$jfs_bin status ${{ inputs.meta_url }} | grep UUID | cut -d '"' -f 4) | ||
| if [ -n "$UUID" ];then | ||
| echo UUID is $UUID | ||
| sudo ./$jfs_bin destroy --force ${{ inputs.meta_url }} $UUID | ||
| fi | ||
| echo "destroy finished" | ||
| ;; | ||
| "write") | ||
| filename=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| size=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $3}') | ||
| dd if=/dev/urandom of=${{ inputs.mount_point }}/tempfile iflag=fullblock,count_bytes bs=4k count=$size | ||
| md5_write="$filename:$(cat ${{ inputs.mount_point }}/tempfile | md5sum)" | ||
| echo "write: filename is: " $filename "md5_write is: " $md5_write | ||
| cp ${{inputs.mount_point}}/tempfile ${{ inputs.mount_point }}/$filename | ||
| echo "write finished" | ||
| ;; | ||
| "read") | ||
| filename=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| read_md5=$(cat ${{ inputs.mount_point }}/$filename | md5sum) | ||
| written_filename=$(echo $md5_write | awk -F: '{print $1}') | ||
| written_md5=$(echo $md5_write | awk -F: '{print $2}') | ||
| echo "read: filename is: " $filename "read_md5 is: " $read_md5, "written_filename is:", $written_filename, "written_md5 is:", $written_md5, | ||
| if [ "$filename" == "$wrtten_filename" ] && [ "$read_md5" != "$written_md5" ];then | ||
| echo "FATAL: the md5 does not match for: ", $filename, "read_md5:", $read_md5, "written_md5:", $written_md5 | ||
| exit 1 | ||
| fi | ||
| echo "read finished" | ||
| ;; | ||
| "dump") | ||
| filename=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| sudo ./$jfs_bin dump ${{ inputs.meta_url }} $filename | ||
| echo "dump finished" | ||
| ;; | ||
| "load") | ||
| filename=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| echo filename is : $filename | ||
| sudo ./$jfs_bin load ${{inputs.meta_url}} $filename | ||
| echo "load finished" | ||
| ;; | ||
| "config") | ||
| encrypt=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| ./mc admin user remove minio juicedata || true | ||
| ./mc admin user add minio juicedata juicedata1 | ||
| ./mc admin policy set minio consoleAdmin user=juicedata | ||
| if [ -n "$encrypt" ]; then | ||
| sudo ./$jfs_bin config ${{inputs.meta_url}} --secret-key juicedata1 --encrypt-secret | ||
| else | ||
| sudo ./$jfs_bin config ${{inputs.meta_url}} --secret-key juicedata1 | ||
| fi | ||
| sleep 60 | ||
| echo "config finished" | ||
| ;; | ||
| "load") | ||
| filename=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}') | ||
| sudo ./$jfs_bin load ${{ inputs.meta_url }} $filename | ||
| echo "load finished" | ||
| ;; | ||
| "status") | ||
| sudo ./$jfs_bin status ${{ inputs.meta_url }} | ||
| echo "status finished" | ||
| ;; | ||
| "fsck") | ||
| sudo ./$jfs_bin fsck ${{ inputs.meta_url }} | ||
| echo "fsck finished" | ||
| ;; | ||
| "flushmeta") | ||
| meta_url=${{inputs.meta_url}} | ||
| if [[ "$meta_url" == redis* ]]; then | ||
| redis-cli flushall | ||
| elif [[ "$meta_url" == mysql* ]]; then | ||
| db_name=$(basename $meta_url | awk -F? '{print $1}') | ||
| user=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $1}') | ||
| password=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $2}') | ||
| test -n "$password" && password="-p$password" || password="" | ||
| host=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $1}') | ||
| port=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $2}') | ||
| test -z "$port" && port="3306" | ||
| mysql -u$user $password -h $host -P $port -e "drop database if exists $db_name; create database $db_name;" | ||
| elif [[ "$meta_url" == postgres* ]]; then | ||
| db_name=$(basename $meta_url | awk -F? '{print $1}') | ||
| export PGPASSWORD="postgres" | ||
| printf "\set AUTOCOMMIT on\ndrop database if exists $db_name; create database $db_name; " | psql -U postgres -h localhost | ||
| fi | ||
| echo "fsck finished" | ||
| ;; | ||
| esac | ||
| done | ||
| done | ||
| done | ||
| - name: log | ||
| if: ${{ always() }} | ||
| shell: bash | ||
| run: | | ||
| tail -300 /var/log/juicefs.log | ||
| grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.