Skip to content
Longevity Testing

Longevity Testing #96

Sign in to view logs

Longevity Testing

Longevity Testing #96

Workflow file for this run

.github/workflows/longevity_test.yml at c0460e3
name: Longevity Testing
on:
schedule:
# Run every day at 04:00 GMT (roughly 8pm PST)
- cron: "0 4 * * *"
workflow_dispatch:
inputs:
user:
description: 'User to run longevity test on'
permissions:
# required to retrieve AWS credentials
id-token: write
contents: write
# cancel currently running jobs if a new version of the branch is pushed
concurrency:
group: longevity_testing-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
SetM365App:
uses: alcionai/corso/.github/workflows/accSelector.yaml@main
Longevity-Tests:
needs: [ SetM365App ]
environment: Testing
runs-on: ubuntu-latest
env:
# Need these in the local env so that corso can read them
AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
# re-used values
CORSO_LOG_DIR: ${{ github.workspace }}/src/testlog
CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-longevity.log
RESTORE_DEST_PFX: Corso_Test_Longevity_
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
PREFIX: 'longevity'
# Options for retention.
RETENTION_MODE: GOVERNANCE
# Time to retain blobs for in hours.
RETENTION_DURATION: 168
defaults:
run:
working-directory: src
##########################################################################################################################################
# setup
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # needed to get latest tag
- name: Setup Golang with cache
uses: magnetikonline/action-golang-cache@v4
with:
go-version-file: src/go.mod
- run: |
go build -o longevity-test ./cmd/longevity_test
go build -o s3checker ./cmd/s3checker
- name: Get version string
id: version
run: |
echo version=$(git describe --tags --abbrev=0) | tee -a $GITHUB_OUTPUT
# Checkout the .github directory at the original branch's ref so we have a
# stable view of the actions.
- name: Code Checkout
working-directory: ${{ github.workspace }}
run: |
git checkout ${{ steps.version.outputs.version }}
git checkout ${{ github.ref }} -- .github
- run: go build -o corso
- run: mkdir ${CORSO_LOG_DIR}
# Use shorter-lived credentials obtained from assume-role since these
# runs haven't been taking long.
- name: Configure AWS credentials from Test account
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
role-session-name: integration-testing
aws-region: us-east-1
##########################################################################################################################################
# Repository commands
- name: Version Test
run: |
./corso --version | grep -c 'Corso version:'
- name: Repo init test
id: repo-init
run: |
set -euo pipefail
echo -e "\nRepo init test\n" >> ${{ env.CORSO_LOG_FILE }}
./corso repo init s3 \
--no-stats \
--hide-progress \
--prefix ${{ env.PREFIX }} \
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
--succeed-if-exists \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
if grep -q 'Failed to' ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
then
echo "Repo could not be initialized"
exit 1
fi
- name: Repo connect test
run: |
set -euo pipefail
echo -e "\nRepo connect test\n" >> ${{ env.CORSO_LOG_FILE }}
./corso repo connect s3 \
--no-stats \
--hide-progress \
--prefix ${{ env.PREFIX }} \
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
if ! grep -q 'Connected to S3 bucket' ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
then
echo "Repo could not be connected"
exit 1
fi
##########################################################################################################################################
# Exchange
- name: Backup exchange test
id: exchange-test
run: |
echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE}
./corso backup create exchange \
--no-stats \
--mailbox "${TEST_USER}" \
--hide-progress \
--json \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt )
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
echo "backup was not successful"
exit 1
fi
data=$( echo $resultjson | jq -r '.[0] | .id' )
echo result=$data >> $GITHUB_OUTPUT
##########################################################################################################################################
# Onedrive
- name: Backup onedrive test
id: onedrive-test
run: |
set -euo pipefail
echo -e "\nBackup OneDrive test\n" >> ${CORSO_LOG_FILE}
./corso backup create onedrive \
--no-stats \
--hide-progress \
--user "${TEST_USER}" \
--json \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt )
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
echo "backup was not successful"
exit 1
fi
data=$( echo $resultjson | jq -r '.[0] | .id' )
echo result=$data >> $GITHUB_OUTPUT
##########################################################################################################################################
# Sharepoint test
- name: Backup sharepoint test
id: sharepoint-test
run: |
set -euo pipefail
echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE}
./corso backup create sharepoint \
--no-stats \
--hide-progress \
--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}" \
--json \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt )
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
echo "backup was not successful"
exit 1
fi
data=$( echo $resultjson | jq -r '.[0] | .id' )
echo result=$data >> $GITHUB_OUTPUT
##########################################################################################################################################
# Backup Exchange Deletion test
- name: Backup Delete exchange test
id: delete-exchange-test
env:
SERVICE: "exchange"
DELETION_DAYS: 10
run: |
set -euo pipefail
echo -e "\nDelete Backup exchange \n" >> ${CORSO_LOG_FILE}
./longevity-test
##########################################################################################################################################
# Backup Onedrive Deletion test
- name: Backup Delete onedrive test
id: delete-onedrive-test
env:
SERVICE: "onedrive"
DELETION_DAYS: 10
run: |
set -euo pipefail
echo -e "\nDelete Backup onedrive \n" >> ${CORSO_LOG_FILE}
./longevity-test
##########################################################################################################################################
# Backup Sharepoint Deletion test
- name: Backup Delete Sharepoint test
id: delete-sharepoint-test
env:
SERVICE: "sharepoint"
DELETION_DAYS: 5
run: |
set -euo pipefail
echo -e "\nDelete Backup sharepoint \n" >> ${CORSO_LOG_FILE}
./longevity-test
##########################################################################################################################################
# TODO(ashmrtn) Testing only, remove before merging.
- name: S3 retention check
id: s3-retention-check
run: |
# TODO(ashmrtn): We can also check that non-current versions of
# blobs don't have their retention extended if we want.
#
# Objects in S3 should have longer locking retention times now that
# we can explicitly check for.
#
# Blob prefixes are as follows:
# - kopia.blobcfg - repo-wide config
# - kopia.repository - repo-wide config
# - p - data pack blobs (i.e. file data)
# - q - metadata pack blobs (i.e. manifests, directory listings, etc)
# - x - index blobs
./s3checker \
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
--bucket-prefix ${{ env.PREFIX }} \
--retention-mode ${{ env.RETENTION_MODE }} \
--live-retention-duration "$((${{ env.RETENTION_DURATION}}-1))h" \
--prefix "kopia.blobcfg" \
--prefix "kopia.repository" \
--prefix "p" \
--prefix "q" \
--prefix "x"
# Export OneDrive Test
- name: OneDrive Export test
run: |
set -euo pipefail
echo -e "\Export OneDrive test\n" >> ${CORSO_LOG_FILE}
echo -e "\Export OneDrive test - first entry\n" >> ${CORSO_LOG_FILE}
./corso backup list onedrive 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
while read -r line; do
./corso export onedrive \
"/tmp/corso-export--$line" \
--no-stats \
--backup "$line" \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_first.txt
done
echo -e "\Export OneDrive test - last entry\n" >> ${CORSO_LOG_FILE}
./corso backup list onedrive 2>/dev/null | tail -n1 | awk '{print $1}' |
while read -r line; do
./corso export onedrive \
"/tmp/corso-export--$line" \
--no-stats \
--backup "$line" \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_last.txt
done
##########################################################################################################################################
# Export SharePoint Test
- name: SharePoint Export test
run: |
set -euo pipefail
echo -e "\Export SharePoint test\n" >> ${CORSO_LOG_FILE}
echo -e "\Export SharePoint test - first entry\n" >> ${CORSO_LOG_FILE}
./corso backup list sharepoint 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
while read -r line; do
./corso export sharepoint \
"/tmp/corso-export--$line" \
--no-stats \
--backup "$line" \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_first.txt
done
echo -e "\Export SharePoint test - last entry\n" >> ${CORSO_LOG_FILE}
./corso backup list sharepoint 2>/dev/null | tail -n1 | awk '{print $1}' |
while read -r line; do
./corso export sharepoint \
"/tmp/corso-export--$line" \
--no-stats \
--backup "$line" \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_last.txt
done
##########################################################################################################################################
# Maintenance test
- name: Maintenance test Daily
id: maintenance-test-daily
run: |
set -euo pipefail
echo -e "\n Maintenance test Daily\n" >> ${CORSO_LOG_FILE}
# Run with the force flag so it doesn't fail if the github runner
# hostname isn't what's expected. This is only safe because we can
# guarantee only one runner will be executing maintenance at a time.
./corso repo maintenance --mode metadata \
--no-stats \
--hide-progress \
--force \
--json \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_metadata.txt
- name: Maintenance test Weekly
id: maintenance-test-weekly
run: |
if [[ $(date +%A) == "Saturday" ]]; then
set -euo pipefail
echo -e "\n Maintenance test Weekly\n" >> ${CORSO_LOG_FILE}
./corso repo maintenance --mode complete \
--no-stats \
--hide-progress \
--force \
--json \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_complete.txt
# TODO(ashmrtn): We can also check that non-current versions of
# blobs don't have their retention extended if we want.
#
# Objects in S3 should have longer locking retention times now that
# we can explicitly check for.
#
# Blob prefixes are as follows:
# - kopia.blobcfg - repo-wide config
# - kopia.repository - repo-wide config
# - p - data pack blobs (i.e. file data)
# - q - metadata pack blobs (i.e. manifests, directory listings, etc)
# - x - index blobs
./s3checker \
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
--bucket-prefix ${{ env.PREFIX }} \
--retention-mode ${{ env.RETENTION_MODE }} \
--live-retention-duration "$((${{ env.RETENTION_DURATION}}-1))h" \
--prefix "kopia.blobcfg" \
--prefix "kopia.repository" \
--prefix "p" \
--prefix "q" \
--prefix "x"
fi
##########################################################################################################################################
# Logging & Notifications
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: always()
uses: actions/upload-artifact@v3
with:
name: longevity-test-log
path: src/testlog/*
if-no-files-found: error
retention-days: 14
- name: Notify failure in slack
if: failure()
uses: ./.github/actions/slack-message
with:
msg: "[FAILED] Longevity Test"
slack_url: ${{ secrets.SLACK_WEBHOOK_URL }}