diff --git a/hadoop-tools/hadoop-azure/.gitignore b/hadoop-tools/hadoop-azure/.gitignore index a7d3296d0fff3..4ca164c0865be 100644 --- a/hadoop-tools/hadoop-azure/.gitignore +++ b/hadoop-tools/hadoop-azure/.gitignore @@ -5,3 +5,4 @@ src/test/resources/abfs-combination-test-configs.xml dev-support/testlogs src/test/resources/accountSettings/* !src/test/resources/accountSettings/accountName_settings.xml.template +dev-support/testrun-scripts/cronjob.sh diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh index a0039ced06d70..5672c8e2420cb 100755 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh @@ -117,46 +117,61 @@ runNonHNSOAuthDFSIngressBlobTest() runTest=false cleanUpTestContainers=false -echo 'Ensure below are complete before running script:' -echo '1. Account specific settings file is present.' -echo ' Copy accountName_settings.xml.template to accountName_settings.xml' -echo ' where accountName in copied file name should be the test account name without domain' -echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings' -echo ' folder. New account settings file to be added to same folder.)' -echo ' Follow instructions in the template to populate settings correctly for the account' -echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName' -echo ' where accountNames should be the test account names without domain' -echo ' ' -echo ' ' -echo 'Choose action:' -echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]' -select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN -do - case $scriptMode in - SET_ACTIVE_TEST_CONFIG) - runTest=false - break - ;; - RUN_TEST) - runTest=true - read -r -p "Enter parallel test run process count [default - 8]: " processCount - processCount=${processCount:-8} - break - ;; - CLEAN_UP_OLD_TEST_CONTAINERS) - runTest=false - cleanUpTestContainers=true - break - ;; - SET_OR_CHANGE_TEST_ACCOUNT) - runTest=false - cleanUpTestContainers=false - accountSettingsFile="src/test/resources/azure-auth-keys.xml" - if [[ ! -f "$accountSettingsFile" ]]; - then - logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template" - cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile - fi + +if [ "$isCronJob" = "true" ]; then + runTest=true + runHNSOAuthDFSTest + runHNSSharedKeyDFSTest + runNonHNSSharedKeyDFSTest + runAppendBlobHNSOAuthDFSTest + runNonHNSSharedKeyBlobTest + runNonHNSOAuthDFSTest + runNonHNSOAuthBlobTest + runAppendBlobNonHNSOAuthBlobTest + runHNSOAuthDFSIngressBlobTest + runNonHNSOAuthDFSIngressBlobTest + uploadToAzure +else + + echo 'Ensure below are complete before running script:' + echo '1. Account specific settings file is present.' + echo ' Copy accountName_settings.xml.template to accountName_settings.xml' + echo ' where accountName in copied file name should be the test account name without domain' + echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings' + echo ' folder. New account settings file to be added to same folder.)' + echo ' Follow instructions in the template to populate settings correctly for the account' + echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName' + echo ' where accountNames should be the test account names without domain' + echo ' ' + echo ' ' + echo 'Choose action:' + echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]' + select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN; do + case $scriptMode in + SET_ACTIVE_TEST_CONFIG) + runTest=false + break + ;; + RUN_TEST) + runTest=true + read -r -p "Enter parallel test run process count [default - 8]: " processCount + processCount=${processCount:-8} + break + ;; + CLEAN_UP_OLD_TEST_CONTAINERS) + runTest=false + cleanUpTestContainers=true + break + ;; + SET_OR_CHANGE_TEST_ACCOUNT) + runTest=false + cleanUpTestContainers=false + accountSettingsFile="src/test/resources/azure-auth-keys.xml" + if [[ ! -f "$accountSettingsFile" ]]; + then + logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template" + cp "$accountSettingsFile.template" $accountSettingsFile + fi vi $accountSettingsFile exit 0 @@ -247,10 +262,11 @@ do ;; *) logOutput "ERROR: Invalid selection" ;; - esac -done + esac + done +fi -if [ $runTest == true ] +if [[ $runTest && $"$isCronJob" != "true" ]] then printAggregate fi diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh index 8862942a1119d..0f0a044cecb52 100644 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh @@ -64,6 +64,44 @@ fnsBlobConfigFileCheck() { fi } +checkCronjobDependencies() { + if ! [ "$(command -v az)" ]; then + echo "Azure CLI (az) could not be found. Installing Azure CLI..." + if ! sudo apt update || ! sudo apt install -y azure-cli; then + echo "Failed to install Azure CLI. Exiting..." + exit 1 + fi + echo "Azure CLI installed successfully." + fi +} + +uploadToAzure() { + azureConfigFilePath="${accountSettingsDir}runresult${accountConfigFileSuffix}" + testResultsAccountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.test.results.account.name"]/value' -n $azureConfigFilePath) + testResultsAccountKey=$(xmlstarlet sel -t -v '//property[name = "fs.azure.test.results.account.key"]/value' -n $azureConfigFilePath) + branchName="${branchName,,}" + containerName="$(xmlstarlet sel -t -v '//property[name = "fs.azure.container.name"]/value' -n $azureConfigFilePath)" + printAggregate + + year=$(date +"%Y") + month=$(date +"%m") + day=$(date +"%d") + + directoryStructure="$year-$month-$day/$branchName" + AggregatedTestFolder="$testOutputLogFolder" + + checkCronjobDependencies + if ! az storage container create --name $containerName --account-name $testResultsAccountName --account-key "$testResultsAccountKey"; then + echo "Failed to create container. Exiting..." + exit 1 + fi + if ! az storage blob upload-batch --destination "$containerName/$directoryStructure" --source $AggregatedTestFolder --account-name $testResultsAccountName --account-key "$testResultsAccountKey"; then + echo "Failed upload test results in the destination. Exiting..." + exit 1 + fi + echo "Upload complete." +} + triggerRun() { echo ' ' @@ -214,14 +252,17 @@ init() { aggregatedTestResult="$testOutputLogFolder/Test-Results.txt" } - printAggregate() { - echo :::: AGGREGATED TEST RESULT :::: - cat "$aggregatedTestResult" +printAggregate() { + branchName=$(git rev-parse --abbrev-ref HEAD) + commitHash=$(git rev-parse HEAD) + + echo "Branch: $branchName, Commit: $commitHash" >> "$aggregatedTestResult" + fullRunEndTime=$(date +%s) fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime)) mins=$((fullRunTimeInSecs / 60)) secs=$((fullRunTimeInSecs % 60)) - printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs" + printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs" >> "$aggregatedTestResult" } logOutput() {