[POC] CI automation scripts

hadoop-tools/hadoop-azure/.gitignore

@@ -5,3 +5,4 @@ src/test/resources/abfs-combination-test-configs.xml
 dev-support/testlogs
 src/test/resources/accountSettings/*
 !src/test/resources/accountSettings/accountName_settings.xml.template
+dev-support/testrun-scripts/cronjob.sh

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

@@ -117,46 +117,61 @@ runNonHNSOAuthDFSIngressBlobTest()
 
 runTest=false
 cleanUpTestContainers=false
-echo 'Ensure below are complete before running script:'
-echo '1. Account specific settings file is present.'
-echo '   Copy accountName_settings.xml.template to accountName_settings.xml'
-echo '   where accountName in copied file name should be the test account name without domain'
-echo '   (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
-echo '   folder. New account settings file to be added to same folder.)'
-echo '   Follow instructions in the template to populate settings correctly for the account'
-echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
-echo '   where accountNames should be the test account names without domain'
-echo ' '
-echo ' '
-echo 'Choose action:'
-echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
-select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
-do
-  case $scriptMode in
-  SET_ACTIVE_TEST_CONFIG)
-    runTest=false
-    break
-    ;;
-  RUN_TEST)
-    runTest=true
-    read -r -p "Enter parallel test run process count [default - 8]: " processCount
-    processCount=${processCount:-8}
-    break
-    ;;
-  CLEAN_UP_OLD_TEST_CONTAINERS)
-    runTest=false
-    cleanUpTestContainers=true
-    break
-    ;;
-  SET_OR_CHANGE_TEST_ACCOUNT)
-    runTest=false
-    cleanUpTestContainers=false
-    accountSettingsFile="src/test/resources/azure-auth-keys.xml"
-    if [[ ! -f "$accountSettingsFile" ]];
-    then
-      logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
-      cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
-    fi
+
+if [ "$isCronJob" = "true" ]; then
+  runTest=true
+  runHNSOAuthDFSTest
+  runHNSSharedKeyDFSTest
+  runNonHNSSharedKeyDFSTest
+  runAppendBlobHNSOAuthDFSTest
+  runNonHNSSharedKeyBlobTest
+  runNonHNSOAuthDFSTest
+  runNonHNSOAuthBlobTest
+  runAppendBlobNonHNSOAuthBlobTest
+  runHNSOAuthDFSIngressBlobTest
+  runNonHNSOAuthDFSIngressBlobTest
+  uploadToAzure
+else
+
+  echo 'Ensure below are complete before running script:'
+  echo '1. Account specific settings file is present.'
+  echo '   Copy accountName_settings.xml.template to accountName_settings.xml'
+  echo '   where accountName in copied file name should be the test account name without domain'
+  echo '   (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+  echo '   folder. New account settings file to be added to same folder.)'
+  echo '   Follow instructions in the template to populate settings correctly for the account'
+  echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+  echo '   where accountNames should be the test account names without domain'
+  echo ' '
+  echo ' '
+  echo 'Choose action:'
+  echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+  select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN; do
+    case $scriptMode in
+    SET_ACTIVE_TEST_CONFIG)
+      runTest=false
+      break
+      ;;
+    RUN_TEST)
+      runTest=true
+      read -r -p "Enter parallel test run process count [default - 8]: " processCount
+      processCount=${processCount:-8}
+      break
+      ;;
+    CLEAN_UP_OLD_TEST_CONTAINERS)
+      runTest=false
+      cleanUpTestContainers=true
+      break
+      ;;
+    SET_OR_CHANGE_TEST_ACCOUNT)
+      runTest=false
+      cleanUpTestContainers=false
+      accountSettingsFile="src/test/resources/azure-auth-keys.xml"
+      if [[ ! -f "$accountSettingsFile" ]];
+      then
+        logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
+        cp "$accountSettingsFile.template" $accountSettingsFile
+      fi
 
     vi $accountSettingsFile
     exit 0
@@ -247,10 +262,11 @@ do
          ;;
       *) logOutput "ERROR: Invalid selection"
       ;;
-   esac
-done
+    esac
+  done
+fi
 
-if [ $runTest == true ]
+if [[ $runTest && $"$isCronJob" != "true" ]]
 then
   printAggregate
 fi

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh

@@ -64,6 +64,44 @@ fnsBlobConfigFileCheck() {
   fi
 }
 
+checkCronjobDependencies() {
+  if ! [ "$(command -v az)" ]; then
+    echo "Azure CLI (az) could not be found. Installing Azure CLI..."
+    if ! sudo apt update || ! sudo apt install -y azure-cli; then
+      echo "Failed to install Azure CLI. Exiting..."
+      exit 1
+    fi
+    echo "Azure CLI installed successfully."
+  fi
+}
+
+uploadToAzure() {
+  azureConfigFilePath="${accountSettingsDir}runresult${accountConfigFileSuffix}"
+  testResultsAccountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.test.results.account.name"]/value' -n $azureConfigFilePath)
+  testResultsAccountKey=$(xmlstarlet sel -t -v '//property[name = "fs.azure.test.results.account.key"]/value' -n $azureConfigFilePath)
+  branchName="${branchName,,}"
+  containerName="$(xmlstarlet sel -t -v '//property[name = "fs.azure.container.name"]/value' -n $azureConfigFilePath)"
+  printAggregate
+
+  year=$(date +"%Y")
+  month=$(date +"%m")
+  day=$(date +"%d")
+
+  directoryStructure="$year-$month-$day/$branchName"
+  AggregatedTestFolder="$testOutputLogFolder"
+
+  checkCronjobDependencies
+  if ! az storage container create --name $containerName --account-name $testResultsAccountName --account-key "$testResultsAccountKey"; then
+    echo "Failed to create container. Exiting..."
+    exit 1
+  fi
+  if ! az storage blob upload-batch --destination "$containerName/$directoryStructure" --source $AggregatedTestFolder --account-name $testResultsAccountName --account-key "$testResultsAccountKey"; then
+    echo "Failed upload test results in the destination. Exiting..."
+    exit 1
+  fi
+  echo "Upload complete."
+}
+
 triggerRun()
 {
   echo ' '
@@ -214,14 +252,17 @@ init() {
   aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
  }
 
- printAggregate() {
-   echo  :::: AGGREGATED TEST RESULT ::::
-   cat "$aggregatedTestResult"
+printAggregate() {
+  branchName=$(git rev-parse --abbrev-ref HEAD)
+  commitHash=$(git rev-parse HEAD)
+
+  echo "Branch: $branchName, Commit: $commitHash" >> "$aggregatedTestResult"
+
   fullRunEndTime=$(date +%s)
   fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
   mins=$((fullRunTimeInSecs / 60))
   secs=$((fullRunTimeInSecs % 60))
-  printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
+  printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs" >> "$aggregatedTestResult"
  }
 
 logOutput() {