[POC] internal ref pr for checking tests diff.

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2319,6 +2319,12 @@
   <description>The AbstractFileSystem for gs: uris.</description>
 </property>
 
+<property>
+  <name>fs.azure.enable.readahead</name>
+  <value>true</value>
+  <description>Enabled readahead/prefetching in AbfsInputStream.</description>
+</property>
+
 <property>
   <name>io.seqfile.compress.blocksize</name>
   <value>1000000</value>

hadoop-tools/hadoop-azure/.gitignore

@@ -1,5 +1,6 @@
 .checkstyle
 bin/
-src/test/resources/combinationConfigFiles
 src/test/resources/abfs-combination-test-configs.xml
 dev-support/testlogs
+src/test/resources/accountSettings/*
+!src/test/resources/accountSettings/accountName_settings.xml.template

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

@@ -2,7 +2,7 @@
 
 # shellcheck disable=SC2034
 # unused variables are global in nature and used in testsupport.sh
-
+test
 set -eo pipefail
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,36 +22,167 @@ set -eo pipefail
 
 # shellcheck disable=SC1091
 . dev-support/testrun-scripts/testsupport.sh
+init
+
+resourceDir=src/test/resources/
+logdir=dev-support/testlogs/
+azureTestXml=azure-auth-keys.xml
+azureTestXmlPath=$resourceDir$azureTestXml
+processCount=8
+
+## SECTION: TEST COMBINATION METHODS
 
-begin
+runHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
-### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
+runHNSSharedKeyTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("SharedKey")
+  triggerRun "HNS-SharedKey" "$accountName"  "$runTest" $processCount "$cleanUpTestContainers"
+}
 
+runNonHNSSharedKeyTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("SharedKey")
+  triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth")
-generateconfigs
+runNonHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "NonHNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=AppendBlob-HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
-generateconfigs
+runAppendBlobHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+  VALUES=("OAuth" "true")
+  triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
-generateconfigs
+runTest=false
+cleanUpTestContainers=false
+echo 'Ensure below are complete before running script:'
+echo '1. Account specific settings file is present.'
+echo '   Copy accountName_settings.xml.template to accountName_settings.xml'
+echo '   where accountName in copied file name should be the test account name without domain'
+echo '   (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+echo '   folder. New account settings file to be added to same folder.)'
+echo '   Follow instructions in the template to populate settings correctly for the account'
+echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+echo '   where accountNames should be the test account names without domain'
+echo ' '
+echo ' '
+echo 'Choose action:'
+echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+do
+  case $scriptMode in
+  SET_ACTIVE_TEST_CONFIG)
+    runTest=false
+    break
+    ;;
+  RUN_TEST)
+    runTest=true
+    read -r -p "Enter parallel test run process count [default - 8]: " processCount
+    processCount=${processCount:-8}
+    break
+    ;;
+  CLEAN_UP_OLD_TEST_CONTAINERS)
+    runTest=false
+    cleanUpTestContainers=true
+    break
+    ;;
+  SET_OR_CHANGE_TEST_ACCOUNT)
+    runTest=false
+    cleanUpTestContainers=false
+    accountSettingsFile="src/test/resources/azure-auth-keys.xml"
+    if [[ ! -f "$accountSettingsFile" ]];
+    then
+      logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
+      cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
+    fi
 
-combination=NonHNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
-generateconfigs
+    vi $accountSettingsFile
+    exit 0
+    break
+    ;;
+  PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
+    runTest=false
+    cleanUpTestContainers=false
+    logFilePaths=/tmp/logPaths
+    find target/ -name "*output.txt" > $logFilePaths
+    logOutput "$(cat $logFilePaths)"
+    rm $logFilePaths
+    exit 0
+    break
+    ;;
+  *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
 
+## SECTION: COMBINATION DEFINITIONS AND TRIGGER
 
-### DO NOT EDIT THE LINES BELOW.
+echo ' '
+echo 'Set the active test combination to run the action:'
+select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey NonHNS-OAuth AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
+do
+   case $combo in
+      HNS-OAuth)
+         runHNSOAuthTest
+         break
+         ;;
+      HNS-SharedKey)
+         runHNSSharedKeyTest
+         break
+         ;;
+      nonHNS-SharedKey)
+         runNonHNSSharedKeyTest
+         break
+         ;;
+       NonHNS-OAuth)
+          runNonHNSOAuthTest
+          break
+          ;;
+       AppendBlob-HNS-OAuth)
+         runAppendBlobHNSOAuthTest
+         break
+         ;;
+      AllCombinationsTestRun)
+        if [ $runTest == false ]
+        then
+          logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
+          break
+        fi
+         runHNSOAuthTest
+         runHNSSharedKeyTest
+         runNonHNSSharedKeyTest
+         runNonHNSOAuthTest
+         runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
+         break
+         ;;
+      Quit)
+         exit 0
+         ;;
+      *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
 
-runtests "$@"
+if [ $runTest == true ]
+then
+  printAggregate
+fi