From ff7fbef169ff9529b1ba6f27220ac0220c8bacbe Mon Sep 17 00:00:00 2001
From: Raymond Xu <2701446+xushiyan@users.noreply.github.com>
Date: Wed, 15 Dec 2021 12:43:30 -0800
Subject: [PATCH 1/2] [HUDI-3028] Use blob storage to speed up CI downloads

---
 azure-pipelines.yml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2249328ff9957..8f250e7fdd84c 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -32,6 +32,7 @@ stages:
   - stage: test
     jobs:
       - job: UT_FT_1
+        condition: false
         displayName: UT FT common & flink & UT client/spark-client
         steps:
           - task: Cache@2
@@ -70,6 +71,7 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_2
+        condition: false
         displayName: FT client/spark-client
         steps:
           - task: Cache@2
@@ -99,6 +101,7 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_3
+        condition: false
         displayName: UT FT clients & cli & utilities & sync/hive-sync
         steps:
           - task: Cache@2
@@ -137,6 +140,7 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_4
+        condition: false
         displayName: UT FT other modules
         steps:
           - task: Cache@2
@@ -176,10 +180,19 @@ stages:
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: IT
         steps:
+          - task: AzureCLI@2
+            displayName: Azure CLI
+            inputs:
+              azureSubscription: apachehudici-service-connection
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                az --version
+                az account show
+                echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
+                az storage blob download -c ci-caches -n spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -f $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz --account-name apachehudici
+                tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -C $(Pipeline.Workspace)/
           - script: |
-              echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
-              wget https://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz
-              tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/
               mkdir /tmp/spark-events/
               mvn $(MAVEN_OPTS) -Pintegration-tests verify
             displayName: IT

From 202e3525f5fcf564ea511199ae15fd3c263a0086 Mon Sep 17 00:00:00 2001
From: Sivabalan Narayanan <n.siva.b@gmail.com>
Date: Wed, 15 Dec 2021 13:58:25 -0800
Subject: [PATCH 2/2] fixing pipline yaml

---
 azure-pipelines.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 8f250e7fdd84c..6d96ed51f242d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -32,7 +32,6 @@ stages:
   - stage: test
     jobs:
       - job: UT_FT_1
-        condition: false
         displayName: UT FT common & flink & UT client/spark-client
         steps:
           - task: Cache@2
@@ -71,7 +70,6 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_2
-        condition: false
         displayName: FT client/spark-client
         steps:
           - task: Cache@2
@@ -101,7 +99,6 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_3
-        condition: false
         displayName: UT FT clients & cli & utilities & sync/hive-sync
         steps:
           - task: Cache@2
@@ -140,7 +137,6 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: UT_FT_4
-        condition: false
         displayName: UT FT other modules
         steps:
           - task: Cache@2