Merge branch 'main' into automation/add-new-python-versions-2023-12-08

build/__dotNetCoreRunTimeVersions.sh

@@ -1,6 +1,6 @@
 # This file was auto-generated from 'constants.yaml'. Changes may be overridden.
 
-DOT_NET_CORE_RUNTIME_BASE_TAG='20231121.1'
+DOT_NET_CORE_RUNTIME_BASE_TAG='20231211.2'
 NET_CORE_APP_10='1.0.16'
 NET_CORE_APP_11='1.1.13'
 NET_CORE_APP_20='2.0.9'

build/__pythonVersions.sh

@@ -8,5 +8,5 @@ PYTHON37_VERSION='3.7.15'
 PYTHON38_VERSION='3.8.18'
 PYTHON39_VERSION='3.9.18'
 PYTHON310_VERSION='3.10.13'
-PYTHON311_VERSION='3.11.6'
+PYTHON311_VERSION='3.11.7'
 PYTHON312_VERSION='3.12.1'

build/__stagingRuntimeConstants.sh

@@ -1,6 +1,6 @@
 # This file was auto-generated from 'constants.yaml'. Changes may be overridden.
 
-DOTNETCORE_STAGING_RUNTIME_VERSIONS=("8.0")
+DOTNETCORE_STAGING_RUNTIME_VERSIONS=("")
 PYTHON_STAGING_RUNTIME_VERSIONS=("")
 NODE_STAGING_RUNTIME_VERSIONS=("")
 JAVA_STAGING_RUNTIME_VERSIONS=("")

build/constants.yaml

@@ -69,7 +69,7 @@
       file-name-prefix: __
 - name: dot-net-core-run-time-versions
   constants:
-    dot-net-core-runtime-base-tag: 20231121.1
+    dot-net-core-runtime-base-tag: 20231211.2
     net-core-app-10: 1.0.16
     net-core-app-11: 1.1.13
     net-core-app-20: 2.0.9
@@ -157,7 +157,7 @@
     python38-version: 3.8.18
     python39-version: 3.9.18
     python310-version: 3.10.13
-    python311-version: 3.11.6
+    python311-version: 3.11.7
     python312-version: 3.12.1
     runtime-versions:
       - 3.7-debian-bullseye
@@ -479,7 +479,7 @@
 - name: staging-runtime-constants
   constants:
     dotnetcore-staging-runtime-versions:
-      - 8.0
+      -
     python-staging-runtime-versions:
       -
     node-staging-runtime-versions:

builders/container-apps-internal-registry-demo/Dockerfile

@@ -38,6 +38,7 @@ RUN set -eux; \
 		sudo \
 		curl \
 		unzip \
+		grep \
 	; \
 	rm -rf /var/lib/apt/lists/*
 

builders/container-apps-internal-registry-demo/startup-script.sh

@@ -11,6 +11,7 @@ file_upload_endpoint="$FILE_UPLOAD_CONTAINER_URL/$FILE_UPLOAD_BLOB_NAME"
 
 temp_app_source_dir="/tmp/appsource"
 temp_app_source_path="$temp_app_source_dir/$FILE_UPLOAD_BLOB_NAME"
+temp_app_header_path="$temp_app_source_dir/header.txt"
 mkdir $temp_app_source_dir
 
 # List all the environment variables and filter the environment variable with prefix "ACA_CLOUD_BUILD_USER_ENV_", then write them to folder "/platform/env", 
@@ -20,26 +21,45 @@ env | grep -E '^ACA_CLOUD_BUILD_USER_ENV_' | while read -r line; do
   key=$(echo "$line" | cut -d= -f1)
   value=$(echo "$line" | cut -d= -f2-)
   filename="${key#ACA_CLOUD_BUILD_USER_ENV_}"
-  echo "$value" > "$build_env_dir/$filename"
+  echo -n "$value" > "$build_env_dir/$filename"
 done
 
 # write environment variable CORRELATION_ID to folder "/platform/env", 
 if [ -n "$CORRELATION_ID" ]; then
-  echo "$CORRELATION_ID" > "$build_env_dir/CORRELATION_ID"
+  echo -n "$CORRELATION_ID" > "$build_env_dir/CORRELATION_ID"
 fi
 
+file_extension=""
 while [[ ! -f "$temp_app_source_path" || ! "$(file $temp_app_source_path)" =~ "compressed data" ]]
 do
   echo "Waiting for app source to be uploaded. Please upload the app source to the endpoint specified in the Build resource's 'uploadEndpoint' property."
-  curl -H "$auth_header" -H "$version_header" -H "$date_header" -X GET "$file_upload_endpoint" -o "$temp_app_source_path" -s
+  curl -H "$auth_header" -H "$version_header" -H "$date_header" -X GET "$file_upload_endpoint" -o "$temp_app_source_path" -D "$temp_app_header_path" -s
+  if [[ -f "$temp_app_header_path" ]]; then
+    file_extension=$(grep -i x-ms-meta-FileExtension "$temp_app_header_path" | cut -d ' ' -f2)
+    # Check if the original file extension is .jar, .war, .zip or .tar.gz
+    if [[ "$file_extension" =~ ".tar.gz" 
+          || "$file_extension" =~ ".jar" 
+          || "$file_extension" =~ ".war" 
+          || "$file_extension" =~ ".zip"  ]]; then
+      break
+    fi
+  fi
   sleep 5
 done
 
 # Extract app code to CNB_APP_DIR directory.
 echo "Found app source at '$temp_app_source_path'. Extracting to $CNB_APP_DIR"
 mkdir -p $CNB_APP_DIR
 cd $CNB_APP_DIR
-tar -xzf "$temp_app_source_path"
+
+if [[ "$file_extension" =~ ".jar" 
+      || "$file_extension" =~ ".war" 
+      || "$file_extension" =~ ".zip"  ]]; then
+  unzip -qq "$temp_app_source_path"
+else
+# Keep compatibility with old logic
+  tar -xzf "$temp_app_source_path"
+fi
 
 fileCount=$(ls | wc -l)
 if [ "$fileCount" = "1" ]; then

doc/supportedPlatformVersions.md

@@ -1234,6 +1234,7 @@
 - 3.11.0b1
 - 3.11.0
 - 3.11.6
+- 3.11.7
 - 3.12.0
 - 3.12.1
 
@@ -1260,6 +1261,7 @@
 - 3.10.8
 - 3.10.13
 - 3.11.6
+- 3.11.7
 - 3.12.1
 
 ### focal-scm
@@ -1286,6 +1288,7 @@
 - 3.10.13
 - 3.11.6
 - 3.12.1
+- 3.11.7
 
 ### stretch
 
@@ -1336,6 +1339,7 @@
 - 3.10.8
 - 3.10.13
 - 3.11.6
+- 3.11.7
 - 3.12.1
 
 

platforms/dotnet/versions/buster/versionsToBuild.txt

@@ -5,7 +5,7 @@
 2.1.812, 09fb6fe64434aa83084c1cd17d632648d4be3146a9a6c0e8e50f574748adcac8c1dadb0c49b3e909252223166fedef97173552331052fce4051b216c9f8497d8,
 2.1.813, fbf4bbe5ff88bd12ef9da097839e9b426ce6d03b5ebff4a32f8cbc8b082b1086d5b6daacbe6c38e581d606ca2e4d7da98bac86ed608ecbee35616f9b9adbfa53,
 2.1.814, 79408996f53650d0c3ac39348fa102537d14190ba5dcc4b9152cdb8fc72566608ad7430f196731eeb62dcfacdb0f2fa37577b5d51e165afd9dd9ae15f9d2aabc,
-2.1.816, 58f0bc1f67de034ffd0dafb9c0fdb082786fc5057e89396ff574428d57331cd8d5b3e944e103918e05f7b66e354d56cdb242350a6ef932906c9c3d4b08d177e9, 
+2.1.816, 58f0bc1f67de034ffd0dafb9c0fdb082786fc5057e89396ff574428d57331cd8d5b3e944e103918e05f7b66e354d56cdb242350a6ef932906c9c3d4b08d177e9,
 2.1.817, 1bdc0f0bba9a0fbdf0546ff5296d6198aec34b96f5a771ef54f905678ce78de3b1abeac92f42f2118043a69271d268cba196ff3958a37094e5f2e9703966c300,
 2.1.818, 0975301378d1238e55285d37aed4ab10df242e0884c0e3bb2eaf2e96af7bf6d554f5df1e653abddabf23ed8f1ea1665c452de42ad912ca84cd71760171416ecd,
 3.1.404, 94d8eca3b4e2e6c36135794330ab196c621aee8392c2545a19a991222e804027f300d8efd152e9e4893c4c610d6be8eef195e30e6f6675285755df1ea49d3605,
@@ -26,7 +26,7 @@
 5.0.100, BEC37BFB327C45CC01FD843EF93B22B556F753B04724BBA501622DF124E7E144C303A4D7E931B5DBADBD4F7B39E5ADB8F601CB6293E317AD46D8FE7D52AA9A09,
 5.0.102, 0ce2d5365ca39808fb71baec4584d4ec786491c3735543dc93244604ea97e242377d0987cd8b1e529258dee68f203b5780559201e7ea6d84487d6d8d433329b3,
 5.0.103, bf452dbdaec7a82835cfc7c698d2558e7ac4b092e7ef35092796ba5440eb45dd880e86c1e61f8e170ac4eb813240cf83f9fc2e342dfa8b37e45afdf5cd82fb8e,
-5.0.104, 7b76e0b3a67e0febdc04085b3b73c6a8799290ec935bafefe699c43f8a499c1d3d80d18090e62410d45e49620cfbb1c898259139a0f34e4a9c4db6307324cf7e, 
+5.0.104, 7b76e0b3a67e0febdc04085b3b73c6a8799290ec935bafefe699c43f8a499c1d3d80d18090e62410d45e49620cfbb1c898259139a0f34e4a9c4db6307324cf7e,
 5.0.202, 01ed59f236184987405673d24940d55ce29d830e7dbbc19556fdc03893039e6046712de6f901dc9911047a0dee4fd15319b7e94f8a31df6b981fa35bd93d9838,
 5.0.203, 49d8f0414806a9c938192ed13e7707ac2609ca6c2dc408d616e56e98fc0a954b1aa3f569858f7ba38fb79b2ee36dc1920c7f08d1ba4f93da501542b1c8a1320f,
 5.0.302, c80df1f2208f07a76e6bc123aa0fe94879e202dbab9006a5bcd6ec8508e58aa9da5e2c7736f45a2c307016bb6356e8c5ce39cd4ca65d732091d7f46c7eacc397,
@@ -59,7 +59,6 @@
 6.0.410, 8c85f5b10eb786c8cf31bf268131a2345a295d88d318310dc8457d831f0a587ec1600e43beb7f55aec2248483b9a95e905a468b592f0c910443b4aaa9baeb2e3,
 6.0.411, dc8aa1f84ad97cf25a979bfc243c200b7a8e73b930b68d5eed782743d88aad823c32c267c83d7a19d3c4f910a8fae7f12d07ea5a35a1d3a97e13a8674d29037b,
 6.0.412, ee97aa5258e05aecadc66e844fa8eef97d51e5035492999b974cc8272e4db1a862a1f88a925f38be9e95c71d2e961b56459dcd614475895df608945c8057c311,
-6.0.417, 997caff60dbad7259db7e3dd89886fc86b733fa6c1bd3864c8199f704eb24ee59395e327c43bb7c0ed74e57ec412bd616ea26f02f8f8668d04423d6f8e0a8a33,
 7.0.100-preview.1.22110.4, 54488a911172f059e3823d6bf52e1fa87305eb09e84d97f81a40e0815fc8a73a480b149023283f557a672ef0341f022b8ca16ebec92264ee16a56fac8f35e2e2,
 7.0.100-preview.5.22307.18, 691c0d8917bc9848a08707b7fa22da05228dad0ba6335ff06c6d80f9a95349307572ff45c0b088d9fc199c40a1784ff314e1a8735d0366bd3aa06eb8dfa2b7d5,
 7.0.100-preview.6.22352.1, e49a2119021e4069f1193898536cc59628336e656f2f7e49d454a593a330d8e437acf2f4efb70925bc16a9c900c2e49f4a6c2fb5f69e696b09a91ebccd2c9307,
@@ -76,7 +75,6 @@
 7.0.304, f4b7d0cde432bd37f445363b3937ad483e5006794886941e43124de051475925b3cd11313b73d2cae481ee9b8f131394df0873451f6088ffdbe73f150b1ed727,
 7.0.305, c63e6baf3760a9949089d78920531a1140f4f31fffe13069b2011c61c3d583d585f5ec8cca973b414fa35d48ccbfea9c1ec1c88222b53afd2af5974be3b5cb1b,
 7.0.306, 62df9bca9492b3273830e098e787ec3664243989ac03550534599fc331693553660d3cf8bca655f2d1326070dbb7b20b04743eaba77fa9cc69f6f0fddfdebd06,
-7.0.404, f5c122044e9a107968af1a534051e28242f45307c3db760fbb4f3a003d92d8ea5a856ad4c4e8e4b88a3b6a825fe5e3c9e596c9d2cfa0eca8d5d9ee2c5dad0053,
 8.0.100-preview.1.23115.2, 23a14c92e402161ed8d42ec9cb25a97868a1b72348195d28cffa00a12815f019308b56485e4375c0d0a33d9a683d83cc1e1a2a517eea44af8fb353171b6c3f64,
 8.0.100-preview.2.23157.25, 97302c3600af7787fb136b226ca7e2a0a22241aa93dcffc70010b475bf6f8c4ff74a363d94949e1b64a91032b57a58a7065d7c6b2177696d8e78504ef4f1280f,
 8.0.100-preview.3.23178.7, 3b5d72979831256b9340a01db23d3b2dca801672546eeed04385949ed5f4363d3c731f31477ec82c7200ce88502dc45e03986c8acc8f2fc611b0343af5f1c488,

platforms/python/versions/bullseye/versionsToBuild.txt

@@ -9,6 +9,7 @@
 3.11.0, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.11.0b1, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.11.6, A035C8C19219BA821ECEA86B64E628F8D684696D,
+3.11.7, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.12.0, 7169605F62C751356D054A26A821E680E5FA6305,
 3.12.1, 7169605F62C751356D054A26A821E680E5FA6305,
 3.7.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

platforms/python/versions/buster/versionsToBuild.txt

@@ -7,6 +7,7 @@
 3.10.4, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.10.8, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.11.6, A035C8C19219BA821ECEA86B64E628F8D684696D,
+3.11.7, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.12.1, 7169605F62C751356D054A26A821E680E5FA6305,
 3.6.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,
 3.6.15, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

platforms/python/versions/focal-scm/versionsToBuild.txt

@@ -7,6 +7,7 @@
 3.10.4, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.10.8, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.11.6, A035C8C19219BA821ECEA86B64E628F8D684696D,
+3.11.7, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.12.1, 7169605F62C751356D054A26A821E680E5FA6305,
 3.6.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,
 3.6.15, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

platforms/python/versions/stretch/versionsToBuild.txt

@@ -7,6 +7,7 @@
 3.10.4, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.10.8, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.11.6, A035C8C19219BA821ECEA86B64E628F8D684696D,
+3.11.7, A035C8C19219BA821ECEA86B64E628F8D684696D,
 3.12.1, 7169605F62C751356D054A26A821E680E5FA6305,
 3.6.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,
 3.6.15, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

src/BuildScriptGenerator/DotNetCore/DotNetCoreRunTimeVersions.cs

@@ -4,7 +4,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
 {
     public static class DotNetCoreRunTimeVersions
     {
-        public const string DotNetCoreRuntimeBaseTag = "20231121.1";
+        public const string DotNetCoreRuntimeBaseTag = "20231211.2";
         public const string NetCoreApp10 = "1.0.16";
         public const string NetCoreApp11 = "1.1.13";
         public const string NetCoreApp20 = "2.0.9";

src/BuildScriptGenerator/PythonVersions.cs

@@ -14,7 +14,7 @@ public static class PythonVersions
         public const string Python38Version = "3.8.18";
         public const string Python39Version = "3.9.18";
         public const string Python310Version = "3.10.13";
-        public const string Python311Version = "3.11.6";
+        public const string Python311Version = "3.11.7";
         public const string Python312Version = "3.12.1";
         public static readonly List<string> RuntimeVersions = new List<string> { "3.7-debian-bullseye", "3.7-debian-buster", "3.8-debian-bullseye", "3.8-debian-buster", "3.9-debian-bullseye", "3.9-debian-buster", "3.10-debian-bullseye", "3.10-debian-buster", "3.11-debian-bullseye", "3.12-debian-bullseye", "dynamic-debian-buster" };
     }

src/BuildScriptGenerator/StagingRuntimeConstants.cs

@@ -6,7 +6,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Common
 {
     public static class StagingRuntimeConstants
     {
-        public static readonly List<string> DotnetcoreStagingRuntimeVersions = new List<string> { "8.0" };
+        public static readonly List<string> DotnetcoreStagingRuntimeVersions = new List<string> { };
         public static readonly List<string> PythonStagingRuntimeVersions = new List<string> { };
         public static readonly List<string> NodeStagingRuntimeVersions = new List<string> { };
         public static readonly List<string> JavaStagingRuntimeVersions = new List<string> { };

tests/Oryx.Integration.Tests/StagingStorageAccountSanityTest.cs

@@ -11,7 +11,7 @@
 
 namespace Oryx.Integration.Tests
 {
-    [Trait("StorageAccountTests", "Dev")]
+    [Trait("StorageAccountTests", "Staging")]
     public class StagingStorageAccountSanityTest : StorageAccountSanityTestBase
     {
         public StagingStorageAccountSanityTest(

vsts/pipelines/ci.yml

@@ -8,40 +8,40 @@ parameters:
   - name: buildImages
     type: object
     default:
-      - 
+      -
         key: Latest
         value: latest
-      - 
+      -
         key: LtsVersions
         value: ltsversions
-      - 
+      -
         key: Jamstack
         value: jamstack
-      - 
+      -
         key: GithubActions
         value: githubactions
-      - 
+      -
         key: VsoFocal
         value: vso-focal
-      - 
+      -
         key: VsoBullseye
         value: vso-bullseye
-      - 
+      -
         key: Full
         value: full
-      - 
+      -
         key: Cli
         value: cli-stretch
-      - 
+      -
         key: CliBuster
         value: cli-buster
-      - 
+      -
         key: CliBullseye
         value: cli-bullseye
-      - 
+      -
         key: CliBuilderBullseye
         value: cli-builder-bullseye
-      - 
+      -
         key: Buildpack
         value: buildpack
 
@@ -65,11 +65,11 @@ stages:
       steps:
       - checkout: self
         clean: true
-        
+
       - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
         displayName: 'Component Detection - OSS Compliance'
         inputs:
-          ignoreDirectories: '$(Build.SourcesDirectory)/tests'  
+          ignoreDirectories: '$(Build.SourcesDirectory)/tests'
 
       - task: ShellScript@2
         inputs:
@@ -124,7 +124,7 @@ stages:
           }
         displayName: 'Set release tag name as environment variable'
       - template: templates/_signBinary.yml
-    
+
       # Job loop for BuildImages
     - ${{ each buildImage in parameters.buildImages }}:
       - job: Job_BuildImage_${{ buildImage.key }}
@@ -264,11 +264,33 @@ stages:
       - template: templates/_buildTemplate.yml
         parameters:
             imageType: bookworm
-            
+
     - template: templates/_integrationJobTemplate.yml
       parameters:
           storageAccountUrl: ${{ parameters.storageAccountUrl }}
 
+    - job: Job_ProductionStorageAccountTest
+      displayName: Test Production Storage Account
+      pool:
+        name: AzurePipelines-EO
+        demands:
+          - ImageOverride -equals AzurePipelinesUbuntu20.04compliant
+      variables:
+        skipComponentGovernanceDetection: true
+      steps:
+      - task: UseDotNet@2
+        displayName: 'Use .NET Core sdk 7.x'
+        inputs:
+          version: 7.0.306
+
+      - task: ShellScript@2
+        displayName: 'Test production storage account'
+        env:
+          ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }}
+        inputs:
+          scriptPath: ./build/testIntegration.sh
+          args: StorageAccountTests=Prod
+
   - stage: Release
     displayName: Release Stage
     dependsOn: Build

vsts/pipelines/nightly.yml

@@ -203,9 +203,32 @@ stages:
       - template: templates/_buildTemplate.yml
         parameters:
             imageType: bookworm
-            
+
     - template: templates/_integrationJobTemplate.yml
       parameters:
         storageAccountUrl: ${{ parameters.storageAccountUrl }}
 
+    - job: Job_StagingStorageAccountTest
+      displayName: Test Staging Storage Account
+      pool:
+        name: AzurePipelines-EO
+        demands:
+          - ImageOverride -equals AzurePipelinesUbuntu20.04compliant
+      variables:
+        skipComponentGovernanceDetection: true
+      steps:
+      - task: UseDotNet@2
+        displayName: 'Use .NET Core sdk 7.x'
+        inputs:
+          version: 7.0.306
+
+      - task: ShellScript@2
+        displayName: 'Test staging storage account'
+        env:
+          ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }}
+          ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN)
+        inputs:
+          scriptPath: ./build/testIntegration.sh
+          args: StorageAccountTests=Staging
+
 trigger: none