[Yamato] Auto version and preview promotion of 10.x.x stream [skip ci] (#815)

* set up auto version for master

* create "promote all - nightly" job and edit dependencies

* add nightly trigger to preview promote job

* Update pb_promote.py

Author: sophiaaar

.yamato/_preview_publish.yml

@@ -16,6 +16,10 @@ auto-version:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci utils auto-version bump --package-path com.unity.render-pipelines.core --package-path com.unity.render-pipelines.universal --package-path com.unity.render-pipelines.high-definition --package-path com.unity.shadergraph --package-path com.unity.visualeffectgraph --package-path com.unity.render-pipelines.high-definition-config --package-path com.unity.render-pipelines.lightweight
       -  upm-ci utils auto-version commit --push
+    triggers:
+        branches:
+            only:
+              -  master
     artifacts:
         packages:
             paths:
@@ -30,6 +34,20 @@ publish_all_preview:
       -  .yamato/_preview_publish.yml#publish_vfx
       -  .yamato/_preview_publish.yml#publish_config
       -  .yamato/_preview_publish.yml#publish_lwrp
+promote_all_preview:
+    name: Promote all preview packages - nightly
+    dependencies:
+      -  .yamato/_preview_publish.yml#promote_core
+      -  .yamato/_preview_publish.yml#promote_universal
+      -  .yamato/_preview_publish.yml#promote_hdrp
+      -  .yamato/_preview_publish.yml#promote_shadergraph
+      -  .yamato/_preview_publish.yml#promote_vfx
+      -  .yamato/_preview_publish.yml#promote_config
+      -  .yamato/_preview_publish.yml#promote_lwrp
+    triggers:
+        recurring:
+          -  branch: master
+             frequency: daily
 wait_for_nightly:
     name: Wait for nightly
     dependencies:
@@ -82,7 +100,10 @@ promote_core:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_core
       -  .yamato/_packages.yml#pack_core
+      -  .yamato/_packages.yml#test_core_OSX_trunk
+      -  .yamato/_packages.yml#test_core_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.render-pipelines.core
@@ -117,7 +138,10 @@ promote_universal:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_universal
       -  .yamato/_packages.yml#pack_universal
+      -  .yamato/_packages.yml#test_universal_OSX_trunk
+      -  .yamato/_packages.yml#test_universal_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.render-pipelines.universal
@@ -152,7 +176,10 @@ promote_hdrp:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_hdrp
       -  .yamato/_packages.yml#pack_hdrp
+      -  .yamato/_packages.yml#test_hdrp_OSX_trunk
+      -  .yamato/_packages.yml#test_hdrp_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.render-pipelines.high-definition
@@ -187,7 +214,10 @@ promote_shadergraph:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_shadergraph
       -  .yamato/_packages.yml#pack_shadergraph
+      -  .yamato/_packages.yml#test_shadergraph_OSX_trunk
+      -  .yamato/_packages.yml#test_shadergraph_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.shadergraph
@@ -222,7 +252,10 @@ promote_vfx:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_vfx
       -  .yamato/_packages.yml#pack_vfx
+      -  .yamato/_packages.yml#test_vfx_OSX_trunk
+      -  .yamato/_packages.yml#test_vfx_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.visualeffectgraph
@@ -257,7 +290,10 @@ promote_config:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_config
       -  .yamato/_packages.yml#pack_config
+      -  .yamato/_packages.yml#test_config_OSX_trunk
+      -  .yamato/_packages.yml#test_config_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.render-pipelines.high-definition-config
@@ -292,7 +328,10 @@ promote_lwrp:
     variables:
         UPMCI_PROMOTION: 1
     dependencies:
+      -  .yamato/_preview_publish.yml#publish_lwrp
       -  .yamato/_packages.yml#pack_lwrp
+      -  .yamato/_packages.yml#test_lwrp_OSX_trunk
+      -  .yamato/_packages.yml#test_lwrp_Win_trunk
     commands:
       -  npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
       -  upm-ci package promote --package-path com.unity.render-pipelines.lightweight

.yamato/ruamel/build.py

@@ -21,6 +21,7 @@
 from jobs.preview_publish.pb_promote import PreviewPublish_PromoteJob
 from jobs.preview_publish.pb_auto_version import PreviewPublish_AutoVersionJob
 from jobs.preview_publish.pb_publish_all_preview import PreviewPublish_PublishAllPreviewJob
+from jobs.preview_publish.pb_promote_all_preview import PreviewPublish_PromoteAllPreviewJob
 from jobs.preview_publish.pb_wait_for_nightly import PreviewPublish_WaitForNightlyJob
 from jobs.templates.template_pack import Template_PackJob
 from jobs.templates.template_test import Template_TestJob
@@ -224,6 +225,9 @@ def create_preview_publish_jobs(metafile_name):
     job = PreviewPublish_PublishAllPreviewJob(metafile["packages"], target_branch, metafile["publishing"]["auto_publish"])
     yml[job.job_id] = job.yml
 
+    job = PreviewPublish_PromoteAllPreviewJob(metafile["packages"], target_branch, metafile["publishing"]["auto_publish"])
+    yml[job.job_id] = job.yml
+
     job = PreviewPublish_WaitForNightlyJob(metafile["packages"],  metafile["platforms"], target_editor)
     yml[job.job_id] = job.yml
 
@@ -233,7 +237,7 @@ def create_preview_publish_jobs(metafile_name):
             job = PreviewPublish_PublishJob(get_agent(metafile["agent_publish"]), package, metafile["platforms"], target_editor)
             yml[job.job_id] = job.yml
 
-            job = PreviewPublish_PromoteJob(get_agent(metafile["agent_promote"]), package)
+            job = PreviewPublish_PromoteJob(get_agent(metafile["agent_promote"]), package,  metafile["platforms"], target_editor)
             yml[job.job_id] = job.yml
 
     dump_yml(pb_filepath(), yml)

.yamato/ruamel/jobs/preview_publish/pb_auto_version.py

@@ -23,6 +23,7 @@ def get_job_definition(self, agent, packages, target_branch, auto_version):
                 f'npm install upm-ci-utils@stable -g --registry {NPM_UPMCI_INSTALL_URL}',
                 f'upm-ci utils auto-version bump {bump_packages_args}',
                 f'upm-ci utils auto-version commit --push'])
+        job.add_trigger_integration_branch(target_branch)
         job.add_artifacts_packages()
         # if auto_version is True:
         #     job.add_trigger_integration_branch(target_branch)

.yamato/ruamel/jobs/preview_publish/pb_promote.py

@@ -5,22 +5,29 @@
 
 class PreviewPublish_PromoteJob():
 
-    def __init__(self, agent, package):
+    def __init__(self, agent, package, platforms, target_editor):
         self.job_id = pb_job_id_promote(package["name"])
-        self.yml = self.get_job_definition(agent, package).get_yml()
+        self.yml = self.get_job_definition(agent, package, platforms, target_editor).get_yml()
 
 
-    def get_job_definition(self, agent, package):
-
+    def get_job_definition(self, agent, package, platforms, target_editor):
         if package["publish_source"] != True:
             raise Exception('Tried to promote package for which "publish_source" set to false.')
+
+        # define dependencies
+        dependencies = [
+            f'{pb_filepath()}#{pb_job_id_publish(package["name"])}',
+            f'{packages_filepath()}#{package_job_id_pack(package["name"])}']
+            
+        for platform in platforms:
+            dependencies.append(f'{packages_filepath()}#{package_job_id_test(package["name"],  platform["os"], target_editor)}')
 
         # construct job
         job = YMLJob()
         job.set_name(f'[{package["name"]}] Production Promote')
         job.set_agent(agent)
         job.add_var_custom('UPMCI_PROMOTION', 1)
-        job.add_dependencies([f'{packages_filepath()}#{package_job_id_pack(package["name"])}'])
+        job.add_dependencies(dependencies)
         job.add_commands([
                 f'npm install upm-ci-utils@stable -g --registry {NPM_UPMCI_INSTALL_URL}',
                 f'upm-ci {package["type"]} promote --{package["type"]}-path {package["path"]}'])

.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py

@@ -0,0 +1,22 @@
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString as dss
+from ..shared.namer import *
+from ..shared.yml_job import YMLJob
+
+class PreviewPublish_PromoteAllPreviewJob():
+    
+    def __init__(self, packages, target_branch, auto_publish):
+        self.job_id = pb_job_id_promote_all_preview()
+        self.yml = self.get_job_definition(packages, target_branch, auto_publish).get_yml()
+
+
+    def get_job_definition(self, packages, target_branch, auto_publish):
+
+        # construct job
+        job = YMLJob()
+        job.set_name(f'Promote all preview packages - nightly')
+        job.add_dependencies([f'{pb_filepath()}#{pb_job_id_promote(package["name"])}' for package in packages])
+        if auto_publish is True:
+            job.add_trigger_recurrent(target_branch, 'daily')
+        return job
+    
+    

.yamato/ruamel/jobs/shared/namer.py

@@ -103,4 +103,7 @@ def pb_job_id_wait_for_nightly():
     return f'wait_for_nightly'
 
 def pb_job_id_publish_all_preview():
-    return f'publish_all_preview'
+    return f'publish_all_preview'
+
+def pb_job_id_promote_all_preview():
+    return f'promote_all_preview'