Publish on Maven via beetmover

Co-authored-by: Mihai Tabara <[email protected]>

.taskcluster.yml

@@ -40,11 +40,6 @@ tasks:
       $if: 'tasks_for == "github-pull-request"'
       then: ${event.pull_request.head.repo.html_url}
       else: ${event.repository.html_url}
-
-    clone_url:
-      $if: 'tasks_for == "github-pull-request"'
-      then: ${event.pull_request.head.repo.clone_url}
-      else: ${event.repository.clone_url}
   in:
     $let:
       default_task_definition:
@@ -75,6 +70,8 @@ tasks:
         metadata:
           owner: &task_owner ${user}@users.noreply.github.com
           source: &task_source ${repository}/raw/${head_rev}/.taskcluster.yml
+        extra:
+          tasks_for: ${tasks_for}
         payload:
           artifacts:
             public/task-graph.json:
@@ -101,13 +98,13 @@ tasks:
               python3 -m pip install pyyaml &&
               git init repo &&
               cd repo &&
-              git fetch --tags ${clone_url} ${head_branch} &&
+              git fetch --tags ${repository} ${head_branch} &&
               git reset --hard ${head_rev} &&
               python3 automation/taskcluster/decision_task.py
           env:
-            GIT_URL: ${clone_url}
-            GIT_REF: ${head_branch}
-            GIT_SHA: ${head_rev}
+            APPSERVICES_HEAD_REPOSITORY: ${repository}
+            APPSERVICES_HEAD_BRANCH: ${head_branch}
+            APPSERVICES_HEAD_REV: ${head_rev}
             TASK_FOR: ${tasks_for}
             TASK_OWNER: *task_owner
             TASK_SOURCE: *task_source
@@ -137,18 +134,32 @@ tasks:
                 description: Schedules the build and test tasks for Application Services.
         "tasks_for == 'github-release'":
           $let:
+            beetmover_worker_type: appsv-beetmover-v1
+            beetmover_bucket: maven-production
+            beetmover_bucket_public_url: https://maven.mozilla.org/
             tag: ${event.release.tag_name}
             release_task_definition:
+              payload:
+                features:
+                  chainOfTrust: true
               scopes:
-                # So that we can publish to nalexander@'s personal bintray
-                # at https://bintray.com/ncalexander/application-services.
-                - "secrets:get:project/application-services/publish"
+                # So that we can publish on Maven using beetmover
+                - project:mozilla:application-services:releng:beetmover:action:push-to-maven
                 # So that we can upload symbols to Socorro
                 - "secrets:get:project/application-services/symbols-token"
           in:
             $mergeDeep:
               - {$eval: 'default_task_definition'}
               - {$eval: 'release_task_definition'}
+              - payload:
+                  env:
+                    BEETMOVER_WORKER_TYPE: ${beetmover_worker_type}
+                    BEETMOVER_BUCKET: ${beetmover_bucket}
+                    BEETMOVER_BUCKET_PUBLIC_URL: ${beetmover_bucket_public_url}
+              - scopes:
+                # So that we can publish on Maven using beetmover
+                - project:mozilla:application-services:releng:beetmover:bucket:${beetmover_bucket}
+                - queue:create-task:${tasks_priority}:scriptworker-prov-v1/${beetmover_worker_type}
               - metadata:
                   name: Application Services - Decision task (${tag})
                   description: Build and publish release versions.

automation/symbols-generation/symbolstore.py

@@ -121,8 +121,8 @@ class GitHubRepoInfo:
     """
     def __init__(self, path):
         self.path = path
-        if 'GIT_URL' in os.environ:
-            remote_url = os.environ['GIT_URL']
+        if 'APPSERVICES_HEAD_REPOSITORY' in os.environ:
+            remote_url = os.environ['APPSERVICES_HEAD_REPOSITORY']
         else:
             remote_url = read_output('git', '-C', path, 'remote', 'get-url', 'origin')
         match = githubRegex.match(remote_url)

automation/taskcluster/decision_task.py

@@ -55,10 +55,11 @@ def main(task_for):
 # Calls "$PLATFORM_libs" functions and returns
 # their tasks IDs.
 def libs_for(*platforms):
-    return list(map(lambda p: globals()[p + "_libs"](), platforms))
+    is_release = os.environ["TASK_FOR"] == "github-release"
+    return list(map(lambda p: globals()[p + "_libs"](is_release), platforms))
 
-def android_libs():
-    return (
+def android_libs(is_release):
+    task = (
         linux_build_task("Android libs (all architectures): build")
         .with_script("""
             pushd libs
@@ -69,11 +70,14 @@ def android_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.android." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.android." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_linux_libs():
-    return (
+def desktop_linux_libs(is_release):
+    task = (
         linux_build_task("Desktop libs (Linux): build")
         .with_script("""
             pushd libs
@@ -84,11 +88,14 @@ def desktop_linux_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.linux." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.linux." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_macos_libs():
-    return (
+def desktop_macos_libs(is_release):
+    task = (
         linux_cross_compile_build_task("Desktop libs (macOS): build")
         .with_script("""
             pushd libs
@@ -99,11 +106,14 @@ def desktop_macos_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.macos." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.macos." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_win32_x86_64_libs():
-    return (
+def desktop_win32_x86_64_libs(is_release):
+    task = (
         linux_build_task("Desktop libs (win32-x86-64): build")
         .with_script("""
             apt-get install --quiet --yes --no-install-recommends mingw-w64
@@ -115,8 +125,11 @@ def desktop_win32_x86_64_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.win32-x86-64." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.win32-x86-64." + CONFIG.git_sha_for_directory("libs"))
 
 def android_task(task_name, libs_tasks):
     task = linux_cross_compile_build_task(task_name)
@@ -156,11 +169,7 @@ def gradle_module_task_name(module, gradle_task_name):
 
 def gradle_module_task(libs_tasks, module_info, is_release):
     module = module_info['name']
-    if is_release:
-        task_title = "{} - Build, test and upload to bintray".format(module)
-    else:
-        task_title = "{} - Build and test".format(module)
-    task = android_task(task_title, libs_tasks)
+    task = android_task("{} - Build and test".format(module), libs_tasks)
     # This is important as by default the Rust plugin will only cross-compile for Android + host platform.
     task.with_script('echo "rust.targets=arm,arm64,x86_64,x86,darwin,linux-x86-64,win32-x86-64-gnu\n" > local.properties')
     if not is_release: # Makes builds way faster.
@@ -179,13 +188,9 @@ def gradle_module_task(libs_tasks, module_info, is_release):
     )
     for artifact_info in module_info['artifacts']:
         task.with_artifacts(artifact_info['artifact'])
-    if is_release:
-        if module_info['uploadSymbols']:
-            task.with_scopes("secrets:get:project/application-services/symbols-token")
-            task.with_script("./automation/upload_android_symbols.sh {}".format(module_info['path']))
-        task.with_scopes("secrets:get:project/application-services/publish")
-        task.with_script("python automation/taskcluster/release/fetch-bintray-api-key.py")
-        task.with_script('./gradlew --no-daemon {} --debug -PvcsTag="$GIT_SHA"'.format(gradle_module_task_name(module, "bintrayUpload")))
+    if is_release and module_info['uploadSymbols']:
+        task.with_scopes("secrets:get:project/application-services/symbols-token")
+        task.with_script("./automation/upload_android_symbols.sh {}".format(module_info['path']))
     return task.create()
 
 def build_gradle_modules_tasks(is_release):
@@ -201,40 +206,54 @@ def android_multiarch():
 
 def android_multiarch_release():
     module_build_tasks = build_gradle_modules_tasks(True)
-    return (
-        linux_build_task("All modules - Publish via bintray")
-        .with_dependencies(*module_build_tasks.values())
-        # Our -unpublished- artifacts were uploaded in build_gradle_modules_tasks(),
-        # however there is not way to just trigger a bintray publish from gradle without
-        # uploading anything, so we do it manually using curl :(
-        # We COULD publish each artifact individually, however that would mean if
-        # a build task fails we end up with a partial release.
-        # Since we manipulate secrets, we also disable bash debug mode.
-        .with_script("""
-            python automation/taskcluster/release/fetch-bintray-api-key.py
-            set +x
-            BINTRAY_USER=$(grep 'bintray.user=' local.properties | cut -d'=' -f2)
-            BINTRAY_APIKEY=$(grep 'bintray.apikey=' local.properties | cut -d'=' -f2)
-            PUBLISH_URL=https://api.bintray.com/content/mozilla-appservices/application-services/org.mozilla.appservices/{}/publish
-            echo "Publishing on $PUBLISH_URL"
-            curl -X POST -u $BINTRAY_USER:$BINTRAY_APIKEY $PUBLISH_URL
-            echo "Success!"
-            set -x
-        """.format(appservices_version()))
-        .with_scopes("secrets:get:project/application-services/publish")
-        .with_features('taskclusterProxy') # So we can fetch the bintray secret.
-        .create()
-    )
+
+    version = appservices_version()
+    worker_type = os.environ['BEETMOVER_WORKER_TYPE']
+    bucket_name = os.environ['BEETMOVER_BUCKET']
+    bucket_public_url = os.environ['BEETMOVER_BUCKET_PUBLIC_URL']
+
+    for module_info in module_definitions():
+        module = module_info['name']
+        build_task = module_build_tasks[module]
+        for artifact in module_info['artifacts']:
+            artifact_name = artifact['name']
+            artifact_path = artifact['path']
+            (
+                BeetmoverTask("Publish Android module: {} via beetmover".format(artifact_name))
+                .with_description("Publish release module {} to {}".format(artifact_name, bucket_public_url))
+                .with_worker_type(worker_type)
+                # We want to make sure ALL builds succeeded before doing a release.
+                .with_dependencies(*module_build_tasks.values())
+                .with_upstream_artifact({
+                    "paths": [artifact_path],
+                    "taskId": build_task,
+                    "taskType": "build",
+                    "zipExtract": True,
+                })
+                .with_app_name("appservices")
+                .with_artifact_id(artifact_name)
+                .with_app_version(version)
+                .with_scopes(
+                    "project:mozilla:application-services:releng:beetmover:bucket:{}".format(bucket_name),
+                    "project:mozilla:application-services:releng:beetmover:action:push-to-maven"
+                )
+                .create()
+            )
 
 def dockerfile_path(name):
     return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile")
 
-
 def linux_task(name):
-    return DockerWorkerTask(name).with_worker_type("application-services-r")
-
+    task = (
+        DockerWorkerTask(name)
+        .with_worker_type("application-services-r")
+    )
+    if os.environ["TASK_FOR"] == "github-release":
+        task.with_features("chainOfTrust")
+    return task
 
 def linux_build_task(name):
+    use_indexed_docker_image = os.environ["TASK_FOR"] != "github-release"
     task = (
         linux_task(name)
         # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
@@ -251,7 +270,7 @@ def linux_build_task(name):
         .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
         .with_artifacts("/build/sccache.log")
         .with_max_run_time_minutes(120)
-        .with_dockerfile(dockerfile_path("build"))
+        .with_dockerfile(dockerfile_path("build"), use_indexed_docker_image)
         .with_env(**build_env, **linux_build_env)
         .with_script("""
             rustup toolchain install stable