ci: support optional jobs

Author: marcoieni

src/ci/citool/src/jobs.rs

@@ -66,6 +66,8 @@ pub struct JobDatabase {
     pub try_jobs: Vec<Job>,
     #[serde(rename = "auto")]
     pub auto_jobs: Vec<Job>,
+    #[serde(rename = "optional")]
+    pub optional_jobs: Vec<Job>,
 
     /// Shared environments for the individual run types.
     envs: JobEnvironments,
@@ -75,9 +77,10 @@ impl JobDatabase {
     /// Find `auto` jobs that correspond to the passed `pattern`.
     /// Patterns are matched using the glob syntax.
     /// For example `dist-*` matches all jobs starting with `dist-`.
-    fn find_auto_jobs_by_pattern(&self, pattern: &str) -> Vec<Job> {
+    fn find_auto_or_optional_jobs_by_pattern(&self, pattern: &str) -> Vec<Job> {
         self.auto_jobs
             .iter()
+            .chain(self.optional_jobs.iter())
             .filter(|j| glob_match::glob_match(pattern, &j.name))
             .cloned()
             .collect()
@@ -181,7 +184,7 @@ fn calculate_jobs(
                 let mut jobs: Vec<Job> = vec![];
                 let mut unknown_patterns = vec![];
                 for pattern in patterns {
-                    let matched_jobs = db.find_auto_jobs_by_pattern(pattern);
+                    let matched_jobs = db.find_auto_or_optional_jobs_by_pattern(pattern);
                     if matched_jobs.is_empty() {
                         unknown_patterns.push(pattern.clone());
                     } else {

src/ci/citool/src/jobs/tests.rs

@@ -46,6 +46,13 @@ auto:
     - name: test-msvc-i686-2
       os: ubuntu
       env: {}
+optional:
+    - name: optional-job-1
+      os: ubuntu
+      env: {}
+    - name: optional-dist-x86_64
+      os: ubuntu
+      env: {}
 "#,
     )
     .unwrap();
@@ -57,12 +64,18 @@ auto:
         "*i686*",
         &["test-i686", "dist-i686", "test-msvc-i686-1", "test-msvc-i686-2"],
     );
+    // Test that optional jobs are found
+    check_pattern(&db, "optional-*", &["optional-job-1", "optional-dist-x86_64"]);
+    check_pattern(&db, "*optional*", &["optional-job-1", "optional-dist-x86_64"]);
 }
 
 #[track_caller]
 fn check_pattern(db: &JobDatabase, pattern: &str, expected: &[&str]) {
-    let jobs =
-        db.find_auto_jobs_by_pattern(pattern).into_iter().map(|j| j.name).collect::<Vec<_>>();
+    let jobs = db
+        .find_auto_or_optional_jobs_by_pattern(pattern)
+        .into_iter()
+        .map(|j| j.name)
+        .collect::<Vec<_>>();
 
     assert_eq!(jobs, expected);
 }
@@ -116,8 +129,13 @@ fn validate_jobs() {
         load_job_db(&db_str).expect("Failed to load job database")
     };
 
-    let all_jobs =
-        db.pr_jobs.iter().chain(db.try_jobs.iter()).chain(db.auto_jobs.iter()).collect::<Vec<_>>();
+    let all_jobs = db
+        .pr_jobs
+        .iter()
+        .chain(db.try_jobs.iter())
+        .chain(db.auto_jobs.iter())
+        .chain(db.optional_jobs.iter())
+        .collect::<Vec<_>>();
 
     let errors: Vec<anyhow::Error> =
         all_jobs.into_iter().filter_map(|job| validate_codebuild_image(job).err()).collect();

src/ci/citool/tests/test-jobs.yml

@@ -139,3 +139,8 @@ auto:
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift
     <<: *job-windows
+
+# Jobs that only run when explicitly invoked via `@bors try`.
+optional:
+  - name: test-optional-job
+    <<: *job-linux-4c

src/ci/github-actions/jobs.yml

@@ -160,6 +160,17 @@ pr:
 try:
   - <<: *job-dist-x86_64-linux
 
+# Jobs that only run when explicitly invoked in one of the following ways:
+# - comment `@bors2 try jobs=<job-name>`
+# - `try-job: <job-name>` in the PR description and comment `@bors try` or `@bors2 try`.
+optional:
+  # This job is used just to test optional jobs.
+  # It will be replaced by tier 2 and tier 3 jobs in the future.
+  - name: optional-mingw-check-1
+    env:
+      IMAGE: mingw-check-1
+    <<: *job-linux-4c
+
 # Main CI jobs that have to be green to merge a commit into master
 # These jobs automatically inherit envs.auto, to avoid repeating
 # it in each job definition.