manifest: Add lockfile-repos field

docs/manual/treefile.md

@@ -17,7 +17,7 @@ It supports the following parameters:
    secret key must be in the home directory of the building user.  Defaults to
    none.
 
- * `repos` array of strings, mandatory: Names of yum repositories to
+ * `repos`: array of strings, mandatory: Names of yum repositories to
    use, from any files that end in `.repo`, in the same directory as
    the treefile.  `rpm-ostree compose tree` does not use the system
    `/etc/yum.repos.d`, because it's common to want to compose a target
@@ -304,3 +304,9 @@ version of `rpm-ostree`.
 
  * `rojig`: Object, optional.  Sub-keys are `name`, `summary`, `license`,
    and `description`.  Of those, `name` and `license` are mandatory.
+
+ * `lockfile-repos`: array of strings, optional: Semantically similar to
+   `repo`, but these repos will only be used to fetch packages locked
+   via lockfiles. This is useful when locked packages are kept
+   separately from the primary repos and one wants to ensure that
+   rpm-ostree will otherwise not select unlocked packages from them.

rust/src/treefile.rs

@@ -313,6 +313,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     );
     merge_vecs!(
         repos,
+        lockfile_repos,
         packages,
         bootstrap_packages,
         exclude_packages,
@@ -646,6 +647,9 @@ struct TreeComposeConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     repos: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "lockfile-repos")]
+    lockfile_repos: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     selinux: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(rename = "gpg-key")]

src/app/rpmostree-composeutil.c

@@ -251,6 +251,8 @@ rpmostree_composeutil_get_treespec (RpmOstreeContext  *ctx,
     return FALSE;
   if (!treespec_bind_array (treedata, treespec, "repos", NULL, TRUE, error))
     return FALSE;
+  if (!treespec_bind_array (treedata, treespec, "lockfile-repos", NULL, FALSE, error))
+    return FALSE;
   if (!treespec_bind_bool (treedata, treespec, "documentation", TRUE, error))
     return FALSE;
   if (!treespec_bind_bool (treedata, treespec, "recommends", TRUE, error))

src/libpriv/rpmostree-core.c

@@ -268,6 +268,10 @@ rpmostree_treespec_new_from_keyfile (GKeyFile   *keyfile,
     if (val && *val)
       add_canonicalized_string_array (&builder, "repos", NULL, keyfile);
   }
+  { g_auto(GStrv) val = g_key_file_get_string_list (keyfile, "tree", "lockfile-repos", NULL, NULL);
+    if (val && *val)
+      add_canonicalized_string_array (&builder, "lockfile-repos", NULL, keyfile);
+  }
   add_canonicalized_string_array (&builder, "instlangs", "instlangs-all", keyfile);
 
   if (g_key_file_get_boolean (keyfile, "tree", "skip-sanity-check", NULL))
@@ -758,15 +762,41 @@ rpmostree_context_setup (RpmOstreeContext    *self,
     }
   else
     {
+      /* Makes sure we only disable all repos once. This is more for future proofing against
+       * refactors for now since we don't support `lockfile-repos` on the client-side and on
+       * the server-side we always require `repos` anyway. */
+      gboolean disabled_all_repos = FALSE;
+
       /* NB: missing "repos" --> let libdnf figure it out for itself (we're likely doing a
        * client-side compose where we want to use /etc/yum.repos.d/) */
       g_autofree char **enabled_repos = NULL;
       if (g_variant_dict_lookup (self->spec->dict, "repos", "^a&s", &enabled_repos))
         {
-          disable_all_repos (self);
+          if (!disabled_all_repos)
+            {
+              disable_all_repos (self);
+              disabled_all_repos = TRUE;
+            }
           if (!enable_repos (self, (const char *const*)enabled_repos, error))
             return FALSE;
         }
+
+      /* only enable lockfile-repos if we actually have a lockfile so we don't even waste
+       * time fetching metadata */
+      if (self->vlockmap)
+        {
+          g_autofree char **enabled_lockfile_repos = NULL;
+          if (g_variant_dict_lookup (self->spec->dict, "lockfile-repos", "^a&s", &enabled_lockfile_repos))
+            {
+              if (!disabled_all_repos)
+                {
+                  disable_all_repos (self);
+                  disabled_all_repos = TRUE;
+                }
+              if (!enable_repos (self, (const char *const*)enabled_lockfile_repos, error))
+                return FALSE;
+            }
+        }
     }
 
   g_autoptr(GPtrArray) repos =
@@ -2048,6 +2078,21 @@ rpmostree_context_prepare (RpmOstreeContext *self,
         }
       else
         {
+          /* Exclude all the packages in lockfile repos except locked packages. */
+          g_autofree char **lockfile_repos = NULL;
+          g_variant_dict_lookup (self->spec->dict, "lockfile-repos", "^a&s", &lockfile_repos);
+          for (char **it = lockfile_repos; it && *it; it++)
+            {
+              const char *repo = *it;
+              hy_autoquery HyQuery query = hy_query_create (sack);
+              hy_query_filter (query, HY_PKG_REPONAME, HY_EQ, repo);
+              DnfPackageSet *pset = hy_query_run_set (query);
+              Map *map = dnf_packageset_get_map (pset);
+              map_subtract (map, dnf_packageset_get_map (locked_pset));
+              dnf_sack_add_excludes (sack, pset);
+              dnf_packageset_free (pset);
+            }
+
           /* In relaxed mode, we allow packages to be added or removed without having to
            * edit lockfiles. However, we still want to make sure that if a package does get
            * installed which is in the lockfile, it can only pick that NEVRA. To do this, we

tests/compose.sh

@@ -126,6 +126,7 @@ EOF
 import sys, json
 y = json.load(sys.stdin)
 y["repos"] = ["cache"]
+y.pop("lockfile-repos", None)
 json.dump(y, sys.stdout)' < manifest.json > manifest.json.new
   mv manifest.json{.new,}
   git add .

tests/compose/test-lockfile.sh

@@ -151,3 +151,37 @@ if runcompose \
 fi
 assert_file_has_content err.txt "Couldn't find locked package 'unmatched-pkg-1.0-1.x86_64'"
 echo "ok strict mode locked pkg missing from rpmmd"
+
+# test lockfile-repos, i.e. check that a pkg in a lockfile repo with higher
+# NEVRA isn't picked unless if it's not in the lockfile
+
+# some file shuffling to get a separate yumrepo-locked/ which has foobar-2.0
+build_rpm foobar
+mv yumrepo yumrepo.bak
+build_rpm foobar version 2.0
+mv yumrepo yumrepo-locked
+mv yumrepo.bak yumrepo
+sed -e 's/test-repo/test-lockfile-repo/g' < yumrepo.repo > yumrepo-locked.repo
+sed -e 's/yumrepo/yumrepo-locked/g' < yumrepo-locked.repo > yumrepo-locked.repo.new
+mv yumrepo-locked.repo.new yumrepo-locked.repo
+ln "$PWD/yumrepo-locked.repo" config/yumrepo-locked.repo
+treefile_append "packages" '["foobar"]'
+
+# try first as a regular repo, to make sure it's functional
+treefile_append "repos" '["test-lockfile-repo"]'
+runcompose \
+  --ex-lockfile="$PWD/versions.lock" \
+  --ex-write-lockfile-to="$PWD/versions.lock.new" \
+  --dry-run "${treefile}" |& tee out.txt
+assert_file_has_content out.txt 'foobar-2.0-1.x86_64'
+
+# ok, now as a lockfile repo
+treefile_remove "repos" '"test-lockfile-repo"'
+treefile_append "lockfile-repos" '["test-lockfile-repo"]'
+runcompose \
+  --ex-lockfile="$PWD/versions.lock" \
+  --ex-write-lockfile-to="$PWD/versions.lock.new" \
+  --dry-run "${treefile}" |& tee out.txt
+assert_file_has_content out.txt 'foobar-1.0-1.x86_64'
+treefile_remove "packages" '"foobar"'
+echo "ok lockfile-repos"