Skip to content

Commit

Permalink
Merge pull request #764 from flyingcircusio/PL-131682-update-nixpkgs
Browse files Browse the repository at this point in the history 
Update nixpkgs
  • Loading branch information
@dpausp
dpausp authored Aug 10, 2023
2 parents 513e4d9 + 8f1ba66 commit ef0aecc
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 50 deletions.
13 changes: 7 additions & 6 deletions important_packages.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@
"gnumake",
"gnupg",
"go",
"go_1_17",
"go_1_18",
"go_1_19",
"go_1_20",
"grafana",
"haproxy",
"imagemagick",
Expand All @@ -62,7 +63,6 @@
"jre",
"k3s",
"keycloak",
"kibana6-oss",
"kubernetes-helm",
"libgcrypt",
"libmodsecurity",
Expand All @@ -85,21 +85,23 @@
"mysql80",
"nfs-utils",
"nginx",
"nginxModules.modsecurity",
"nginxMainline",
"nginxModules.modsecurity",
"nginxStable",
"nix",
"nodejs",
"nodejs_14",
"nodejs_16",
"nodejs_18",
"nodejs_19",
"nodejs_20",
"nspr",
"nss_latest",
"openjdk",
"openjpeg",
"openldap_2_4",
"openssh",
"openssl",
"openssl_3",
"openvpn",
"pcre",
"pcre2",
Expand Down Expand Up @@ -156,9 +158,8 @@
"tcpdump",
"telegraf",
"tmux",
"tomcat",
"tomcat9",
"tomcat10",
"tomcat9",
"unzip",
"util-linux",
"varnish",
Expand Down
8 changes: 0 additions & 8 deletions nixos/roles/gitlab.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,16 +213,8 @@ in

};


# Needed for Git via SSH.
users.users.gitlab.extraGroups = [ "login" ];

# ensure that gitlab is restarted again, when stopped due to a dependency
# (e.g. postgresql) being stopped and started again
systemd.services.gitlab = {
wantedBy = lib.mkForce [];
requiredBy = [ "gitlab.target" ];
};
})

(lib.mkIf (cfg.enable && cfg.extraSecrets != []) {
Expand Down
2 changes: 1 addition & 1 deletion nixpkgs-config.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"imagemagick-6.9.12-68" # Legacy, but gets updates. Customer still needs it.
"nodejs-14.21.3" # Needed for opensearch-dashboards.
"nodejs-16.20.1" # EOL 2023-09-11, needed for discourse and some customers.
"openssl-1.1.1u" # EOL 2023-09-11, needed for Percona and older PHP versions.
"openssl-1.1.1v" # EOL 2023-09-11, needed for Percona and older PHP versions.
"python-2.7.18.6" # Needed for some legacy customer applications.
"ruby-2.7.8" # EOL 2023-03-31, needed for Sensu checks
];
Expand Down
73 changes: 47 additions & 26 deletions package-versions.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,14 @@
"version": "2.40.1"
},
"github-runner": {
"name": "github-runner-2.306.0",
"name": "github-runner-2.307.1",
"pname": "github-runner",
"version": "2.306.0"
"version": "2.307.1"
},
"gitlab": {
"name": "gitlab-16.1.2",
"name": "gitlab-16.1.3",
"pname": "gitlab",
"version": "16.1.2"
"version": "16.1.3"
},
"glibc": {
"name": "glibc-2.37-8",
Expand All @@ -219,36 +219,45 @@
"pname": "go",
"version": "1.20.6"
},
"go_1_17": {},
"go_1_18": {
"name": "go-1.18.10",
"pname": "go",
"version": "1.18.10"
},
"go_1_19": {
"name": "go-1.19.10",
"pname": "go",
"version": "1.19.10"
},
"go_1_20": {
"name": "go-1.20.6",
"pname": "go",
"version": "1.20.6"
},
"grafana": {
"name": "grafana-9.5.6",
"name": "grafana-9.5.7",
"pname": "grafana",
"version": "9.5.6"
"version": "9.5.7"
},
"haproxy": {
"name": "haproxy-2.7.8",
"pname": "haproxy",
"version": "2.7.8"
},
"imagemagick": {
"name": "imagemagick-7.1.1-14",
"name": "imagemagick-7.1.1-15",
"pname": "imagemagick",
"version": "7.1.1-14"
"version": "7.1.1-15"
},
"imagemagick6": {
"name": "imagemagick-6.9.12-68",
"pname": "imagemagick",
"version": "6.9.12-68"
},
"imagemagick7": {
"name": "imagemagick-7.1.1-14",
"name": "imagemagick-7.1.1-15",
"pname": "imagemagick",
"version": "7.1.1-14"
"version": "7.1.1-15"
},
"inetutils": {
"name": "inetutils-2.4",
Expand Down Expand Up @@ -305,7 +314,6 @@
"pname": "keycloak",
"version": "21.1.2"
},
"kibana6-oss": {},
"kubernetes-helm": {
"name": "kubernetes-helm-3.11.3",
"pname": "kubernetes-helm",
Expand Down Expand Up @@ -337,9 +345,9 @@
"version": "2.10.4"
},
"linux": {
"name": "linux-6.1.41",
"name": "linux-6.1.43",
"pname": "linux",
"version": "6.1.41"
"version": "6.1.43"
},
"logrotate": {
"name": "logrotate-3.21.0",
Expand All @@ -362,19 +370,19 @@
"version": "10.6.14"
},
"mastodon": {
"name": "mastodon-4.1.4",
"name": "mastodon-4.1.6",
"pname": "mastodon",
"version": "4.1.4"
"version": "4.1.6"
},
"matomo": {
"name": "matomo-4.14.2",
"pname": "matomo",
"version": "4.14.2"
},
"matrix-synapse": {
"name": "matrix-synapse-1.88.0",
"name": "matrix-synapse-1.89.0",
"pname": "matrix-synapse",
"version": "1.88.0"
"version": "1.89.0"
},
"mcpp": {
"name": "mcpp-2.7.2.1",
Expand All @@ -397,9 +405,9 @@
"version": "10.6.14"
},
"mysql80": {
"name": "mysql-8.0.33",
"name": "mysql-8.0.34",
"pname": "mysql",
"version": "8.0.33"
"version": "8.0.34"
},
"nfs-utils": {
"name": "nfs-utils-2.6.2",
Expand Down Expand Up @@ -431,6 +439,11 @@
"pname": "nix",
"version": "2.13.3"
},
"nodejs": {
"name": "nodejs-18.16.1",
"pname": "nodejs",
"version": "18.16.1"
},
"nodejs_14": {
"name": "nodejs-14.21.3",
"pname": "nodejs",
Expand All @@ -446,16 +459,20 @@
"pname": "nodejs",
"version": "18.16.1"
},
"nodejs_19": {},
"nodejs_20": {
"name": "nodejs-20.3.1",
"pname": "nodejs",
"version": "20.3.1"
},
"nspr": {
"name": "nspr-4.35",
"pname": "nspr",
"version": "4.35"
},
"nss_latest": {
"name": "nss-3.91",
"name": "nss-3.92",
"pname": "nss",
"version": "3.91"
"version": "3.92"
},
"openjdk": {
"name": "openjdk-19.0.2+7",
Expand All @@ -482,6 +499,11 @@
"pname": "openssl",
"version": "3.0.9"
},
"openssl_3": {
"name": "openssl-3.0.9",
"pname": "openssl",
"version": "3.0.9"
},
"openvpn": {
"name": "openvpn-2.5.8",
"pname": "openvpn",
Expand Down Expand Up @@ -743,9 +765,9 @@
"version": "12.6.2"
},
"systemd": {
"name": "systemd-253.5",
"name": "systemd-253.6",
"pname": "systemd",
"version": "253.5"
"version": "253.6"
},
"tcpdump": {
"name": "tcpdump-4.99.4",
Expand All @@ -762,7 +784,6 @@
"pname": "tmux",
"version": "3.3a"
},
"tomcat": {},
"tomcat10": {
"name": "apache-tomcat-10.0.27",
"pname": "apache-tomcat",
Expand Down
66 changes: 59 additions & 7 deletions update-nixpkgs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,20 +79,35 @@ def format_as_msg(self):


def rebase_nixpkgs(nixpkgs_repo: Repo, nixos_version: NixOSVersion):
print("Fetching origin remote...")
nixpkgs_repo.git.fetch("origin")
origin_ref_id = f"origin/{nixos_version}"
origin_ref = nixpkgs_repo.refs[origin_ref_id]

if nixpkgs_repo.head.commit != origin_ref.commit:
do_reset = confirm(
f"local HEAD differs from {origin_ref_id}, hard-reset to origin?",
default=True,
)
if do_reset:
nixpkgs_repo.git.reset(hard=True)

print("Fetching upstream remote...")
nixpkgs_repo.git.fetch("upstream")
upstream_ref_id = f"upstream/{nixos_version}"

print(f"Using upstream ref {upstream_ref_id}")
old_rev = str(nixpkgs_repo.head.ref.commit)
upstream_ref = f"upstream/{nixos_version}"
print(f"Using upstream ref {upstream_ref}")
nixpkgs_repo.git.rebase(upstream_ref)
nixpkgs_repo.git.rebase(upstream_ref_id)
new_rev = str(nixpkgs_repo.head.ref.commit)

version_range = f"{old_rev}..{new_rev}"
do_push = confirm(
f"nixpkgs rebased: {version_range}. Push now?",
default=True,
)
if do_push:
nixpkgs_repo.git.push(force=True)
nixpkgs_repo.git.push(force_with_lease=True)


def prefetch_nixpkgs(nixos_version: str) -> dict[str, str]:
Expand All @@ -111,8 +126,35 @@ def prefetch_nixpkgs(nixos_version: str) -> dict[str, str]:
return prefetch_result


def get_package_list():
run_on_hydra("")
def update_package_list(local_path: Path):
basedir = "$XDG_RUNTIME_DIR"
subprocess.run(
["rsync", "-a", local_path, f"hydra01:{basedir}"], check=True
)
dest = f"{basedir}/{local_path.name}/"
proc = run_on_hydra(
f"(cd {dest}; eval $(./dev-setup); set pipefail; nix-build ./get-package-versions.nix | xargs cat)"
)
versions_path = Path("package-versions.json")
old_versions = json.loads(versions_path.read_text())
new_versions = json.loads(proc.stdout)
print("Versions diffs:")
for pkg_name in old_versions:
old = old_versions[pkg_name].get("version")
new = new_versions[pkg_name].get("version")

if not old:
print(f"(old version missing for {pkg_name})")
continue

if not new:
print(f"(new version missing for {pkg_name})")
continue

if old != new:
print(f"{pkg_name}: {old} -> {new}")

versions_path.write_text(json.dumps(new_versions, indent=2) + "\n")


def get_interesting_commit_msgs(nixpkgs_repo, old_rev, new_rev):
Expand Down Expand Up @@ -215,7 +257,8 @@ def update_fc_nixos(
ticket_number: str,
prefetch_json: dict[str, str],
):
versions_json_path = Path(fc_nixos_repo.working_dir) / "versions.json"
workdir_path = Path(fc_nixos_repo.working_dir)
versions_json_path = workdir_path / "versions.json"

with open(versions_json_path) as f:
versions_json = json.load(f)
Expand All @@ -224,6 +267,9 @@ def update_fc_nixos(
new_rev = str(nixpkgs_repo.head.commit)

update_versions_json(fc_nixos_repo, new_rev, prefetch_json["sha256"])
print()
print("-" * 80)
update_package_list(workdir_path)

interesting_msgs = get_interesting_commit_msgs(
nixpkgs_repo, old_rev, new_rev
Expand All @@ -236,6 +282,7 @@ def update_fc_nixos(
final_msgs = filter_and_merge_commit_msgs(interesting_msgs)
commit_msg = format_fcio_commit_msg(final_msgs, ticket_number)
print()
print("-" * 80)
print("Commit message:")
print()
print(commit_msg)
Expand Down Expand Up @@ -289,6 +336,11 @@ def nixpkgs():
rebase_nixpkgs(nixpkgs_repo, context.nixos_version)


@app.command()
def package_versions():
update_package_list(context.fc_nixos_path)


@app.command()
def prefetch(
nixos_version: str = Option("nixos-23.05"),
Expand Down
Loading

0 comments on commit ef0aecc

Please sign in to comment.