From 5cf5ba3f8707699a8345cd2aaf8d2c7b6615bc47 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 10 Sep 2020 12:54:02 -0700 Subject: [PATCH 1/2] core-services/prow/02_config/generate-boskos: For spitting out lots of names Quotas for some providers depend on both account-scoped resources (e.g. AWS elastic IPs default to five per account [1]), region-scoped resources (e.g. AWS VPCs default to five per region [1]), potentialy other scoping (e.g. AWS NAT gateways default to five per availability zone [1]). This commit moves us away from account-scoped leases and towards region-scoped leases in account-scoped buckets. That allows us to say things like "on Azure, we have more capacity in centralus than we do in our other regions". The generating script is because typing out "us-east-1" 50 times is tedious and hard to review, and today there is apparently no support in the Boskos config directly to ask for $N copies of a given name. I'm also adding myself as an owner, so DPTP doesn't have to bother with lease adjustment. Other CI-platform maintainers should feel free to add themselves as well, if they want to approve lease adjustments to their platforms. Currently a mostly-no-op reshuffling of the exisiting Boskos config. The python-validation image follows the pattern set by the existing prow-config-semantics job. This script works with both Python 2 and Python 3, but I'm using python3 in the shebang because the verification image currently has 'python' pointed at Python 2 but only installs PyYAML for Python 3, and I've been unable to land a pivot to point it at Python 3 [2]. [1]: https://docs.openshift.com/container-platform/4.5/installing/installing_aws/installing-aws-account.html#installation-aws-limits_installing-aws-account [2]: https://github.com/openshift/release/pull/11869 --- Makefile | 13 +- .../openshift-release-master-presubmits.yaml | 22 ++++ core-services/prow/02_config/OWNERS | 3 + core-services/prow/02_config/_boskos.yaml | 118 +++++++++--------- .../prow/02_config/generate-boskos.py | 75 +++++++++++ hack/validate-boskos.sh | 30 +++++ 6 files changed, 200 insertions(+), 61 deletions(-) create mode 100644 core-services/prow/02_config/OWNERS create mode 100755 core-services/prow/02_config/generate-boskos.py create mode 100755 hack/validate-boskos.sh diff --git a/Makefile b/Makefile index d938c8e6c7331..71aa5cf099cec 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL=/usr/bin/env bash -o errexit -.PHONY: help check check-core check-services dry-core core dry-services services all +.PHONY: help check check-boskos check-core check-services dry-core core dry-services services all CONTAINER_ENGINE ?= docker @@ -9,9 +9,13 @@ help: all: core services -check: check-core check-services +check: check-core check-services check-boskos @echo "Service config check: PASS" +check-boskos: + hack/validate-boskos.sh + @echo "Boskos config check: PASS" + check-core: core-services/_hack/validate-core-services.sh core-services @echo "Core service config check: PASS" @@ -38,6 +42,7 @@ services: update: $(MAKE) jobs $(MAKE) ci-operator-config + $(MAKE) boskos-config $(MAKE) prow-config $(MAKE) registry-metadata @@ -59,6 +64,10 @@ registry-metadata: $(CONTAINER_ENGINE) pull registry.svc.ci.openshift.org/ci/generate-registry-metadata:latest $(CONTAINER_ENGINE) run --rm -v "$(CURDIR)/ci-operator/step-registry:/ci-operator/step-registry:z" registry.svc.ci.openshift.org/ci/generate-registry-metadata:latest --registry /ci-operator/step-registry +boskos-config: + cd core-services/prow/02_config && ./generate-boskos.py +.PHONY: boskos-config + prow-config: $(CONTAINER_ENGINE) pull registry.svc.ci.openshift.org/ci/determinize-prow-config:latest $(CONTAINER_ENGINE) run --rm -v "$(CURDIR)/core-services/prow/02_config:/config:z" registry.svc.ci.openshift.org/ci/determinize-prow-config:latest --prow-config-dir /config diff --git a/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml b/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml index c22739dc55a49..6a644344db2a1 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml @@ -38,6 +38,28 @@ presubmits: cpu: 10m serviceAccountName: config-updater trigger: (?m)^/test( | .* )app-ci-config-dry,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - master + cluster: build01 + context: ci/prow/boskos-config + decorate: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-release-master-boskos-config + rerun_command: /test boskos-config + spec: + containers: + - command: + - hack/validate-boskos.sh + image: registry.svc.ci.openshift.org/ci/python-validation + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + trigger: (?m)^/test( | .* )boskos-config,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/core-services/prow/02_config/OWNERS b/core-services/prow/02_config/OWNERS new file mode 100644 index 0000000000000..7b77ff680b351 --- /dev/null +++ b/core-services/prow/02_config/OWNERS @@ -0,0 +1,3 @@ +approvers: + - dptp + - wking diff --git a/core-services/prow/02_config/_boskos.yaml b/core-services/prow/02_config/_boskos.yaml index e57145199b8d6..4235a69e8a668 100644 --- a/core-services/prow/02_config/_boskos.yaml +++ b/core-services/prow/02_config/_boskos.yaml @@ -1,78 +1,78 @@ ---- +# generated with generate-boskos.py; do not edit directly resources: -- type: aws-quota-slice - state: free +- max-count: 150 min-count: 150 - max-count: 150 -- type: azure4-quota-slice state: free + type: aws-quota-slice +- max-count: 30 min-count: 30 - max-count: 30 -- type: gcp-quota-slice state: free + type: azure4-quota-slice +- max-count: 120 min-count: 120 - max-count: 120 -- type: libvirt-s390x-quota-slice state: free - names: - - "libvirt-s390x-0-0" - - "libvirt-s390x-0-1" - - "libvirt-s390x-0-2" - - "libvirt-s390x-0-3" - - "libvirt-s390x-0-4" -- type: libvirt-ppc64le-quota-slice + type: gcp-quota-slice +- names: + - libvirt-ppc64le-0-0 + - libvirt-ppc64le-0-1 + - libvirt-ppc64le-0-2 + - libvirt-ppc64le-0-3 + - libvirt-ppc64le-1-0 + - libvirt-ppc64le-1-1 + - libvirt-ppc64le-1-2 + - libvirt-ppc64le-1-3 state: free - names: - - "libvirt-ppc64le-0-0" - - "libvirt-ppc64le-0-1" - - "libvirt-ppc64le-0-2" - - "libvirt-ppc64le-0-3" - - "libvirt-ppc64le-1-0" - - "libvirt-ppc64le-1-1" - - "libvirt-ppc64le-1-2" - - "libvirt-ppc64le-1-3" -- type: metal-quota-slice + type: libvirt-ppc64le-quota-slice +- names: + - libvirt-s390x-0-0 + - libvirt-s390x-0-1 + - libvirt-s390x-0-2 + - libvirt-s390x-0-3 + - libvirt-s390x-0-4 state: free + type: libvirt-s390x-quota-slice +- max-count: 1000 min-count: 1000 - max-count: 1000 -- type: openstack-quota-slice state: free + type: metal-quota-slice +- names: + - openstack-osuosl-ppc64le-01 + - openstack-osuosl-ppc64le-02 + - openstack-osuosl-ppc64le-03 + - openstack-osuosl-ppc64le-04 + state: free + type: openstack-osuosl-ppc64le-quota-slice +- names: + - openstack-ppc64le-00 + - openstack-ppc64le-01 + - openstack-ppc64le-02 + - openstack-ppc64le-03 + state: free + type: openstack-ppc64le-quota-slice +- max-count: 7 min-count: 7 - max-count: 7 -- type: openstack-vexxhost-quota-slice state: free + type: openstack-quota-slice +- max-count: 3 min-count: 3 - max-count: 3 -- type: openstack-ppc64le-quota-slice state: free - names: - - "openstack-ppc64le-00" - - "openstack-ppc64le-01" - - "openstack-ppc64le-02" - - "openstack-ppc64le-03" -- type: openstack-osuosl-ppc64le-quota-slice + type: openstack-vexxhost-quota-slice +- names: + - ovirt-10 + - ovirt-11 + - ovirt-12 + - ovirt-13 + - ovirt-14 + - ovirt-15 + - ovirt-16 + - ovirt-17 state: free - names: - - "openstack-osuosl-ppc64le-01" - - "openstack-osuosl-ppc64le-02" - - "openstack-osuosl-ppc64le-03" - - "openstack-osuosl-ppc64le-04" -- type: vsphere-quota-slice + type: ovirt-quota-slice +- max-count: 20 + min-count: 20 state: free + type: packet-quota-slice +- max-count: 10 min-count: 10 - max-count: 10 -- type: ovirt-quota-slice - state: free - names: - - "ovirt-10" - - "ovirt-11" - - "ovirt-12" - - "ovirt-13" - - "ovirt-14" - - "ovirt-15" - - "ovirt-16" - - "ovirt-17" -- type: packet-quota-slice state: free - min-count: 20 - max-count: 20 + type: vsphere-quota-slice diff --git a/core-services/prow/02_config/generate-boskos.py b/core-services/prow/02_config/generate-boskos.py new file mode 100755 index 0000000000000..758152b114c04 --- /dev/null +++ b/core-services/prow/02_config/generate-boskos.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import yaml + + +CONFIG = { + 'aws-quota-slice': { + 'default': 150, + }, + 'azure4-quota-slice': { + 'default': 30, + }, + 'gcp-quota-slice': { + 'default': 120, + }, + 'libvirt-s390x-quota-slice': {}, + 'libvirt-ppc64le-quota-slice': {}, + 'metal-quota-slice': { + # Wild guesses. We'll see when we hit quota issues + 'default': 1000, + }, + 'openstack-osuosl-ppc64le-quota-slice': {}, + 'openstack-quota-slice': { + 'default': 7, + }, + 'openstack-vexxhost-quota-slice': { + 'default': 3, + }, + 'openstack-ppc64le-quota-slice': {}, + 'ovirt-quota-slice': {}, + 'packet-quota-slice': { + 'default': 20, + }, + 'vsphere-quota-slice': { + 'default': 10, + }, +} + +for i in range(1): + for j in range(5): + CONFIG['libvirt-s390x-quota-slice']['libvirt-s390x-{}-{}'.format(i, j)] = 1 + +for i in range(2): + for j in range(4): + CONFIG['libvirt-ppc64le-quota-slice']['libvirt-ppc64le-{}-{}'.format(i, j)] = 1 + +for i in range(1, 5): + CONFIG['openstack-osuosl-ppc64le-quota-slice']['openstack-osuosl-ppc64le-{0:0>2}'.format(i)] = 1 + +for i in range(4): + CONFIG['openstack-ppc64le-quota-slice']['openstack-ppc64le-{0:0>2}'.format(i)] = 1 + +for i in range(10, 18): + CONFIG['ovirt-quota-slice']['ovirt-{}'.format(i)] = 1 + +config = { + 'resources': [], +} + +for typeName, data in sorted(CONFIG.items()): + resource = { + 'type': typeName, + 'state': 'free', + } + if set(data.keys()) == {'default'}: + resource['min-count'] = resource['max-count'] = data['default'] + else: + resource['names'] = [] + for name, count in sorted(data.items()): + resource['names'].extend([name]*count) + config['resources'].append(resource) + +with open('_boskos.yaml', 'w') as f: + f.write('# generated with generate-boskos.py; do not edit directly\n') + yaml.dump(config, f, default_flow_style=False) diff --git a/hack/validate-boskos.sh b/hack/validate-boskos.sh new file mode 100755 index 0000000000000..d097755d68265 --- /dev/null +++ b/hack/validate-boskos.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script ensures that the Boskos configuration checked into git is up-to-date +# with the generator. If it is not, re-generate the configuration to update it. + +set -o errexit +set -o nounset +set -o pipefail + +base_dir=. + +cd "${base_dir}/core-services/prow/02_config" +./generate-boskos.py +DIFF="$(git diff)" +if test -n "${DIFF}" +then + cat << EOF +ERROR: This check enforces that the Boskos configuration is generated +ERROR: correctly. We have automation in place that updates the configuration and +ERROR: new changes to the configuration should be followed with a re-generation. + +ERROR: Run the following command to re-generate the Boskos configuration: +ERROR: $ make boskos-config + +ERROR: The following errors were found: + +EOF + echo "${DIFF}" + exit 1 +fi From 204f9a738ac1ee824ccaaf808a7b04582d40e7b6 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 8 Oct 2020 14:29:19 -0700 Subject: [PATCH 2/2] hack/validate-boskos: Do not require Git Seems like our python-validation image does not include it [1]: hack/validate-boskos.sh: line 14: git: command not found 'diff -u A B' is in POSIX [2]. The '|| true' catches the exit status, which is when differences were found [2]. The chance of a >1 exit status for other errors seems small enough that I haven't bothered to handle it, although ideally we'd exit out in that situation. [1]: https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_release/11752/rehearse-11752-pull-ci-openshift-release-master-boskos-config/1314310508639162368 [2]: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/diff.html --- hack/validate-boskos.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/validate-boskos.sh b/hack/validate-boskos.sh index d097755d68265..cfcf2185478b4 100755 --- a/hack/validate-boskos.sh +++ b/hack/validate-boskos.sh @@ -10,8 +10,9 @@ set -o pipefail base_dir=. cd "${base_dir}/core-services/prow/02_config" +ORIGINAL="$(cat _boskos.yaml)" ./generate-boskos.py -DIFF="$(git diff)" +DIFF="$(diff -u <(echo "${ORIGINAL}") _boskos.yaml || true)" if test -n "${DIFF}" then cat << EOF