diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 9069a12b..55b1c648 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -4,94 +4,62 @@ The CI Operator consumes a configuration file that describes how release artifac are built from a repository's branch. An example file is shown below with all optional fields present: -```json -{ - "tag_specification": { - "cluster": "", - "namespace": "", - "name": "", - "tag": "", - "tag_overrides": { - - } - }, - "base_images": { - "": { - "cluster": "", - "namespace": "", - "name": "", - "tag": "" - } - }, - "test_base_image": { - "cluster": "", - "namespace": "", - "name": "", - "tag": "" - }, - - "canonical_go_repository": "", - - "binary_build_commands": "", - "test_binary_build_commands": "", - "rpm_build_commands": "", - "rpm_build_location": "", - - "images": [ - { - "from": "", - "to": "", - "context_dir": "", - "dockerfile_path": "", - "inputs": { - "": { - "as": "", - "paths": [ - { - "source_path": "", - "destination_dir": "" - } - ] - } - }, - "optional": false - } - ], - - "tests": [ - { - "as": "", - "from": "", - "commands": "", - "artifact_dir": "" - } - ], - - "raw_steps": [], - - "promotion": { - "namespace": "", - "name": "", - "tag": "", - "name_prefix": "", - "additional_images": { - "": "" - } - }, - - "resources": { - "": { - "requests": { - "cpu": "", - "memory": "" - }, - "limits": { - "cpu": "", - "memory": "" - } - } - } -} +```yaml +base_images: + : + cluster: '' + name: '' + namespace: '' + tag: '' +binary_build_commands: '' +canonical_go_repository: '' +images: +- context_dir: '' + dockerfile_path: '' + from: '' + inputs: + : + as: '' + paths: + - destination_dir: '' + source_path: '' + optional: false + to: '' +promotion: + additional_images: + : '' + name: '' + name_prefix: '' + namespace: '' + tag: '' +raw_steps: [] +resources: + : + limits: + cpu: '' + memory: '' + requests: + cpu: '' + memory: '' +rpm_build_commands: '' +rpm_build_location: '' +tag_specification: + cluster: '' + name: '' + namespace: '' + tag: '' + tag_overrides: {} +test_base_image: + cluster: '' + name: '' + namespace: '' + tag: '' +test_binary_build_commands: '' +tests: +- artifact_dir: '' + as: '' + commands: '' + from: '' ``` # `tag_specification` @@ -100,14 +68,13 @@ tagged into tests for the repository. The Origin CI assembles latest releases for all components using one `ImageStream` and many tags. To use these releases, use the following specification: -```json -"tag_specification": { - "cluster": "https://api.ci.openshift.org", - "namespace": "openshift", - "name": "origin-v3.11", - "tag": "", - "tag_overrides": {} -} +```yaml +tag_specification: + cluster: https://api.ci.openshift.org + name: origin-v3.11 + namespace: openshift + tag: '' + tag_overrides: {} ``` There are two primary modes for assembling a release: @@ -151,15 +118,13 @@ streams and one tag are used to assemble a release. from the repository. The field is a mapping from pipeline image name to remote `ImageStream` specification. A common base image might be an operating system: -```json -"base_images": { - "os": { - "cluster": "https://api.ci.openshift.org", - "namespace": "openshift", - "name": "centos", - "tag": "7" - } -} +```yaml +base_images: + os: + cluster: https://api.ci.openshift.org + name: centos + namespace: openshift + tag: '7' ``` The key in this mapping is the name that can be used in `"from"` fields elsewhere @@ -178,13 +143,12 @@ be used as the build environment for the source code cloning and any downstream builds like compilation or unit tests. Commonly, the `openshift/release` image is used: -```json -"test_base_image": { - "cluster": "https://api.ci.openshift.org", - "namespace": "openshift", - "name": "release", - "tag": "golang-1.10" -} +```yaml +test_base_image: + cluster: https://api.ci.openshift.org + name: release + namespace: openshift + tag: golang-1.10 ``` # `canonical_go_repository` @@ -361,4 +325,4 @@ the test fails to `malloc()`. `cpu` is the maximum CPU, with the unit, often in millicores (`100m`). ### `resources.$name.limits.memory` -`memory` is the maximum RAM, with the unit, often in MiB (`200Mi`). \ No newline at end of file +`memory` is the maximum RAM, with the unit, often in MiB (`200Mi`). diff --git a/ONBOARD.md b/ONBOARD.md index 9f678200..b305c984 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -23,7 +23,7 @@ as expected (you need to be logged in to a cluster, e.g. to [api.ci](https://api.ci.openshift.org)): ``` -./ci-operator --config config.json --git-ref openshift/@ +./ci-operator --config config.yaml --git-ref openshift/@ ``` After you make sure everything works, you need to create a subdirectory @@ -42,22 +42,19 @@ and injects the source code into the base image specified by the `test_base_image` key. The base image should contain all build dependencies of the tested component, so the it will often be a `openshift/release:` image. -```json -{ - "test_base_image": { - "cluster": "https://api.ci.openshift.org", - "namespace": "openshift", - "name": "release", - "tag": "golang-1.10" - } -} +```yaml +test_base_image: + cluster: https://api.ci.openshift.org + name: release + namespace: openshift + tag: golang-1.10 ``` Given your component can be built in the context of the `openshift/release` image, you can test building the `src` target: ``` -$ ./ci-operator --config example.json --git-ref=openshift/@ --target=src +$ ./ci-operator --config example.yaml --git-ref=openshift/@ --target=src ``` ### Test targets @@ -68,21 +65,14 @@ example of two test targets, each performing a different test by calling different `make` target in a `src` image (of course, a `Makefile` in your component repository would need to have these targets for this to work). -```json -{ - "tests": [ - { - "as": "unit", - "from": "src", - "commands": "make test-unit" - }, - { - "as": "performance", - "from": "src", - "commands": "make test-performance" - } - ] -} +```yaml +tests: +- as: unit + commands: make test-unit + from: src +- as: performance + commands: make test-performance + from: src ``` By default, ci-operator runs all specified test targets, building all their @@ -105,28 +95,19 @@ Here, `unit` and `integration` targets will both be built from a `test-bin` image, which will be a result of running `make instrumented-build` over a `src` image, while `performance` test target will be run from a `bin` image: -```json -{ - "binary_build_commands": "make build", - "test_binary_builds_commands": "make instrumented-build", - "tests": [ - { - "as": "unit", - "from": "test-bin", - "commands": "make test-unit" - }, - { - "as": "integration", - "from": "test-bin", - "commands": "make test-integration", - }, - { - "as": "performance", - "from": "bin", - "commands": "make test-performance" - } - ] -} +```yaml +binary_build_commands: make build +test_binary_builds_commands: make instrumented-build +tests: +- as: unit + commands: make test-unit + from: test-bin +- as: integration + commands: make test-integration + from: test-bin +- as: performance + commands: make test-performance + from: bin ``` ### Using Separate Build Environment and Release Environment Images diff --git a/README.md b/README.md index 0c50bf24..60f49c97 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ ci-operator is mainly intended to be run automatically by the CI system, but after you build it, you can also run it locally: ``` -./ci-operator --config component.json --git-ref=openshift/{repo}@master +./ci-operator --config component.yaml --git-ref=openshift/{repo}@master ``` For more information about ci-operator options, use the `--help` parameter: diff --git a/cmd/ci-operator-checkconfig/main.go b/cmd/ci-operator-checkconfig/main.go index c70b712e..fa310ace 100644 --- a/cmd/ci-operator-checkconfig/main.go +++ b/cmd/ci-operator-checkconfig/main.go @@ -1,13 +1,14 @@ package main import ( - "encoding/json" "flag" "fmt" "io/ioutil" "os" "path/filepath" + "github.com/ghodss/yaml" + "github.com/openshift/ci-operator/pkg/api" ) @@ -26,8 +27,8 @@ func main() { fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", configDir, err) return err } - if filepath.Ext(path) == ".json" { - // we assume any JSON in the config dir is a CI Operator config + if filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".json" { + // we assume any JSON or YAML in the config dir is a CI Operator config name, err := filepath.Rel(configDir, path) if err != nil { return fmt.Errorf("could not determine relative path name for %s: %v", path, err) @@ -39,7 +40,7 @@ func main() { } var config api.ReleaseBuildConfiguration - if err := json.Unmarshal(data, &config); err != nil { + if err := yaml.Unmarshal(data, &config); err != nil { return fmt.Errorf("invalid configuration from %s: %v\nvalue:%s", name, err, string(data)) } diff --git a/cmd/ci-operator/main.go b/cmd/ci-operator/main.go index b97a0ee5..8ecbff11 100644 --- a/cmd/ci-operator/main.go +++ b/cmd/ci-operator/main.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/base32" - "encoding/json" "encoding/xml" "errors" "flag" @@ -29,6 +28,8 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" + "github.com/ghodss/yaml" + imageapi "github.com/openshift/api/image/v1" projectapi "github.com/openshift/api/project/v1" templateapi "github.com/openshift/api/template/v1" @@ -44,7 +45,7 @@ import ( const usage = `Orchestrate multi-stage image-based builds -The ci-operator reads a declarative configuration JSON file and executes a set of build +The ci-operator reads a declarative configuration YAML file and executes a set of build steps on an OpenShift cluster for image-based components. By default, all steps are run, but a caller may select one or more targets (image names or test names) to limit to only steps that those targets depend on. The build creates a new project to run the builds in @@ -258,7 +259,7 @@ func (o *options) Complete() error { return fmt.Errorf("CONFIG_SPEC environment variable is not set or empty and no --config file was set") } } - if err := json.Unmarshal([]byte(configSpec), &o.configSpec); err != nil { + if err := yaml.Unmarshal([]byte(configSpec), &o.configSpec); err != nil { return fmt.Errorf("invalid configuration: %v\nvalue:\n%s", err, string(configSpec)) } @@ -274,7 +275,7 @@ func (o *options) Complete() error { overrideSpec = []byte(os.Getenv("OVERRIDE_SPEC")) } if len(overrideSpec) > 0 { - if err := json.Unmarshal(overrideSpec, &o.configSpec); err != nil { + if err := yaml.Unmarshal(overrideSpec, &o.configSpec); err != nil { return fmt.Errorf("invalid configuration: %v\nvalue:\n%s", err, string(overrideSpec)) } } @@ -307,9 +308,9 @@ func (o *options) Complete() error { o.jobSpec = jobSpec if o.dry { - config, _ := json.MarshalIndent(o.configSpec, "", " ") + config, _ := yaml.Marshal(o.configSpec) log.Printf("Resolved configuration:\n%s", string(config)) - job, _ := json.MarshalIndent(o.jobSpec, "", " ") + job, _ := yaml.Marshal(o.jobSpec) log.Printf("Resolved job spec:\n%s", string(job)) } refs := o.jobSpec.Refs @@ -492,7 +493,7 @@ func (o *options) resolveInputs(ctx context.Context, steps []api.Step) error { } // a change in the config for the build changes the output - configSpec, err := json.Marshal(o.configSpec) + configSpec, err := yaml.Marshal(o.configSpec) if err != nil { panic(err) } diff --git a/glide.yaml b/glide.yaml index 416e505f..59c10ff1 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,3 +10,5 @@ import: version: release-1.9 - package: k8s.io/client-go version: kubernetes-1.9.1 +- package: github.com/ghodss/yaml + version: master diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index c732a942..408598c2 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -1,9 +1,10 @@ package api import ( - "encoding/json" "reflect" "testing" + + "github.com/ghodss/yaml" ) func TestOverlay(t *testing.T) { @@ -115,13 +116,13 @@ func TestOverlay(t *testing.T) { t.Run(tt.name, func(t *testing.T) { config := &ReleaseBuildConfiguration{} input := &InputConfiguration{} - if err := json.Unmarshal([]byte(tt.base), config); err != nil { + if err := yaml.Unmarshal([]byte(tt.base), config); err != nil { t.Fatal(err) } - if err := json.Unmarshal([]byte(tt.overlay), config); err != nil { + if err := yaml.Unmarshal([]byte(tt.overlay), config); err != nil { t.Fatal(err) } - if err := json.Unmarshal([]byte(tt.overlay), input); err != nil { + if err := yaml.Unmarshal([]byte(tt.overlay), input); err != nil { t.Fatal(err) } if got := input; !reflect.DeepEqual(got, tt.wantInput) { diff --git a/test/config.json b/test/config.json index 646ef6ca..a5332b7c 100644 --- a/test/config.json +++ b/test/config.json @@ -1,37 +1,25 @@ -{ - "test_base_image": { - "namespace": "ci", - "name": "release-with-clonerefs", - "tag": "golang-1.9" - }, - "binary_build_commands": "make build", - "test_binary_build_commands": "OS_GOFLAGS='-race' make build", - "rpm_build_commands": "make build-rpms", - "base_rpm_images": { - "base": { - "namespace": "openshift", - "name": "origin-v3.10", - "tag": "base" - } - }, - "images": [ - { - "from": "base", - "to": "docker-registry", - "context_dir": "images/dockerregistry/" - } - ], - "tag_specification": { - "namespace": "openshift", - "name": "origin-v3.10", - "tag": "", - "tag_overrides": {} - }, - "tests": [ - { - "as": "unit", - "from": "test-bin", - "commands": "hack/test-go.sh" - } - ] -} +base_rpm_images: + base: + name: origin-v3.10 + namespace: openshift + tag: base +binary_build_commands: make build +images: +- context_dir: images/dockerregistry/ + from: base + to: docker-registry +rpm_build_commands: make build-rpms +tag_specification: + name: origin-v3.10 + namespace: openshift + tag: '' + tag_overrides: {} +test_base_image: + name: release-with-clonerefs + namespace: ci + tag: golang-1.9 +test_binary_build_commands: OS_GOFLAGS='-race' make build +tests: +- as: unit + commands: hack/test-go.sh + from: test-bin