Skip to content

rhcos: Embed full build metadata in binary #1423

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 1 commit into from
Apr 4, 2019
Merged

rhcos: Embed full build metadata in binary #1423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions data/data/rhcos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"amis": {
"ap-northeast-1": {
"hvm": "ami-0fbc0018a8310e53e"
},
"ap-northeast-2": {
"hvm": "ami-0ba6edf20991dee91"
},
"ap-south-1": {
"hvm": "ami-0acf1668760f8f8e9"
},
"ap-southeast-1": {
"hvm": "ami-0bcecfdf9ff5fd5ca"
},
"ap-southeast-2": {
"hvm": "ami-0480abd0220d56ae2"
},
"ca-central-1": {
"hvm": "ami-0de4e822461cb671b"
},
"eu-central-1": {
"hvm": "ami-056c9291dce6d5023"
},
"eu-west-1": {
"hvm": "ami-0f0159b00648b0bf4"
},
"eu-west-2": {
"hvm": "ami-044f299a3abcb4d96"
},
"eu-west-3": {
"hvm": "ami-095776c2e71c62b2f"
},
"sa-east-1": {
"hvm": "ami-00aefabf6d653ff5d"
},
"us-east-1": {
"hvm": "ami-07f604c5f18a1e7a9"
},
"us-east-2": {
"hvm": "ami-0eef624367320ec26"
},
"us-west-1": {
"hvm": "ami-0f89dba68d747846b"
},
"us-west-2": {
"hvm": "ami-0eac581fbaa9fa9c6"
}
},
"baseURI": "https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190325.0/",
"buildid": "410.8.20190325.0",
"images": {
"metal-bios": {
"path": "rhcos-410.8.20190325.0-metal-bios.raw",
"sha256": "3caaf8d714ac8bb820af6b31d595b8afae5ef951c68f8cee3701e236e9600f30",
"size": "735841103",
"uncompressed-sha256": "32d19b94cfb2aa45799caed2dda70f4e7ac1d0fca2b834a1cbd4459a6b0d056a",
"uncompressed-size": "17179869184"
},
"metal-uefi": {
"path": "rhcos-410.8.20190325.0-metal-uefi.raw",
"sha256": "1686bb2e3b01873804325b50e286d858435afe1fbc3e88479d54b7a21e58d0f1",
"size": "732781221",
"uncompressed-sha256": "4c9a34f8b30ff7c5b107c87faac0f34a6b591d07a43ae729c45d98d15f3a8fa1",
"uncompressed-size": "17179869184"
},
"openstack": {
"path": "rhcos-410.8.20190325.0-openstack.qcow2",
"sha256": "1b16214e59d38b5c1e6d291aea35f2539845232ca3300462104d022f9877fede",
"size": "721092654",
"uncompressed-sha256": "f2f1362c155d8d331387ce759a46b16c83c8a454f59d2ab5087d9781cdc29c97",
"uncompressed-size": "2014380032"
},
"qemu": {
"path": "rhcos-410.8.20190325.0-qemu.qcow2",
"sha256": "12b2db0cae8ea4019f24183dfc905609ab27e8a0fa01bf3631dc6ad7d2afe664",
"size": "721096885",
"uncompressed-sha256": "3b288cf2e02b63f8852c731836d5923a75230a836c63814a3988a96c26268257",
"uncompressed-size": "2014314496"
},
"vmware": {
"path": "rhcos-410.8.20190325.0-vmware.ova",
"sha256": "d8c78e90f3a6fccd21c848a53b669c1c51080972377ad3f6bc81cdcc2cb48300",
"size": "710257080",
"uncompressed-sha256": "f73f5ef77095b0c79a648e2957a965f60544da08c78176028393948500da0dfc",
"uncompressed-size": "743802880"
}
},
"oscontainer": {
"digest": "sha256:007512169406dfa78f2eefdcd1553d94c399340bd1e871b4c884baf609a967ba",
"image": "docker-registry-default.cloud.registry.upshift.redhat.com/redhat-coreos/ootpa"
},
"ostree-commit": "ed3e5f10b22db9db37e48b72e907ee60113ed259eeef5c019faa0109feeeccf8",
"ostree-version": "410.8.20190325.0"
}
6 changes: 0 additions & 6 deletions hack/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

set -ex

RHCOS_BUILD_NAME="${RHCOS_BUILD_NAME:-410.8.20190325.0}"

# shellcheck disable=SC2068
version() { IFS="."; printf "%03d%03d%03d\\n" $@; unset IFS;}

Expand Down Expand Up @@ -46,10 +44,6 @@ release)
then
LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/asset/ignition/bootstrap.defaultReleaseImage=${RELEASE_IMAGE}"
fi
if test -n "${RHCOS_BUILD_NAME}"
then
LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/rhcos.buildName=${RHCOS_BUILD_NAME}"
fi
if test "${SKIP_GENERATION}" != y
then
go generate ./data
Expand Down
28 changes: 28 additions & 0 deletions hack/update-rhcos-bootimage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/python3
# Usage: ./hack/update-rhcos-bootimage.py https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190401.0/meta.json
import codecs,os,sys,json,argparse
import urllib.parse
import urllib.request

dn = os.path.abspath(os.path.dirname(sys.argv[0]))

parser = argparse.ArgumentParser()
parser.add_argument("meta", action='store')
args = parser.parse_args()

with urllib.request.urlopen(args.meta) as f:
string_f = codecs.getreader('utf-8')(f) # support for Python < 3.6
meta = json.load(string_f)
newmeta = {}
for k in ['images', 'buildid', 'oscontainer',
'ostree-commit', 'ostree-version']:
newmeta[k] = meta[k]
newmeta['amis'] = {
entry['name']: {
'hvm': entry['hvm'],
}
for entry in meta['amis']
}
newmeta['baseURI'] = urllib.parse.urljoin(args.meta, '.')
with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f:
json.dump(newmeta, f, sort_keys=True, indent=4)
4 changes: 2 additions & 2 deletions pkg/asset/rhcos/image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ func (i *Image) Generate(p asset.Parents) error {
defer cancel()
switch config.Platform.Name() {
case aws.Name:
osimage, err = rhcos.AMI(ctx, rhcos.DefaultChannel, config.Platform.AWS.Region)
osimage, err = rhcos.AMI(ctx, config.Platform.AWS.Region)
case libvirt.Name:
osimage, err = rhcos.QEMU(ctx, rhcos.DefaultChannel)
osimage, err = rhcos.QEMU(ctx)
case openstack.Name:
osimage = "rhcos"
case none.Name:
Expand Down
15 changes: 7 additions & 8 deletions pkg/rhcos/ami.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,17 @@ import (
"github.com/pkg/errors"
)

// AMI fetches the HVM AMI ID of the latest Red Hat Enterprise Linux CoreOS release.
func AMI(ctx context.Context, channel, region string) (string, error) {
meta, err := fetchLatestMetadata(ctx, channel)
// AMI fetches the HVM AMI ID of the Red Hat Enterprise Linux CoreOS release.
func AMI(ctx context.Context, region string) (string, error) {
meta, err := fetchRHCOSBuild(ctx)
if err != nil {
return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
}

for _, ami := range meta.AMIs {
if ami.Name == region {
return ami.HVM, nil
}
ami, ok := meta.AMIs[region]
if !ok {
return "", errors.Errorf("no RHCOS AMIs found in %s", region)
}

return "", errors.Errorf("no RHCOS AMIs found in %s", region)
return ami.HVM, nil
}
98 changes: 13 additions & 85 deletions pkg/rhcos/builds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,18 @@ package rhcos
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"

"github.com/openshift/installer/data"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

var (
// DefaultChannel is the default RHCOS channel for the cluster.
DefaultChannel = "ootpa"

// buildName is the name of the build in the channel that will be picked up
// empty string means the first one in the build list (latest) will be used
buildName = ""

baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
)

type metadata struct {
AMIs []struct {
HVM string `json:"hvm"`
Name string `json:"name"`
AMIs map[string]struct {
HVM string `json:"hvm"`
} `json:"amis"`
Images struct {
BaseURI string `json:"baseURI"`
Images struct {
QEMU struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Expand All @@ -36,81 +23,22 @@ type metadata struct {
OSTreeVersion string `json:"ostree-version"`
}

func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) {
build := buildName
var err error
if build == "" {
build, err = fetchLatestBuild(ctx, channel)
if err != nil {
return metadata{}, errors.Wrap(err, "failed to fetch latest build")
}
}

url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build)
logrus.Debugf("Fetching RHCOS metadata from %q", url)
req, err := http.NewRequest("GET", url, nil)
func fetchRHCOSBuild(ctx context.Context) (*metadata, error) {
file, err := data.Assets.Open("rhcos.json")
if err != nil {
return metadata{}, errors.Wrap(err, "failed to build request")
return nil, err
}
defer file.Close()

client := &http.Client{}
resp, err := client.Do(req.WithContext(ctx))
body, err := ioutil.ReadAll(file)
if err != nil {
return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
return nil, err
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return metadata{}, errors.Wrap(err, "failed to read HTTP response")
}

var meta metadata
var meta *metadata
if err := json.Unmarshal(body, &meta); err != nil {
return meta, errors.Wrap(err, "failed to parse HTTP response")
return meta, errors.Wrap(err, "failed to parse RHCOS build metadata")
}

return meta, nil
}

func fetchLatestBuild(ctx context.Context, channel string) (string, error) {
url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel)
logrus.Debugf("Fetching RHCOS builds from %q", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", errors.Wrap(err, "failed to build request")
}

client := &http.Client{}
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
return "", errors.Wrap(err, "failed to fetch builds")
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status)
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "failed to read HTTP response")
}

var builds struct {
Builds []string `json:"builds"`
}
if err := json.Unmarshal(body, &builds); err != nil {
return "", errors.Wrap(err, "failed to parse HTTP response")
}

if len(builds.Builds) == 0 {
return "", errors.Errorf("no builds found")
}

return builds.Builds[0], nil
}
20 changes: 15 additions & 5 deletions pkg/rhcos/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,27 @@ package rhcos

import (
"context"
"fmt"
"net/url"

"github.com/pkg/errors"
)

// QEMU fetches the URL of the latest Red Hat Enterprise Linux CoreOS release.
func QEMU(ctx context.Context, channel string) (string, error) {
meta, err := fetchLatestMetadata(ctx, channel)
// QEMU fetches the URL of the Red Hat Enterprise Linux CoreOS release.
func QEMU(ctx context.Context) (string, error) {
meta, err := fetchRHCOSBuild(ctx)
if err != nil {
return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
}

return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
base, err := url.Parse(meta.BaseURI)
if err != nil {
return "", err
}

relQEMU, err := url.Parse(meta.Images.QEMU.Path)
if err != nil {
return "", err
}

return base.ResolveReference(relQEMU).String(), nil
}