Skip to content

osbuild-worker: use the new ostree resolver API (HMS-4773) #4412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
achilleas-k merged 1 commit into from
Nov 7, 2024
Merged

osbuild-worker: use the new ostree resolver API (HMS-4773) #4412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/osbuild-koji-tests/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/platform"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/osbuild-composer/internal/upload/koji"
)
Expand Down Expand Up @@ -182,7 +182,7 @@ func TestKojiImport(t *testing.T) {
Extra: &koji.BuildOutputExtra{
ImageOutput: koji.ImageExtraInfo{
Arch: "noarch",
BootMode: distro.BOOT_LEGACY.String(),
BootMode: platform.BOOT_LEGACY.String(),
},
},
},
Expand Down
4 changes: 2 additions & 2 deletions cmd/osbuild-koji/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"time"

"github.com/google/uuid"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/platform"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/osbuild-composer/internal/upload/koji"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -101,7 +101,7 @@ func main() {
Extra: &koji.BuildOutputExtra{
ImageOutput: koji.ImageExtraInfo{
Arch: arch,
BootMode: distro.BOOT_NONE.String(), // TODO: put the correct boot mode here
BootMode: platform.BOOT_NONE.String(), // TODO: put the correct boot mode here
},
},
},
Expand Down
5 changes: 5 additions & 0 deletions cmd/osbuild-worker/jobimpl-osbuild.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,14 +516,19 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
}
}

// Both curl and ostree input share the same MTLS config
if impl.RepositoryMTLSConfig != nil {
if impl.RepositoryMTLSConfig.CA != "" {
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CA_CERT=%s", impl.RepositoryMTLSConfig.CA))
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CA_CERT=%s", impl.RepositoryMTLSConfig.CA))
}
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY=%s", impl.RepositoryMTLSConfig.MTLSClientKey))
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT=%s", impl.RepositoryMTLSConfig.MTLSClientCert))
Comment thread
lzap marked this conversation as resolved.
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CLIENT_KEY=%s", impl.RepositoryMTLSConfig.MTLSClientKey))
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CLIENT_CERT=%s", impl.RepositoryMTLSConfig.MTLSClientCert))
if impl.RepositoryMTLSConfig.Proxy != nil {
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_PROXY=%s", impl.RepositoryMTLSConfig.Proxy.String()))
extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_PROXY=%s", impl.RepositoryMTLSConfig.Proxy.String()))
}
}

Expand Down
42 changes: 41 additions & 1 deletion cmd/osbuild-worker/jobimpl-ostree-resolve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package main

import (
"fmt"
"net/url"
"strings"

"github.com/sirupsen/logrus"

Expand All @@ -11,6 +13,26 @@ import (
)

type OSTreeResolveJobImpl struct {
RepositoryMTLSConfig *RepositoryMTLSConfig
}

func (job *OSTreeResolveJobImpl) CompareBaseURL(baseURLStr string) (bool, error) {
baseURL, err := url.Parse(baseURLStr)
if err != nil {
return false, err
}

if baseURL.Scheme != job.RepositoryMTLSConfig.BaseURL.Scheme {
return false, nil
}
if baseURL.Host != job.RepositoryMTLSConfig.BaseURL.Host {
return false, nil
}
if !strings.HasPrefix(baseURL.Path, job.RepositoryMTLSConfig.BaseURL.Path) {
return false, nil
}

return true, nil
}

func setError(err error, result *worker.OSTreeResolveJobResult) {
Expand Down Expand Up @@ -51,7 +73,25 @@ func (impl *OSTreeResolveJobImpl) Run(job worker.Job) error {
logWithId.Infof("Resolving (%d) ostree commits", len(args.Specs))

for i, s := range args.Specs {
reqParams := ostree.SourceSpec(s)
reqParams := ostree.SourceSpec{}
reqParams.URL = s.URL
reqParams.Ref = s.Ref
if match, err := impl.CompareBaseURL(s.URL); match && err == nil {
reqParams.Proxy = impl.RepositoryMTLSConfig.Proxy.String()
reqParams.MTLS = &ostree.MTLS{
CA: impl.RepositoryMTLSConfig.CA,
ClientCert: impl.RepositoryMTLSConfig.MTLSClientCert,
ClientKey: impl.RepositoryMTLSConfig.MTLSClientKey,
}
} else if err != nil {
logWithId.Errorf("Error comparing base URL: %v", err)
result.JobError = clienterrors.New(
clienterrors.ErrorInvalidRepositoryURL,
"Repository URL is malformed",
err.Error(),
)
break
}
commitSpec, err := ostree.Resolve(reqParams)
Comment thread
croissanne marked this conversation as resolved.
if err != nil {
logWithId.Infof("Resolving ostree params failed: %v", err)
Expand Down
9 changes: 6 additions & 3 deletions cmd/osbuild-worker/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ import (
"errors"
"flag"
"fmt"
slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger"
"net/url"
"os"
"path"
"strings"
"time"

slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger"

"github.com/BurntSushi/toml"
"github.com/sirupsen/logrus"

Expand Down Expand Up @@ -508,8 +509,10 @@ func main() {
worker.JobTypeContainerResolve: &ContainerResolveJobImpl{
AuthFilePath: containersAuthFilePath,
},
worker.JobTypeOSTreeResolve: &OSTreeResolveJobImpl{},
worker.JobTypeFileResolve: &FileResolveJobImpl{},
worker.JobTypeOSTreeResolve: &OSTreeResolveJobImpl{
RepositoryMTLSConfig: repositoryMTLSConfig,
},
worker.JobTypeFileResolve: &FileResolveJobImpl{},
worker.JobTypeAWSEC2Copy: &AWSEC2CopyJobImpl{
AWSCreds: awsCredentials,
},
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ require (
github.com/labstack/gommon v0.4.2
github.com/openshift-online/ocm-sdk-go v0.1.438
github.com/oracle/oci-go-sdk/v54 v54.0.0
github.com/osbuild/images v0.95.0
github.com/osbuild/images v0.96.0
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d
github.com/osbuild/pulp-client v0.1.0
github.com/prometheus/client_golang v1.20.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,8 +534,8 @@ github.com/openshift-online/ocm-sdk-go v0.1.438 h1:tsLCCUzbLCTL4RZG02y9RuopmGCXp
github.com/openshift-online/ocm-sdk-go v0.1.438/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y=
github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXchUUZ+LS4=
github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc=
github.com/osbuild/images v0.95.0 h1:WWxYEQKD9wFGs/zkWF4wd3IDwNColZwzKsQh/+dwvUw=
github.com/osbuild/images v0.95.0/go.mod h1:4bNmMQOVadIKVC1q8zsLO8tdEQFH90zIp+MQBQUnCiE=
github.com/osbuild/images v0.96.0 h1:ZieK4i5pyKTdLaA/EwxeNEQsWBLEkX3FsZVyIaYCJKI=
github.com/osbuild/images v0.96.0/go.mod h1:4bNmMQOVadIKVC1q8zsLO8tdEQFH90zIp+MQBQUnCiE=
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d h1:r9BFPDv0uuA9k1947Jybcxs36c/pTywWS1gjeizvtcQ=
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d/go.mod h1:zR1iu/hOuf+OQNJlk70tju9IqzzM4ycq0ectkFBm94U=
github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8=
Expand Down
7 changes: 4 additions & 3 deletions internal/cloudapi/v2/imagerequest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/osbuild/images/pkg/disk"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/platform"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
"github.com/osbuild/osbuild-composer/internal/common"
Expand Down Expand Up @@ -56,11 +57,11 @@ func newAWSTarget(options UploadOptions, imageType distro.ImageType) (*target.Ta

var amiBootMode *string
switch imageType.BootMode() {
case distro.BOOT_HYBRID:
case platform.BOOT_HYBRID:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefiPreferred))
case distro.BOOT_UEFI:
case platform.BOOT_UEFI:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefi))
case distro.BOOT_LEGACY:
case platform.BOOT_LEGACY:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesLegacyBios))
}

Expand Down
13 changes: 11 additions & 2 deletions internal/cloudapi/v2/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,12 @@ func (s *Server) enqueueCompose(irs []imageRequest, channel string) (uuid.UUID,
workerResolveSpecs := make([]worker.OSTreeResolveSpec, len(sources))
for idx, source := range sources {
// ostree.SourceSpec is directly convertible to worker.OSTreeResolveSpec
workerResolveSpecs[idx] = worker.OSTreeResolveSpec(source)
workerResolveSpecs[idx] = worker.OSTreeResolveSpec{
URL: source.URL,
Ref: source.Ref,
RHSM: source.RHSM,
}

}
jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{Specs: workerResolveSpecs}, channel)
if err != nil {
Expand Down Expand Up @@ -356,7 +361,11 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
workerResolveSpecs := make([]worker.OSTreeResolveSpec, len(sources))
for idx, source := range sources {
// ostree.SourceSpec is directly convertible to worker.OSTreeResolveSpec
workerResolveSpecs[idx] = worker.OSTreeResolveSpec(source)
workerResolveSpecs[idx] = worker.OSTreeResolveSpec{
URL: source.URL,
Ref: source.Ref,
RHSM: source.RHSM,
}
}
jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{Specs: workerResolveSpecs}, channel)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions internal/weldr/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2393,6 +2393,7 @@ func (api *API) resolveOSTreeCommits(sourceSpecs map[string][]ostree.SourceSpec,
Checksum: checksum,
}
} else {
// MTLS not supported on-prem
commit, err := ostree.Resolve(source)
if err != nil {
return nil, err
Expand Down
7 changes: 4 additions & 3 deletions internal/weldr/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/platform"
"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -283,11 +284,11 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.

var amiBootMode *string
switch imageType.BootMode() {
case distro.BOOT_HYBRID:
case platform.BOOT_HYBRID:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefiPreferred))
case distro.BOOT_UEFI:
case platform.BOOT_UEFI:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefi))
case distro.BOOT_LEGACY:
case platform.BOOT_LEGACY:
amiBootMode = common.ToPtr(string(ec2types.BootModeValuesLegacyBios))
}

Expand Down
70 changes: 70 additions & 0 deletions vendor/github.com/osbuild/images/pkg/disk/disk.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading