Skip to content

Commit

Permalink
feat(api): Refactor API server to introduce support for Docker image …
Browse files Browse the repository at this point in the history 
…registries and s3 bucket support (#394)

* Update config files and image builder to configure kaniko push registry and artifact service type

* Update docker files for ensembler job

* Update docker files for ensembler service

* Add increased permissions to dockerfile to install deps

* Fix s3 copy command destination

* Fix broken unit tests

* Fix broken e2e test config

* Add new fields to example.yaml file

* Fix config validator tags and remove redundant config values

* Pin max version of setuptools

* Refactor image builder

* Remove code to parse env vars into kaniko arguments

* Fix incorrect if-else handling when configuring volume and volume mounts

* Fix incorrect if-else handling when configuring volume and volume mounts

* Reorder dockerfile layers
  • Loading branch information
@deadlycoconuts
deadlycoconuts authored Nov 7, 2024
1 parent 358bea8 commit 100c478
Show file tree
Hide file tree
Showing 19 changed files with 280 additions and 56 deletions.
2 changes: 1 addition & 1 deletion api/config-dev.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ BatchEnsemblingConfig:
Limits:
CPU: "1"
Memory: 1Gi
PushRegistryType: docker
EnsemblerServiceBuilderConfig:
ClusterName: k3s
ImageBuildingConfig:
Expand Down Expand Up @@ -98,5 +99,4 @@ OpenapiConfig:
ServingPath: /api-docs/

MlflowConfig:
TrackingURL: ""
ArtifactServiceType: "nop"
24 changes: 22 additions & 2 deletions api/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/rs/cors v1.8.2
github.com/spf13/viper v1.13.0
github.com/stretchr/testify v1.8.4
github.com/stretchr/testify v1.9.0
github.com/xanzy/go-gitlab v0.32.0
go.uber.org/zap v1.26.0
golang.org/x/oauth2 v0.18.0
Expand Down Expand Up @@ -66,6 +66,24 @@ require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2 v1.30.6-0.20240906182417-827d25db0048 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.17.8 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect
github.com/aws/smithy-go v1.20.4 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blendle/zapdriver v1.3.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
Expand Down Expand Up @@ -140,6 +158,7 @@ require (
github.com/jedib0t/go-pretty/v6 v6.5.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
Expand Down Expand Up @@ -185,7 +204,7 @@ require (
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
Expand Down Expand Up @@ -248,6 +267,7 @@ replace (

github.com/caraml-dev/merlin => github.com/caraml-dev/merlin/api v0.0.0-20240313065547-6778bd14c119
github.com/caraml-dev/merlin-pyspark-app => github.com/caraml-dev/merlin/python/batch-predictor v0.0.0-20240313065547-6778bd14c119
github.com/caraml-dev/mlp => github.com/deadlycoconuts/mlp v0.0.0-20240917090435-d94d92572eac

github.com/caraml-dev/turing/engines/experiment => ../engines/experiment
github.com/caraml-dev/turing/engines/router => ../engines/router
Expand Down
56 changes: 51 additions & 5 deletions api/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,47 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2 v1.30.6-0.20240906182417-827d25db0048 h1:wXvkIvYQ3EPVO5MhCoEv2u5LDwfWp+kLTQMIGyyvi/0=
github.com/aws/aws-sdk-go-v2 v1.30.6-0.20240906182417-827d25db0048/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE=
github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM=
github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 h1:OwhhKc1P9ElfWbMKPIbMMZBV6hzJlL2JKD76wNNVzgQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM=
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
Expand All @@ -84,8 +125,6 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/caraml-dev/merlin/api v0.0.0-20240313065547-6778bd14c119 h1:8zt/6B7ySOokLa7WDpjaybBOL8x5C35nbMIrKFMsZcg=
github.com/caraml-dev/merlin/api v0.0.0-20240313065547-6778bd14c119/go.mod h1:b2MbvZaVUSJQaoF0AuEgwTVVYK8lPHCIvnYpfvnjkQY=
github.com/caraml-dev/mlp v1.12.0 h1:yb/EgMh+83oKj0C1AhH3R6xt0eSd9GX+incZEOwM7oo=
github.com/caraml-dev/mlp v1.12.0/go.mod h1:Zdz4bALO9WOHXhOgsoLmCjMCJnDVEZEnQFg8rk+u2cE=
github.com/caraml-dev/universal-prediction-interface v0.3.6 h1:G/D4aukfjLECl8armJqFy/R2+0u/f4AiurSFqAo33uQ=
github.com/caraml-dev/universal-prediction-interface v0.3.6/go.mod h1:e0qmFOXQxx8HFg5ObYyQO3WVnrqsr5v5JApFmeF7eJo=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down Expand Up @@ -131,6 +170,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deadlycoconuts/mlp v0.0.0-20240917090435-d94d92572eac h1:M6dR1d3O+xY4suQPumkbNO1Ie70M6WoYFaoW+cjKT+8=
github.com/deadlycoconuts/mlp v0.0.0-20240917090435-d94d92572eac/go.mod h1:9kPooDSYsVu5q/z2K4T9uu08RGyiFNbCAFnQVBMJxOk=
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs=
github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE=
Expand Down Expand Up @@ -465,6 +506,10 @@ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
Expand Down Expand Up @@ -692,8 +737,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
Expand All @@ -705,8 +751,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
Expand Down
2 changes: 2 additions & 0 deletions api/turing/api/appcontext.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ func NewAppContext(
ensemblingImageBuilder, err = imagebuilder.NewEnsemblerJobImageBuilder(
imageBuildingController,
*cfg.BatchEnsemblingConfig.ImageBuildingConfig,
cfg.MlflowConfig.ArtifactServiceType,
)
if err != nil {
return nil, errors.Wrapf(err, "Failed initializing ensembling image builder")
Expand Down Expand Up @@ -158,6 +159,7 @@ func NewAppContext(
ensemblerServiceImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder(
clusterControllers[cfg.EnsemblerServiceBuilderConfig.ClusterName],
*cfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig,
cfg.MlflowConfig.ArtifactServiceType,
)
if err != nil {
return nil, errors.Wrapf(err, "Failed initializing ensembler service builder")
Expand Down
2 changes: 2 additions & 0 deletions api/turing/api/appcontext_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ func TestNewAppContext(t *testing.T) {
ensemblingImageBuilder, err := imagebuilder.NewEnsemblerJobImageBuilder(
nil,
*testCfg.BatchEnsemblingConfig.ImageBuildingConfig,
testCfg.MlflowConfig.ArtifactServiceType,
)
assert.Nil(t, err)

Expand Down Expand Up @@ -342,6 +343,7 @@ func TestNewAppContext(t *testing.T) {
ensemblerImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder(
nil,
*testCfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig,
testCfg.MlflowConfig.ArtifactServiceType,
)

assert.NoError(t, err)
Expand Down
15 changes: 13 additions & 2 deletions api/turing/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,16 @@ type KanikoConfig struct {
Image string `validate:"required"`
// ImageVersion is the version tag of the Kaniko image
ImageVersion string `validate:"required"`
// AdditionalArgs allows platform-level additional arguments to be configured for Kaniko jobs
AdditionalArgs []string
// Kaniko kubernetes service account
ServiceAccount string
// ResourceRequestsLimits is the resources required by Kaniko executor.
ResourceRequestsLimits ResourceRequestsLimits `validate:"required"`
// Kaniko push registry type
PushRegistryType string `validate:"required,oneof=docker gcr"`
// Kaniko docker credential secret name for pushing to docker registries
DockerCredentialSecretName string
}

// SparkAppConfig contains the infra configurations that is unique to the user's Kubernetes
Expand Down Expand Up @@ -426,8 +432,13 @@ type MLPConfig struct {
}

type MlflowConfig struct {
TrackingURL string
ArtifactServiceType string
TrackingURL string `validate:"required_if=ArtifactServiceType gcs ArtifactServiceType s3"`
// Note that the Kaniko image builder needs to be configured correctly to have the necessary credentials to download
// the artifacts from the blob storage tool depending on the artifact service type selected (gcs/s3). For gcs, the
// credentials can be provided via a k8s service account or a secret but for s3, the credentials can be provided via
// additional arguments in the config KanikoConfig.AdditionalArgs e.g.
// --build-arg=[AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_DEFAULT_REGION/AWS_ENDPOINT_URL]=xxx
ArtifactServiceType string `validate:"required,oneof=nop gcs s3"`
}

// OpenapiConfig contains the settings for the OpenAPI specs used for validation and Swagger UI
Expand Down
6 changes: 6 additions & 0 deletions api/turing/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,7 @@ func TestConfigValidate(t *testing.T) {
Memory: "1Gi",
},
},
PushRegistryType: "docker",
},
},
},
Expand Down Expand Up @@ -870,6 +871,7 @@ func TestConfigValidate(t *testing.T) {
Memory: "1Gi",
},
},
PushRegistryType: "docker",
},
},
},
Expand All @@ -889,6 +891,10 @@ func TestConfigValidate(t *testing.T) {
MaxMemory: config.Quantity(resource.MustParse("8Gi")),
MaxAllowedReplica: 30,
},
MlflowConfig: &config.MlflowConfig{
TrackingURL: "http://localhost:8081",
ArtifactServiceType: "gcs",
},
SparkAppConfig: &config.SparkAppConfig{
NodeSelector: map[string]string{
"node-workload-type": "batch",
Expand Down
4 changes: 4 additions & 0 deletions api/turing/config/example.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ BatchEnsemblingConfig:
Limits:
CPU: "1"
Memory: 1Gi
PushRegistryType: docker

# PyFunc ensembler service image builder configurations (this is largely similar to the batch ensembler job config)
EnsemblerServiceBuilderConfig:
Expand Down Expand Up @@ -219,6 +220,9 @@ MLPConfig:
MerlinURL: http://api.merlin.example.com
MLPEncryptionKey: secret

MlflowConfig:
ArtifactServiceType: "nop"

# Turing UI configuration to set how the UI assets will be served
TuringUIConfig:
ServingDirectory: /ui
Expand Down
4 changes: 4 additions & 0 deletions api/turing/imagebuilder/ensembler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import (
func NewEnsemblerJobImageBuilder(
clusterController cluster.Controller,
imageBuildingConfig config.ImageBuildingConfig,
artifactServiceType string,
) (ImageBuilder, error) {
return newImageBuilder(
clusterController,
imageBuildingConfig,
&ensemblerJobNameGenerator{registry: imageBuildingConfig.DestinationRegistry},
models.EnsemblerRunnerTypeJob,
artifactServiceType,
)
}

Expand Down Expand Up @@ -48,12 +50,14 @@ func (n *ensemblerJobNameGenerator) generateDockerImageName(projectName string,
func NewEnsemblerServiceImageBuilder(
clusterController cluster.Controller,
imageBuildingConfig config.ImageBuildingConfig,
artifactServiceType string,
) (ImageBuilder, error) {
return newImageBuilder(
clusterController,
imageBuildingConfig,
&ensemblerServiceNameGenerator{registry: imageBuildingConfig.DestinationRegistry},
models.EnsemblerRunnerTypeService,
artifactServiceType,
)
}

Expand Down
Loading

0 comments on commit 100c478

Please sign in to comment.