diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 273a2aea7..431a93272 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -26,7 +26,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10"] - + defaults: run: working-directory: ./sdk @@ -53,6 +53,12 @@ jobs: - name: Run unit tests run: make test + - uses: codecov/codecov-action@v4 + with: + flags: sdk-test-${{ matrix.python-version }} + name: sdk-test-${{ matrix.python-version }} + token: ${{ secrets.CODECOV_TOKEN }} + release-rules: runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/turing.yaml b/.github/workflows/turing.yaml index 67d97c9c3..65b89bb4c 100644 --- a/.github/workflows/turing.yaml +++ b/.github/workflows/turing.yaml @@ -276,6 +276,13 @@ jobs: working-directory: api args: --timeout 3m --verbose + - uses: codecov/codecov-action@v4 + with: + flags: api-test + name: api-test + token: ${{ secrets.CODECOV_TOKEN }} + working-directory: api + test-engines-router: runs-on: ubuntu-latest defaults: diff --git a/api/api/openapi-sdk.yaml b/api/api/openapi-sdk.yaml index 6c6755dcf..96e35387a 100644 --- a/api/api/openapi-sdk.yaml +++ b/api/api/openapi-sdk.yaml @@ -16,6 +16,10 @@ paths: "/projects/{project_id}/ensemblers/{ensembler_id}": $ref: "specs/ensemblers.yaml#/paths/~1projects~1{project_id}~1ensemblers~1{ensembler_id}" + # E N S E M B L E R I M A G E S + "/projects/{project_id}/ensemblers/{ensembler_id}/images": + $ref: "specs/ensembler-images.yaml#/paths/~1projects~1{project_id}~1ensemblers~1{ensembler_id}~1images" + # J O B S "/projects/{project_id}/jobs": $ref: "specs/jobs.yaml#/paths/~1projects~1{project_id}~1jobs" @@ -40,4 +44,4 @@ paths: "/projects/{project_id}/routers/{router_id}/events": $ref: "specs/routers.yaml#/paths/~1projects~1{project_id}~1routers~1{router_id}~1events" "/projects/{project_id}/router-versions": - $ref: "specs/routers.yaml#/paths/~1projects~1{project_id}~1router-versions" \ No newline at end of file + $ref: "specs/routers.yaml#/paths/~1projects~1{project_id}~1router-versions" diff --git a/api/api/openapi.bundle.yaml b/api/api/openapi.bundle.yaml index dd2462b8d..ab625aef3 100644 --- a/api/api/openapi.bundle.yaml +++ b/api/api/openapi.bundle.yaml @@ -183,6 +183,77 @@ paths: summary: Updates existing Ensembler with the data provided in the payload tags: - Ensembler + /projects/{project_id}/ensemblers/{ensembler_id}/images: + get: + operationId: ListEnsemblerImages + parameters: + - in: path + name: project_id + required: true + schema: + format: int32 + type: integer + - in: path + name: ensembler_id + required: true + schema: + format: int32 + type: integer + - in: query + name: runner_type + required: false + schema: + $ref: '#/components/schemas/EnsemblerImageRunnerType' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/EnsemblerImages' + description: A JSON object + "400": + description: Invalid request body + "404": + description: Ensembler not found + "500": + description: Unable to list ensembler images + summary: Returns a list of ensembler images that belong to the ensembler + tags: + - Ensembler Images + put: + operationId: CreateEnsemblerImage + parameters: + - in: path + name: project_id + required: true + schema: + format: int32 + type: integer + - in: path + name: ensembler_id + required: true + schema: + format: int32 + type: integer + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BuildEnsemblerImageRequest' + description: A JSON object containing information about the ensembler + required: true + responses: + "202": + description: Accepted + "400": + description: Invalid request body + "404": + description: Ensembler not found + "500": + description: Unable to build ensembler image + summary: Creates a new ensembler image + tags: + - Ensembler Images /projects/{project_id}/jobs: get: operationId: ListEnsemblingJobs @@ -1441,6 +1512,64 @@ components: Id: format: int32 type: integer + EnsemblerImages: + items: + $ref: '#/components/schemas/EnsemblerImage' + type: array + EnsemblerImage: + example: + image_ref: image_ref + project_id: 0 + image_building_job_status: + message: message + exists: true + ensembler_id: 6 + properties: + project_id: + format: int32 + type: integer + ensembler_id: + format: int32 + type: integer + runner_type: + $ref: '#/components/schemas/EnsemblerImageRunnerType' + image_ref: + type: string + exists: + type: boolean + image_building_job_status: + $ref: '#/components/schemas/ImageBuildingJobStatus' + type: object + EnsemblerImageRunnerType: + enum: + - job + - service + nullable: true + type: string + ImageBuildingJobStatus: + example: + message: message + properties: + state: + $ref: '#/components/schemas/ImageBuildingJobState' + message: + type: string + type: object + ImageBuildingJobState: + enum: + - active + - succeeded + - failed + - unknown + type: string + BuildEnsemblerImageRequest: + example: {} + properties: + runner_type: + $ref: '#/components/schemas/EnsemblerImageRunnerType' + required: + - runner_type + type: object EnsemblingJobPaginatedResults: allOf: - $ref: '#/components/schemas/EnsemblersPaginatedResults_allOf' diff --git a/api/api/openapi.yaml b/api/api/openapi.yaml index a1de5e0f3..91d34624b 100644 --- a/api/api/openapi.yaml +++ b/api/api/openapi.yaml @@ -20,6 +20,10 @@ paths: "/projects/{project_id}/ensemblers/{ensembler_id}": $ref: "specs/ensemblers.yaml#/paths/~1projects~1{project_id}~1ensemblers~1{ensembler_id}" + # E N S E M B L E R I M A G E S + "/projects/{project_id}/ensemblers/{ensembler_id}/images": + $ref: "specs/ensembler-images.yaml#/paths/~1projects~1{project_id}~1ensemblers~1{ensembler_id}~1images" + # J O B S "/projects/{project_id}/jobs": $ref: "specs/jobs.yaml#/paths/~1projects~1{project_id}~1jobs" diff --git a/api/api/specs/ensembler-images.yaml b/api/api/specs/ensembler-images.yaml new file mode 100644 index 000000000..f6104cf48 --- /dev/null +++ b/api/api/specs/ensembler-images.yaml @@ -0,0 +1,136 @@ +openapi: 3.0.3 +info: + title: Endpoints and schemas of Turing ensembler images + version: 0.0.1 + +.tags: &tags + - "Ensembler Images" + +.id: &id + type: "integer" + format: "int32" + +paths: + "/projects/{project_id}/ensemblers/{ensembler_id}/images": + get: + tags: *tags + operationId: "ListEnsemblerImages" + summary: Returns a list of ensembler images that belong to the ensembler + parameters: + - in: path + name: project_id + schema: + <<: *id + required: true + - in: path + name: ensembler_id + schema: + <<: *id + required: true + - in: query + name: runner_type + schema: + "$ref": "#/components/schemas/EnsemblerImageRunnerType" + required: false + responses: + "200": + description: A JSON object + content: + application/json: + schema: + $ref: "#/components/schemas/EnsemblerImages" + "400": + description: Invalid request body + "404": + description: Ensembler not found + "500": + description: Unable to list ensembler images + put: + tags: *tags + operationId: "CreateEnsemblerImage" + summary: Creates a new ensembler image + parameters: + - in: path + name: project_id + schema: + <<: *id + required: true + - in: path + name: ensembler_id + schema: + <<: *id + required: true + requestBody: + description: A JSON object containing information about the ensembler + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BuildEnsemblerImageRequest" + responses: + "202": + description: Accepted + "400": + description: Invalid request body + "404": + description: Ensembler not found + "500": + description: Unable to build ensembler image + +components: + schemas: + EnsemblerId: + $ref: "./common.yaml#/components/schemas/IdObject" + + EnsemblerImages: + type: array + items: + $ref: "#/components/schemas/EnsemblerImage" + + EnsemblerImage: + type: object + properties: + project_id: + type: integer + format: int32 + ensembler_id: + type: integer + format: int32 + runner_type: + "$ref": "#/components/schemas/EnsemblerImageRunnerType" + image_ref: + type: string + exists: + type: boolean + image_building_job_status: + "$ref": "#/components/schemas/ImageBuildingJobStatus" + + EnsemblerImageRunnerType: + type: string + nullable: true + enum: + - job + - service + + ImageBuildingJobStatus: + type: object + properties: + state: + "$ref": "#/components/schemas/ImageBuildingJobState" + message: + type: string + ImageBuildingJobState: + type: string + enum: + - active + - succeeded + - failed + - unknown + + BuildEnsemblerImageRequest: + type: object + properties: + runner_type: + "$ref": "#/components/schemas/EnsemblerImageRunnerType" + required: + - runner_type diff --git a/api/go.mod b/api/go.mod index 629ddbfa4..520579927 100644 --- a/api/go.mod +++ b/api/go.mod @@ -137,6 +137,7 @@ require ( github.com/jackc/pgtype v1.12.0 // indirect github.com/jackc/pgx/v4 v4.17.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jedib0t/go-pretty/v6 v6.5.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/api/go.sum b/api/go.sum index 487b7ed91..6a160ffa9 100644 --- a/api/go.sum +++ b/api/go.sum @@ -453,6 +453,8 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jedib0t/go-pretty/v6 v6.5.3 h1:GIXn6Er/anHTkVUoufs7ptEvxdD6KIhR7Axa2wYCPF0= +github.com/jedib0t/go-pretty/v6 v6.5.3/go.mod h1:5LQIxa52oJ/DlDSLv0HEkWOFMDGoWkJb9ss5KqPpJBg= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b h1:izTof8BKh/nE1wrKOrloNA5q4odOarjf+Xpe+4qow98= diff --git a/api/turing/api/appcontext.go b/api/turing/api/appcontext.go index 6939bfd88..c56efe93c 100644 --- a/api/turing/api/appcontext.go +++ b/api/turing/api/appcontext.go @@ -23,13 +23,14 @@ import ( // AppContext stores the entities relating to the application's context type AppContext struct { // DAO - DeploymentService service.DeploymentService - RoutersService service.RoutersService - RouterVersionsService service.RouterVersionsService - EventService service.EventService - EnsemblersService service.EnsemblersService - EnsemblingJobService service.EnsemblingJobService - AlertService service.AlertService + DeploymentService service.DeploymentService + RoutersService service.RoutersService + RouterVersionsService service.RouterVersionsService + EventService service.EventService + EnsemblersService service.EnsemblersService + EnsemblerImagesService service.EnsemblerImagesService + EnsemblingJobService service.EnsemblingJobService + AlertService service.AlertService // Default configuration for routers RouterDefaults *config.RouterDefaults @@ -86,6 +87,7 @@ func NewAppContext( // Initialise Batch components // Since there is only the default environment, we will not create multiple batch runners. var batchJobRunners []batchrunner.BatchJobRunner + var ensemblingImageBuilder imagebuilder.ImageBuilder var ensemblingJobService service.EnsemblingJobService // Init ensemblers service @@ -117,7 +119,8 @@ func NewAppContext( if !ok { return nil, errors.Wrapf(err, "Failed getting the image building controller") } - ensemblingImageBuilder, err := imagebuilder.NewEnsemblerJobImageBuilder( + + ensemblingImageBuilder, err = imagebuilder.NewEnsemblerJobImageBuilder( imageBuildingController, *cfg.BatchEnsemblingConfig.ImageBuildingConfig, ) @@ -129,6 +132,7 @@ func NewAppContext( if !ok { return nil, fmt.Errorf("Failed getting the batch ensembling job controller") } + batchEnsemblingController := batchensembling.NewBatchEnsemblingController( batchClusterController, mlpSvc, @@ -168,16 +172,17 @@ func NewAppContext( } appContext := &AppContext{ - DeploymentService: service.NewDeploymentService(cfg, clusterControllers, ensemblerServiceImageBuilder), - RoutersService: service.NewRoutersService(db, mlpSvc, cfg.RouterDefaults.MonitoringURLFormat), - EnsemblersService: ensemblersService, - EnsemblingJobService: ensemblingJobService, - RouterVersionsService: service.NewRouterVersionsService(db, mlpSvc, cfg.RouterDefaults.MonitoringURLFormat), - EventService: service.NewEventService(db), - RouterDefaults: cfg.RouterDefaults, - CryptoService: cryptoService, - MLPService: mlpSvc, - ExperimentsService: expSvc, + DeploymentService: service.NewDeploymentService(cfg, clusterControllers, ensemblerServiceImageBuilder), + RoutersService: service.NewRoutersService(db, mlpSvc, cfg.RouterDefaults.MonitoringURLFormat), + EnsemblersService: ensemblersService, + EnsemblerImagesService: service.NewEnsemblerImagesService(ensemblingImageBuilder, ensemblerServiceImageBuilder), + EnsemblingJobService: ensemblingJobService, + RouterVersionsService: service.NewRouterVersionsService(db, mlpSvc, cfg.RouterDefaults.MonitoringURLFormat), + EventService: service.NewEventService(db), + RouterDefaults: cfg.RouterDefaults, + CryptoService: cryptoService, + MLPService: mlpSvc, + ExperimentsService: expSvc, PodLogService: service.NewPodLogService( clusterControllers, ), diff --git a/api/turing/api/appcontext_test.go b/api/turing/api/appcontext_test.go index adfac40d1..7528f5f62 100644 --- a/api/turing/api/appcontext_test.go +++ b/api/turing/api/appcontext_test.go @@ -360,15 +360,16 @@ func TestNewAppContext(t *testing.T) { }, ensemblerImageBuilder, ), - RoutersService: service.NewRoutersService(nil, mlpSvc, testCfg.RouterDefaults.MonitoringURLFormat), - EnsemblersService: ensemblersService, - EnsemblingJobService: ensemblingJobService, - RouterVersionsService: service.NewRouterVersionsService(nil, mlpSvc, testCfg.RouterDefaults.MonitoringURLFormat), - EventService: service.NewEventService(nil), - RouterDefaults: testCfg.RouterDefaults, - CryptoService: service.NewCryptoService(testCfg.TuringEncryptionKey), - MLPService: mlpService, - ExperimentsService: experimentService, + RoutersService: service.NewRoutersService(nil, mlpSvc, testCfg.RouterDefaults.MonitoringURLFormat), + EnsemblersService: ensemblersService, + EnsemblerImagesService: service.NewEnsemblerImagesService(ensemblingImageBuilder, ensemblerImageBuilder), + EnsemblingJobService: ensemblingJobService, + RouterVersionsService: service.NewRouterVersionsService(nil, mlpSvc, testCfg.RouterDefaults.MonitoringURLFormat), + EventService: service.NewEventService(nil), + RouterDefaults: testCfg.RouterDefaults, + CryptoService: service.NewCryptoService(testCfg.TuringEncryptionKey), + MLPService: mlpService, + ExperimentsService: experimentService, PodLogService: service.NewPodLogService( map[string]cluster.Controller{ defaultEnvironment: nil, diff --git a/api/turing/api/ensembler_images_api.go b/api/turing/api/ensembler_images_api.go new file mode 100644 index 000000000..94f11970d --- /dev/null +++ b/api/turing/api/ensembler_images_api.go @@ -0,0 +1,116 @@ +package api + +import ( + "fmt" + "net/http" + + "github.com/caraml-dev/turing/api/turing/api/request" + "github.com/caraml-dev/turing/api/turing/log" + "github.com/caraml-dev/turing/api/turing/models" + "github.com/caraml-dev/turing/api/turing/service" +) + +type EnsemblerImagesController struct { + BaseController +} + +func (c EnsemblerImagesController) ListImages( + _ *http.Request, + vars RequestVars, + _ interface{}, +) *Response { + options := service.EnsemblerImagesListOptions{} + if err := c.ParseVars(&options, vars); err != nil { + return BadRequest("failed to list ensembler images", + fmt.Sprintf("failed to parse query string: %s", err)) + } + + project, err := c.MLPService.GetProject(options.ProjectID) + if err != nil { + return InternalServerError("unable to get MLP project for the router", err.Error()) + } + + ensembler, err := c.EnsemblersService.FindByID( + options.EnsemblerID, + service.EnsemblersFindByIDOptions{ + ProjectID: &options.ProjectID, + }) + if err != nil { + return NotFound("ensembler not found", err.Error()) + } + + pyFuncEnsembler, isPyfunc := ensembler.(*models.PyFuncEnsembler) + if !isPyfunc { + return InternalServerError("unable to list ensembler images", "ensembler is not a PyFuncEnsembler") + } + + images, err := c.EnsemblerImagesService.ListImages(project, pyFuncEnsembler, options.EnsemblerRunnerType) + if err != nil { + return InternalServerError("unable to list ensembler images", err.Error()) + } + + return Ok(images) +} + +func (c EnsemblerImagesController) BuildImage( + _ *http.Request, + vars RequestVars, + body interface{}, +) *Response { + options := EnsemblerImagesPathOptions{} + if err := c.ParseVars(&options, vars); err != nil { + return BadRequest("failed to build ensembler image", + fmt.Sprintf("failed to parse query string: %s", err)) + } + + project, err := c.MLPService.GetProject(*options.ProjectID) + if err != nil { + return InternalServerError("unable to get MLP project for the router", err.Error()) + } + log.Infof("project: %v", project) + + ensembler, err := c.EnsemblersService.FindByID( + *options.EnsemblerID, + service.EnsemblersFindByIDOptions{ + ProjectID: options.ProjectID, + }) + if err != nil { + return NotFound("ensembler not found", err.Error()) + } + + pyFuncEnsembler, isPyfunc := ensembler.(*models.PyFuncEnsembler) + if !isPyfunc { + return InternalServerError("unable to build ensembler image", "ensembler is not a PyFuncEnsembler") + } + + request := body.(*request.BuildEnsemblerImageRequest) + + go func() { + if err := c.EnsemblerImagesService.BuildImage(project, pyFuncEnsembler, request.RunnerType); err != nil { + log.Errorf("unable to build ensembler image", err.Error()) + } + }() + + return Accepted(nil) +} + +func (c EnsemblerImagesController) Routes() []Route { + return []Route{ + { + method: http.MethodGet, + path: "/projects/{project_id}/ensemblers/{ensembler_id}/images", + handler: c.ListImages, + }, + { + method: http.MethodPut, + path: "/projects/{project_id}/ensemblers/{ensembler_id}/images", + body: request.BuildEnsemblerImageRequest{}, + handler: c.BuildImage, + }, + } +} + +type EnsemblerImagesPathOptions struct { + ProjectID *models.ID `schema:"project_id" validate:"required"` + EnsemblerID *models.ID `schema:"ensembler_id" validate:"required"` +} diff --git a/api/turing/api/ensembler_images_api_test.go b/api/turing/api/ensembler_images_api_test.go new file mode 100644 index 000000000..2428a70c5 --- /dev/null +++ b/api/turing/api/ensembler_images_api_test.go @@ -0,0 +1,837 @@ +package api + +import ( + "fmt" + "net/http" + "reflect" + "testing" + + "github.com/caraml-dev/mlp/api/client" + "github.com/caraml-dev/turing/api/turing/api/request" + "github.com/caraml-dev/turing/api/turing/imagebuilder" + "github.com/caraml-dev/turing/api/turing/models" + "github.com/caraml-dev/turing/api/turing/service" + "github.com/caraml-dev/turing/api/turing/service/mocks" + "github.com/caraml-dev/turing/api/turing/validation" + "github.com/stretchr/testify/mock" +) + +func TestEnsemblerImagesController_ListImages(t *testing.T) { + projectID := models.ID(1) + + type args struct { + in0 *http.Request + vars RequestVars + in2 interface{} + } + tests := []struct { + name string + mlpService func() *mocks.MLPService + ensemblerService func() *mocks.EnsemblersService + ensemblerImageService func() *mocks.EnsemblerImagesService + args args + want *Response + }{ + { + name: "success - ensembler job image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("ListImages", mock.Anything, mock.Anything, mock.Anything). + Return( + []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, nil, + ) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {"job"}, + }, + in2: nil, + }, + want: &Response{ + code: 200, + data: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + }, + }, + { + name: "success - ensembler service image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("ListImages", mock.Anything, mock.Anything, mock.Anything). + Return( + []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, nil, + ) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {"service"}, + }, + in2: nil, + }, + want: &Response{ + code: 200, + data: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + }, + }, + { + name: "success - both ensembler job & service image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("ListImages", mock.Anything, mock.Anything, mock.Anything). + Return( + []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, nil, + ) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {""}, + }, + in2: nil, + }, + want: &Response{ + code: 200, + data: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + }, + }, + { + name: "failed - invalid query string", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "X-project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {""}, + }, + in2: nil, + }, + want: &Response{ + code: 400, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "failed to list ensembler images", + Message: "failed to parse query string: Key: 'EnsemblerImagesListOptions.ProjectID' Error:Field validation for 'ProjectID' failed on the 'required' tag", // nolint: lll + }, + }, + }, + { + name: "failed - timeout getting MLP project", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(nil, fmt.Errorf("timeout")) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {""}, + }, + in2: nil, + }, + want: &Response{ + code: 500, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "unable to get MLP project for the router", + Message: "timeout", + }, + }, + }, + { + name: "failed - ensembler not found", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(100), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(nil, fmt.Errorf("not found")) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"100"}, + "runner_type": {""}, + }, + in2: nil, + }, + want: &Response{ + code: 404, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "ensembler not found", + Message: "not found", + }, + }, + }, + { + name: "failed - invalid ensembler object", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(2), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"2"}, + "runner_type": {""}, + }, + in2: nil, + }, + want: &Response{ + code: 500, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "unable to list ensembler images", + Message: "ensembler is not a PyFuncEnsembler", + }, + }, + }, + { + name: "failed - error listing images", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("ListImages", mock.Anything, mock.Anything, mock.Anything). + Return( + nil, fmt.Errorf("timeout"), + ) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + "runner_type": {"job"}, + }, + in2: nil, + }, + want: &Response{ + code: 500, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "unable to list ensembler images", + Message: "timeout", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + validator, _ := validation.NewValidator(nil) + + c := EnsemblerImagesController{ + BaseController: NewBaseController( + &AppContext{ + MLPService: tt.mlpService(), + EnsemblersService: tt.ensemblerService(), + EnsemblerImagesService: tt.ensemblerImageService(), + }, + validator, + ), + } + if got := c.ListImages(tt.args.in0, tt.args.vars, tt.args.in2); !reflect.DeepEqual(got, tt.want) { + t.Errorf("EnsemblerImagesController.ListImages() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEnsemblerImagesController_BuildImage(t *testing.T) { + projectID := models.ID(1) + + type args struct { + in0 *http.Request + vars RequestVars + body interface{} + } + tests := []struct { + name string + mlpService func() *mocks.MLPService + ensemblerService func() *mocks.EnsemblersService + ensemblerImageService func() *mocks.EnsemblerImagesService + args args + want *Response + }{ + { + name: "success - build ensembler job image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("BuildImage", mock.Anything, mock.Anything, mock.Anything). + Return(nil) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 202, + data: nil, + }, + }, + { + name: "success - build ensembler service image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(1), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("BuildImage", mock.Anything, mock.Anything, mock.Anything). + Return(nil) + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeService, + }, + }, + want: &Response{ + code: 202, + data: nil, + }, + }, + { + name: "failed - invalid query string", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "X-project_id": {"1"}, + "ensembler_id": {"1"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 400, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "failed to build ensembler image", + Message: "failed to parse query string: Key: 'EnsemblerImagesPathOptions.ProjectID' Error:Field validation for 'ProjectID' failed on the 'required' tag", // nolint: lll + }, + }, + }, + { + name: "failed - timeout getting MLP project", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(nil, fmt.Errorf("timeout")) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"1"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 500, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "unable to get MLP project for the router", + Message: "timeout", + }, + }, + }, + { + name: "failed - ensembler not found", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(100), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(nil, fmt.Errorf("not found")) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"100"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 404, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "ensembler not found", + Message: "not found", + }, + }, + }, + { + name: "failed - invalid ensembler object", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(2), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"2"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 500, + data: struct { + Description string `json:"description"` + Message string `json:"error"` + }{ + Description: "unable to build ensembler image", + Message: "ensembler is not a PyFuncEnsembler", + }, + }, + }, + { + name: "failed - error building image", + mlpService: func() *mocks.MLPService { + s := &mocks.MLPService{} + s.On("GetProject", models.ID(1)).Return(&client.Project{ + ID: int32(projectID), + Name: "myproject", + }, nil) + return s + }, + ensemblerService: func() *mocks.EnsemblersService { + s := &mocks.EnsemblersService{} + s.On("FindByID", models.ID(2), service.EnsemblersFindByIDOptions{ + ProjectID: &projectID, + }).Return(&models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, nil) + return s + }, + ensemblerImageService: func() *mocks.EnsemblerImagesService { + s := &mocks.EnsemblerImagesService{} + s.On("BuildImage", mock.Anything, mock.Anything, mock.Anything). + Return(fmt.Errorf("timeout")) + + return s + }, + args: args{ + in0: &http.Request{}, + vars: RequestVars{ + "project_id": {"1"}, + "ensembler_id": {"2"}, + }, + body: &request.BuildEnsemblerImageRequest{ + RunnerType: models.EnsemblerRunnerTypeJob, + }, + }, + want: &Response{ + code: 202, + data: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + validator, _ := validation.NewValidator(nil) + + c := EnsemblerImagesController{ + BaseController: NewBaseController( + &AppContext{ + MLPService: tt.mlpService(), + EnsemblersService: tt.ensemblerService(), + EnsemblerImagesService: tt.ensemblerImageService(), + }, + validator, + ), + } + if got := c.BuildImage(tt.args.in0, tt.args.vars, tt.args.body); !reflect.DeepEqual(got, tt.want) { + t.Errorf("EnsemblerImagesController.BuildImage() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/api/turing/api/request/ensembler_images.go b/api/turing/api/request/ensembler_images.go new file mode 100644 index 000000000..8fed33bce --- /dev/null +++ b/api/turing/api/request/ensembler_images.go @@ -0,0 +1,7 @@ +package request + +import "github.com/caraml-dev/turing/api/turing/models" + +type BuildEnsemblerImageRequest struct { + RunnerType models.EnsemblerRunnerType `json:"runner_type" validate:"required"` +} diff --git a/api/turing/batch/ensembling/runner.go b/api/turing/batch/ensembling/runner.go index 54c97a070..99103d78d 100644 --- a/api/turing/batch/ensembling/runner.go +++ b/api/turing/batch/ensembling/runner.go @@ -206,14 +206,14 @@ func (r *ensemblingJobRunner) processBuildingImage( ensemblingJob *models.EnsemblingJob, mlpProject *mlp.Project, ) { - status, err := r.imageBuilder.GetImageBuildingJobStatus( + status := r.imageBuilder.GetImageBuildingJobStatus( mlpProject.Name, *ensemblingJob.InfraConfig.EnsemblerName, ensemblingJob.EnsemblerID, *ensemblingJob.InfraConfig.RunId, ) - if status == imagebuilder.JobStatusActive { + if status.IsActive() { // Do nothing return } @@ -228,7 +228,7 @@ func (r *ensemblingJobRunner) processBuildingImage( ensemblingJob.RetryCount++ saveErr := r.ensemblingJobService.Save(ensemblingJob) if saveErr != nil { - log.Errorf("Unable to save ensemblingJob %d: %v", ensemblingJob.ID, err) + log.Errorf("Unable to save ensemblingJob %d: %v", ensemblingJob.ID, saveErr) } } diff --git a/api/turing/batch/ensembling/runner_test.go b/api/turing/batch/ensembling/runner_test.go index 7cf94acee..846dad8ad 100644 --- a/api/turing/batch/ensembling/runner_test.go +++ b/api/turing/batch/ensembling/runner_test.go @@ -27,7 +27,7 @@ func TestRun(t *testing.T) { svc.On("FindByID", mock.Anything, mock.Anything).Return(&models.PyFuncEnsembler{}, nil) return svc } - var tests = map[string]struct { + tests := map[string]struct { ensemblingController func() EnsemblingController imageBuilder func() imagebuilder.ImageBuilder ensemblingJobService func() service.EnsemblingJobService @@ -130,7 +130,9 @@ func TestRun(t *testing.T) { mock.Anything, mock.Anything, mock.Anything, - ).Return(imagebuilder.JobStatusFailed, nil) + ).Return(imagebuilder.JobStatus{ + State: imagebuilder.JobStateFailed, + }, nil) return ib }, ensemblingJobService: func() service.EnsemblingJobService { diff --git a/api/turing/imagebuilder/ensembler.go b/api/turing/imagebuilder/ensembler.go index 03b2cfa0b..4aaad05e0 100644 --- a/api/turing/imagebuilder/ensembler.go +++ b/api/turing/imagebuilder/ensembler.go @@ -17,6 +17,7 @@ func NewEnsemblerJobImageBuilder( clusterController, imageBuildingConfig, &ensemblerJobNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, + models.EnsemblerRunnerTypeJob, ) } @@ -28,19 +29,19 @@ type ensemblerJobNameGenerator struct { // generateBuilderJobName generate pod name that will be used to build docker image of the ensembling job func (n *ensemblerJobNameGenerator) generateBuilderName( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, ) string { // Creates a unique resource name with partial versioning (part of the versionID hash) as max char count is limited // by k8s job name length (63) partialVersionID := getPartialVersionID(versionID, 5) - return fmt.Sprintf("batch-%s-%s-%d-%s", projectName, modelName, modelID, partialVersionID) + return fmt.Sprintf("batch-%s-%s-%d-%s", projectName, ensemblerName, ensemblerID, partialVersionID) } // generateDockerImageName generate the name of docker image of prediction job that will be created from given model -func (n *ensemblerJobNameGenerator) generateDockerImageName(projectName string, modelName string) string { - return fmt.Sprintf("%s/%s/ensembler-jobs/%s", n.registry, projectName, modelName) +func (n *ensemblerJobNameGenerator) generateDockerImageName(projectName string, ensemblerName string) string { + return fmt.Sprintf("%s/%s/ensembler-jobs/%s", n.registry, projectName, ensemblerName) } // NewEnsemblerServiceImageBuilder create ImageBuilder for building docker image of the ensembling service (real-time) @@ -52,6 +53,7 @@ func NewEnsemblerServiceImageBuilder( clusterController, imageBuildingConfig, &ensemblerServiceNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, + models.EnsemblerRunnerTypeService, ) } @@ -63,20 +65,20 @@ type ensemblerServiceNameGenerator struct { // generateBuilderServiceName generate pod name that will be used to build docker image of the ensembling service func (n *ensemblerServiceNameGenerator) generateBuilderName( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, ) string { // Creates a unique resource name with partial versioning (part of the versionID hash) as max char count is limited // by k8s pod name length (63) partialVersionID := getPartialVersionID(versionID, 5) - return fmt.Sprintf("service-%s-%s-%d-%s", projectName, modelName, modelID, partialVersionID) + return fmt.Sprintf("service-%s-%s-%d-%s", projectName, ensemblerName, ensemblerID, partialVersionID) } // generateServiceImageName generate the name of docker image of the ensembling service that will be created from given // model -func (n *ensemblerServiceNameGenerator) generateDockerImageName(projectName string, modelName string) string { - return fmt.Sprintf("%s/%s/ensembler-services/%s", n.registry, projectName, modelName) +func (n *ensemblerServiceNameGenerator) generateDockerImageName(projectName string, ensemblerName string) string { + return fmt.Sprintf("%s/%s/ensembler-services/%s", n.registry, projectName, ensemblerName) } func getPartialVersionID(versionID string, numChar int) string { diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 189bc7119..ae8d15ec9 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sort" "strings" "time" @@ -18,6 +19,8 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" + "github.com/caraml-dev/merlin/utils" + mlp "github.com/caraml-dev/mlp/api/client" "github.com/caraml-dev/turing/api/turing/cluster" "github.com/caraml-dev/turing/api/turing/config" "github.com/caraml-dev/turing/api/turing/log" @@ -38,8 +41,17 @@ const ( kanikoSecretMountpath = "/secret" ) -// JobStatus is the current state of the image building job. -type JobStatus int +// JobStatus is the current status of the image building job. +type JobStatus struct { + State JobState `json:"state"` + Message string `json:"message,omitempty"` +} + +func (js JobStatus) IsActive() bool { + return js.State == JobStateActive +} + +type JobState int const ( // jobDeletionTimeoutInSeconds is the maximum time to wait for a job to be deleted from a cluster @@ -48,14 +60,14 @@ const ( jobDeletionTickDurationInMilliseconds = 100 // jobCompletionTickDurationInSeconds is the interval at which the API server checks if a job has completed jobCompletionTickDurationInSeconds = 5 - // JobStatusActive is the status of the image building job is active - JobStatusActive = JobStatus(iota) - // JobStatusFailed is when the image building job has failed - JobStatusFailed - // JobStatusSucceeded is when the image building job has succeeded - JobStatusSucceeded - // JobStatusUnknown is when the image building job status is unknown - JobStatusUnknown + // JobStateActive is the status of the image building job is active + JobStateActive = JobState(iota) + // JobStateFailed is when the image building job has failed + JobStateFailed + // JobStateSucceeded is when the image building job has succeeded + JobStateSucceeded + // JobStateUnknown is when the image building job status is unknown + JobStateUnknown ) // BuildImageRequest contains the information needed to build the OCI image @@ -70,35 +82,46 @@ type BuildImageRequest struct { BaseImageRefTag string } +type EnsemblerImage struct { + ProjectID models.ID `json:"project_id"` + EnsemblerID models.ID `json:"ensembler_id"` + EnsemblerRunnerType models.EnsemblerRunnerType `json:"runner_type"` + ImageRef string `json:"image_ref"` + Exists bool `json:"exists"` + JobStatus JobStatus `json:"image_building_job_status"` +} + // ImageBuilder defines the operations on building and publishing OCI images. type ImageBuilder interface { // Build OCI image based on a Dockerfile BuildImage(request BuildImageRequest) (string, error) + GetEnsemblerImage(project *mlp.Project, ensembler *models.PyFuncEnsembler) (EnsemblerImage, error) GetImageBuildingJobStatus( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, - ) (JobStatus, error) + ) JobStatus DeleteImageBuildingJob( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, ) error } type nameGenerator interface { // generateBuilderJobName generate kaniko job name that will be used to build a docker image - generateBuilderName(projectName string, modelName string, modelID models.ID, versionID string) string + generateBuilderName(projectName string, ensemblerName string, ensemblerID models.ID, versionID string) string // generateDockerImageName generate image name based on project and model - generateDockerImageName(projectName string, modelName string) string + generateDockerImageName(projectName string, ensemblerName string) string } type imageBuilder struct { clusterController cluster.Controller imageBuildingConfig config.ImageBuildingConfig nameGenerator nameGenerator + runnerType models.EnsemblerRunnerType } // NewImageBuilder creates a new ImageBuilder @@ -106,6 +129,7 @@ func newImageBuilder( clusterController cluster.Controller, imageBuildingConfig config.ImageBuildingConfig, nameGenerator nameGenerator, + runnerType models.EnsemblerRunnerType, ) (ImageBuilder, error) { err := checkParseResources(imageBuildingConfig.KanikoConfig.ResourceRequestsLimits) if err != nil { @@ -116,6 +140,7 @@ func newImageBuilder( clusterController: clusterController, imageBuildingConfig: imageBuildingConfig, nameGenerator: nameGenerator, + runnerType: runnerType, }, nil } @@ -427,50 +452,99 @@ func checkParseResources(resourceRequestsLimits config.ResourceRequestsLimits) e func (ib *imageBuilder) GetImageBuildingJobStatus( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, -) (JobStatus, error) { +) (status JobStatus) { + status.State = JobStateUnknown + + jobConditionTable := "" + podContainerTable := "" + podLastTerminationMessage := "" + podLastTerminationReason := "" + + defer func() { + if jobConditionTable != "" { + status.Message = fmt.Sprintf("%s\n\nJob conditions:\n%s", status.Message, jobConditionTable) + } + + if podContainerTable != "" { + status.Message = fmt.Sprintf("%s\n\nPod container status:\n%s", status.Message, podContainerTable) + } + + if podLastTerminationMessage != "" { + status.Message = fmt.Sprintf("%s\n\nPod last termination message:\n%s", status.Message, podLastTerminationMessage) + } + }() + kanikoJobName := ib.nameGenerator.generateBuilderName( projectName, - modelName, - modelID, + ensemblerName, + ensemblerID, versionID, ) + log.Infof("Checking status of image building job %s", kanikoJobName) job, err := ib.clusterController.GetJob( context.Background(), ib.imageBuildingConfig.BuildNamespace, kanikoJobName, ) - if err != nil { - return JobStatusUnknown, err + if err != nil && !kerrors.IsNotFound(err) { + status.Message = err.Error() + return } if job.Status.Active != 0 { - return JobStatusActive, nil + status.State = JobStateActive + return } if job.Status.Succeeded != 0 { - return JobStatusSucceeded, nil + status.State = JobStateSucceeded + return } if job.Status.Failed != 0 { - return JobStatusFailed, nil + status.State = JobStateFailed + } + + if len(job.Status.Conditions) > 0 { + jobConditionTable, err = parseJobConditions(job.Status.Conditions) + status.Message = err.Error() + } + + pods, err := ib.clusterController.ListPods( + context.Background(), + ib.imageBuildingConfig.BuildNamespace, + fmt.Sprintf("job-name=%s", kanikoJobName), + ) + if err != nil && !kerrors.IsNotFound(err) { + status.Message = err.Error() + return + } + + for _, pod := range pods.Items { + if len(pod.Status.ContainerStatuses) > 0 { + podContainerTable, podLastTerminationMessage, + podLastTerminationReason = utils.ParsePodContainerStatuses(pod.Status.ContainerStatuses) + status.Message = podLastTerminationReason + break + } } - return JobStatusUnknown, nil + return } func (ib *imageBuilder) DeleteImageBuildingJob( projectName string, - modelName string, - modelID models.ID, + ensemblerName string, + ensemblerID models.ID, versionID string, ) error { kanikoJobName := ib.nameGenerator.generateBuilderName( projectName, - modelName, - modelID, + ensemblerName, + ensemblerID, versionID, ) job, err := ib.clusterController.GetJob( @@ -486,3 +560,50 @@ func (ib *imageBuilder) DeleteImageBuildingJob( err = ib.clusterController.DeleteJob(context.Background(), ib.imageBuildingConfig.BuildNamespace, job.Name) return err } + +func (ib *imageBuilder) GetEnsemblerImage( + project *mlp.Project, + ensembler *models.PyFuncEnsembler, +) (EnsemblerImage, error) { + imageName := ib.nameGenerator.generateDockerImageName(project.Name, ensembler.Name) + imageExists, err := ib.checkIfImageExists(imageName, ensembler.RunID) + if err != nil { + return EnsemblerImage{}, err + } + + imageRef := fmt.Sprintf("%s:%s", imageName, ensembler.RunID) + + image := EnsemblerImage{ + ProjectID: models.ID(project.ID), + EnsemblerID: models.ID(ensembler.GetID()), + EnsemblerRunnerType: ib.runnerType, + ImageRef: imageRef, + Exists: imageExists, + } + return image, nil +} + +func parseJobConditions(jobConditions []apibatchv1.JobCondition) (string, error) { + var err error + + jobConditionHeaders := []string{"TIMESTAMP", "TYPE", "REASON", "MESSAGE"} + jobConditionRows := [][]string{} + + sort.Slice(jobConditions, func(i, j int) bool { + return jobConditions[i].LastProbeTime.Before(&jobConditions[j].LastProbeTime) + }) + + for _, condition := range jobConditions { + jobConditionRows = append(jobConditionRows, []string{ + condition.LastProbeTime.Format(time.RFC1123), + string(condition.Type), + condition.Reason, + condition.Message, + }) + + err = errors.New(condition.Reason) + } + + jobTable := utils.LogTable(jobConditionHeaders, jobConditionRows) + return jobTable, err +} diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 28c4cd83a..bfc07e59f 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" apibatchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -18,9 +19,7 @@ import ( "github.com/caraml-dev/turing/api/turing/models" ) -var ( - timeout, _ = time.ParseDuration("10s") -) +var timeout, _ = time.ParseDuration("10s") const ( projectName = "test-project" @@ -63,7 +62,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { }, }, } - var tests = map[string]struct { + tests := map[string]struct { name string expected string projectName string @@ -339,7 +338,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { }, }, } - var tests = map[string]struct { + tests := map[string]struct { name string expectedImage string expectedImageBuildingError string @@ -628,7 +627,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { } func TestParseResources(t *testing.T) { - var tests = map[string]struct { + tests := map[string]struct { name string expected bool resourceRequestsLimits config.ResourceRequestsLimits @@ -756,8 +755,10 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: false, - expected: JobStatusActive, + hasErr: false, + expected: JobStatus{ + State: JobStateActive, + }, }, "success | succeeded": { imageBuildingConfig: imageBuildingConfig, @@ -773,8 +774,10 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: false, - expected: JobStatusSucceeded, + hasErr: false, + expected: JobStatus{ + State: JobStateSucceeded, + }, }, "success | Failed": { imageBuildingConfig: imageBuildingConfig, @@ -782,16 +785,79 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { ctlr := &clustermock.Controller{} ctlr.On("GetJob", mock.Anything, mock.Anything, mock.Anything).Return( &apibatchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "batch-test-project-mymodel-1-abc12", + Namespace: imageBuildingConfig.BuildNamespace, + }, Status: apibatchv1.JobStatus{ Failed: 1, + Conditions: []apibatchv1.JobCondition{ + { + LastProbeTime: metav1.Date(2024, 4, 29, 0o0, 0o0, 0o0, 0, time.UTC), + Type: apibatchv1.JobFailed, + Reason: "BackoffLimitExceeded", + Message: "Job has reached the specified backoff limit", + }, + }, + }, + }, + nil, + ) + ctlr.On("ListPods", mock.Anything, mock.Anything, mock.Anything).Return( + &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "batch-test-project-mymodel-1-abc12-123", + Namespace: imageBuildingConfig.BuildNamespace, + Labels: map[string]string{ + "job-name": "batch-test-project-mymodel-1-abc12", + }, + }, + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "kaniko-builder", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + Message: "CondaEnvException: Pip failed", + }, + }, + }, + }, + }, + }, }, }, nil, ) return ctlr }, - hasErr: false, - expected: JobStatusFailed, + hasErr: false, + expected: JobStatus{ + State: JobStateFailed, + Message: `Error + +Job conditions: +┌───────────────────────────────┬────────┬──────────────────────┬─────────────────────────────────────────────┐ +│ TIMESTAMP │ TYPE │ REASON │ MESSAGE │ +├───────────────────────────────┼────────┼──────────────────────┼─────────────────────────────────────────────┤ +│ Mon, 29 Apr 2024 00:00:00 UTC │ Failed │ BackoffLimitExceeded │ Job has reached the specified backoff limit │ +└───────────────────────────────┴────────┴──────────────────────┴─────────────────────────────────────────────┘ + +Pod container status: +┌────────────────┬────────────┬───────────┬────────┐ +│ CONTAINER NAME │ STATUS │ EXIT CODE │ REASON │ +├────────────────┼────────────┼───────────┼────────┤ +│ kaniko-builder │ Terminated │ 1 │ Error │ +└────────────────┴────────────┴───────────┴────────┘ + +Pod last termination message: +CondaEnvException: Pip failed`, + }, }, "success | Unknown": { imageBuildingConfig: imageBuildingConfig, @@ -801,10 +867,18 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { &apibatchv1.Job{}, nil, ) + ctlr.On("ListPods", mock.Anything, mock.Anything, mock.Anything).Return( + &v1.PodList{ + Items: []v1.Pod{}, + }, + nil, + ) return ctlr }, - hasErr: false, - expected: JobStatusUnknown, + hasErr: false, + expected: JobStatus{ + State: JobStateUnknown, + }, }, "failure | Unknown": { imageBuildingConfig: imageBuildingConfig, @@ -816,21 +890,18 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: true, - expected: JobStatusUnknown, + hasErr: true, + expected: JobStatus{ + State: JobStateUnknown, + Message: "hello", + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { clusterController := tt.clusterController() ib, _ := NewEnsemblerJobImageBuilder(clusterController, tt.imageBuildingConfig) - status, err := ib.GetImageBuildingJobStatus("", "", models.ID(1), runID) - - if tt.hasErr { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - } + status := ib.GetImageBuildingJobStatus(projectName, modelName, models.ID(1), runID) assert.Equal(t, tt.expected, status) }) } @@ -881,8 +952,10 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: false, - expected: JobStatusActive, + hasErr: false, + expected: JobStatus{ + State: JobStateActive, + }, }, "success | succeeded": { imageBuildingConfig: imageBuildingConfig, @@ -898,8 +971,10 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: false, - expected: JobStatusSucceeded, + hasErr: false, + expected: JobStatus{ + State: JobStateSucceeded, + }, }, "success | Failed": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -936,10 +1011,18 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { }, nil, ) + ctlr.On("ListPods", mock.Anything, mock.Anything, mock.Anything).Return( + &v1.PodList{ + Items: []v1.Pod{}, + }, + nil, + ) return ctlr }, - hasErr: false, - expected: JobStatusFailed, + hasErr: false, + expected: JobStatus{ + State: JobStateFailed, + }, }, "success | Unknown": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -972,10 +1055,18 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { &apibatchv1.Job{}, nil, ) + ctlr.On("ListPods", mock.Anything, mock.Anything, mock.Anything).Return( + &v1.PodList{ + Items: []v1.Pod{}, + }, + nil, + ) return ctlr }, - hasErr: false, - expected: JobStatusUnknown, + hasErr: false, + expected: JobStatus{ + State: JobStateUnknown, + }, }, "failure | Unknown": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -1010,21 +1101,18 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { ) return ctlr }, - hasErr: true, - expected: JobStatusUnknown, + hasErr: true, + expected: JobStatus{ + State: JobStateUnknown, + Message: "hello", + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { clusterController := tt.clusterController() ib, _ := NewEnsemblerServiceImageBuilder(clusterController, tt.imageBuildingConfig) - status, err := ib.GetImageBuildingJobStatus("", "", models.ID(1), runID) - - if tt.hasErr { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - } + status := ib.GetImageBuildingJobStatus("", "", models.ID(1), runID) assert.Equal(t, tt.expected, status) }) } diff --git a/api/turing/imagebuilder/mocks/image_builder.go b/api/turing/imagebuilder/mocks/image_builder.go index c8ca3813d..55dc1c2ae 100644 --- a/api/turing/imagebuilder/mocks/image_builder.go +++ b/api/turing/imagebuilder/mocks/image_builder.go @@ -1,11 +1,11 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - + client "github.com/caraml-dev/mlp/api/client" imagebuilder "github.com/caraml-dev/turing/api/turing/imagebuilder" + mock "github.com/stretchr/testify/mock" models "github.com/caraml-dev/turing/api/turing/models" ) @@ -20,13 +20,16 @@ func (_m *ImageBuilder) BuildImage(request imagebuilder.BuildImageRequest) (stri ret := _m.Called(request) var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(imagebuilder.BuildImageRequest) (string, error)); ok { + return rf(request) + } if rf, ok := ret.Get(0).(func(imagebuilder.BuildImageRequest) string); ok { r0 = rf(request) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(imagebuilder.BuildImageRequest) error); ok { r1 = rf(request) } else { @@ -36,14 +39,13 @@ func (_m *ImageBuilder) BuildImage(request imagebuilder.BuildImageRequest) (stri return r0, r1 } -// DeleteImageBuildingJob provides a mock function with given fields: projectName, modelName, versionID -func (_m *ImageBuilder) DeleteImageBuildingJob(projectName string, modelName string, modelID models.ID, - versionID string) error { - ret := _m.Called(projectName, modelName, modelID, versionID) +// DeleteImageBuildingJob provides a mock function with given fields: projectName, ensemblerName, ensemblerID, versionID +func (_m *ImageBuilder) DeleteImageBuildingJob(projectName string, ensemblerName string, ensemblerID models.ID, versionID string) error { + ret := _m.Called(projectName, ensemblerName, ensemblerID, versionID) var r0 error if rf, ok := ret.Get(0).(func(string, string, models.ID, string) error); ok { - r0 = rf(projectName, modelName, modelID, versionID) + r0 = rf(projectName, ensemblerName, ensemblerID, versionID) } else { r0 = ret.Error(0) } @@ -51,24 +53,55 @@ func (_m *ImageBuilder) DeleteImageBuildingJob(projectName string, modelName str return r0 } -// GetImageBuildingJobStatus provides a mock function with given fields: projectName, modelName, versionID -func (_m *ImageBuilder) GetImageBuildingJobStatus(projectName string, modelName string, modelID models.ID, - versionID string) (imagebuilder.JobStatus, error) { - ret := _m.Called(projectName, modelName, modelID, versionID) +// GetEnsemblerImage provides a mock function with given fields: project, ensembler +func (_m *ImageBuilder) GetEnsemblerImage(project *client.Project, ensembler *models.PyFuncEnsembler) (imagebuilder.EnsemblerImage, error) { + ret := _m.Called(project, ensembler) - var r0 imagebuilder.JobStatus - if rf, ok := ret.Get(0).(func(string, string, models.ID, string) imagebuilder.JobStatus); ok { - r0 = rf(projectName, modelName, modelID, versionID) + var r0 imagebuilder.EnsemblerImage + var r1 error + if rf, ok := ret.Get(0).(func(*client.Project, *models.PyFuncEnsembler) (imagebuilder.EnsemblerImage, error)); ok { + return rf(project, ensembler) + } + if rf, ok := ret.Get(0).(func(*client.Project, *models.PyFuncEnsembler) imagebuilder.EnsemblerImage); ok { + r0 = rf(project, ensembler) } else { - r0 = ret.Get(0).(imagebuilder.JobStatus) + r0 = ret.Get(0).(imagebuilder.EnsemblerImage) } - var r1 error - if rf, ok := ret.Get(1).(func(string, string, models.ID, string) error); ok { - r1 = rf(projectName, modelName, modelID, versionID) + if rf, ok := ret.Get(1).(func(*client.Project, *models.PyFuncEnsembler) error); ok { + r1 = rf(project, ensembler) } else { r1 = ret.Error(1) } return r0, r1 } + +// GetImageBuildingJobStatus provides a mock function with given fields: projectName, ensemblerName, ensemblerID, versionID +func (_m *ImageBuilder) GetImageBuildingJobStatus(projectName string, ensemblerName string, ensemblerID models.ID, versionID string) imagebuilder.JobStatus { + ret := _m.Called(projectName, ensemblerName, ensemblerID, versionID) + + var r0 imagebuilder.JobStatus + if rf, ok := ret.Get(0).(func(string, string, models.ID, string) imagebuilder.JobStatus); ok { + r0 = rf(projectName, ensemblerName, ensemblerID, versionID) + } else { + r0 = ret.Get(0).(imagebuilder.JobStatus) + } + + return r0 +} + +type mockConstructorTestingTNewImageBuilder interface { + mock.TestingT + Cleanup(func()) +} + +// NewImageBuilder creates a new instance of ImageBuilder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewImageBuilder(t mockConstructorTestingTNewImageBuilder) *ImageBuilder { + mock := &ImageBuilder{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/models/ensembler.go b/api/turing/models/ensembler.go index c45501c47..f0cf9f7f8 100644 --- a/api/turing/models/ensembler.go +++ b/api/turing/models/ensembler.go @@ -79,6 +79,13 @@ type ExperimentMapping struct { Route string `json:"route" validate:"required"` // Route ID to select for the experiment treatment } +type EnsemblerRunnerType string + +const ( + EnsemblerRunnerTypeJob EnsemblerRunnerType = "job" + EnsemblerRunnerTypeService EnsemblerRunnerType = "service" +) + // Value implements sql.driver.Valuer interface so database tools like go-orm knows how to serialize the struct object // for storage in the database func (c EnsemblerStandardConfig) Value() (driver.Value, error) { diff --git a/api/turing/server/api.go b/api/turing/server/api.go index 77689fc83..e8ca1b865 100644 --- a/api/turing/server/api.go +++ b/api/turing/server/api.go @@ -37,6 +37,7 @@ func AddAPIRoutesHandler(r *mux.Router, path string, appCtx *api.AppContext, cfg controllers := []api.Controller{ api.AlertsController{BaseController: baseController}, api.EnsemblersController{BaseController: baseController}, + api.EnsemblerImagesController{BaseController: baseController}, api.ExperimentsController{BaseController: baseController}, api.PodLogController{BaseController: baseController}, api.ProjectsController{BaseController: baseController}, diff --git a/api/turing/service/ensembler_image_service.go b/api/turing/service/ensembler_image_service.go new file mode 100644 index 000000000..398087dd7 --- /dev/null +++ b/api/turing/service/ensembler_image_service.go @@ -0,0 +1,122 @@ +package service + +import ( + "fmt" + + mlp "github.com/caraml-dev/mlp/api/client" + "github.com/caraml-dev/turing/api/turing/cluster/labeller" + "github.com/caraml-dev/turing/api/turing/imagebuilder" + "github.com/caraml-dev/turing/api/turing/models" +) + +type EnsemblerImagesService interface { + ListImages( + project *mlp.Project, + ensembler *models.PyFuncEnsembler, + runnerType models.EnsemblerRunnerType, + ) ([]imagebuilder.EnsemblerImage, error) + BuildImage( + project *mlp.Project, + ensembler *models.PyFuncEnsembler, + runnerType models.EnsemblerRunnerType, + ) error +} + +type EnsemblerImagesListOptions struct { + ProjectID models.ID `schema:"project_id" validate:"required"` + EnsemblerID models.ID `schema:"ensembler_id" validate:"required"` + EnsemblerRunnerType models.EnsemblerRunnerType `schema:"runner_type"` +} + +type ensemblerImagesService struct { + ensemblerJobImageBuilder imagebuilder.ImageBuilder + ensemblerServiceImageBuilder imagebuilder.ImageBuilder +} + +func NewEnsemblerImagesService(ensemblerJobImageBuilder imagebuilder.ImageBuilder, + ensemblerServiceImageBuilder imagebuilder.ImageBuilder, +) EnsemblerImagesService { + return &ensemblerImagesService{ + ensemblerJobImageBuilder: ensemblerJobImageBuilder, + ensemblerServiceImageBuilder: ensemblerServiceImageBuilder, + } +} + +func (s *ensemblerImagesService) ListImages( + project *mlp.Project, + ensembler *models.PyFuncEnsembler, + runnerType models.EnsemblerRunnerType, +) ([]imagebuilder.EnsemblerImage, error) { + builders := []imagebuilder.ImageBuilder{} + + if runnerType == models.EnsemblerRunnerTypeJob { + builders = append(builders, s.ensemblerJobImageBuilder) + } else if runnerType == models.EnsemblerRunnerTypeService { + builders = append(builders, s.ensemblerServiceImageBuilder) + } else { + builders = append(builders, s.ensemblerJobImageBuilder, s.ensemblerServiceImageBuilder) + } + + images := []imagebuilder.EnsemblerImage{} + for _, builder := range builders { + image, err := builder.GetEnsemblerImage(project, ensembler) + if err != nil { + return nil, err + } + + jobStatus := builder.GetImageBuildingJobStatus(project.Name, ensembler.Name, ensembler.ID, ensembler.RunID) + image.JobStatus = jobStatus + + images = append(images, image) + } + + return images, nil +} + +func (s *ensemblerImagesService) BuildImage( + project *mlp.Project, + ensembler *models.PyFuncEnsembler, + runnerType models.EnsemblerRunnerType, +) error { + ib, err := s.getImageBuilder(runnerType) + if err != nil { + return err + } + + request := imagebuilder.BuildImageRequest{ + ProjectName: project.Name, + ResourceName: ensembler.Name, + ResourceID: ensembler.ID, + VersionID: ensembler.RunID, + ArtifactURI: ensembler.ArtifactURI, + BuildLabels: labeller.BuildLabels( + labeller.KubernetesLabelsRequest{ + Stream: project.Stream, + Team: project.Team, + App: ensembler.Name, + Labels: project.Labels, + }, + ), + EnsemblerFolder: EnsemblerFolder, + BaseImageRefTag: ensembler.PythonVersion, + } + + if _, err := ib.BuildImage(request); err != nil { + return err + } + + return nil +} + +func (s *ensemblerImagesService) getImageBuilder( + runnerType models.EnsemblerRunnerType, +) (imagebuilder.ImageBuilder, error) { + switch runnerType { + case models.EnsemblerRunnerTypeJob: + return s.ensemblerJobImageBuilder, nil + case models.EnsemblerRunnerTypeService: + return s.ensemblerServiceImageBuilder, nil + default: + return nil, fmt.Errorf("runner type %s is not supported", runnerType) + } +} diff --git a/api/turing/service/ensembler_image_service_test.go b/api/turing/service/ensembler_image_service_test.go new file mode 100644 index 000000000..f989884ac --- /dev/null +++ b/api/turing/service/ensembler_image_service_test.go @@ -0,0 +1,342 @@ +package service + +import ( + "reflect" + "testing" + + mlp "github.com/caraml-dev/mlp/api/client" + "github.com/caraml-dev/turing/api/turing/imagebuilder" + mockImgBuilder "github.com/caraml-dev/turing/api/turing/imagebuilder/mocks" + "github.com/caraml-dev/turing/api/turing/models" + mock "github.com/stretchr/testify/mock" +) + +func Test_ensemblerImagesService_ListImages(t *testing.T) { + type args struct { + project *mlp.Project + ensembler *models.PyFuncEnsembler + runnerType models.EnsemblerRunnerType + } + tests := []struct { + name string + ensemblerJobImageBuilder func() *mockImgBuilder.ImageBuilder + ensemblerServiceImageBuilder func() *mockImgBuilder.ImageBuilder + args args + want []imagebuilder.EnsemblerImage + wantErr bool + }{ + { + name: "success - ensembler job image", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("GetEnsemblerImage", mock.Anything, mock.Anything). + Return(imagebuilder.EnsemblerImage{ + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + }, nil) + ib.On("GetImageBuildingJobStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, nil) + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + runnerType: models.EnsemblerRunnerTypeJob, + }, + want: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + wantErr: false, + }, + { + name: "success - ensembler service image", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("GetEnsemblerImage", mock.Anything, mock.Anything). + Return(imagebuilder.EnsemblerImage{ + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + }, nil) + ib.On("GetImageBuildingJobStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, nil) + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + runnerType: models.EnsemblerRunnerTypeService, + }, + want: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + wantErr: false, + }, + { + name: "success - both ensembler job and service image", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("GetEnsemblerImage", mock.Anything, mock.Anything). + Return(imagebuilder.EnsemblerImage{ + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + }, nil) + ib.On("GetImageBuildingJobStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, nil) + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("GetEnsemblerImage", mock.Anything, mock.Anything). + Return(imagebuilder.EnsemblerImage{ + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + }, nil) + ib.On("GetImageBuildingJobStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, nil) + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + }, + want: []imagebuilder.EnsemblerImage{ + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeJob, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + { + ProjectID: models.ID(1), + EnsemblerID: models.ID(1), + EnsemblerRunnerType: models.EnsemblerRunnerTypeService, + ImageRef: "ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", + Exists: true, + JobStatus: imagebuilder.JobStatus{ + State: imagebuilder.JobStateSucceeded, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &ensemblerImagesService{ + ensemblerJobImageBuilder: tt.ensemblerJobImageBuilder(), + ensemblerServiceImageBuilder: tt.ensemblerServiceImageBuilder(), + } + got, err := s.ListImages(tt.args.project, tt.args.ensembler, tt.args.runnerType) + if (err != nil) != tt.wantErr { + t.Errorf("ensemblerImagesService.ListImages() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ensemblerImagesService.ListImages() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_ensemblerImagesService_BuildImage(t *testing.T) { + type args struct { + project *mlp.Project + ensembler *models.PyFuncEnsembler + runnerType models.EnsemblerRunnerType + } + tests := []struct { + name string + ensemblerJobImageBuilder func() *mockImgBuilder.ImageBuilder + ensemblerServiceImageBuilder func() *mockImgBuilder.ImageBuilder + args args + wantErr bool + }{ + { + name: "success - build ensembler job image", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("BuildImage", mock.Anything). + Return("ghcr.io/caraml-dev/turing/ensembler-jobs/myproject/myensembler-1:abc123", nil) + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + runnerType: models.EnsemblerRunnerTypeJob, + }, + wantErr: false, + }, + { + name: "success - build ensembler service image", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + ib.On("BuildImage", mock.Anything). + Return("ghcr.io/caraml-dev/turing/ensembler-services/myproject/myensembler-1:abc123", nil) + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + runnerType: models.EnsemblerRunnerTypeService, + }, + wantErr: false, + }, + { + name: "invalid runner type", + ensemblerJobImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + ensemblerServiceImageBuilder: func() *mockImgBuilder.ImageBuilder { + ib := &mockImgBuilder.ImageBuilder{} + return ib + }, + args: args{ + project: &mlp.Project{ + ID: 1, + Name: "myproject", + }, + ensembler: &models.PyFuncEnsembler{ + GenericEnsembler: &models.GenericEnsembler{ + Model: models.Model{ + ID: models.ID(1), + }, + ProjectID: 1, + Name: "myensembler", + }, + RunID: "abc123", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &ensemblerImagesService{ + ensemblerJobImageBuilder: tt.ensemblerJobImageBuilder(), + ensemblerServiceImageBuilder: tt.ensemblerServiceImageBuilder(), + } + if err := s.BuildImage(tt.args.project, tt.args.ensembler, tt.args.runnerType); (err != nil) != tt.wantErr { + t.Errorf("ensemblerImagesService.BuildImage() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/api/turing/service/mocks/alert_service.go b/api/turing/service/mocks/alert_service.go index ba93ca570..b2553fc51 100644 --- a/api/turing/service/mocks/alert_service.go +++ b/api/turing/service/mocks/alert_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks @@ -35,6 +35,10 @@ func (_m *AlertService) FindByID(id models.ID) (*models.Alert, error) { ret := _m.Called(id) var r0 *models.Alert + var r1 error + if rf, ok := ret.Get(0).(func(models.ID) (*models.Alert, error)); ok { + return rf(id) + } if rf, ok := ret.Get(0).(func(models.ID) *models.Alert); ok { r0 = rf(id) } else { @@ -43,7 +47,6 @@ func (_m *AlertService) FindByID(id models.ID) (*models.Alert, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID) error); ok { r1 = rf(id) } else { @@ -58,13 +61,16 @@ func (_m *AlertService) GetDashboardURL(alert *models.Alert, project *client.Pro ret := _m.Called(alert, project, environment, router, routerVersion) var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(*models.Alert, *client.Project, *merlinclient.Environment, *models.Router, *models.RouterVersion) (string, error)); ok { + return rf(alert, project, environment, router, routerVersion) + } if rf, ok := ret.Get(0).(func(*models.Alert, *client.Project, *merlinclient.Environment, *models.Router, *models.RouterVersion) string); ok { r0 = rf(alert, project, environment, router, routerVersion) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(*models.Alert, *client.Project, *merlinclient.Environment, *models.Router, *models.RouterVersion) error); ok { r1 = rf(alert, project, environment, router, routerVersion) } else { @@ -79,6 +85,10 @@ func (_m *AlertService) List(_a0 string) ([]*models.Alert, error) { ret := _m.Called(_a0) var r0 []*models.Alert + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*models.Alert, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(string) []*models.Alert); ok { r0 = rf(_a0) } else { @@ -87,7 +97,6 @@ func (_m *AlertService) List(_a0 string) ([]*models.Alert, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(_a0) } else { @@ -102,6 +111,10 @@ func (_m *AlertService) Save(alert models.Alert, authorEmail string, dashboardUR ret := _m.Called(alert, authorEmail, dashboardURL) var r0 *models.Alert + var r1 error + if rf, ok := ret.Get(0).(func(models.Alert, string, string) (*models.Alert, error)); ok { + return rf(alert, authorEmail, dashboardURL) + } if rf, ok := ret.Get(0).(func(models.Alert, string, string) *models.Alert); ok { r0 = rf(alert, authorEmail, dashboardURL) } else { @@ -110,7 +123,6 @@ func (_m *AlertService) Save(alert models.Alert, authorEmail string, dashboardUR } } - var r1 error if rf, ok := ret.Get(1).(func(models.Alert, string, string) error); ok { r1 = rf(alert, authorEmail, dashboardURL) } else { @@ -133,3 +145,18 @@ func (_m *AlertService) Update(alert models.Alert, authorEmail string, dashboard return r0 } + +type mockConstructorTestingTNewAlertService interface { + mock.TestingT + Cleanup(func()) +} + +// NewAlertService creates a new instance of AlertService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewAlertService(t mockConstructorTestingTNewAlertService) *AlertService { + mock := &AlertService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/crypto_service.go b/api/turing/service/mocks/crypto_service.go index b02b3fae9..3c60b268c 100644 --- a/api/turing/service/mocks/crypto_service.go +++ b/api/turing/service/mocks/crypto_service.go @@ -1,32 +1,73 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + package mocks -import "github.com/stretchr/testify/mock" +import mock "github.com/stretchr/testify/mock" -// CryptoService implements CryptoService interface +// CryptoService is an autogenerated mock type for the CryptoService type type CryptoService struct { mock.Mock } -// Encrypt implements CryptoService.Encrypt -func (cs *CryptoService) Encrypt(text string) (string, error) { - ret := cs.Called(text) +// Decrypt provides a mock function with given fields: ciphertext +func (_m *CryptoService) Decrypt(ciphertext string) (string, error) { + ret := _m.Called(ciphertext) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string) (string, error)); ok { + return rf(ciphertext) + } + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(ciphertext) + } else { + r0 = ret.Get(0).(string) + } - var err error - if ret[1] != nil { - err = (ret[1]).(error) + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(ciphertext) + } else { + r1 = ret.Error(1) } - return (ret[0]).(string), err + return r0, r1 } -// Decrypt implements CryptoService.Decrypt -func (cs *CryptoService) Decrypt(cipher string) (string, error) { - ret := cs.Called(cipher) +// Encrypt provides a mock function with given fields: plaintext +func (_m *CryptoService) Encrypt(plaintext string) (string, error) { + ret := _m.Called(plaintext) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string) (string, error)); ok { + return rf(plaintext) + } + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(plaintext) + } else { + r0 = ret.Get(0).(string) + } - var err error - if ret[1] != nil { - err = (ret[1]).(error) + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(plaintext) + } else { + r1 = ret.Error(1) } - return (ret[0]).(string), err + return r0, r1 +} + +type mockConstructorTestingTNewCryptoService interface { + mock.TestingT + Cleanup(func()) +} + +// NewCryptoService creates a new instance of CryptoService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCryptoService(t mockConstructorTestingTNewCryptoService) *CryptoService { + mock := &CryptoService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/api/turing/service/mocks/ensembler_images_service.go b/api/turing/service/mocks/ensembler_images_service.go new file mode 100644 index 000000000..75d7292a9 --- /dev/null +++ b/api/turing/service/mocks/ensembler_images_service.go @@ -0,0 +1,71 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + client "github.com/caraml-dev/mlp/api/client" + imagebuilder "github.com/caraml-dev/turing/api/turing/imagebuilder" + mock "github.com/stretchr/testify/mock" + + models "github.com/caraml-dev/turing/api/turing/models" +) + +// EnsemblerImagesService is an autogenerated mock type for the EnsemblerImagesService type +type EnsemblerImagesService struct { + mock.Mock +} + +// BuildImage provides a mock function with given fields: project, ensembler, runnerType +func (_m *EnsemblerImagesService) BuildImage(project *client.Project, ensembler *models.PyFuncEnsembler, runnerType models.EnsemblerRunnerType) error { + ret := _m.Called(project, ensembler, runnerType) + + var r0 error + if rf, ok := ret.Get(0).(func(*client.Project, *models.PyFuncEnsembler, models.EnsemblerRunnerType) error); ok { + r0 = rf(project, ensembler, runnerType) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ListImages provides a mock function with given fields: project, ensembler, runnerType +func (_m *EnsemblerImagesService) ListImages(project *client.Project, ensembler *models.PyFuncEnsembler, runnerType models.EnsemblerRunnerType) ([]imagebuilder.EnsemblerImage, error) { + ret := _m.Called(project, ensembler, runnerType) + + var r0 []imagebuilder.EnsemblerImage + var r1 error + if rf, ok := ret.Get(0).(func(*client.Project, *models.PyFuncEnsembler, models.EnsemblerRunnerType) ([]imagebuilder.EnsemblerImage, error)); ok { + return rf(project, ensembler, runnerType) + } + if rf, ok := ret.Get(0).(func(*client.Project, *models.PyFuncEnsembler, models.EnsemblerRunnerType) []imagebuilder.EnsemblerImage); ok { + r0 = rf(project, ensembler, runnerType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]imagebuilder.EnsemblerImage) + } + } + + if rf, ok := ret.Get(1).(func(*client.Project, *models.PyFuncEnsembler, models.EnsemblerRunnerType) error); ok { + r1 = rf(project, ensembler, runnerType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewEnsemblerImagesService interface { + mock.TestingT + Cleanup(func()) +} + +// NewEnsemblerImagesService creates a new instance of EnsemblerImagesService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEnsemblerImagesService(t mockConstructorTestingTNewEnsemblerImagesService) *EnsemblerImagesService { + mock := &EnsemblerImagesService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/ensemblers_service.go b/api/turing/service/mocks/ensemblers_service.go index caee2579e..db9335125 100644 --- a/api/turing/service/mocks/ensemblers_service.go +++ b/api/turing/service/mocks/ensemblers_service.go @@ -1,12 +1,11 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - models "github.com/caraml-dev/turing/api/turing/models" service "github.com/caraml-dev/turing/api/turing/service" + mock "github.com/stretchr/testify/mock" ) // EnsemblersService is an autogenerated mock type for the EnsemblersService type diff --git a/api/turing/service/mocks/ensembling_job_service.go b/api/turing/service/mocks/ensembling_job_service.go index 5556ddf98..75f34f4a7 100644 --- a/api/turing/service/mocks/ensembling_job_service.go +++ b/api/turing/service/mocks/ensembling_job_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks @@ -21,6 +21,10 @@ func (_m *EnsemblingJobService) CreateEnsemblingJob(job *models.EnsemblingJob, p ret := _m.Called(job, projectID, ensembler) var r0 *models.EnsemblingJob + var r1 error + if rf, ok := ret.Get(0).(func(*models.EnsemblingJob, models.ID, *models.PyFuncEnsembler) (*models.EnsemblingJob, error)); ok { + return rf(job, projectID, ensembler) + } if rf, ok := ret.Get(0).(func(*models.EnsemblingJob, models.ID, *models.PyFuncEnsembler) *models.EnsemblingJob); ok { r0 = rf(job, projectID, ensembler) } else { @@ -29,7 +33,6 @@ func (_m *EnsemblingJobService) CreateEnsemblingJob(job *models.EnsemblingJob, p } } - var r1 error if rf, ok := ret.Get(1).(func(*models.EnsemblingJob, models.ID, *models.PyFuncEnsembler) error); ok { r1 = rf(job, projectID, ensembler) } else { @@ -74,6 +77,10 @@ func (_m *EnsemblingJobService) FindByID(id models.ID, options service.Ensemblin ret := _m.Called(id, options) var r0 *models.EnsemblingJob + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, service.EnsemblingJobFindByIDOptions) (*models.EnsemblingJob, error)); ok { + return rf(id, options) + } if rf, ok := ret.Get(0).(func(models.ID, service.EnsemblingJobFindByIDOptions) *models.EnsemblingJob); ok { r0 = rf(id, options) } else { @@ -82,7 +89,6 @@ func (_m *EnsemblingJobService) FindByID(id models.ID, options service.Ensemblin } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID, service.EnsemblingJobFindByIDOptions) error); ok { r1 = rf(id, options) } else { @@ -97,13 +103,16 @@ func (_m *EnsemblingJobService) FormatLoggingURL(ensemblerName string, namespace ret := _m.Called(ensemblerName, namespace, componentType) var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (string, error)); ok { + return rf(ensemblerName, namespace, componentType) + } if rf, ok := ret.Get(0).(func(string, string, string) string); ok { r0 = rf(ensemblerName, namespace, componentType) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(string, string, string) error); ok { r1 = rf(ensemblerName, namespace, componentType) } else { @@ -146,6 +155,10 @@ func (_m *EnsemblingJobService) List(options service.EnsemblingJobListOptions) ( ret := _m.Called(options) var r0 *service.PaginatedResults + var r1 error + if rf, ok := ret.Get(0).(func(service.EnsemblingJobListOptions) (*service.PaginatedResults, error)); ok { + return rf(options) + } if rf, ok := ret.Get(0).(func(service.EnsemblingJobListOptions) *service.PaginatedResults); ok { r0 = rf(options) } else { @@ -154,7 +167,6 @@ func (_m *EnsemblingJobService) List(options service.EnsemblingJobListOptions) ( } } - var r1 error if rf, ok := ret.Get(1).(func(service.EnsemblingJobListOptions) error); ok { r1 = rf(options) } else { @@ -191,3 +203,18 @@ func (_m *EnsemblingJobService) Save(ensemblingJob *models.EnsemblingJob) error return r0 } + +type mockConstructorTestingTNewEnsemblingJobService interface { + mock.TestingT + Cleanup(func()) +} + +// NewEnsemblingJobService creates a new instance of EnsemblingJobService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEnsemblingJobService(t mockConstructorTestingTNewEnsemblingJobService) *EnsemblingJobService { + mock := &EnsemblingJobService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/event_service.go b/api/turing/service/mocks/event_service.go index dbf83f4d8..bd56ab435 100644 --- a/api/turing/service/mocks/event_service.go +++ b/api/turing/service/mocks/event_service.go @@ -1,11 +1,10 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - models "github.com/caraml-dev/turing/api/turing/models" + mock "github.com/stretchr/testify/mock" ) // EventService is an autogenerated mock type for the EventService type @@ -32,6 +31,10 @@ func (_m *EventService) ListEvents(routerID int) ([]*models.Event, error) { ret := _m.Called(routerID) var r0 []*models.Event + var r1 error + if rf, ok := ret.Get(0).(func(int) ([]*models.Event, error)); ok { + return rf(routerID) + } if rf, ok := ret.Get(0).(func(int) []*models.Event); ok { r0 = rf(routerID) } else { @@ -40,7 +43,6 @@ func (_m *EventService) ListEvents(routerID int) ([]*models.Event, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(int) error); ok { r1 = rf(routerID) } else { @@ -63,3 +65,18 @@ func (_m *EventService) Save(event *models.Event) error { return r0 } + +type mockConstructorTestingTNewEventService interface { + mock.TestingT + Cleanup(func()) +} + +// NewEventService creates a new instance of EventService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEventService(t mockConstructorTestingTNewEventService) *EventService { + mock := &EventService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/experiments_service.go b/api/turing/service/mocks/experiments_service.go index 52035cfdb..e28f92fd2 100644 --- a/api/turing/service/mocks/experiments_service.go +++ b/api/turing/service/mocks/experiments_service.go @@ -1,13 +1,12 @@ -// Code generated by mockery v2.9.4. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( json "encoding/json" - mock "github.com/stretchr/testify/mock" - manager "github.com/caraml-dev/turing/engines/experiment/manager" + mock "github.com/stretchr/testify/mock" ) // ExperimentsService is an autogenerated mock type for the ExperimentsService type @@ -20,6 +19,10 @@ func (_m *ExperimentsService) GetExperimentRunnerConfig(engine string, cfg json. ret := _m.Called(engine, cfg) var r0 json.RawMessage + var r1 error + if rf, ok := ret.Get(0).(func(string, json.RawMessage) (json.RawMessage, error)); ok { + return rf(engine, cfg) + } if rf, ok := ret.Get(0).(func(string, json.RawMessage) json.RawMessage); ok { r0 = rf(engine, cfg) } else { @@ -28,7 +31,6 @@ func (_m *ExperimentsService) GetExperimentRunnerConfig(engine string, cfg json. } } - var r1 error if rf, ok := ret.Get(1).(func(string, json.RawMessage) error); ok { r1 = rf(engine, cfg) } else { @@ -43,13 +45,16 @@ func (_m *ExperimentsService) IsClientSelectionEnabled(engine string) (bool, err ret := _m.Called(engine) var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { + return rf(engine) + } if rf, ok := ret.Get(0).(func(string) bool); ok { r0 = rf(engine) } else { r0 = ret.Get(0).(bool) } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(engine) } else { @@ -78,6 +83,10 @@ func (_m *ExperimentsService) ListClients(engine string) ([]manager.Client, erro ret := _m.Called(engine) var r0 []manager.Client + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]manager.Client, error)); ok { + return rf(engine) + } if rf, ok := ret.Get(0).(func(string) []manager.Client); ok { r0 = rf(engine) } else { @@ -86,7 +95,6 @@ func (_m *ExperimentsService) ListClients(engine string) ([]manager.Client, erro } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(engine) } else { @@ -117,6 +125,10 @@ func (_m *ExperimentsService) ListExperiments(engine string, clientID string) ([ ret := _m.Called(engine, clientID) var r0 []manager.Experiment + var r1 error + if rf, ok := ret.Get(0).(func(string, string) ([]manager.Experiment, error)); ok { + return rf(engine, clientID) + } if rf, ok := ret.Get(0).(func(string, string) []manager.Experiment); ok { r0 = rf(engine, clientID) } else { @@ -125,7 +137,6 @@ func (_m *ExperimentsService) ListExperiments(engine string, clientID string) ([ } } - var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(engine, clientID) } else { @@ -140,13 +151,16 @@ func (_m *ExperimentsService) ListVariables(engine string, clientID string, expe ret := _m.Called(engine, clientID, experimentIDs) var r0 manager.Variables + var r1 error + if rf, ok := ret.Get(0).(func(string, string, []string) (manager.Variables, error)); ok { + return rf(engine, clientID, experimentIDs) + } if rf, ok := ret.Get(0).(func(string, string, []string) manager.Variables); ok { r0 = rf(engine, clientID, experimentIDs) } else { r0 = ret.Get(0).(manager.Variables) } - var r1 error if rf, ok := ret.Get(1).(func(string, string, []string) error); ok { r1 = rf(engine, clientID, experimentIDs) } else { @@ -169,3 +183,18 @@ func (_m *ExperimentsService) ValidateExperimentConfig(engine string, cfg json.R return r0 } + +type mockConstructorTestingTNewExperimentsService interface { + mock.TestingT + Cleanup(func()) +} + +// NewExperimentsService creates a new instance of ExperimentsService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewExperimentsService(t mockConstructorTestingTNewExperimentsService) *ExperimentsService { + mock := &ExperimentsService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/mlp_service.go b/api/turing/service/mocks/mlp_service.go index 28aa6f9ac..2c37de184 100644 --- a/api/turing/service/mocks/mlp_service.go +++ b/api/turing/service/mocks/mlp_service.go @@ -1,10 +1,9 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( client "github.com/caraml-dev/merlin/client" - apiclient "github.com/caraml-dev/mlp/api/client" mock "github.com/stretchr/testify/mock" @@ -22,6 +21,10 @@ func (_m *MLPService) GetEnvironment(name string) (*client.Environment, error) { ret := _m.Called(name) var r0 *client.Environment + var r1 error + if rf, ok := ret.Get(0).(func(string) (*client.Environment, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) *client.Environment); ok { r0 = rf(name) } else { @@ -30,7 +33,6 @@ func (_m *MLPService) GetEnvironment(name string) (*client.Environment, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -45,6 +47,10 @@ func (_m *MLPService) GetEnvironments() ([]client.Environment, error) { ret := _m.Called() var r0 []client.Environment + var r1 error + if rf, ok := ret.Get(0).(func() ([]client.Environment, error)); ok { + return rf() + } if rf, ok := ret.Get(0).(func() []client.Environment); ok { r0 = rf() } else { @@ -53,7 +59,6 @@ func (_m *MLPService) GetEnvironments() ([]client.Environment, error) { } } - var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { @@ -68,6 +73,10 @@ func (_m *MLPService) GetProject(id models.ID) (*apiclient.Project, error) { ret := _m.Called(id) var r0 *apiclient.Project + var r1 error + if rf, ok := ret.Get(0).(func(models.ID) (*apiclient.Project, error)); ok { + return rf(id) + } if rf, ok := ret.Get(0).(func(models.ID) *apiclient.Project); ok { r0 = rf(id) } else { @@ -76,7 +85,6 @@ func (_m *MLPService) GetProject(id models.ID) (*apiclient.Project, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID) error); ok { r1 = rf(id) } else { @@ -91,6 +99,10 @@ func (_m *MLPService) GetProjects(name string) ([]apiclient.Project, error) { ret := _m.Called(name) var r0 []apiclient.Project + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]apiclient.Project, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []apiclient.Project); ok { r0 = rf(name) } else { @@ -99,7 +111,6 @@ func (_m *MLPService) GetProjects(name string) ([]apiclient.Project, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -114,13 +125,16 @@ func (_m *MLPService) GetSecret(projectID models.ID, name string) (string, error ret := _m.Called(projectID, name) var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, string) (string, error)); ok { + return rf(projectID, name) + } if rf, ok := ret.Get(0).(func(models.ID, string) string); ok { r0 = rf(projectID, name) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(models.ID, string) error); ok { r1 = rf(projectID, name) } else { @@ -129,3 +143,18 @@ func (_m *MLPService) GetSecret(projectID models.ID, name string) (string, error return r0, r1 } + +type mockConstructorTestingTNewMLPService interface { + mock.TestingT + Cleanup(func()) +} + +// NewMLPService creates a new instance of MLPService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMLPService(t mockConstructorTestingTNewMLPService) *MLPService { + mock := &MLPService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/pod_log_service.go b/api/turing/service/mocks/pod_log_service.go index c710f2274..4739b8336 100644 --- a/api/turing/service/mocks/pod_log_service.go +++ b/api/turing/service/mocks/pod_log_service.go @@ -1,11 +1,10 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - service "github.com/caraml-dev/turing/api/turing/service" + mock "github.com/stretchr/testify/mock" ) // PodLogService is an autogenerated mock type for the PodLogService type @@ -18,6 +17,10 @@ func (_m *PodLogService) ListPodLogs(request service.PodLogRequest) ([]*service. ret := _m.Called(request) var r0 []*service.PodLog + var r1 error + if rf, ok := ret.Get(0).(func(service.PodLogRequest) ([]*service.PodLog, error)); ok { + return rf(request) + } if rf, ok := ret.Get(0).(func(service.PodLogRequest) []*service.PodLog); ok { r0 = rf(request) } else { @@ -26,7 +29,6 @@ func (_m *PodLogService) ListPodLogs(request service.PodLogRequest) ([]*service. } } - var r1 error if rf, ok := ret.Get(1).(func(service.PodLogRequest) error); ok { r1 = rf(request) } else { @@ -35,3 +37,18 @@ func (_m *PodLogService) ListPodLogs(request service.PodLogRequest) ([]*service. return r0, r1 } + +type mockConstructorTestingTNewPodLogService interface { + mock.TestingT + Cleanup(func()) +} + +// NewPodLogService creates a new instance of PodLogService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewPodLogService(t mockConstructorTestingTNewPodLogService) *PodLogService { + mock := &PodLogService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/router_monitoring_service.go b/api/turing/service/mocks/router_monitoring_service.go new file mode 100644 index 000000000..b92be5032 --- /dev/null +++ b/api/turing/service/mocks/router_monitoring_service.go @@ -0,0 +1,52 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + models "github.com/caraml-dev/turing/api/turing/models" + mock "github.com/stretchr/testify/mock" +) + +// RouterMonitoringService is an autogenerated mock type for the RouterMonitoringService type +type RouterMonitoringService struct { + mock.Mock +} + +// GenerateMonitoringURL provides a mock function with given fields: projectID, environmentName, routerName, routerVersion +func (_m *RouterMonitoringService) GenerateMonitoringURL(projectID models.ID, environmentName string, routerName string, routerVersion *uint) (string, error) { + ret := _m.Called(projectID, environmentName, routerName, routerVersion) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, string, string, *uint) (string, error)); ok { + return rf(projectID, environmentName, routerName, routerVersion) + } + if rf, ok := ret.Get(0).(func(models.ID, string, string, *uint) string); ok { + r0 = rf(projectID, environmentName, routerName, routerVersion) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(models.ID, string, string, *uint) error); ok { + r1 = rf(projectID, environmentName, routerName, routerVersion) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewRouterMonitoringService interface { + mock.TestingT + Cleanup(func()) +} + +// NewRouterMonitoringService creates a new instance of RouterMonitoringService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRouterMonitoringService(t mockConstructorTestingTNewRouterMonitoringService) *RouterMonitoringService { + mock := &RouterMonitoringService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/router_versions_service.go b/api/turing/service/mocks/router_versions_service.go index 8774d5a9a..0c4487d43 100644 --- a/api/turing/service/mocks/router_versions_service.go +++ b/api/turing/service/mocks/router_versions_service.go @@ -1,12 +1,11 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - models "github.com/caraml-dev/turing/api/turing/models" service "github.com/caraml-dev/turing/api/turing/service" + mock "github.com/stretchr/testify/mock" ) // RouterVersionsService is an autogenerated mock type for the RouterVersionsService type diff --git a/api/turing/service/mocks/routers_service.go b/api/turing/service/mocks/routers_service.go index 8c1cd4720..ad8fb6c73 100644 --- a/api/turing/service/mocks/routers_service.go +++ b/api/turing/service/mocks/routers_service.go @@ -1,11 +1,10 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - models "github.com/caraml-dev/turing/api/turing/models" + mock "github.com/stretchr/testify/mock" ) // RoutersService is an autogenerated mock type for the RoutersService type @@ -32,6 +31,10 @@ func (_m *RoutersService) FindByID(routerID models.ID) (*models.Router, error) { ret := _m.Called(routerID) var r0 *models.Router + var r1 error + if rf, ok := ret.Get(0).(func(models.ID) (*models.Router, error)); ok { + return rf(routerID) + } if rf, ok := ret.Get(0).(func(models.ID) *models.Router); ok { r0 = rf(routerID) } else { @@ -40,7 +43,6 @@ func (_m *RoutersService) FindByID(routerID models.ID) (*models.Router, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID) error); ok { r1 = rf(routerID) } else { @@ -55,6 +57,10 @@ func (_m *RoutersService) FindByProjectAndEnvironmentAndName(projectID models.ID ret := _m.Called(projectID, environmentName, routerName) var r0 *models.Router + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, string, string) (*models.Router, error)); ok { + return rf(projectID, environmentName, routerName) + } if rf, ok := ret.Get(0).(func(models.ID, string, string) *models.Router); ok { r0 = rf(projectID, environmentName, routerName) } else { @@ -63,7 +69,6 @@ func (_m *RoutersService) FindByProjectAndEnvironmentAndName(projectID models.ID } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID, string, string) error); ok { r1 = rf(projectID, environmentName, routerName) } else { @@ -78,6 +83,10 @@ func (_m *RoutersService) FindByProjectAndName(projectID models.ID, routerName s ret := _m.Called(projectID, routerName) var r0 *models.Router + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, string) (*models.Router, error)); ok { + return rf(projectID, routerName) + } if rf, ok := ret.Get(0).(func(models.ID, string) *models.Router); ok { r0 = rf(projectID, routerName) } else { @@ -86,7 +95,6 @@ func (_m *RoutersService) FindByProjectAndName(projectID models.ID, routerName s } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID, string) error); ok { r1 = rf(projectID, routerName) } else { @@ -101,6 +109,10 @@ func (_m *RoutersService) ListRouters(projectID models.ID, environmentName strin ret := _m.Called(projectID, environmentName) var r0 []*models.Router + var r1 error + if rf, ok := ret.Get(0).(func(models.ID, string) ([]*models.Router, error)); ok { + return rf(projectID, environmentName) + } if rf, ok := ret.Get(0).(func(models.ID, string) []*models.Router); ok { r0 = rf(projectID, environmentName) } else { @@ -109,7 +121,6 @@ func (_m *RoutersService) ListRouters(projectID models.ID, environmentName strin } } - var r1 error if rf, ok := ret.Get(1).(func(models.ID, string) error); ok { r1 = rf(projectID, environmentName) } else { @@ -124,6 +135,10 @@ func (_m *RoutersService) Save(router *models.Router) (*models.Router, error) { ret := _m.Called(router) var r0 *models.Router + var r1 error + if rf, ok := ret.Get(0).(func(*models.Router) (*models.Router, error)); ok { + return rf(router) + } if rf, ok := ret.Get(0).(func(*models.Router) *models.Router); ok { r0 = rf(router) } else { @@ -132,7 +147,6 @@ func (_m *RoutersService) Save(router *models.Router) (*models.Router, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(*models.Router) error); ok { r1 = rf(router) } else { @@ -141,3 +155,18 @@ func (_m *RoutersService) Save(router *models.Router) (*models.Router, error) { return r0, r1 } + +type mockConstructorTestingTNewRoutersService interface { + mock.TestingT + Cleanup(func()) +} + +// NewRoutersService creates a new instance of RoutersService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRoutersService(t mockConstructorTestingTNewRoutersService) *RoutersService { + mock := &RoutersService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/api/turing/service/mocks/u_func.go b/api/turing/service/mocks/u_func.go new file mode 100644 index 000000000..af6ae61fb --- /dev/null +++ b/api/turing/service/mocks/u_func.go @@ -0,0 +1,37 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + service "github.com/caraml-dev/turing/api/turing/service" + mock "github.com/stretchr/testify/mock" + + sync "sync" +) + +// uFunc is an autogenerated mock type for the uFunc type +type uFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *uFunc) Execute(_a0 context.Context, _a1 interface{}, _a2 *sync.WaitGroup, _a3 chan<- error, _a4 *service.EventChannel) { + _m.Called(_a0, _a1, _a2, _a3, _a4) +} + +type mockConstructorTestingTnewUFunc interface { + mock.TestingT + Cleanup(func()) +} + +// newUFunc creates a new instance of uFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func newUFunc(t mockConstructorTestingTnewUFunc) *uFunc { + mock := &uFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engines/pyfunc-ensembler-job/requirements.dev.txt b/engines/pyfunc-ensembler-job/requirements.dev.txt index 0cb035580..101a896ef 100644 --- a/engines/pyfunc-ensembler-job/requirements.dev.txt +++ b/engines/pyfunc-ensembler-job/requirements.dev.txt @@ -1,6 +1,7 @@ black==22.6.0 chispa mypy>=0.910 -pytest +# The next release 8.2.0 of pytest breaks the unit tests +pytest<=8.1.2 pytest-cov -pylint \ No newline at end of file +pylint diff --git a/engines/pyfunc-ensembler-service/requirements.dev.txt b/engines/pyfunc-ensembler-service/requirements.dev.txt index dffdea880..54192dc07 100644 --- a/engines/pyfunc-ensembler-service/requirements.dev.txt +++ b/engines/pyfunc-ensembler-service/requirements.dev.txt @@ -2,4 +2,4 @@ black==22.6.0 # The next release 8.2.0 of pytest breaks the unit tests pytest<=8.1.2 pytest-cov -pylint \ No newline at end of file +pylint diff --git a/sdk/Makefile b/sdk/Makefile index de9325971..c184c02ed 100644 --- a/sdk/Makefile +++ b/sdk/Makefile @@ -30,7 +30,7 @@ build: version .PHONY: test test: - @python -m pytest --cov=turing tests/ + @python -m pytest --cov=turing --cov-report xml --cov-report= tests/ e2e-sdk: @python -m pytest -s -v e2e diff --git a/sdk/requirements.dev.txt b/sdk/requirements.dev.txt index ad7cbd6af..0a1078de0 100644 --- a/sdk/requirements.dev.txt +++ b/sdk/requirements.dev.txt @@ -2,8 +2,8 @@ black==22.6.0 setuptools>=21.0.0 wheel twine -# The next release 8.0.0 of pytest breaks the unit tests -pytest<=7.4.4 +# The next release 8.2.0 of pytest breaks the unit tests +pytest<=8.1.2 pytest-cov urllib3-mock>=0.3.3 caraml-upi-protos diff --git a/sdk/requirements.txt b/sdk/requirements.txt index 0e228746e..e94845513 100644 --- a/sdk/requirements.txt +++ b/sdk/requirements.txt @@ -11,4 +11,5 @@ protobuf>=3.12.0,<5.0.0 # Determined by the mlflow dependency python_dateutil>=2.5.3 requests urllib3>=1.25.3 -caraml-auth-google==0.0.0.post7 \ No newline at end of file +caraml-auth-google==0.0.0.post7 +PyPrind>=2.11.2 diff --git a/sdk/tests/conftest.py b/sdk/tests/conftest.py index 08ebe6e1f..5899a7114 100644 --- a/sdk/tests/conftest.py +++ b/sdk/tests/conftest.py @@ -1,33 +1,34 @@ import json -from datetime import datetime, timedelta -import pytest import random -from sys import version_info import uuid +from datetime import datetime, timedelta +from sys import version_info + +import pytest import tests -from turing.ensembler import PyFuncEnsembler -import turing.generated.models import turing.batch.config -import turing.batch.config.source import turing.batch.config.sink -from turing.router.config.route import Route -from turing.router.config.router_config import RouterConfig, Protocol +import turing.batch.config.source +import turing.generated.models +from tests.fixtures.gcs import mock_gcs +from tests.fixtures.mlflow import mock_mlflow +from turing.ensembler import PyFuncEnsembler from turing.router.config.autoscaling_policy import AutoscalingPolicy -from turing.router.config.traffic_rule import DefaultTrafficRule -from turing.router.config.resource_request import ResourceRequest -from turing.router.config.log_config import LogConfig, ResultLoggerType +from turing.router.config.common.env_var import EnvVar from turing.router.config.enricher import Enricher +from turing.router.config.experiment_config import ExperimentConfig +from turing.router.config.log_config import LogConfig, ResultLoggerType +from turing.router.config.resource_request import ResourceRequest +from turing.router.config.route import Route +from turing.router.config.router_config import Protocol, RouterConfig from turing.router.config.router_ensembler_config import ( + DockerRouterEnsemblerConfig, EnsemblerNopConfig, EnsemblerStandardConfig, - DockerRouterEnsemblerConfig, PyfuncRouterEnsemblerConfig, ) -from turing.router.config.common.env_var import EnvVar -from turing.router.config.experiment_config import ExperimentConfig -from tests.fixtures.mlflow import mock_mlflow -from tests.fixtures.gcs import mock_gcs +from turing.router.config.traffic_rule import DefaultTrafficRule @pytest.fixture diff --git a/sdk/tests/ensembler_image_test.py b/sdk/tests/ensembler_image_test.py new file mode 100644 index 000000000..21f46f023 --- /dev/null +++ b/sdk/tests/ensembler_image_test.py @@ -0,0 +1,142 @@ +import json +import logging + +import pytest +from urllib3_mock import Responses + +import tests +import turing.ensembler +import turing.generated +import turing.generated.models +from turing.ensembler_image import EnsemblerImage + +responses = Responses("requests.packages.urllib3") + + +@pytest.fixture(scope="module", name="responses") +def _responses(): + return responses + + +@responses.activate +@pytest.mark.parametrize("ensembler_name", ["ensembler_1"]) +def test_list_images(turing_api, use_google_oauth, active_project, pyfunc_ensembler): + turing.set_url(turing_api, use_google_oauth) + turing.set_project(active_project.name) + + ensembler_id = 1 + + responses.add( + method="GET", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}", + body=json.dumps(pyfunc_ensembler, default=tests.json_serializer), + status=200, + content_type="application/json", + ) + + ensembler = turing.PyFuncEnsembler.get_by_id(ensembler_id) + + job_image = turing.generated.models.EnsemblerImage( + project_id=active_project.id, + ensembler_id=ensembler_id, + runner_type=turing.generated.models.EnsemblerImageRunnerType("job"), + image_ref=f"ghcr.io/caraml-dev/turing/ensembler-jobs/{active_project.name}-ensembler-1:latest", + exists=True, + image_building_job_status=turing.generated.models.ImageBuildingJobStatus( + state=turing.generated.models.ImageBuildingJobState("succeeded"), + ), + ) + + service_image = turing.generated.models.EnsemblerImage( + project_id=active_project.id, + ensembler_id=ensembler_id, + runner_type=turing.generated.models.EnsemblerImageRunnerType("service"), + image_ref=f"ghcr.io/caraml-dev/turing/ensembler-services/{active_project.name}-ensembler-1:latest", + exists=True, + image_building_job_status=turing.generated.models.ImageBuildingJobStatus( + state=turing.generated.models.ImageBuildingJobState("succeeded"), + ), + ) + + responses.add( + method="GET", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}/images?runner_type=job", + match_querystring=True, + body=json.dumps([job_image], default=tests.json_serializer), + status=200, + content_type="application/json", + ) + + images = turing.EnsemblerImage.list(ensembler=ensembler, runner_type="job") + assert len(images) == 1 + assert EnsemblerImage.from_open_api(job_image) in images + assert EnsemblerImage.from_open_api(service_image) not in images + + responses.add( + method="GET", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}/images", + body=json.dumps([job_image, service_image], default=tests.json_serializer), + status=200, + content_type="application/json", + ) + + images = turing.EnsemblerImage.list(ensembler=ensembler) + assert len(images) == 2 + assert EnsemblerImage.from_open_api(job_image) in images + assert EnsemblerImage.from_open_api(service_image) in images + + +@responses.activate +@pytest.mark.parametrize("ensembler_name", ["ensembler_1"]) +def test_create_image(turing_api, use_google_oauth, active_project, pyfunc_ensembler): + turing.set_url(turing_api, use_google_oauth) + turing.set_project(active_project.name) + + ensembler_id = 1 + + responses.add( + method="GET", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}", + body=json.dumps(pyfunc_ensembler, default=tests.json_serializer), + status=200, + content_type="application/json", + ) + + ensembler = turing.PyFuncEnsembler.get_by_id(ensembler_id) + + responses.add( + method="PUT", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}/images", + body=json.dumps(pyfunc_ensembler, default=tests.json_serializer), + status=201, + content_type="application/json", + ) + + job_image = turing.generated.models.EnsemblerImage( + project_id=active_project.id, + ensembler_id=ensembler_id, + runner_type=turing.generated.models.EnsemblerImageRunnerType("job"), + image_ref=f"ghcr.io/caraml-dev/turing/ensembler-jobs/{active_project.name}-ensembler-1:latest", + exists=True, + image_building_job_status=turing.generated.models.ImageBuildingJobStatus( + state=turing.generated.models.ImageBuildingJobState("succeeded"), + ), + ) + + responses.add( + method="GET", + url=f"/v1/projects/{active_project.id}/ensemblers/{ensembler_id}/images?runner_type=job", + match_querystring=True, + body=json.dumps([job_image], default=tests.json_serializer), + status=200, + content_type="application/json", + ) + + turing.EnsemblerImage.create( + ensembler=ensembler, + runner_type=turing.generated.models.EnsemblerImageRunnerType("job"), + ) + + images = turing.EnsemblerImage.list(ensembler=ensembler, runner_type="job") + assert len(images) == 1 + assert EnsemblerImage.from_open_api(job_image) in images diff --git a/sdk/tests/ensembler_test.py b/sdk/tests/ensembler_test.py index 78843ddd6..a824be575 100644 --- a/sdk/tests/ensembler_test.py +++ b/sdk/tests/ensembler_test.py @@ -1,12 +1,13 @@ import json import os.path import random + import pandas import pytest -import re +from urllib3_mock import Responses + import tests import turing.ensembler -from urllib3_mock import Responses import turing.generated.models responses = Responses("requests.packages.urllib3") @@ -54,12 +55,6 @@ def test_predict(): def test_list_ensemblers( turing_api, active_project, generic_ensemblers, use_google_oauth ): - with pytest.raises( - Exception, - match=re.escape("Active project isn't set, use set_project(...) to set it"), - ): - turing.PyFuncEnsembler.list() - turing.set_url(turing_api, use_google_oauth) turing.set_project(active_project.name) @@ -307,7 +302,6 @@ def test_update_ensembler_existing_job( def test_delete_ensembler( turing_api, active_project, use_google_oauth, actual, expected ): - turing.set_url(turing_api, use_google_oauth) turing.set_project(active_project.name) diff --git a/sdk/turing/__init__.py b/sdk/turing/__init__.py index b1e6979fc..51e49d2e5 100644 --- a/sdk/turing/__init__.py +++ b/sdk/turing/__init__.py @@ -1,4 +1,5 @@ from turing.ensembler import Ensembler, EnsemblerType, PyFuncEnsembler +from turing.ensembler_image import EnsemblerImage from turing.project import Project from turing.router.router import Router from turing.session import TuringSession diff --git a/sdk/turing/ensembler_image.py b/sdk/turing/ensembler_image.py new file mode 100644 index 000000000..13620f9ad --- /dev/null +++ b/sdk/turing/ensembler_image.py @@ -0,0 +1,89 @@ +import logging +from typing import List + +import turing +import turing.generated.models +from turing._base_types import ApiObject, ApiObjectSpec + + +@ApiObjectSpec(turing.generated.models.EnsemblerImage) +class EnsemblerImage(ApiObject): + """ + API entity for Ensembler Image + """ + + def __init__( + self, + project_id: int, + ensembler_id: int, + runner_type: turing.generated.models.EnsemblerImageRunnerType, + image_ref: str, + exists: bool, + image_building_job_status: turing.generated.models.ImageBuildingJobStatus, + **kwargs, + ): + super(EnsemblerImage, self).__init__(**kwargs) + self._project_id = project_id + self._ensembler_id = ensembler_id + self._runner_type = runner_type + self._image_ref = image_ref + self._exists = exists + self._image_building_job_status = image_building_job_status + + @property + def project_id(self) -> int: + return self._project_id + + @property + def ensembler_id(self) -> int: + return self._ensembler_id + + @property + def runner_type(self) -> turing.generated.models.EnsemblerImageRunnerType: + return self._runner_type + + @property + def image_ref(self) -> str: + return self._image_ref + + @property + def exists(self) -> bool: + return self._exists + + @property + def image_building_job_status( + self, + ) -> turing.generated.models.ImageBuildingJobStatus: + return self._image_building_job_status + + def list( + ensembler: turing.generated.models.Ensembler, + runner_type: turing.generated.models.EnsemblerImageRunnerType = None, + ) -> List["EnsemblerImage"]: + """ + List all Docker images for the ensembler + + :param ensembler: Ensembler object + :param runner_type: (optional) Runner type of image building job used to filter the images. (default: None, options: [None, 'job', 'service']) + + :return: List of EnsemblerImage objects + """ + response = turing.active_session.list_ensembler_images( + ensembler=ensembler, runner_type=runner_type + ) + return [EnsemblerImage.from_open_api(item) for item in response.value] + + def create( + ensembler: turing.generated.models.Ensembler, + runner_type: turing.generated.models.EnsemblerImageRunnerType, + ) -> "EnsemblerImage": + """ + Create a new Docker image for the ensembler + + :param ensembler: Ensembler object + :param runner_type: Runner type of image building job (options: ['job', 'service']) + """ + image = turing.active_session.create_ensembler_image( + ensembler=ensembler, runner_type=runner_type + ) + return EnsemblerImage.from_open_api(image) diff --git a/sdk/turing/generated/api/ensembler_images_api.py b/sdk/turing/generated/api/ensembler_images_api.py new file mode 100644 index 000000000..e574e74fc --- /dev/null +++ b/sdk/turing/generated/api/ensembler_images_api.py @@ -0,0 +1,306 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.api_client import ApiClient, Endpoint as _Endpoint +from turing.generated.model_utils import ( # noqa: F401 + check_allowed_values, + check_validations, + date, + datetime, + file_type, + none_type, + validate_and_convert_types +) +from turing.generated.model.build_ensembler_image_request import BuildEnsemblerImageRequest +from turing.generated.model.ensembler_image_runner_type import EnsemblerImageRunnerType +from turing.generated.model.ensembler_images import EnsemblerImages + + +class EnsemblerImagesApi(object): + """NOTE: This class is auto generated by OpenAPI Generator + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + def __init__(self, api_client=None): + if api_client is None: + api_client = ApiClient() + self.api_client = api_client + + def __create_ensembler_image( + self, + project_id, + ensembler_id, + build_ensembler_image_request, + **kwargs + ): + """Creates a new ensembler image # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async_req=True + + >>> thread = api.create_ensembler_image(project_id, ensembler_id, build_ensembler_image_request, async_req=True) + >>> result = thread.get() + + Args: + project_id (int): + ensembler_id (int): + build_ensembler_image_request (BuildEnsemblerImageRequest): A JSON object containing information about the ensembler + + Keyword Args: + _return_http_data_only (bool): response data without head status + code and headers. Default is True. + _preload_content (bool): if False, the urllib3.HTTPResponse object + will be returned without reading/decoding response data. + Default is True. + _request_timeout (float/tuple): timeout setting for this request. If one + number provided, it will be total request timeout. It can also + be a pair (tuple) of (connection, read) timeouts. + Default is None. + _check_input_type (bool): specifies if type checking + should be done one the data sent to the server. + Default is True. + _check_return_type (bool): specifies if type checking + should be done one the data received from the server. + Default is True. + _host_index (int/None): specifies the index of the server + that we want to use. + Default is read from the configuration. + async_req (bool): execute request asynchronously + + Returns: + None + If the method is called asynchronously, returns the request + thread. + """ + kwargs['async_req'] = kwargs.get( + 'async_req', False + ) + kwargs['_return_http_data_only'] = kwargs.get( + '_return_http_data_only', True + ) + kwargs['_preload_content'] = kwargs.get( + '_preload_content', True + ) + kwargs['_request_timeout'] = kwargs.get( + '_request_timeout', None + ) + kwargs['_check_input_type'] = kwargs.get( + '_check_input_type', True + ) + kwargs['_check_return_type'] = kwargs.get( + '_check_return_type', True + ) + kwargs['_host_index'] = kwargs.get('_host_index') + kwargs['project_id'] = \ + project_id + kwargs['ensembler_id'] = \ + ensembler_id + kwargs['build_ensembler_image_request'] = \ + build_ensembler_image_request + return self.call_with_http_info(**kwargs) + + self.create_ensembler_image = _Endpoint( + settings={ + 'response_type': None, + 'auth': [], + 'endpoint_path': '/projects/{project_id}/ensemblers/{ensembler_id}/images', + 'operation_id': 'create_ensembler_image', + 'http_method': 'PUT', + 'servers': None, + }, + params_map={ + 'all': [ + 'project_id', + 'ensembler_id', + 'build_ensembler_image_request', + ], + 'required': [ + 'project_id', + 'ensembler_id', + 'build_ensembler_image_request', + ], + 'nullable': [ + ], + 'enum': [ + ], + 'validation': [ + ] + }, + root_map={ + 'validations': { + }, + 'allowed_values': { + }, + 'openapi_types': { + 'project_id': + (int,), + 'ensembler_id': + (int,), + 'build_ensembler_image_request': + (BuildEnsemblerImageRequest,), + }, + 'attribute_map': { + 'project_id': 'project_id', + 'ensembler_id': 'ensembler_id', + }, + 'location_map': { + 'project_id': 'path', + 'ensembler_id': 'path', + 'build_ensembler_image_request': 'body', + }, + 'collection_format_map': { + } + }, + headers_map={ + 'accept': [], + 'content_type': [ + 'application/json' + ] + }, + api_client=api_client, + callable=__create_ensembler_image + ) + + def __list_ensembler_images( + self, + project_id, + ensembler_id, + **kwargs + ): + """Returns a list of ensembler images that belong to the ensembler # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async_req=True + + >>> thread = api.list_ensembler_images(project_id, ensembler_id, async_req=True) + >>> result = thread.get() + + Args: + project_id (int): + ensembler_id (int): + + Keyword Args: + runner_type (EnsemblerImageRunnerType): [optional] + _return_http_data_only (bool): response data without head status + code and headers. Default is True. + _preload_content (bool): if False, the urllib3.HTTPResponse object + will be returned without reading/decoding response data. + Default is True. + _request_timeout (float/tuple): timeout setting for this request. If one + number provided, it will be total request timeout. It can also + be a pair (tuple) of (connection, read) timeouts. + Default is None. + _check_input_type (bool): specifies if type checking + should be done one the data sent to the server. + Default is True. + _check_return_type (bool): specifies if type checking + should be done one the data received from the server. + Default is True. + _host_index (int/None): specifies the index of the server + that we want to use. + Default is read from the configuration. + async_req (bool): execute request asynchronously + + Returns: + EnsemblerImages + If the method is called asynchronously, returns the request + thread. + """ + kwargs['async_req'] = kwargs.get( + 'async_req', False + ) + kwargs['_return_http_data_only'] = kwargs.get( + '_return_http_data_only', True + ) + kwargs['_preload_content'] = kwargs.get( + '_preload_content', True + ) + kwargs['_request_timeout'] = kwargs.get( + '_request_timeout', None + ) + kwargs['_check_input_type'] = kwargs.get( + '_check_input_type', True + ) + kwargs['_check_return_type'] = kwargs.get( + '_check_return_type', True + ) + kwargs['_host_index'] = kwargs.get('_host_index') + kwargs['project_id'] = \ + project_id + kwargs['ensembler_id'] = \ + ensembler_id + return self.call_with_http_info(**kwargs) + + self.list_ensembler_images = _Endpoint( + settings={ + 'response_type': (EnsemblerImages,), + 'auth': [], + 'endpoint_path': '/projects/{project_id}/ensemblers/{ensembler_id}/images', + 'operation_id': 'list_ensembler_images', + 'http_method': 'GET', + 'servers': None, + }, + params_map={ + 'all': [ + 'project_id', + 'ensembler_id', + 'runner_type', + ], + 'required': [ + 'project_id', + 'ensembler_id', + ], + 'nullable': [ + ], + 'enum': [ + ], + 'validation': [ + ] + }, + root_map={ + 'validations': { + }, + 'allowed_values': { + }, + 'openapi_types': { + 'project_id': + (int,), + 'ensembler_id': + (int,), + 'runner_type': + (EnsemblerImageRunnerType,), + }, + 'attribute_map': { + 'project_id': 'project_id', + 'ensembler_id': 'ensembler_id', + 'runner_type': 'runner_type', + }, + 'location_map': { + 'project_id': 'path', + 'ensembler_id': 'path', + 'runner_type': 'query', + }, + 'collection_format_map': { + } + }, + headers_map={ + 'accept': [ + 'application/json' + ], + 'content_type': [], + }, + api_client=api_client, + callable=__list_ensembler_images + ) diff --git a/sdk/turing/generated/apis/__init__.py b/sdk/turing/generated/apis/__init__.py index 5c5cdd7e2..3f6519b11 100644 --- a/sdk/turing/generated/apis/__init__.py +++ b/sdk/turing/generated/apis/__init__.py @@ -15,6 +15,7 @@ # Import APIs into API package: from turing.generated.api.ensembler_api import EnsemblerApi +from turing.generated.api.ensembler_images_api import EnsemblerImagesApi from turing.generated.api.ensembling_job_api import EnsemblingJobApi from turing.generated.api.project_api import ProjectApi from turing.generated.api.router_api import RouterApi diff --git a/sdk/turing/generated/model/build_ensembler_image_request.py b/sdk/turing/generated/model/build_ensembler_image_request.py new file mode 100644 index 000000000..1ec747595 --- /dev/null +++ b/sdk/turing/generated/model/build_ensembler_image_request.py @@ -0,0 +1,174 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + +def lazy_import(): + from turing.generated.model.ensembler_image_runner_type import EnsemblerImageRunnerType + globals()['EnsemblerImageRunnerType'] = EnsemblerImageRunnerType + + +class BuildEnsemblerImageRequest(ModelNormal): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + lazy_import() + return { + 'runner_type': (EnsemblerImageRunnerType,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'runner_type': 'runner_type', # noqa: E501 + } + + _composed_schemas = {} + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, runner_type, *args, **kwargs): # noqa: E501 + """BuildEnsemblerImageRequest - a model defined in OpenAPI + + Args: + runner_type (EnsemblerImageRunnerType): + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + self.runner_type = runner_type + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) diff --git a/sdk/turing/generated/model/ensembler_image.py b/sdk/turing/generated/model/ensembler_image.py new file mode 100644 index 000000000..2bc248dc1 --- /dev/null +++ b/sdk/turing/generated/model/ensembler_image.py @@ -0,0 +1,188 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + +def lazy_import(): + from turing.generated.model.ensembler_image_runner_type import EnsemblerImageRunnerType + from turing.generated.model.image_building_job_status import ImageBuildingJobStatus + globals()['EnsemblerImageRunnerType'] = EnsemblerImageRunnerType + globals()['ImageBuildingJobStatus'] = ImageBuildingJobStatus + + +class EnsemblerImage(ModelNormal): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + lazy_import() + return { + 'project_id': (int,), # noqa: E501 + 'ensembler_id': (int,), # noqa: E501 + 'runner_type': (EnsemblerImageRunnerType,), # noqa: E501 + 'image_ref': (str,), # noqa: E501 + 'exists': (bool,), # noqa: E501 + 'image_building_job_status': (ImageBuildingJobStatus,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'project_id': 'project_id', # noqa: E501 + 'ensembler_id': 'ensembler_id', # noqa: E501 + 'runner_type': 'runner_type', # noqa: E501 + 'image_ref': 'image_ref', # noqa: E501 + 'exists': 'exists', # noqa: E501 + 'image_building_job_status': 'image_building_job_status', # noqa: E501 + } + + _composed_schemas = {} + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): # noqa: E501 + """EnsemblerImage - a model defined in OpenAPI + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + project_id (int): [optional] # noqa: E501 + ensembler_id (int): [optional] # noqa: E501 + runner_type (EnsemblerImageRunnerType): [optional] # noqa: E501 + image_ref (str): [optional] # noqa: E501 + exists (bool): [optional] # noqa: E501 + image_building_job_status (ImageBuildingJobStatus): [optional] # noqa: E501 + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) diff --git a/sdk/turing/generated/model/ensembler_image_runner_type.py b/sdk/turing/generated/model/ensembler_image_runner_type.py new file mode 100644 index 000000000..dc0dc5bbd --- /dev/null +++ b/sdk/turing/generated/model/ensembler_image_runner_type.py @@ -0,0 +1,184 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + + +class EnsemblerImageRunnerType(ModelSimple): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + ('value',): { + 'None': None, + 'JOB': "job", + 'SERVICE': "service", + }, + } + + validations = { + } + + additional_properties_type = None + + _nullable = True + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + return { + 'value': (str,), + } + + @cached_property + def discriminator(): + return None + + + attribute_map = {} + + _composed_schemas = None + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): + """EnsemblerImageRunnerType - a model defined in OpenAPI + + Note that value can be passed either in args or in kwargs, but not in both. + + Args: + args[0] (str):, must be one of ["job", "service", ] # noqa: E501 + + Keyword Args: + value (str):, must be one of ["job", "service", ] # noqa: E501 + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + # required up here when default value is not given + _path_to_item = kwargs.pop('_path_to_item', ()) + + if 'value' in kwargs: + value = kwargs.pop('value') + elif args: + args = list(args) + value = args.pop(0) + else: + raise ApiTypeError( + "value is required, but not passed in args or kwargs and doesn't have default", + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + self.value = value + if kwargs: + raise ApiTypeError( + "Invalid named arguments=%s passed to %s. Remove those invalid named arguments." % ( + kwargs, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) diff --git a/sdk/turing/generated/model/ensembler_images.py b/sdk/turing/generated/model/ensembler_images.py new file mode 100644 index 000000000..160de1773 --- /dev/null +++ b/sdk/turing/generated/model/ensembler_images.py @@ -0,0 +1,184 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + +def lazy_import(): + from turing.generated.model.ensembler_image import EnsemblerImage + globals()['EnsemblerImage'] = EnsemblerImage + + +class EnsemblerImages(ModelSimple): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + lazy_import() + return { + 'value': ([EnsemblerImage],), + } + + @cached_property + def discriminator(): + return None + + + attribute_map = {} + + _composed_schemas = None + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): + """EnsemblerImages - a model defined in OpenAPI + + Note that value can be passed either in args or in kwargs, but not in both. + + Args: + args[0] ([EnsemblerImage]): # noqa: E501 + + Keyword Args: + value ([EnsemblerImage]): # noqa: E501 + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + # required up here when default value is not given + _path_to_item = kwargs.pop('_path_to_item', ()) + + if 'value' in kwargs: + value = kwargs.pop('value') + elif args: + args = list(args) + value = args.pop(0) + else: + raise ApiTypeError( + "value is required, but not passed in args or kwargs and doesn't have default", + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + self.value = value + if kwargs: + raise ApiTypeError( + "Invalid named arguments=%s passed to %s. Remove those invalid named arguments." % ( + kwargs, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) diff --git a/sdk/turing/generated/model/image_building_job_state.py b/sdk/turing/generated/model/image_building_job_state.py new file mode 100644 index 000000000..2fa921984 --- /dev/null +++ b/sdk/turing/generated/model/image_building_job_state.py @@ -0,0 +1,185 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + + +class ImageBuildingJobState(ModelSimple): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + ('value',): { + 'ACTIVE': "active", + 'SUCCEEDED': "succeeded", + 'FAILED': "failed", + 'UNKNOWN': "unknown", + }, + } + + validations = { + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + return { + 'value': (str,), + } + + @cached_property + def discriminator(): + return None + + + attribute_map = {} + + _composed_schemas = None + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): + """ImageBuildingJobState - a model defined in OpenAPI + + Note that value can be passed either in args or in kwargs, but not in both. + + Args: + args[0] (str):, must be one of ["active", "succeeded", "failed", "unknown", ] # noqa: E501 + + Keyword Args: + value (str):, must be one of ["active", "succeeded", "failed", "unknown", ] # noqa: E501 + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + # required up here when default value is not given + _path_to_item = kwargs.pop('_path_to_item', ()) + + if 'value' in kwargs: + value = kwargs.pop('value') + elif args: + args = list(args) + value = args.pop(0) + else: + raise ApiTypeError( + "value is required, but not passed in args or kwargs and doesn't have default", + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + self.value = value + if kwargs: + raise ApiTypeError( + "Invalid named arguments=%s passed to %s. Remove those invalid named arguments." % ( + kwargs, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) diff --git a/sdk/turing/generated/model/image_building_job_status.py b/sdk/turing/generated/model/image_building_job_status.py new file mode 100644 index 000000000..9c865b262 --- /dev/null +++ b/sdk/turing/generated/model/image_building_job_status.py @@ -0,0 +1,174 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + +def lazy_import(): + from turing.generated.model.image_building_job_state import ImageBuildingJobState + globals()['ImageBuildingJobState'] = ImageBuildingJobState + + +class ImageBuildingJobStatus(ModelNormal): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + lazy_import() + return { + 'state': (ImageBuildingJobState,), # noqa: E501 + 'message': (str,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'state': 'state', # noqa: E501 + 'message': 'message', # noqa: E501 + } + + _composed_schemas = {} + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): # noqa: E501 + """ImageBuildingJobStatus - a model defined in OpenAPI + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + state (ImageBuildingJobState): [optional] # noqa: E501 + message (str): [optional] # noqa: E501 + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) diff --git a/sdk/turing/generated/models/__init__.py b/sdk/turing/generated/models/__init__.py index a3de7d315..8fba95939 100644 --- a/sdk/turing/generated/models/__init__.py +++ b/sdk/turing/generated/models/__init__.py @@ -17,6 +17,7 @@ from turing.generated.model.big_query_sink import BigQuerySink from turing.generated.model.big_query_sink_all_of import BigQuerySinkAllOf from turing.generated.model.big_query_sink_config import BigQuerySinkConfig +from turing.generated.model.build_ensembler_image_request import BuildEnsemblerImageRequest from turing.generated.model.dataset import Dataset from turing.generated.model.default_traffic_rule import DefaultTrafficRule from turing.generated.model.enricher import Enricher @@ -25,6 +26,9 @@ from turing.generated.model.ensembler_config_kind import EnsemblerConfigKind from turing.generated.model.ensembler_docker_config import EnsemblerDockerConfig from turing.generated.model.ensembler_id import EnsemblerId +from turing.generated.model.ensembler_image import EnsemblerImage +from turing.generated.model.ensembler_image_runner_type import EnsemblerImageRunnerType +from turing.generated.model.ensembler_images import EnsemblerImages from turing.generated.model.ensembler_infra_config import EnsemblerInfraConfig from turing.generated.model.ensembler_job_status import EnsemblerJobStatus from turing.generated.model.ensembler_pyfunc_config import EnsemblerPyfuncConfig @@ -55,6 +59,8 @@ from turing.generated.model.generic_ensembler import GenericEnsembler from turing.generated.model.generic_sink import GenericSink from turing.generated.model.id_object import IdObject +from turing.generated.model.image_building_job_state import ImageBuildingJobState +from turing.generated.model.image_building_job_status import ImageBuildingJobStatus from turing.generated.model.job_id import JobId from turing.generated.model.kafka_config import KafkaConfig from turing.generated.model.label import Label diff --git a/sdk/turing/session.py b/sdk/turing/session.py index 508a1837d..3483a0787 100644 --- a/sdk/turing/session.py +++ b/sdk/turing/session.py @@ -1,28 +1,40 @@ import os -import mlflow +from time import sleep from typing import List, Optional +import mlflow +import pyprind + import turing.generated.models from turing.ensembler import EnsemblerType from turing.generated import ApiClient, Configuration -from turing.generated.apis import EnsemblerApi, EnsemblingJobApi, ProjectApi, RouterApi +from turing.generated.apis import ( + EnsemblerApi, + EnsemblerImagesApi, + EnsemblingJobApi, + ProjectApi, + RouterApi, +) +from turing.generated.model.ensembler_image_runner_type import EnsemblerImageRunnerType from turing.generated.models import ( - Project, Ensembler, - EnsemblingJob, + EnsemblerId, + EnsemblerImage, + EnsemblerImages, EnsemblerJobStatus, EnsemblersPaginatedResults, + EnsemblingJob, EnsemblingJobPaginatedResults, JobId, - RouterId, - RouterIdObject, - RouterIdAndVersion, + Project, Router, - RouterDetails, RouterConfig, + RouterDetails, + RouterId, + RouterIdAndVersion, + RouterIdObject, RouterVersion, RouterVersionConfig, - EnsemblerId, RouterVersionStatus, ) @@ -59,7 +71,7 @@ def __init__( get_default_id_token_credentials, ) from google.auth.transport.requests import Request - from google.auth.transport.urllib3 import urllib3, AuthorizedHttp + from google.auth.transport.urllib3 import AuthorizedHttp, urllib3 credentials = get_default_id_token_credentials(target_audience="sdk.caraml") # Refresh credentials, in case it's coming from Compute Engine. @@ -183,6 +195,88 @@ def delete_ensembler(self, ensembler_id: int) -> EnsemblerId: project_id=self.active_project.id, ensembler_id=ensembler_id ) + @require_active_project + def list_ensembler_images( + self, + ensembler: Ensembler, + runner_type: EnsemblerImageRunnerType = None, + ) -> EnsemblerImages: + """ + List ensembler images + """ + return EnsemblerImagesApi(self._api_client).list_ensembler_images( + project_id=self.active_project.id, + ensembler_id=ensembler.id, + runner_type=runner_type, + ) + + @require_active_project + def create_ensembler_image( + self, + ensembler: Ensembler, + runner_type: EnsemblerImageRunnerType = None, + ) -> EnsemblerImage: + """ + Create ensembler image + """ + build_ensembler_image_request = ( + turing.generated.models.BuildEnsemblerImageRequest( + runner_type=runner_type, + ) + ) + + EnsemblerImagesApi(self._api_client).create_ensembler_image( + project_id=self.active_project.id, + ensembler_id=ensembler.id, + build_ensembler_image_request=build_ensembler_image_request, + ) + + bar = pyprind.ProgBar( + 100, + track_time=True, + title=f"Building Docker image for ensembler {runner_type} of {ensembler.name}", + ) + bar.update() + sleep(10) + + while bar.active: + images = self.list_ensembler_images(ensembler, runner_type) + + if len(images.value) != 1: + break + + image = images.value[0] + + if image.exists: + break + + if ( + image.image_building_job_status is not None + and image.image_building_job_status.state.value != "active" + ): + break + + bar.update() + sleep(10) + bar.stop() + + if image.exists: + print( + f"Succefully built Docker image for ensembler {runner_type} of {ensembler.name}." + f"\nDocker image ref: {image.image_ref}" + ) + else: + print( + f"Failed to build Docker image for model {runner_type} of {ensembler.name}" + ) + if ( + image.image_building_job_status is not None + and image.image_building_job_status.message != "" + ): + print(f"{image.image_building_job_status.message}") + + return image + @require_active_project def list_ensembling_jobs( self,