Skip to content

Commit bb116b3

Browse files
authored
Performs Code Structure/Naming changes for Orchestration Refactor (#207)
1 parent 856ecad commit bb116b3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+197
-195
lines changed

src/golang/cmd/server/handler/create_table.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ import (
99

1010
"github.com/aqueducthq/aqueduct/cmd/server/routes"
1111
"github.com/aqueducthq/aqueduct/lib/collections/integration"
12+
"github.com/aqueducthq/aqueduct/lib/collections/operator/connector"
1213
"github.com/aqueducthq/aqueduct/lib/collections/shared"
1314
aq_context "github.com/aqueducthq/aqueduct/lib/context"
1415
"github.com/aqueducthq/aqueduct/lib/database"
1516
"github.com/aqueducthq/aqueduct/lib/job"
1617
"github.com/aqueducthq/aqueduct/lib/storage"
1718
"github.com/aqueducthq/aqueduct/lib/vault"
18-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector"
1919
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/auth"
2020
"github.com/aqueducthq/aqueduct/lib/workflow/utils"
2121
"github.com/dropbox/godropbox/errors"

src/golang/cmd/server/handler/discover.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ import (
99
"github.com/aqueducthq/aqueduct/cmd/server/queries"
1010
"github.com/aqueducthq/aqueduct/cmd/server/routes"
1111
"github.com/aqueducthq/aqueduct/lib/collections/integration"
12+
"github.com/aqueducthq/aqueduct/lib/collections/operator/connector"
1213
"github.com/aqueducthq/aqueduct/lib/collections/shared"
1314
aq_context "github.com/aqueducthq/aqueduct/lib/context"
1415
"github.com/aqueducthq/aqueduct/lib/database"
1516
"github.com/aqueducthq/aqueduct/lib/job"
1617
"github.com/aqueducthq/aqueduct/lib/vault"
17-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector"
1818
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/auth"
1919
workflow_utils "github.com/aqueducthq/aqueduct/lib/workflow/utils"
2020
"github.com/dropbox/godropbox/errors"

src/golang/cmd/server/handler/get_artifact_versions.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ import (
55
"net/http"
66

77
"github.com/aqueducthq/aqueduct/cmd/server/queries"
8+
"github.com/aqueducthq/aqueduct/lib/collections/operator/connector"
89
"github.com/aqueducthq/aqueduct/lib/collections/shared"
910
aq_context "github.com/aqueducthq/aqueduct/lib/context"
1011
"github.com/aqueducthq/aqueduct/lib/database"
11-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector"
1212
"github.com/dropbox/godropbox/errors"
1313
"github.com/google/uuid"
1414
)

src/golang/cmd/server/handler/get_workflow.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ type getWorkflowArgs struct {
4040

4141
type getWorkflowResponse struct {
4242
// a map of workflow dags keyed by their IDs
43-
WorkflowDags map[uuid.UUID]*workflow_dag.WorkflowDag `json:"workflow_dags"`
43+
WorkflowDags map[uuid.UUID]*workflow_dag.DBWorkflowDag `json:"workflow_dags"`
4444
// a list of dag results. Each result's `workflow_dag_id` field correspond to the
4545
WorkflowDagResults []workflowDagResult `json:"workflow_dag_results"`
4646
// a list of auth0Ids associated with workflow watchers
@@ -116,7 +116,7 @@ func (h *GetWorkflowHandler) Perform(ctx context.Context, interfaceArgs interfac
116116
return emptyResp, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error occurred when retrieving workflow.")
117117
}
118118

119-
workflowDags := make(map[uuid.UUID]*workflow_dag.WorkflowDag, len(dbWorkflowDags))
119+
workflowDags := make(map[uuid.UUID]*workflow_dag.DBWorkflowDag, len(dbWorkflowDags))
120120
for _, dbWorkflowDag := range dbWorkflowDags {
121121
constructedDag, err := workflow_utils.ReadWorkflowDagFromDatabase(
122122
ctx,

src/golang/cmd/server/handler/preview.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ func deserializeArtifactResponses(
226226
ctx context.Context,
227227
workflowStoragePaths *utils.WorkflowStoragePaths,
228228
storageConfig *shared.StorageConfig,
229-
dagArtifacts map[uuid.UUID]artifact.Artifact,
229+
dagArtifacts map[uuid.UUID]artifact.DBArtifact,
230230
artifactsToSkipFetch map[uuid.UUID]bool,
231231
) (map[uuid.UUID]previewArtifactResponse, error) {
232232
responses := make(map[uuid.UUID]previewArtifactResponse, len(workflowStoragePaths.ArtifactPaths))

src/golang/cmd/server/handler/preview_table.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ import (
88

99
"github.com/aqueducthq/aqueduct/cmd/server/routes"
1010
"github.com/aqueducthq/aqueduct/lib/collections/integration"
11+
"github.com/aqueducthq/aqueduct/lib/collections/operator/connector"
1112
"github.com/aqueducthq/aqueduct/lib/collections/shared"
1213
aq_context "github.com/aqueducthq/aqueduct/lib/context"
1314
"github.com/aqueducthq/aqueduct/lib/database"
1415
"github.com/aqueducthq/aqueduct/lib/job"
1516
"github.com/aqueducthq/aqueduct/lib/storage"
1617
"github.com/aqueducthq/aqueduct/lib/vault"
17-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector"
1818
"github.com/aqueducthq/aqueduct/lib/workflow/scheduler"
1919
workflow_utils "github.com/aqueducthq/aqueduct/lib/workflow/utils"
2020
"github.com/dropbox/godropbox/errors"

src/golang/cmd/server/handler/register_workflow.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ type RegisterWorkflowHandler struct {
6262

6363
type registerWorkflowArgs struct {
6464
*aq_context.AqContext
65-
workflowDag *workflow_dag.WorkflowDag
65+
workflowDag *workflow_dag.DBWorkflowDag
6666
operatorIdToFileContents map[uuid.UUID][]byte
6767

6868
// Whether this is a registering a new workflow or updating an existing one.

src/golang/cmd/server/request/dag.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@ import (
77

88
"github.com/aqueducthq/aqueduct/cmd/server/routes"
99
"github.com/aqueducthq/aqueduct/lib/collections/operator"
10+
"github.com/aqueducthq/aqueduct/lib/collections/operator/function"
1011
"github.com/aqueducthq/aqueduct/lib/collections/shared"
1112
"github.com/aqueducthq/aqueduct/lib/collections/workflow_dag"
1213
"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/github"
13-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/function"
1414
"github.com/dropbox/godropbox/errors"
1515
"github.com/google/uuid"
1616
)
1717

1818
const dagKey = "dag"
1919

2020
type DagSummary struct {
21-
Dag *workflow_dag.WorkflowDag
21+
Dag *workflow_dag.DBWorkflowDag
2222

2323
// Extract the operator contents from the request body
2424
FileContentsByOperatorUUID map[uuid.UUID][]byte
@@ -40,7 +40,7 @@ func ParseDagSummaryFromRequest(
4040
return nil, http.StatusBadRequest, errors.Wrap(err, "Serialized dag object not available")
4141
}
4242

43-
var workflowDag workflow_dag.WorkflowDag
43+
var workflowDag workflow_dag.DBWorkflowDag
4444
err = json.Unmarshal(serializedDAGBytes, &workflowDag)
4545
if err != nil {
4646
return nil, http.StatusBadRequest, errors.Wrap(err, "Invalid dag specification.")
@@ -81,7 +81,7 @@ func ParseDagSummaryFromRequest(
8181
// For github contents, retrieve zipball for files and update string contents like sql queries.
8282
func extractOperatorContentsFromRequest(
8383
r *http.Request,
84-
op *operator.Operator,
84+
op *operator.DBOperator,
8585
ghClient github.Client,
8686
) ([]byte, int, error) {
8787
if op.Spec.IsExtract() {

src/golang/lib/collections/artifact/artifact.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"github.com/google/uuid"
88
)
99

10-
type Artifact struct {
10+
type DBArtifact struct {
1111
Id uuid.UUID `db:"id" json:"id"`
1212
Name string `db:"name" json:"name"`
1313
Description string `db:"description" json:"description"`
@@ -16,13 +16,13 @@ type Artifact struct {
1616

1717
type Reader interface {
1818
Exists(ctx context.Context, id uuid.UUID, db database.Database) (bool, error)
19-
GetArtifact(ctx context.Context, id uuid.UUID, db database.Database) (*Artifact, error)
20-
GetArtifacts(ctx context.Context, ids []uuid.UUID, db database.Database) ([]Artifact, error)
19+
GetArtifact(ctx context.Context, id uuid.UUID, db database.Database) (*DBArtifact, error)
20+
GetArtifacts(ctx context.Context, ids []uuid.UUID, db database.Database) ([]DBArtifact, error)
2121
GetArtifactsByWorkflowDagId(
2222
ctx context.Context,
2323
workflowDagId uuid.UUID,
2424
db database.Database,
25-
) ([]Artifact, error)
25+
) ([]DBArtifact, error)
2626
ValidateArtifactOwnership(
2727
ctx context.Context,
2828
organizationId string,
@@ -38,13 +38,13 @@ type Writer interface {
3838
description string,
3939
spec *Spec,
4040
db database.Database,
41-
) (*Artifact, error)
41+
) (*DBArtifact, error)
4242
UpdateArtifact(
4343
ctx context.Context,
4444
id uuid.UUID,
4545
changes map[string]interface{},
4646
db database.Database,
47-
) (*Artifact, error)
47+
) (*DBArtifact, error)
4848
DeleteArtifact(ctx context.Context, id uuid.UUID, db database.Database) error
4949
DeleteArtifacts(ctx context.Context, ids []uuid.UUID, db database.Database) error
5050
}

src/golang/lib/collections/artifact/artifact_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,18 @@ package artifact_test
22

33
import (
44
"encoding/json"
5+
"github.com/aqueducthq/aqueduct/lib/collections/artifact/table"
56
"testing"
67

78
"github.com/aqueducthq/aqueduct/lib/collections/artifact"
8-
"github.com/aqueducthq/aqueduct/lib/workflow/artifact/table"
99
"github.com/google/uuid"
1010
"github.com/stretchr/testify/require"
1111
)
1212

1313
func TestSerializingAndDeserializingArtifact(t *testing.T) {
1414
id := uuid.New()
1515

16-
atf := artifact.Artifact{
16+
atf := artifact.DBArtifact{
1717
Id: id,
1818
Name: "test",
1919
Spec: *artifact.NewSpecFromTable(
@@ -24,7 +24,7 @@ func TestSerializingAndDeserializingArtifact(t *testing.T) {
2424
rawAtf, err := json.Marshal(atf)
2525
require.Nil(t, err)
2626

27-
var reconstructedAtf artifact.Artifact
27+
var reconstructedAtf artifact.DBArtifact
2828
err = json.Unmarshal(rawAtf, &reconstructedAtf)
2929
require.Nil(t, err)
3030
require.True(t, reconstructedAtf.Spec.IsTable())

src/golang/lib/collections/artifact/sqlite.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func (w *sqliteWriterImpl) CreateArtifact(
2929
description string,
3030
spec *Spec,
3131
db database.Database,
32-
) (*Artifact, error) {
32+
) (*DBArtifact, error) {
3333
insertColumns := []string{IdColumn, NameColumn, DescriptionColumn, SpecColumn}
3434
insertArtifactStmt := db.PrepareInsertWithReturnAllStmt(tableName, insertColumns, allColumns())
3535

@@ -40,7 +40,7 @@ func (w *sqliteWriterImpl) CreateArtifact(
4040

4141
args := []interface{}{id, name, description, spec}
4242

43-
var artifact Artifact
43+
var artifact DBArtifact
4444
err = db.Query(ctx, &artifact, insertArtifactStmt, args...)
4545
return &artifact, err
4646
}

src/golang/lib/collections/artifact/standard.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ func (w *standardWriterImpl) CreateArtifact(
2222
description string,
2323
spec *Spec,
2424
db database.Database,
25-
) (*Artifact, error) {
25+
) (*DBArtifact, error) {
2626
insertColumns := []string{NameColumn, DescriptionColumn, SpecColumn}
2727
insertArtifactStmt := db.PrepareInsertWithReturnAllStmt(tableName, insertColumns, allColumns())
2828

2929
args := []interface{}{name, description, spec}
3030

31-
var artifact Artifact
31+
var artifact DBArtifact
3232
err := db.Query(ctx, &artifact, insertArtifactStmt, args...)
3333
return &artifact, err
3434
}
@@ -41,7 +41,7 @@ func (r *standardReaderImpl) GetArtifact(
4141
ctx context.Context,
4242
id uuid.UUID,
4343
db database.Database,
44-
) (*Artifact, error) {
44+
) (*DBArtifact, error) {
4545
artifacts, err := r.GetArtifacts(ctx, []uuid.UUID{id}, db)
4646
if err != nil {
4747
return nil, err
@@ -58,7 +58,7 @@ func (r *standardReaderImpl) GetArtifacts(
5858
ctx context.Context,
5959
ids []uuid.UUID,
6060
db database.Database,
61-
) ([]Artifact, error) {
61+
) ([]DBArtifact, error) {
6262
if len(ids) == 0 {
6363
return nil, errors.New("Provided empty IDs list.")
6464
}
@@ -71,7 +71,7 @@ func (r *standardReaderImpl) GetArtifacts(
7171

7272
args := stmt_preparers.CastIdsListToInterfaceList(ids)
7373

74-
var artifacts []Artifact
74+
var artifacts []DBArtifact
7575
err := db.Query(ctx, &artifacts, getArtifactsQuery, args...)
7676
return artifacts, err
7777
}
@@ -80,7 +80,7 @@ func (r *standardReaderImpl) GetArtifactsByWorkflowDagId(
8080
ctx context.Context,
8181
workflowDagId uuid.UUID,
8282
db database.Database,
83-
) ([]Artifact, error) {
83+
) ([]DBArtifact, error) {
8484
getArtifactsByWorkflowDagIdQuery := fmt.Sprintf(
8585
`SELECT %s FROM artifact WHERE id IN
8686
(SELECT from_id FROM workflow_dag_edge WHERE workflow_dag_id = $1 AND type = '%s'
@@ -91,7 +91,7 @@ func (r *standardReaderImpl) GetArtifactsByWorkflowDagId(
9191
workflow_dag_edge.OperatorToArtifactType,
9292
)
9393

94-
var artifacts []Artifact
94+
var artifacts []DBArtifact
9595
err := db.Query(ctx, &artifacts, getArtifactsByWorkflowDagIdQuery, workflowDagId)
9696
return artifacts, err
9797
}
@@ -101,8 +101,8 @@ func (w *standardWriterImpl) UpdateArtifact(
101101
id uuid.UUID,
102102
changes map[string]interface{},
103103
db database.Database,
104-
) (*Artifact, error) {
105-
var artifact Artifact
104+
) (*DBArtifact, error) {
105+
var artifact DBArtifact
106106
err := utils.UpdateRecordToDest(ctx, &artifact, changes, tableName, IdColumn, id, allColumns(), db)
107107
return &artifact, err
108108
}

src/golang/lib/collections/artifact/types.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ import (
44
"database/sql/driver"
55
"encoding/json"
66

7+
"github.com/aqueducthq/aqueduct/lib/collections/artifact/boolean"
8+
"github.com/aqueducthq/aqueduct/lib/collections/artifact/float"
9+
"github.com/aqueducthq/aqueduct/lib/collections/artifact/jsonable"
10+
"github.com/aqueducthq/aqueduct/lib/collections/artifact/table"
711
"github.com/aqueducthq/aqueduct/lib/collections/utils"
8-
"github.com/aqueducthq/aqueduct/lib/workflow/artifact/boolean"
9-
"github.com/aqueducthq/aqueduct/lib/workflow/artifact/float"
10-
"github.com/aqueducthq/aqueduct/lib/workflow/artifact/jsonable"
11-
"github.com/aqueducthq/aqueduct/lib/workflow/artifact/table"
1212
"github.com/dropbox/godropbox/errors"
1313
)
1414

src/golang/lib/workflow/operator/check/check.go renamed to src/golang/lib/collections/operator/check/check.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package check
22

33
import (
4-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/function"
4+
"github.com/aqueducthq/aqueduct/lib/collections/operator/function"
55
)
66

77
type Level string

src/golang/lib/workflow/operator/connector/extract_params.go renamed to src/golang/lib/collections/operator/connector/extract_params.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package connector
22

3-
import gh_types "github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/github/types"
3+
import (
4+
gh_types "github.com/aqueducthq/aqueduct/lib/collections/operator/connector/github"
5+
)
46

57
type ExtractParams interface {
68
isExtractParams()

src/golang/lib/workflow/operator/connector/github/types/types.go renamed to src/golang/lib/collections/operator/connector/github/types.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package types
1+
package github
22

33
import (
44
"fmt"

src/golang/lib/workflow/operator/function/function.go renamed to src/golang/lib/collections/operator/function/function.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package function
22

33
import (
4-
github "github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/github/types"
4+
"github.com/aqueducthq/aqueduct/lib/collections/operator/connector/github"
55
)
66

77
type Function struct {

src/golang/lib/workflow/operator/metric/metric.go renamed to src/golang/lib/collections/operator/metric/metric.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package metric
22

33
import (
4-
"github.com/aqueducthq/aqueduct/lib/workflow/operator/function"
4+
"github.com/aqueducthq/aqueduct/lib/collections/operator/function"
55
)
66

77
type Metric struct {

src/golang/lib/collections/operator/operator.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"github.com/google/uuid"
88
)
99

10-
type Operator struct {
10+
type DBOperator struct {
1111
Id uuid.UUID `db:"id" json:"id"`
1212
Name string `db:"name" json:"name"`
1313
Description string `db:"description" json:"description"`
@@ -20,13 +20,13 @@ type Operator struct {
2020

2121
type Reader interface {
2222
Exists(ctx context.Context, id uuid.UUID, db database.Database) (bool, error)
23-
GetOperator(ctx context.Context, id uuid.UUID, db database.Database) (*Operator, error)
24-
GetOperators(ctx context.Context, ids []uuid.UUID, db database.Database) ([]Operator, error)
23+
GetOperator(ctx context.Context, id uuid.UUID, db database.Database) (*DBOperator, error)
24+
GetOperators(ctx context.Context, ids []uuid.UUID, db database.Database) ([]DBOperator, error)
2525
GetOperatorsByWorkflowDagId(
2626
ctx context.Context,
2727
workflowDagId uuid.UUID,
2828
db database.Database,
29-
) ([]Operator, error)
29+
) ([]DBOperator, error)
3030
ValidateOperatorOwnership(
3131
ctx context.Context,
3232
organizationId string,
@@ -42,13 +42,13 @@ type Writer interface {
4242
description string,
4343
spec *Spec,
4444
db database.Database,
45-
) (*Operator, error)
45+
) (*DBOperator, error)
4646
UpdateOperator(
4747
ctx context.Context,
4848
id uuid.UUID,
4949
changes map[string]interface{},
5050
db database.Database,
51-
) (*Operator, error)
51+
) (*DBOperator, error)
5252
DeleteOperator(ctx context.Context, id uuid.UUID, db database.Database) error
5353
DeleteOperators(ctx context.Context, ids []uuid.UUID, db database.Database) error
5454
}

0 commit comments

Comments
 (0)