diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c6f6ee7..b9ec8933b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.3.4 +Released on May 24, 2023. + +### Enhancements +* Improves consistency of icon sizing on resources page. +* Merges `flow_id` and `name` into a single argument when retrieving workflows + from the SDK. +* Adds ability to parametrize save operators when using AWS S3. + +### Bugfixes +* Fixes bug where race condition could occur when syncing Airflow workflows to + Aqueduct. +* Fixes bug where listing Snowflake data objects would return an error. + ## 0.3.3 Released on May 17, 2023. diff --git a/sdk/version b/sdk/version index 1c09c74e2..42045acae 100644 --- a/sdk/version +++ b/sdk/version @@ -1 +1 @@ -0.3.3 +0.3.4 diff --git a/src/dockerfiles/Makefile b/src/dockerfiles/Makefile index bf7520cda..a45e5f340 100644 --- a/src/dockerfiles/Makefile +++ b/src/dockerfiles/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL = all -VERSION = 0.3.3 +VERSION = 0.3.4 # By default, turn off the Makefile practice of printing each command before # you run it. diff --git a/src/dockerfiles/connectors/athena.dockerfile b/src/dockerfiles/connectors/athena.dockerfile index abaaf4820..00bc97973 100644 --- a/src/dockerfiles/connectors/athena.dockerfile +++ b/src/dockerfiles/connectors/athena.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/bigquery.dockerfile b/src/dockerfiles/connectors/bigquery.dockerfile index 11d62131d..760a15cde 100644 --- a/src/dockerfiles/connectors/bigquery.dockerfile +++ b/src/dockerfiles/connectors/bigquery.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/mysql.dockerfile b/src/dockerfiles/connectors/mysql.dockerfile index bb751b0d2..3011feba1 100644 --- a/src/dockerfiles/connectors/mysql.dockerfile +++ b/src/dockerfiles/connectors/mysql.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/postgres.dockerfile b/src/dockerfiles/connectors/postgres.dockerfile index 470db57cf..2bf494354 100644 --- a/src/dockerfiles/connectors/postgres.dockerfile +++ b/src/dockerfiles/connectors/postgres.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/s3.dockerfile b/src/dockerfiles/connectors/s3.dockerfile index adee7071f..c3782d1c5 100644 --- a/src/dockerfiles/connectors/s3.dockerfile +++ b/src/dockerfiles/connectors/s3.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/snowflake.dockerfile b/src/dockerfiles/connectors/snowflake.dockerfile index ccf48cac1..be5fad948 100644 --- a/src/dockerfiles/connectors/snowflake.dockerfile +++ b/src/dockerfiles/connectors/snowflake.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/connectors/sqlserver.dockerfile b/src/dockerfiles/connectors/sqlserver.dockerfile index a1a115360..2df231b7f 100644 --- a/src/dockerfiles/connectors/sqlserver.dockerfile +++ b/src/dockerfiles/connectors/sqlserver.dockerfile @@ -1,4 +1,4 @@ -FROM aqueducthq/base_connector:0.3.3 +FROM aqueducthq/base_connector:0.3.4 MAINTAINER Aqueduct version: 0.1 diff --git a/src/dockerfiles/function/function310.dockerfile b/src/dockerfiles/function/function310.dockerfile index 912a099db..a5a51862a 100644 --- a/src/dockerfiles/function/function310.dockerfile +++ b/src/dockerfiles/function/function310.dockerfile @@ -16,7 +16,7 @@ pyarrow==7.0.0 \ boto3==1.18.0 \ pydantic==1.9.0 \ scikit_learn==1.0.2 \ -aqueduct-ml==0.3.3 +aqueduct-ml==0.3.4 ENV PYTHONUNBUFFERED 1 diff --git a/src/dockerfiles/function/function37.dockerfile b/src/dockerfiles/function/function37.dockerfile index 157d5936b..744e477d5 100644 --- a/src/dockerfiles/function/function37.dockerfile +++ b/src/dockerfiles/function/function37.dockerfile @@ -16,7 +16,7 @@ boto3==1.18.0 \ pydantic==1.9.0 \ scikit_learn==1.0.2 \ typing_extensions==4.3.0 \ -aqueduct-ml==0.3.3 +aqueduct-ml==0.3.4 ENV PYTHONUNBUFFERED 1 diff --git a/src/dockerfiles/function/function38.dockerfile b/src/dockerfiles/function/function38.dockerfile index 069e900b8..007f3f627 100644 --- a/src/dockerfiles/function/function38.dockerfile +++ b/src/dockerfiles/function/function38.dockerfile @@ -16,7 +16,7 @@ pyarrow==7.0.0 \ boto3==1.18.0 \ pydantic==1.9.0 \ scikit_learn==1.0.2 \ -aqueduct-ml==0.3.3 +aqueduct-ml==0.3.4 ENV PYTHONUNBUFFERED 1 diff --git a/src/dockerfiles/function/function39.dockerfile b/src/dockerfiles/function/function39.dockerfile index 730efaabe..85e7b3ec2 100644 --- a/src/dockerfiles/function/function39.dockerfile +++ b/src/dockerfiles/function/function39.dockerfile @@ -16,7 +16,7 @@ pyarrow==7.0.0 \ boto3==1.18.0 \ pydantic==1.9.0 \ scikit_learn==1.0.2 \ -aqueduct-ml==0.3.3 +aqueduct-ml==0.3.4 ENV PYTHONUNBUFFERED 1 diff --git a/src/dockerfiles/gpu/py310_env.yml b/src/dockerfiles/gpu/py310_env.yml index 8da566211..42d56cfc7 100644 --- a/src/dockerfiles/gpu/py310_env.yml +++ b/src/dockerfiles/gpu/py310_env.yml @@ -14,4 +14,4 @@ dependencies: - pip: - scikit_learn==1.0.2 - typing_extensions==4.3.0 - - aqueduct-ml==0.3.3 + - aqueduct-ml==0.3.4 diff --git a/src/dockerfiles/gpu/py37_env.yml b/src/dockerfiles/gpu/py37_env.yml index 0de1cddd1..456efef35 100644 --- a/src/dockerfiles/gpu/py37_env.yml +++ b/src/dockerfiles/gpu/py37_env.yml @@ -14,4 +14,4 @@ dependencies: - pip: - scikit_learn==1.0.2 - typing_extensions==4.3.0 - - aqueduct-ml==0.3.3 + - aqueduct-ml==0.3.4 diff --git a/src/dockerfiles/gpu/py38_env.yml b/src/dockerfiles/gpu/py38_env.yml index 8d6b0d89b..284e9edb7 100644 --- a/src/dockerfiles/gpu/py38_env.yml +++ b/src/dockerfiles/gpu/py38_env.yml @@ -14,4 +14,4 @@ dependencies: - pip: - scikit_learn==1.0.2 - typing_extensions==4.3.0 - - aqueduct-ml==0.3.3 \ No newline at end of file + - aqueduct-ml==0.3.4 \ No newline at end of file diff --git a/src/dockerfiles/gpu/py39_env.yml b/src/dockerfiles/gpu/py39_env.yml index 77e76d22e..b07ed5c83 100644 --- a/src/dockerfiles/gpu/py39_env.yml +++ b/src/dockerfiles/gpu/py39_env.yml @@ -14,4 +14,4 @@ dependencies: - pip: - scikit_learn==1.0.2 - typing_extensions==4.3.0 - - aqueduct-ml==0.3.3 \ No newline at end of file + - aqueduct-ml==0.3.4 \ No newline at end of file diff --git a/src/dockerfiles/lambda/function/requirements-37.txt b/src/dockerfiles/lambda/function/requirements-37.txt index 531fb83c8..81fb2487d 100644 --- a/src/dockerfiles/lambda/function/requirements-37.txt +++ b/src/dockerfiles/lambda/function/requirements-37.txt @@ -7,4 +7,4 @@ boto3==1.18.0 pydantic==1.9.0 scikit_learn==1.0.2 typing_extensions==4.3.0 -aqueduct-ml==0.3.3 \ No newline at end of file +aqueduct-ml==0.3.4 \ No newline at end of file diff --git a/src/dockerfiles/lambda/function/requirements.txt b/src/dockerfiles/lambda/function/requirements.txt index 211ba6392..e367fbfdf 100644 --- a/src/dockerfiles/lambda/function/requirements.txt +++ b/src/dockerfiles/lambda/function/requirements.txt @@ -6,4 +6,4 @@ pyarrow==7.0.0 boto3==1.18.0 pydantic==1.9.0 scikit_learn==1.0.2 -aqueduct-ml==0.3.3 \ No newline at end of file +aqueduct-ml==0.3.4 \ No newline at end of file diff --git a/src/dockerfiles/lambda/requirements.txt b/src/dockerfiles/lambda/requirements.txt index da6420451..3a6d439b0 100644 --- a/src/dockerfiles/lambda/requirements.txt +++ b/src/dockerfiles/lambda/requirements.txt @@ -6,4 +6,4 @@ pydantic==1.9.0 pyyaml SQLAlchemy==1.4.30 typing_extensions==4.3.0 -aqueduct-ml==0.3.3 \ No newline at end of file +aqueduct-ml==0.3.4 \ No newline at end of file diff --git a/src/dockerfiles/spark/spark-py310-env.dockerfile b/src/dockerfiles/spark/spark-py310-env.dockerfile index 5f35029a4..f617d65c1 100644 --- a/src/dockerfiles/spark/spark-py310-env.dockerfile +++ b/src/dockerfiles/spark/spark-py310-env.dockerfile @@ -25,7 +25,7 @@ RUN wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws unzip awscliv2.zip && ./aws/install RUN conda activate py310_env -RUN pip install conda-pack aqueduct-ml==0.3.3 +RUN pip install conda-pack aqueduct-ml==0.3.4 COPY ./spark/create-conda-env.sh / diff --git a/src/dockerfiles/spark/spark-py37-env.dockerfile b/src/dockerfiles/spark/spark-py37-env.dockerfile index 3887d1819..6a09a6967 100644 --- a/src/dockerfiles/spark/spark-py37-env.dockerfile +++ b/src/dockerfiles/spark/spark-py37-env.dockerfile @@ -25,7 +25,7 @@ RUN wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws unzip awscliv2.zip && ./aws/install RUN conda activate py37_env -RUN pip install conda-pack aqueduct-ml==0.3.3 +RUN pip install conda-pack aqueduct-ml==0.3.4 COPY ./spark/create-conda-env.sh / diff --git a/src/dockerfiles/spark/spark-py38-env.dockerfile b/src/dockerfiles/spark/spark-py38-env.dockerfile index 3ff58b31f..0fdb6c4c5 100644 --- a/src/dockerfiles/spark/spark-py38-env.dockerfile +++ b/src/dockerfiles/spark/spark-py38-env.dockerfile @@ -25,7 +25,7 @@ RUN wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws unzip awscliv2.zip && ./aws/install RUN conda activate py38_env -RUN pip install conda-pack aqueduct-ml==0.3.3 +RUN pip install conda-pack aqueduct-ml==0.3.4 COPY ./spark/create-conda-env.sh / diff --git a/src/dockerfiles/spark/spark-py39-env.dockerfile b/src/dockerfiles/spark/spark-py39-env.dockerfile index 39cdad260..083ea06e4 100644 --- a/src/dockerfiles/spark/spark-py39-env.dockerfile +++ b/src/dockerfiles/spark/spark-py39-env.dockerfile @@ -25,7 +25,7 @@ RUN wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws unzip awscliv2.zip && ./aws/install RUN conda activate py39_env -RUN pip install conda-pack aqueduct-ml==0.3.3 +RUN pip install conda-pack aqueduct-ml==0.3.4 COPY ./spark/create-conda-env.sh / diff --git a/src/golang/lib/constants.go b/src/golang/lib/constants.go index 6c828773d..0388d699c 100644 --- a/src/golang/lib/constants.go +++ b/src/golang/lib/constants.go @@ -1,3 +1,3 @@ package lib -const ServerVersionNumber = "0.3.3" +const ServerVersionNumber = "0.3.4" diff --git a/src/python/bin/aqueduct b/src/python/bin/aqueduct index 63bdf5573..558a385c6 100755 --- a/src/python/bin/aqueduct +++ b/src/python/bin/aqueduct @@ -38,7 +38,7 @@ base_directory = os.path.join(os.environ["HOME"], ".aqueduct") server_directory = os.path.join(os.environ["HOME"], ".aqueduct", "server") ui_directory = os.path.join(os.environ["HOME"], ".aqueduct", "ui") -package_version = "0.3.3" +package_version = "0.3.4" aws_credentials_path = os.path.join(os.environ["HOME"], ".aws", "credentials") default_server_port = 8080 diff --git a/src/python/version b/src/python/version index 1c09c74e2..42045acae 100644 --- a/src/python/version +++ b/src/python/version @@ -1 +1 @@ -0.3.3 +0.3.4 diff --git a/src/ui/app/package.json b/src/ui/app/package.json index e34e3cf45..183d9bbe1 100644 --- a/src/ui/app/package.json +++ b/src/ui/app/package.json @@ -1,7 +1,7 @@ { "name": "@aqueducthq/ui", "author": "Aqueduct, Inc. ", - "version": "0.3.3", + "version": "0.3.4", "scripts": { "start": "parcel --no-cache index.html", "build": "parcel build --public-url /dist --dist-dir dist/default index.html", @@ -9,7 +9,7 @@ "lint:fix": "eslint '*/**/*.{js,ts,tsx}' --format table --fix" }, "dependencies": { - "@aqueducthq/common": "0.3.3", + "@aqueducthq/common": "0.3.4", "@emotion/react": "^11.10.5", "@emotion/styled": "^11.10.5", "@fortawesome/fontawesome-svg-core": "^6.2.1", diff --git a/src/ui/common/package-lock.json b/src/ui/common/package-lock.json index 87ed9b9df..220c2aa71 100644 --- a/src/ui/common/package-lock.json +++ b/src/ui/common/package-lock.json @@ -1,12 +1,12 @@ { "name": "@aqueducthq/common", - "version": "0.3.3", + "version": "0.3.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@aqueducthq/common", - "version": "0.3.3", + "version": "0.3.4", "hasInstallScript": true, "devDependencies": { "@babel/core": "^7.20.12", diff --git a/src/ui/common/package.json b/src/ui/common/package.json index 219ca9aa9..cbe879460 100644 --- a/src/ui/common/package.json +++ b/src/ui/common/package.json @@ -1,7 +1,7 @@ { "name": "@aqueducthq/common", "author": "Aqueduct ", - "version": "0.3.3", + "version": "0.3.4", "main": "dist/index.js", "types": "dist/index.d.ts", "alias": { diff --git a/src/ui/common/src/components/pages/data/index.tsx b/src/ui/common/src/components/pages/data/index.tsx index b10ef8aaa..6adc913dd 100644 --- a/src/ui/common/src/components/pages/data/index.tsx +++ b/src/ui/common/src/components/pages/data/index.tsx @@ -12,9 +12,7 @@ import getPathPrefix from '../../../utils/getPathPrefix'; import { CheckLevel } from '../../../utils/operators'; import ExecutionStatus from '../../../utils/shared'; import DefaultLayout from '../../layouts/default'; -import PaginatedSearchTable, { - PaginatedSearchTableData, -} from '../../tables/PaginatedSearchTable'; +import PaginatedSearchTable from '../../tables/PaginatedSearchTable'; import { LayoutProps } from '../types'; import CheckItem from '../workflows/components/CheckItem'; import ExecutionStatusLink from '../workflows/components/ExecutionStatusLink'; @@ -50,38 +48,27 @@ const DataPage: React.FC = ({ user, Layout = DefaultLayout }) => { }, []); const onGetColumnValue = (row, column) => { - let value = row[column.name]; - - switch (column.name) { + const value = row[column]; + switch (column) { case 'workflow': case 'name': const { name, url, status } = value; - value = ; - break; + return ; case 'created_at': - value = row[column.name].toLocaleString(); - break; + return value.toLocaleString(); case 'metrics': { - value = ; - break; + return ; } case 'checks': { - value = ; - break; + return ; } case 'type': { - value = ( - {row[column.name]} - ); - break; + return {value}; } default: { - value = row[column.name]; - break; + return value; } } - - return value; }; let tableData = []; @@ -184,21 +171,6 @@ const DataPage: React.FC = ({ user, Layout = DefaultLayout }) => { }, ]; - const artifactList: PaginatedSearchTableData = { - schema: { - fields: [ - { name: 'name', type: 'varchar' }, - { name: 'created_at', displayName: 'Created At', type: 'varchar' }, - { name: 'workflow', type: 'varchar' }, - { name: 'type', type: 'varchar' }, - { name: 'metrics', type: 'varchar' }, - { name: 'checks', type: 'varchar' }, - ], - pandas_version: '1.5.1', - }, - data: tableData, - }; - const noItemsMessage = ( There are no data artifacts created yet. Create one right by running a @@ -252,9 +224,17 @@ const DataPage: React.FC = ({ user, Layout = DefaultLayout }) => { breadcrumbs={[BreadcrumbLink.HOME, BreadcrumbLink.DATA]} user={user} > - {artifactList.data?.length && artifactList.data?.length > 0 ? ( + {tableData?.length && tableData?.length > 0 ? ( void; - value?: EmailConfig; -}; - export const EmailDialog: React.FC = ({ editMode = false, }) => { - // Retrieve the form context. - const { register, setValue, getValues } = useFormContext(); + const [selectedLevel, setSelectedLevel] = useState( + EmailDefaultsOnCreate.level + ); - // Register forms with custom logic. + const [notificationsEnabled, setNotificationsEnabled] = useState( + EmailDefaultsOnCreate.enabled + ); + + const { register, setValue, getValues } = useFormContext(); register('enabled', { value: EmailDefaultsOnCreate.enabled }); register('level', { value: EmailDefaultsOnCreate.level }); register('targets_serialized', { @@ -108,7 +108,6 @@ export const EmailDialog: React.FC = ({ description="The email address(es) of the receiver(s). Use comma to separate different addresses." placeholder={Placeholders.reciever} onChange={(event) => { - //setValue('receivers', event.target.value); const receiversList = event.target.value .split(',') .map((r) => r.trim()); @@ -120,9 +119,10 @@ export const EmailDialog: React.FC = ({ { + setNotificationsEnabled(checked ? 'true' : 'false'); setValue('enabled', checked ? 'true' : 'false'); }} > @@ -134,7 +134,7 @@ export const EmailDialog: React.FC = ({ - {enabled === 'true' && ( + {notificationsEnabled === 'true' && ( @@ -147,11 +147,12 @@ export const EmailDialog: React.FC = ({ { + setSelectedLevel(level); setValue('level', level); }} - enabled={enabled === 'true'} + enabled={notificationsEnabled === 'true'} /> )} @@ -162,12 +163,15 @@ export const EmailDialog: React.FC = ({ export function getEmailValidationSchema() { return Yup.object().shape({ host: Yup.string().required('Please enter a host'), - port: Yup.number().required('Please enter a port'), + port: Yup.string().required('Please enter a port'), user: Yup.string().required('Please enter a sender address'), password: Yup.string().required('Please enter a sender password'), targets_serialized: Yup.string().required( 'Please enter at least one receiver' ), - enabled: Yup.string(), + level: Yup.string().required('Please select a notification level'), + enabled: Yup.string().transform((value) => { + value ? value : EmailDefaultsOnCreate.enabled; + }), }); } diff --git a/src/ui/common/src/components/resources/dialogs/mariadbDialog.tsx b/src/ui/common/src/components/resources/dialogs/mariadbDialog.tsx index 5f4d83c38..4b0938b2e 100644 --- a/src/ui/common/src/components/resources/dialogs/mariadbDialog.tsx +++ b/src/ui/common/src/components/resources/dialogs/mariadbDialog.tsx @@ -88,7 +88,7 @@ export const MariaDbDialog: React.FC = ({ export function getMariaDBValidationSchema() { return Yup.object().shape({ host: Yup.string().required('Please enter a host'), - port: Yup.number().required('Please enter a port'), + port: Yup.string().required('Please enter a port'), database: Yup.string().required('Please enter a database'), username: Yup.string().required('Please enter a username'), password: Yup.string().required('Please enter a password'), diff --git a/src/ui/common/src/components/resources/dialogs/onDemandKubernetesDialog.tsx b/src/ui/common/src/components/resources/dialogs/onDemandKubernetesDialog.tsx index 88e332c02..44569b094 100644 --- a/src/ui/common/src/components/resources/dialogs/onDemandKubernetesDialog.tsx +++ b/src/ui/common/src/components/resources/dialogs/onDemandKubernetesDialog.tsx @@ -271,7 +271,7 @@ const OnDemandK8sAWSStep: React.FC = ({ dispatch( handleConnectToNewResource({ apiKey: user.apiKey, - service: 'Kubernetes', + service: 'AWS', name: name, config: data, }) diff --git a/src/ui/common/src/components/resources/dialogs/s3Dialog.tsx b/src/ui/common/src/components/resources/dialogs/s3Dialog.tsx index b7e203378..85e63a9f5 100644 --- a/src/ui/common/src/components/resources/dialogs/s3Dialog.tsx +++ b/src/ui/common/src/components/resources/dialogs/s3Dialog.tsx @@ -41,7 +41,7 @@ export const S3Dialog: React.FC = ({ const [fileData, setFileData] = useState(null); const { register, setValue } = useFormContext(); - register('use_as_storage'); + register('use_as_storage', { value: 'false' }); const [useAsMetadataStorage, setUseAsMetadataStorage] = useState('false'); @@ -251,6 +251,13 @@ export function getS3ValidationSchema() { type: Yup.string().required('Please select a credential type'), bucket: Yup.string().required('Please enter a bucket name'), region: Yup.string().required('Please enter a region'), + use_as_storage: Yup.string().transform((value) => { + if (value === 'true') { + return 'true'; + } + + return 'false'; + }), access_key_id: Yup.string().when('type', { is: 'access_key', then: Yup.string().required('Please enter an access key id'), @@ -289,12 +296,5 @@ export function getS3ValidationSchema() { .required('Please upload a credentials file'), otherwise: null, }), - use_as_storage: Yup.string().transform((value) => { - if (value === 'true') { - return 'true'; - } - - return 'false'; - }), }); }