Skip to content
This repository was archived by the owner on Mar 10, 2024. It is now read-only.

Commit 0ca6e63

Browse files
tomkittomkit
committed
feat: add s3
1 parent 369a83e commit 0ca6e63

File tree

8 files changed

+409
-6
lines changed

8 files changed

+409
-6
lines changed

Diff for: ‎apps/mgmt-ui/src/components/connectors/destination/DestinationDetailsPanel.tsx

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import BigQueryDestinationDetailsPanel from './BigQueryDestinationDetailsPanel';
22
import PostgresDestinationDetailsPanel from './PostgresDestinationDetailsPanel';
33
import RedshiftDestinationDetailsPanel from './RedshiftDestinationDetailsPanel';
4+
import S3DestinationDetailsPanel from './S3DestinationDetailsPanel';
45
import SnowflakeDestinationDetailsPanel from './SnowflakeDestinationDetailsPanel';
56

67
export type DestinationDetailsPanelProps = {
7-
type: 'postgres' | 'bigquery' | 'snowflake' | 'redshift';
8+
type: 'postgres' | 'bigquery' | 'snowflake' | 'redshift' | 's3';
89
isLoading: boolean;
910
};
1011

@@ -18,6 +19,8 @@ export default function DestinationDetailsPanel({ type, isLoading }: Destination
1819
return <SnowflakeDestinationDetailsPanel isLoading={isLoading} />;
1920
case 'redshift':
2021
return <RedshiftDestinationDetailsPanel isLoading={isLoading} />;
22+
case 's3':
23+
return <S3DestinationDetailsPanel isLoading={isLoading} />;
2124
default:
2225
return null;
2326
}

Diff for: ‎apps/mgmt-ui/src/components/connectors/destination/DestinationTabPanelContainer.tsx

+12-2
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ const ICON_SIZE = 35;
1616
export type DestinationCardInfo = {
1717
icon?: React.ReactNode;
1818
name: string;
19-
type: 'postgres' | 'bigquery' | 'snowflake' | 'redshift' | 'supaglue';
19+
type: 'postgres' | 'bigquery' | 'snowflake' | 'redshift' | 'supaglue' | 's3';
2020
classType: 'analytical' | 'application';
2121
status: string;
2222
description: string;
2323
};
2424

2525
export type AnalyticalDestinationCardInfo = DestinationCardInfo & {
26-
type: 'bigquery' | 'snowflake' | 'redshift';
26+
type: 'bigquery' | 'snowflake' | 'redshift' | 's3';
2727
};
2828

2929
export const postgresDestinationCardInfo: DestinationCardInfo = {
@@ -62,6 +62,15 @@ export const redshiftDestinationCardInfo: DestinationCardInfo = {
6262
description: 'Configure your Redshift destination.',
6363
};
6464

65+
export const s3DestinationCardInfo: DestinationCardInfo = {
66+
icon: <Image alt="s3" src={RedshiftQueryIcon} width={ICON_SIZE} height={ICON_SIZE} />,
67+
name: 'S3',
68+
type: 's3',
69+
classType: 'analytical',
70+
status: '',
71+
description: 'Configure your S3 destination.',
72+
};
73+
6574
export const supaglueDestinationCardInfo: DestinationCardInfo = {
6675
icon: <Image alt="supaglue" src={SupaglueIcon} width={ICON_SIZE} height={ICON_SIZE} />,
6776
name: 'Supaglue',
@@ -77,6 +86,7 @@ export const destinationCardsInfo: DestinationCardInfo[] = [
7786
bigQueryDestinationCardInfo,
7887
snowflakeDestinationCardInfo,
7988
redshiftDestinationCardInfo,
89+
s3DestinationCardInfo,
8090
];
8191

8292
export default function DestinationTabPanelContainer() {

Diff for: ‎apps/mgmt-ui/src/components/connectors/destination/S3DestinationDetailsPanel.tsx

+281
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
/* eslint-disable @typescript-eslint/no-floating-promises */
2+
import { createDestination, testDestination, updateDestination } from '@/client';
3+
import Spinner from '@/components/Spinner';
4+
import { useNotification } from '@/context/notification';
5+
import { useActiveApplicationId } from '@/hooks/useActiveApplicationId';
6+
import { useDestinations } from '@/hooks/useDestinations';
7+
import getIcon from '@/utils/companyToIcon';
8+
import { Button, Stack, TextField, Typography } from '@mui/material';
9+
import Card from '@mui/material/Card';
10+
import type { DestinationSafeAny } from '@supaglue/types';
11+
import { useRouter } from 'next/router';
12+
import { useEffect, useState } from 'react';
13+
import { s3DestinationCardInfo } from './DestinationTabPanelContainer';
14+
import { ExistingPasswordTextField, type KnownOrUnknownValue } from './ExistingPasswordTextField';
15+
16+
export type S3DestinationDetailsPanelProps = {
17+
isLoading: boolean;
18+
};
19+
20+
export default function S3DestinationDetailsPanel({ isLoading }: S3DestinationDetailsPanelProps) {
21+
const activeApplicationId = useActiveApplicationId();
22+
const { addNotification } = useNotification();
23+
24+
const { destinations: existingDestinations = [], mutate } = useDestinations();
25+
26+
const destination = existingDestinations.find((existingDestination) => existingDestination.type === 's3');
27+
28+
const [name, setName] = useState<string>('');
29+
const [region, setRegion] = useState<string>('');
30+
const [bucket, setBucket] = useState<string>('');
31+
const [accessKeyId, setAccessKeyId] = useState<string>('');
32+
const [secretAccessKey, setSecretAccessKey] = useState<KnownOrUnknownValue>({ type: 'known', value: '' });
33+
const [isTesting, setIsTesting] = useState<boolean>(false);
34+
const [isTestSuccessful, setIsTestSuccessful] = useState<boolean>(false);
35+
const [isDirty, setIsDirty] = useState<boolean>(false);
36+
const router = useRouter();
37+
38+
const isNew = !destination?.id;
39+
40+
useEffect(() => {
41+
const timer = setTimeout(() => {
42+
setIsTesting(false);
43+
}, 3000); // sum of connection timeout + query timeout
44+
return () => clearTimeout(timer);
45+
}, [isTesting]);
46+
47+
useEffect(() => {
48+
if (destination?.type !== 's3') {
49+
return;
50+
}
51+
52+
setRegion(destination?.config?.region);
53+
setName(destination?.name);
54+
setBucket(destination?.config?.bucket);
55+
setAccessKeyId(destination?.config?.accessKeyId);
56+
setSecretAccessKey({ type: 'unknown' });
57+
}, [destination?.id]);
58+
59+
const createOrUpdateDestination = async (): Promise<DestinationSafeAny | undefined> => {
60+
if (destination) {
61+
const response = await updateDestination({
62+
...destination,
63+
name,
64+
type: 's3',
65+
config: {
66+
region,
67+
bucket,
68+
accessKeyId,
69+
secretAccessKey: secretAccessKey.type === 'known' ? secretAccessKey.value : undefined,
70+
},
71+
version: destination.version,
72+
});
73+
if (!response.ok) {
74+
addNotification({ message: response.errorMessage, severity: 'error' });
75+
return;
76+
}
77+
return response.data;
78+
}
79+
const response = await createDestination({
80+
applicationId: activeApplicationId,
81+
type: 's3',
82+
name,
83+
config: {
84+
region,
85+
bucket,
86+
accessKeyId,
87+
secretAccessKey: secretAccessKey.type === 'known' ? secretAccessKey.value : '', // TODO: shouldn't allow empty string
88+
},
89+
});
90+
if (!response.ok) {
91+
addNotification({ message: response.errorMessage, severity: 'error' });
92+
return;
93+
}
94+
return response.data;
95+
};
96+
97+
const SaveButton = () => {
98+
return (
99+
<Button
100+
disabled={!isTestSuccessful || name === '' || !isDirty}
101+
variant="contained"
102+
onClick={async () => {
103+
const newDestination = await createOrUpdateDestination();
104+
if (!newDestination) {
105+
return;
106+
}
107+
addNotification({ message: 'Successfully updated destination', severity: 'success' });
108+
const latestDestinations = [
109+
...existingDestinations.filter(({ id }) => id !== newDestination.id),
110+
newDestination,
111+
];
112+
mutate(latestDestinations, {
113+
optimisticData: latestDestinations,
114+
revalidate: false,
115+
populateCache: false,
116+
});
117+
setIsDirty(false);
118+
}}
119+
>
120+
Save
121+
</Button>
122+
);
123+
};
124+
125+
const TestButton = () => {
126+
return (
127+
<Button
128+
disabled={isTesting || !isDirty}
129+
variant="contained"
130+
color="secondary"
131+
onClick={async () => {
132+
setIsTesting(true);
133+
const response = await testDestination({
134+
id: destination?.id,
135+
applicationId: activeApplicationId,
136+
type: 's3',
137+
name,
138+
config: {
139+
region,
140+
bucket,
141+
accessKeyId,
142+
secretAccessKey: secretAccessKey.type === 'known' ? secretAccessKey.value : undefined,
143+
},
144+
} as any); // TODO: fix typing
145+
setIsTesting(false);
146+
if (!response.ok) {
147+
addNotification({ message: response.errorMessage, severity: 'error' });
148+
setIsTestSuccessful(false);
149+
return;
150+
}
151+
if (response.data && response.data.success) {
152+
addNotification({ message: 'Successfully tested destination', severity: 'success' });
153+
setIsTestSuccessful(true);
154+
} else {
155+
addNotification({ message: `Failed testing destination: ${response.data.message}`, severity: 'error' });
156+
setIsTestSuccessful(false);
157+
}
158+
}}
159+
>
160+
{isTesting ? 'Testing...' : 'Test'}
161+
</Button>
162+
);
163+
};
164+
165+
if (isLoading) {
166+
return <Spinner />;
167+
}
168+
169+
return (
170+
<Card>
171+
<Stack direction="column" className="gap-4" sx={{ padding: '2rem' }}>
172+
<Stack direction="row" className="items-center justify-between w-full">
173+
<Stack direction="row" className="items-center justify-center gap-2">
174+
{getIcon(s3DestinationCardInfo.type, 35)}
175+
<Stack direction="column">
176+
<Typography variant="subtitle1">{s3DestinationCardInfo.name}</Typography>
177+
</Stack>
178+
</Stack>
179+
</Stack>
180+
181+
<Stack className="gap-2">
182+
<Typography variant="subtitle1">Destination Name</Typography>
183+
<TextField
184+
required={true}
185+
error={!isNew && name === ''}
186+
value={name}
187+
size="small"
188+
label="Name (must be unique)"
189+
variant="outlined"
190+
onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
191+
setName(event.target.value);
192+
setIsDirty(true);
193+
setIsTestSuccessful(false);
194+
}}
195+
/>
196+
</Stack>
197+
198+
<Stack className="gap-2">
199+
<Typography variant="subtitle1">Credentials</Typography>
200+
<TextField
201+
required={true}
202+
error={!isNew && accessKeyId === ''}
203+
value={accessKeyId}
204+
size="small"
205+
label="Access Key ID"
206+
variant="outlined"
207+
onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
208+
setAccessKeyId(event.target.value);
209+
setIsDirty(true);
210+
setIsTestSuccessful(false);
211+
}}
212+
/>
213+
<ExistingPasswordTextField
214+
required={true}
215+
error={!isNew && secretAccessKey.type === 'known' && secretAccessKey.value === ''}
216+
value={secretAccessKey}
217+
size="small"
218+
label="Secret Access Key"
219+
variant="outlined"
220+
type="password"
221+
onChange={(value: KnownOrUnknownValue) => {
222+
setSecretAccessKey(value);
223+
setIsDirty(true);
224+
setIsTestSuccessful(false);
225+
}}
226+
/>
227+
</Stack>
228+
229+
<Stack className="gap-2">
230+
<Typography variant="subtitle1">Region</Typography>
231+
<TextField
232+
required={true}
233+
error={!isNew && region === ''}
234+
value={region}
235+
size="small"
236+
label="AWS Region"
237+
variant="outlined"
238+
onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
239+
setRegion(event.target.value);
240+
setIsDirty(true);
241+
setIsTestSuccessful(false);
242+
}}
243+
/>
244+
</Stack>
245+
246+
<Stack className="gap-2">
247+
<Typography variant="subtitle1">Bucket</Typography>
248+
<TextField
249+
required={true}
250+
error={!isNew && bucket === ''}
251+
value={bucket}
252+
size="small"
253+
label="Bucket"
254+
variant="outlined"
255+
helperText='Bucket name without "s3://" prefix or trailing slash.'
256+
onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
257+
setBucket(event.target.value);
258+
setIsDirty(true);
259+
setIsTestSuccessful(false);
260+
}}
261+
/>
262+
</Stack>
263+
264+
<Stack direction="row" className="gap-2 justify-between">
265+
<Button
266+
variant="outlined"
267+
onClick={() => {
268+
router.back();
269+
}}
270+
>
271+
Back
272+
</Button>
273+
<Stack direction="row" className="gap-2">
274+
<TestButton />
275+
<SaveButton />
276+
</Stack>
277+
</Stack>
278+
</Stack>
279+
</Card>
280+
);
281+
}

Diff for: ‎docs/docs/destinations/s3.md

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
description: ''
3+
---
4+
5+
# S3
6+
7+
## Overview
8+
9+
| Feature | Available |
10+
| --------------------------------- | --------- |
11+
| Data normalization | No |
12+
| Data invalidation for Unified API | No |
13+
14+
## Setup
15+
16+
1. Go to Connectors -> Destinations.
17+
2. Click the Configure button on the S3 card.
18+
3. Enter your S3 credentials.
19+
20+
## Query patterns
21+
22+
Here are a few high-level best practices when working with tables that Supaglue lands:
23+
24+
- Avoid altering the schema of the existing columns
25+
26+
There are a few patterns (from simplest to more complex) for querying tables that Supaglue writes into your BigQuery dataset:
27+
28+
## Schema Evolution
29+
30+
Supaglue may evolve the destination table schemas from time to time. Drop your destination tables to evolve the schemas, and Supaglue will recreate the tables with the new schemas. Please reach out to ([[email protected]](mailto:[email protected])) if you need support for backward-compatible strategies.
31+
32+
## IP Whitelist
33+
34+
The following are Supaglue's CIDR ranges:
35+
36+
```
37+
54.214.243.61/32
38+
54.201.123.169/32
39+
44.226.37.107/32
40+
```

0 commit comments

Comments
 (0)