Skip to content

Commit d08dcff

Browse files
ArpitShukla12ArpitShukla12
authored
Merge pull request #115 from ArpitShukla12/feat/integration_support
Feat/integration support
2 parents c941618 + 8ff1a35 commit d08dcff

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+27095
-8335
lines changed

‎.github/workflows/test-action.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Test Action
22

33
on:
44
pull_request:
5-
types: [ opened, edited, synchronize, reopened, closed ]
5+
types: [opened, edited, synchronize, reopened, closed]
66

77
jobs:
88
get-downstream-assets:
@@ -28,4 +28,4 @@ jobs:
2828
main: DBT-DEMO-PROD
2929
beta: Wide World Importers PE1
3030
test-action: Wide World Importers PE1
31-
IGNORE_MODEL_ALIAS_MATCHING: true
31+
IGNORE_MODEL_ALIAS_MATCHING: true

‎adapters/api/create-resource.js

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import { v4 as uuidv4 } from "uuid";
2+
import fetch from "node-fetch";
3+
import stringify from "json-stringify-safe";
4+
import {
5+
ATLAN_INSTANCE_URL,
6+
ATLAN_API_TOKEN,
7+
} from "../utils/get-environment-variables.js";
8+
9+
export default async function createResource(
10+
guid,
11+
name,
12+
link,
13+
sendSegmentEventOfIntegration
14+
) {
15+
var myHeaders = {
16+
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
17+
"Content-Type": "application/json",
18+
};
19+
20+
var raw = stringify({
21+
entities: [
22+
{
23+
typeName: "Link",
24+
attributes: {
25+
qualifiedName: uuidv4(),
26+
name,
27+
link,
28+
tenantId: "default",
29+
},
30+
relationshipAttributes: {
31+
asset: {
32+
guid,
33+
},
34+
},
35+
},
36+
],
37+
});
38+
39+
var requestOptions = {
40+
method: "POST",
41+
headers: myHeaders,
42+
body: raw,
43+
};
44+
45+
var response = await fetch(
46+
`${ATLAN_INSTANCE_URL}/api/meta/entity/bulk`,
47+
requestOptions
48+
)
49+
.then((e) => e.json())
50+
.catch((err) => {
51+
console.log(err);
52+
sendSegmentEventOfIntegration({
53+
action: "dbt_ci_action_failure",
54+
properties: {
55+
reason: "failed_to_create_resource",
56+
asset_name: name, // This should change
57+
msg: err,
58+
},
59+
});
60+
});
61+
62+
if (response?.errorCode) return null;
63+
return response;
64+
}

‎adapters/api/get-asset.js

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import fetch from "node-fetch";
2+
import stringify from "json-stringify-safe";
3+
import {
4+
getErrorModelNotFound,
5+
getErrorDoesNotMaterialize,
6+
} from "../templates/atlan.js";
7+
import {
8+
ATLAN_INSTANCE_URL,
9+
ATLAN_API_TOKEN,
10+
} from "../utils/get-environment-variables.js";
11+
12+
export default async function getAsset({
13+
name,
14+
sendSegmentEventOfIntegration,
15+
environment,
16+
integration,
17+
}) {
18+
var myHeaders = {
19+
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
20+
"Content-Type": "application/json",
21+
};
22+
23+
var raw = stringify({
24+
dsl: {
25+
from: 0,
26+
size: 21,
27+
query: {
28+
bool: {
29+
must: [
30+
{
31+
match: {
32+
__state: "ACTIVE",
33+
},
34+
},
35+
{
36+
match: {
37+
"__typeName.keyword": "DbtModel",
38+
},
39+
},
40+
{
41+
match: {
42+
"name.keyword": name,
43+
},
44+
},
45+
...(environment
46+
? [
47+
{
48+
term: {
49+
"assetDbtEnvironmentName.keyword": environment,
50+
},
51+
},
52+
]
53+
: []),
54+
],
55+
},
56+
},
57+
},
58+
attributes: [
59+
"name",
60+
"description",
61+
"userDescription",
62+
"sourceURL",
63+
"qualifiedName",
64+
"connectorName",
65+
"certificateStatus",
66+
"certificateUpdatedBy",
67+
"certificateUpdatedAt",
68+
"ownerUsers",
69+
"ownerGroups",
70+
"classificationNames",
71+
"meanings",
72+
"dbtModelSqlAssets",
73+
],
74+
relationAttributes: [
75+
"name",
76+
"description",
77+
"assetDbtProjectName",
78+
"assetDbtEnvironmentName",
79+
"connectorName",
80+
"certificateStatus",
81+
],
82+
});
83+
84+
var requestOptions = {
85+
method: "POST",
86+
headers: myHeaders,
87+
body: raw,
88+
};
89+
90+
var response = await fetch(
91+
`${ATLAN_INSTANCE_URL}/api/meta/search/indexsearch#findAssetByExactName`,
92+
requestOptions
93+
)
94+
.then((e) => e.json())
95+
.catch((err) => {
96+
sendSegmentEventOfIntegration({
97+
action: "dbt_ci_action_failure",
98+
properties: {
99+
reason: "failed_to_get_asset",
100+
asset_name: name,
101+
msg: err,
102+
},
103+
});
104+
});
105+
106+
if (!response?.entities?.length) {
107+
return {
108+
error: getErrorModelNotFound(name),
109+
};
110+
}
111+
112+
if (Array.isArray(response.entities)) {
113+
response.entities.sort((entityA, entityB) => {
114+
const hasDbtModelSqlAssetsA =
115+
entityA.attributes.dbtModelSqlAssets &&
116+
entityA.attributes.dbtModelSqlAssets.length > 0;
117+
const hasDbtModelSqlAssetsB =
118+
entityB.attributes.dbtModelSqlAssets &&
119+
entityB.attributes.dbtModelSqlAssets.length > 0;
120+
121+
if (hasDbtModelSqlAssetsA && !hasDbtModelSqlAssetsB) {
122+
return -1; // entityA comes before entityB
123+
} else if (!hasDbtModelSqlAssetsA && hasDbtModelSqlAssetsB) {
124+
return 1; // entityB comes before entityA
125+
}
126+
127+
// Primary sorting criterion: Latest createTime comes first
128+
if (entityA.createTime > entityB.createTime) {
129+
return -1;
130+
} else if (entityA.createTime < entityB.createTime) {
131+
return 1;
132+
}
133+
134+
return 0; // No difference in sorting for these two entities
135+
});
136+
}
137+
138+
if (!response?.entities[0]?.attributes?.dbtModelSqlAssets?.length > 0)
139+
return {
140+
error: getErrorDoesNotMaterialize(
141+
name,
142+
ATLAN_INSTANCE_URL,
143+
response,
144+
integration
145+
),
146+
};
147+
148+
return response.entities[0];
149+
}

‎adapters/api/get-classifications.js

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import fetch from "node-fetch";
2+
import {
3+
ATLAN_INSTANCE_URL,
4+
ATLAN_API_TOKEN,
5+
} from "../utils/get-environment-variables.js";
6+
7+
export default async function getClassifications({
8+
sendSegmentEventOfIntegration,
9+
}) {
10+
var myHeaders = {
11+
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
12+
"Content-Type": "application/json",
13+
};
14+
15+
var requestOptions = {
16+
method: "GET",
17+
headers: myHeaders,
18+
redirect: "follow",
19+
};
20+
21+
var response = await fetch(
22+
`${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`,
23+
requestOptions
24+
)
25+
.then((e) => e.json())
26+
.catch((err) => {
27+
sendSegmentEventOfIntegration({
28+
action: "dbt_ci_action_failure",
29+
properties: {
30+
reason: "failed_to_get_classifications",
31+
msg: err,
32+
},
33+
});
34+
});
35+
36+
return response?.classificationDefs;
37+
}

‎adapters/api/get-downstream-assets.js

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import fetch from "node-fetch";
2+
import {
3+
getConnectorImage,
4+
getCertificationImage,
5+
getImageURL,
6+
} from "../utils/index.js";
7+
import stringify from "json-stringify-safe";
8+
import {
9+
ATLAN_INSTANCE_URL,
10+
ATLAN_API_TOKEN,
11+
} from "../utils/get-environment-variables.js";
12+
13+
const ASSETS_LIMIT = 100;
14+
15+
export default async function getDownstreamAssets(
16+
asset,
17+
guid,
18+
totalModifiedFiles,
19+
sendSegmentEventOfIntegration,
20+
integration
21+
) {
22+
var myHeaders = {
23+
authorization: `Bearer ${ATLAN_API_TOKEN}`,
24+
"content-type": "application/json",
25+
};
26+
27+
var raw = stringify({
28+
guid: guid,
29+
size: Math.max(Math.ceil(ASSETS_LIMIT / totalModifiedFiles), 1),
30+
from: 0,
31+
depth: 21,
32+
direction: "OUTPUT",
33+
entityFilters: {
34+
condition: "AND",
35+
criterion: [
36+
{
37+
attributeName: "__typeName",
38+
operator: "not_contains",
39+
attributeValue: "Process",
40+
},
41+
{
42+
attributeName: "__state",
43+
operator: "eq",
44+
attributeValue: "ACTIVE",
45+
},
46+
],
47+
},
48+
attributes: [
49+
"name",
50+
"description",
51+
"userDescription",
52+
"sourceURL",
53+
"qualifiedName",
54+
"connectorName",
55+
"certificateStatus",
56+
"certificateUpdatedBy",
57+
"certificateUpdatedAt",
58+
"ownerUsers",
59+
"ownerGroups",
60+
"classificationNames",
61+
"meanings",
62+
],
63+
excludeMeanings: false,
64+
excludeClassifications: false,
65+
});
66+
67+
var requestOptions = {
68+
method: "POST",
69+
headers: myHeaders,
70+
body: raw,
71+
};
72+
73+
var handleError = (err) => {
74+
const comment = `### ${getConnectorImage(
75+
asset.attributes.connectorName
76+
)} [${asset.displayText}](${ATLAN_INSTANCE_URL}/assets/${
77+
asset.guid
78+
}/overview?utm_source=dbt_${integration}_action) ${
79+
asset.attributes?.certificateStatus
80+
? getCertificationImage(asset.attributes.certificateStatus)
81+
: ""
82+
}
83+
84+
_Failed to fetch impacted assets._
85+
86+
${getImageURL(
87+
"atlan-logo",
88+
15,
89+
15
90+
)} [View lineage in Atlan](${ATLAN_INSTANCE_URL}/assets/${
91+
asset.guid
92+
}/lineage/overview?utm_source=dbt_${integration}_action)`;
93+
94+
sendSegmentEventOfIntegration({
95+
action: "dbt_ci_action_failure",
96+
properties: {
97+
reason: "failed_to_fetch_lineage",
98+
asset_guid: asset.guid,
99+
asset_name: asset.name,
100+
asset_typeName: asset.typeName,
101+
msg: err,
102+
},
103+
});
104+
105+
return comment;
106+
};
107+
108+
var response = await fetch(
109+
`${ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
110+
requestOptions
111+
)
112+
.then((e) => {
113+
if (e.status === 200) {
114+
return e.json();
115+
} else {
116+
throw e;
117+
}
118+
})
119+
.catch((err) => {
120+
return {
121+
error: handleError(err),
122+
};
123+
});
124+
if (response.error) return response;
125+
126+
return response;
127+
}

0 commit comments

Comments
 (0)