Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

C #140

Closed
wants to merge 3 commits into from
Closed

C #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e618946
extract config before running complex regex
sharma-shreyas Oct 15, 2024
d0b2a13
update dist/index file
sharma-shreyas Oct 15, 2024
8bc6250
Add files via upload
sharma-shreyas Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 75 additions & 9 deletions adapters/integrations/github-integration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -683,24 +683,90 @@ export default class GitHubIntegration extends IntegrationInterface {
filePath,
});

logger.withInfo(
`Successfully fetched file contents. File size: ${fileContents.length} bytes`,
integrationName,
headSHA,
"getAssetName"
);

if (fileContents) {
var matches = regExp.exec(fileContents);
if (matches) {
logger.withInfo(
`Found a match: ${matches[1].trim()}`,
integrationName,
headSHA,
"getAssetName"
);
return matches[1].trim();
logger.withInfo(
"Starting regex matching",
integrationName,
headSHA,
"getAssetName"
);
const startRegex =
/{{\s*config\s*\(/im;
const startMatch = fileContents.match(startRegex);
let configSection = ''
if (startMatch) {
const startIndex = startMatch.index;
const openParensIndex = fileContents.indexOf('(', startIndex) + 1;
let openParensCount = 1;
let endIndex = openParensIndex;

while (openParensCount > 0 && endIndex < fileContents.length) {
const char = fileContents[endIndex];

if (char === '(') {
openParensCount++;
} else if (char === ')') {
openParensCount--;
}
endIndex++;
}

const endMarker = '}}';
const finalEndIndex = fileContents.indexOf(endMarker, endIndex) + endMarker.length;

configSection = fileContents.substring(startIndex, finalEndIndex);
logger.withInfo(
"Extracted config section",
integrationName,
headSHA,
"getAssetName"
);

if (configSection){
logger.withInfo(
"Executing final regex",
integrationName,
headSHA,
"getAssetName"
);

var matches = regExp.exec(configSection);

logger.withInfo(
"Successfully executed regex matching",
integrationName,
headSHA,
"getAssetName"
);

}
if (matches) {
logger.withInfo(
`Found a match: ${matches[1].trim()}`,
integrationName,
headSHA,
"getAssetName"
);

return matches[1].trim();
}
}
}

logger.withInfo(
`Using filename as asset name: ${fileName}`,
integrationName,
headSHA,
"getAssetName"
);

return fileName;
} catch (error) {
logger.withError(
Expand Down
84 changes: 75 additions & 9 deletions dist/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26179,24 +26179,90 @@ class GitHubIntegration extends IntegrationInterface {
filePath,
});

logger_logger.withInfo(
`Successfully fetched file contents. File size: ${fileContents.length} bytes`,
integrationName,
headSHA,
"getAssetName"
);

if (fileContents) {
var matches = regExp.exec(fileContents);
if (matches) {
logger_logger.withInfo(
`Found a match: ${matches[1].trim()}`,
integrationName,
headSHA,
"getAssetName"
);
return matches[1].trim();
logger_logger.withInfo(
"Starting regex matching",
integrationName,
headSHA,
"getAssetName"
);
const startRegex =
/{{\s*config\s*\(/im;
const startMatch = fileContents.match(startRegex);
let configSection = ''
if (startMatch) {
const startIndex = startMatch.index;
const openParensIndex = fileContents.indexOf('(', startIndex) + 1;
let openParensCount = 1;
let endIndex = openParensIndex;

while (openParensCount > 0 && endIndex < fileContents.length) {
const char = fileContents[endIndex];

if (char === '(') {
openParensCount++;
} else if (char === ')') {
openParensCount--;
}
endIndex++;
}

const endMarker = '}}';
const finalEndIndex = fileContents.indexOf(endMarker, endIndex) + endMarker.length;

configSection = fileContents.substring(startIndex, finalEndIndex);
logger_logger.withInfo(
"Extracted config section",
integrationName,
headSHA,
"getAssetName"
);

if (configSection){
logger_logger.withInfo(
"Executing final regex",
integrationName,
headSHA,
"getAssetName"
);

var matches = regExp.exec(configSection);

logger_logger.withInfo(
"Successfully executed regex matching",
integrationName,
headSHA,
"getAssetName"
);

}
if (matches) {
logger_logger.withInfo(
`Found a match: ${matches[1].trim()}`,
integrationName,
headSHA,
"getAssetName"
);

return matches[1].trim();
}
}
}

logger_logger.withInfo(
`Using filename as asset name: ${fileName}`,
integrationName,
headSHA,
"getAssetName"
);

return fileName;
} catch (error) {
logger_logger.withError(
Expand Down
66 changes: 66 additions & 0 deletions test/models/staging/src_credit_consent_check_history.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{{config(materialized = 'incremental'
,incremental_strategy = 'merge'
,alias = 'credit_consent_check_history'
,unique_key = ['application_number','bureau_name','pulled_date']
,tags = ['genpact']
)
}}

WITH

/*
* WHEN WHO WHAT
* ------------ -------------- --------------------------------------------------------------------------------------
* 12-SEP-2024 PCROSS SNF-7100 initial
* 18-SEP-2024 JSTOPANSKY SNF-7100 updating reference to external table instead of src table
*
* Note : table credit_consent_check_history is being incrementally updated ('merge') using data
* in external table EXT_CREDIT_CONSENT_CHECK_HISTORY.
* the external table is based on a csv file (genpact.csv) in AWS location "s3://prod-data-services-genpact/"
* the loaded data is type VARIANT -> convert to VARCHAR
*
*/

ext AS (
SELECT
value:c1::VARCHAR AS application_number
,value:c2::VARCHAR AS report_status
,value:c3::VARCHAR AS bureau_name_hierarchy
,value:c4::VARCHAR AS bureau_name
,value:c5::VARCHAR AS score_band
,value:c6::VARCHAR AS application_status
,value:c7::VARCHAR AS pulled_date
,value:c8::VARCHAR AS application_created_date
,value:c9::VARCHAR AS score
,value:c10::VARCHAR AS application_final_decision_date
,value:c11::VARCHAR AS application_last_modified_date
,value:c12::VARCHAR AS state
,value:c13::VARCHAR AS zip
,value:c14::VARCHAR AS bureau_dob_year
,metadata$file_last_modified AS s3_last_modified_at
FROM
{{source('SRC_GENPACT','EXT_CREDIT_CONSENT_CHECK_HISTORY')}}
{% if is_incremental() %}
WHERE metadata$file_last_modified > (SELECT NVL(MAX(s3_last_modified_at), to_date('1999-01-01')) FROM {{this}})
{% endif %}
)

SELECT
ext.application_number
,ext.report_status
,ext.bureau_name_hierarchy
,ext.bureau_name
,ext.score_band
,ext.application_status
,ext.pulled_date
,ext.application_created_date
,ext.score
,ext.application_final_decision_date
,ext.application_last_modified_date
,ext.state
,ext.zip
,ext.bureau_dob_year
,ext.s3_last_modified_at
,current_timestamp() AS loaded_at

FROM ext
Loading