From 78adc2d5a6e9127afa09a2f6ebd8e32c47bfeccb Mon Sep 17 00:00:00 2001
From: katebygrace <kate.ruh@gmail.com>
Date: Tue, 16 Apr 2024 11:31:15 -0400
Subject: [PATCH 1/2] chore: fix refresh job

---
 .../analytics/SnowflakeRefreshSnowpipe.groovy |  1 -
 dataeng/resources/secrets-manager.sh          | 74 +++++++++++++------
 .../resources/snowflake-refresh-snowpipe.sh   | 28 ++++---
 3 files changed, 69 insertions(+), 34 deletions(-)

diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
index 64a8236c7..6c136fe31 100644
--- a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
+++ b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
@@ -82,7 +82,6 @@ class SnowflakeRefreshSnowpipe {
                 }
                 publishers common_publishers(allVars)
                 steps {
-                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
                     shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
                 }
             }
diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh
index 8251ccb55..1a8b8458f 100755
--- a/dataeng/resources/secrets-manager.sh
+++ b/dataeng/resources/secrets-manager.sh
@@ -1,38 +1,66 @@
 #!/bin/bash
 
+# Define the location of the script in the Jenkins workspace
+SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"
+
+echo "running setup"
+
+# Write the script content to the specified location
+cat <<EOF > "$SCRIPT_PATH"
+
+#!/usr/bin/env bash
+
 extract_value_from_json() {
-    local json="$1"
-    local key="$2"
-    local value=$(echo "$json" | jq -r ".$key")
+    local secret_json="\$1"
+    local secret_key="\$2"
+
+    local secret_value=\$(echo "$secret_json" | jq -r ".$secret_key")
 }
 
 fetch_whole_secret() {
-    local secret_name="$1"
-    local variable_name="$2"
-    local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
-    #set whole file as env var
-    declare "${secret_name%=*}=${secret_value}"
+    local secret_name="\$1"
+    local variable_name="\$2"
+    echo "\$secret_name"
+    echo "\$variable_name"
+    SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_name" --region "us-east-1" --output json)
+    echo "\$SECRET_JSON"
+    value=\$(echo "\$SECRET_JSON" | jq -r ".SecretString" 2>/dev/null)
+    echo "\$value"
+    echo "\$value" > "\$WORKSPACE/\$variable_name"
+    # Output the contents of the file to verify
+    cat "\$WORKSPACE/\$variable_name"
+    declare "\${variable_name%=*}=\${value}"    
+
+    #declare "$variable_name=$secret_value"
+    #declare "$variable_name=\"$secret_value\""
+    #what brian said to do
+    #declare "\${variable_name%=*}=\${value}"
 }
 
 fetch_specific_key() {
-    local secret_name="$1"
-    local key="$2"
+    local secret_name="\$1"
+    local key="\$2"
     local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
     local extracted_value=$(extract_value_from_json "$secret_value" "$key")
     declare "${key%=*}=${extracted_value}"
 }
 
-# Main script
-if [[ "$1" == "-w" ]]; then
-    if [ $# -ne 3 ]; then
-        echo "Usage: $0 -w <name_of_file> <name_of_variable>"
-        exit 1
+secret_script() {
+    echo "\$1"
+    echo "\$2"
+    echo "\$3"
+    if [[ "\$1" == "-w" ]]; then
+        if [ \$# -ne 3 ]; then
+            echo "Usage: $0 -w <name_of_file> <name_of_variable>"
+            exit 1
+        fi
+        fetch_whole_secret "\$2" "\$3"
+    else
+        if [ $# -ne 2 ]; then
+            echo "Usage: $0 <name_of_file> <name_of_key>"
+            exit 1
+        fi
+        fetch_specific_key "\$1" "\$2"
     fi
-    fetch_whole_secret "$2" "$3"
-else
-    if [ $# -ne 2 ]; then
-        echo "Usage: $0 <name_of_file> <name_of_key>"
-        exit 1
-    fi
-    fetch_specific_key "$1" "$2"
-fi
+}
+EOF
diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh
index 3cf265aa3..06ede1544 100644
--- a/dataeng/resources/snowflake-refresh-snowpipe.sh
+++ b/dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -ex
 
+
 # Creating Python virtual env
 PYTHON_VENV="python_venv"
 virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}"
@@ -10,16 +11,19 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
-# Source the secrets-manager.sh script to make the function available
-source $WORKSPACE/secrets-manager.sh
-# Fetch the secrets from AWS
-set +x
+# Define the location of the script in the Jenkins workspace
+SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"
+
 
+set +x
 
-secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8
-secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user
+echo "Running secrets manager script"
+python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user
+python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user
+#set -x
 
-set -x
+unset KEY_PATH
+unset PASSPHRASE_PATH
 
 python refresh_snowpipe.py \
     --user 'SNOWPIPE' \
@@ -28,6 +32,10 @@ python refresh_snowpipe.py \
     --pipe_name $PIPE_NAME \
     --table_name $TABLE_NAME \
     --delay $DELAY \
-    --limit $LIMIT
-    --key_file $KEY_PATH \
-    --passphrase_file $PASSPHRASE_PATH
+    --limit $LIMIT \
+    --key_file "$(cat "rsa_key_snowpipe_user")" \
+    --passphrase_file "$(cat "rsa_key_passphrase_snowpipe_user")"
+
+rm rsa_key_snowpipe_user
+rm rsa_key_passphrase_snowpipe_user
+

From 8c2d2fe177286ee98e1e46dedb2f8f4607c73010 Mon Sep 17 00:00:00 2001
From: katebygrace <kate.ruh@gmail.com>
Date: Tue, 7 May 2024 16:10:06 -0400
Subject: [PATCH 2/2] chore: revert secrets manager changes

---
 dataeng/resources/secrets-manager.sh          | 74 ++++++-------------
 .../resources/snowflake-refresh-snowpipe.sh   |  6 --
 2 files changed, 23 insertions(+), 57 deletions(-)
 mode change 100755 => 100644 dataeng/resources/secrets-manager.sh

diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh
old mode 100755
new mode 100644
index 1a8b8458f..5d065b8bf
--- a/dataeng/resources/secrets-manager.sh
+++ b/dataeng/resources/secrets-manager.sh
@@ -1,66 +1,38 @@
 #!/bin/bash
 
-# Define the location of the script in the Jenkins workspace
-SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"
-
-echo "running setup"
-
-# Write the script content to the specified location
-cat <<EOF > "$SCRIPT_PATH"
-
-#!/usr/bin/env bash
-
 extract_value_from_json() {
-    local secret_json="\$1"
-    local secret_key="\$2"
-
-    local secret_value=\$(echo "$secret_json" | jq -r ".$secret_key")
+    local json="$1"
+    local key="$2"
+    local value=$(echo "$json" | jq -r ".$key")
 }
 
 fetch_whole_secret() {
-    local secret_name="\$1"
-    local variable_name="\$2"
-    echo "\$secret_name"
-    echo "\$variable_name"
-    SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_name" --region "us-east-1" --output json)
-    echo "\$SECRET_JSON"
-    value=\$(echo "\$SECRET_JSON" | jq -r ".SecretString" 2>/dev/null)
-    echo "\$value"
-    echo "\$value" > "\$WORKSPACE/\$variable_name"
-    # Output the contents of the file to verify
-    cat "\$WORKSPACE/\$variable_name"
-    declare "\${variable_name%=*}=\${value}"    
-
-    #declare "$variable_name=$secret_value"
-    #declare "$variable_name=\"$secret_value\""
-    #what brian said to do
-    #declare "\${variable_name%=*}=\${value}"
+    local secret_name="$1"
+    local variable_name="$2"
+    local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
+    #set whole file as env var
+    declare "${secret_name%=*}=${secret_value}"
 }
 
 fetch_specific_key() {
-    local secret_name="\$1"
-    local key="\$2"
+    local secret_name="$1"
+    local key="$2"
     local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
     local extracted_value=$(extract_value_from_json "$secret_value" "$key")
     declare "${key%=*}=${extracted_value}"
 }
 
-secret_script() {
-    echo "\$1"
-    echo "\$2"
-    echo "\$3"
-    if [[ "\$1" == "-w" ]]; then
-        if [ \$# -ne 3 ]; then
-            echo "Usage: $0 -w <name_of_file> <name_of_variable>"
-            exit 1
-        fi
-        fetch_whole_secret "\$2" "\$3"
-    else
-        if [ $# -ne 2 ]; then
-            echo "Usage: $0 <name_of_file> <name_of_key>"
-            exit 1
-        fi
-        fetch_specific_key "\$1" "\$2"
+# Main script
+if [[ "$1" == "-w" ]]; then
+    if [ $# -ne 3 ]; then
+        echo "Usage: $0 -w <name_of_file> <name_of_variable>"
+        exit 1
     fi
-}
-EOF
+    fetch_whole_secret "$2" "$3"
+else
+    if [ $# -ne 2 ]; then
+        echo "Usage: $0 <name_of_file> <name_of_key>"
+        exit 1
+    fi
+    fetch_specific_key "$1" "$2"
+fi
\ No newline at end of file
diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh
index 06ede1544..543c0c97f 100644
--- a/dataeng/resources/snowflake-refresh-snowpipe.sh
+++ b/dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-
 # Creating Python virtual env
 PYTHON_VENV="python_venv"
 virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}"
@@ -11,13 +10,8 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
-# Define the location of the script in the Jenkins workspace
-SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"
-
 
-set +x
 
-echo "Running secrets manager script"
 python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user
 python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user
 #set -x