From e8dcb1f7f78fdc67aa5a3163dfbe10e66c523439 Mon Sep 17 00:00:00 2001 From: Sagar Khalasi Date: Thu, 7 Nov 2024 13:26:05 +0530 Subject: [PATCH] new update for failing job --- .../workflows/test-vulnerabilities-data.yml | 18 ++- scripts/scout_vulnerabilities_data.sh | 141 ++++-------------- scripts/trivy_vulnerabilities_data.sh | 63 ++++---- 3 files changed, 69 insertions(+), 153 deletions(-) diff --git a/.github/workflows/test-vulnerabilities-data.yml b/.github/workflows/test-vulnerabilities-data.yml index 2ba1b8ced6f4..333458264cf7 100644 --- a/.github/workflows/test-vulnerabilities-data.yml +++ b/.github/workflows/test-vulnerabilities-data.yml @@ -61,4 +61,20 @@ jobs: "${{ github.event.pull_request.number }}" \ "${{ github.event.pull_request.html_url }}" \ "${{ github.run_id }}" - \ No newline at end of file + + - name: Check for new vulnerabilities in Scout and Trivy files + if: always() + run: | + # Check if Scout vulnerabilities file is not empty + if [ -s "scout_new_vulnerabilities.csv" ]; then + echo "Scout vulnerabilities detected." + cat scout_new_vulnerabilities.csv + exit 1 # Fail the job if data exists + fi + + # Check if Trivy vulnerabilities file is not empty + if [ -s "trivy_new_vulnerabilities.csv" ]; then + echo "Trivy vulnerabilities detected." + cat trivy_new_vulnerabilities.csv + exit 1 # Fail the job if data exists + fi \ No newline at end of file diff --git a/scripts/scout_vulnerabilities_data.sh b/scripts/scout_vulnerabilities_data.sh index 563534ad9b5c..9f2b91b223cb 100755 --- a/scripts/scout_vulnerabilities_data.sh +++ b/scripts/scout_vulnerabilities_data.sh @@ -21,72 +21,11 @@ GITHUB_PR_LINK="$3" GITHUB_RUN_ID="$4" OLD_VULN_FILE="${5:-vulnerability_base_data.csv}" -# Function to install Docker Scout -install_docker_scout() { - echo "Installing Docker Scout..." - local attempts=0 - while [ $attempts -lt 5 ]; do - echo "Attempt $((attempts + 1))..." - curl -fsSL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh -o install-scout.sh - sh install-scout.sh &> install_scout_log.txt - if [ $? -eq 0 ]; then - echo "Docker Scout installed successfully." - return 0 - fi - echo "Attempt $((attempts + 1)) failed. Check install_scout_log.txt for details." - ((attempts++)) - sleep 2 - done - echo "Error: Docker Scout installation failed after $attempts attempts." - exit 1 -} - -# Check if Docker is installed -if ! command -v docker &> /dev/null; then - echo "Error: Docker is not installed. Please install Docker and try again." - exit 1 -fi - -# Ensure Docker is running -if ! systemctl is-active --quiet docker; then - echo "Starting Docker..." - sudo systemctl start docker -fi - -# Check if Docker Scout is installed -if ! command -v scout &> /dev/null; then - install_docker_scout -fi - -# Prepare the output CSV file -CSV_OUTPUT_FILE="scout_vulnerabilities.csv" -rm -f "$CSV_OUTPUT_FILE" - -# Extract the product name from the image name -case "$IMAGE" in - *appsmith/appsmith-ce:*) product_name="CE" ;; - *appsmith/appsmith-ee:*) product_name="EE" ;; - *appsmith/cloud-services:*) product_name="CLOUD" ;; - *) product_name="UNKNOWN" ;; -esac - -# Fetch vulnerabilities and format the output correctly -docker scout cves "$IMAGE" | grep -E "✗ |CVE-" | awk -v product_name="$product_name" -F' ' ' -{ - # Check for valid vulnerability data and format it correctly - if ($2 != "" && $3 ~ /^CVE-/) { - # Extract severity level, CVE ID, and format output correctly - print $3","product_name",""SCOUT"","$2 - } -}' | sort -u > "$CSV_OUTPUT_FILE" - -# Check if the CSV output file is empty -[ -s "$CSV_OUTPUT_FILE" ] || echo "No vulnerabilities found for image: $IMAGE" > "$CSV_OUTPUT_FILE" - -# Insert new vulnerabilities into the PostgreSQL database using psql -insert_vulns_into_db() { - local query_file="insert_vulns.sql" - echo "BEGIN;" > "$query_file" +# Compare each vulnerability with the database and store new ones in a CSV file +compare_and_store_vulns() { + local new_vulns_file="scout_new_vulnerabilities.csv" + + echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header while IFS=, read -r vurn_id product scanner_tool priority; do if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then @@ -94,65 +33,39 @@ insert_vulns_into_db() { continue fi - local pr_id="${GITHUB_PR_ID:-}" - local pr_link="${GITHUB_PR_LINK:-}" - local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - local comments="Initial vulnerability report" - local owner="John Doe" - local pod="Security" + # Clean up and trim spaces from input values + vurn_id=$(echo "$vurn_id" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//') + priority=$(echo "$priority" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//') + product=$(echo "$product" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]') + scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]') - # Clean up input values - vurn_id=$(echo "$vurn_id" | sed "s/'/''/g") - priority=$(echo "$priority" | sed "s/'/''/g") - product=$(echo "$product" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//') - scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//') + # Check if vurn_id exists in the database + existing_entry=$(psql -t -c "SELECT vurn_id FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null) - # Fetch existing values for this vulnerability ID - existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null) - - # Process fetched data if [[ -z "$existing_entry" ]]; then - combined_products="$product" - combined_scanner_tools="$scanner_tool" + # If vurn_id doesn't exist, store data in CSV file + echo "$vurn_id,$product,$scanner_tool,$priority" >> "$new_vulns_file" + echo "New vulnerability detected: $vurn_id" else - IFS='|' read -r existing_product existing_scanner_tool <<< "$existing_entry" - combined_products=$(echo "$existing_product,$product" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//') - combined_scanner_tools=$(echo "$existing_scanner_tool,$scanner_tool" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//') + echo "Skipping existing vulnerability: $vurn_id" fi - # Write the insert query to the SQL file - echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod) - VALUES ('$combined_products', '$combined_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod') - ON CONFLICT (vurn_id) - DO UPDATE SET - product = '$combined_products', - scanner_tool = '$combined_scanner_tools', - priority = EXCLUDED.priority, - pr_id = EXCLUDED.pr_id, - pr_link = EXCLUDED.pr_link, - github_run_id = EXCLUDED.github_run_id, - update_date = EXCLUDED.update_date, - comments = EXCLUDED.comments, - owner = EXCLUDED.owner, - pod = EXCLUDED.pod;" >> "$query_file" - done < "$CSV_OUTPUT_FILE" - echo "COMMIT;" >> "$query_file" - echo "Queries written to $query_file." - - # Execute the SQL file and rollback on failure - if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then - echo "Vulnerabilities successfully inserted into the database." + # Print the contents of new vulnerabilities + if [ -s "$new_vulns_file" ]; then + echo "****************************************************************" + echo "New vulnerabilities stored in $new_vulns_file:" + cat "$new_vulns_file" + echo "****************************************************************" else - echo "Error: Failed to insert vulnerabilities. Performing rollback." - echo "ROLLBACK;" | psql "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" - exit 1 + echo "No new vulnerabilities to store." fi } +# Check if there are vulnerabilities to process if [ -s "$CSV_OUTPUT_FILE" ]; then - insert_vulns_into_db + compare_and_store_vulns else - echo "No new vulnerabilities to insert." -fi \ No newline at end of file + echo "No vulnerabilities to process." +fi diff --git a/scripts/trivy_vulnerabilities_data.sh b/scripts/trivy_vulnerabilities_data.sh index 82a337bb4bb8..72eca96bf8c9 100755 --- a/scripts/trivy_vulnerabilities_data.sh +++ b/scripts/trivy_vulnerabilities_data.sh @@ -89,30 +89,24 @@ else fi -# Insert new vulnerabilities into PostgreSQL -insert_vulns_into_db() { - local query_file="insert_vulns.sql" - echo "BEGIN;" > "$query_file" +# Compare each vulnerability with the database and store new ones in a CSV file +compare_and_store_vulns() { + local new_vulns_file="trivy_new_vulnerabilities.csv" + + echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header while IFS=, read -r vurn_id product scanner_tool priority; do if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then continue fi - local pr_id="${GITHUB_PR_ID:-}" - local pr_link="${GITHUB_PR_LINK:-}" - local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - local comments="Initial vulnerability report" - local owner="John Doe" - local pod="Security" - # Remove spaces and redundant commas, and escape single quotes for SQL vurn_id=$(echo "$vurn_id" | sed "s/'/''/g") priority=$(echo "$priority" | sed "s/'/''/g") product=$(echo "$product" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//') scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//') - # Fetch existing product and scanner_tool values for the vulnerability + # Check if vurn_id exists in the database existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null) if [ -n "$existing_entry" ]; then @@ -131,40 +125,33 @@ insert_vulns_into_db() { unique_scanner_tools="$scanner_tool" fi - # Write the insert query to the SQL file - echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod) - VALUES ('$unique_products', '$unique_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod') - ON CONFLICT (vurn_id) - DO UPDATE SET - product = '$unique_products', - scanner_tool = '$unique_scanner_tools', - priority = EXCLUDED.priority, - pr_id = EXCLUDED.pr_id, - pr_link = EXCLUDED.pr_link, - github_run_id = EXCLUDED.github_run_id, - update_date = EXCLUDED.update_date, - comments = EXCLUDED.comments, - owner = EXCLUDED.owner, - pod = EXCLUDED.pod;" >> "$query_file" - done < "$NEW_VULN_FILE" + # If the vulnerability is new, store it in the CSV file + if [[ -z "$existing_entry" ]]; then + echo "$vurn_id,$unique_products,$unique_scanner_tools,$priority" >> "$new_vulns_file" + echo "New vulnerability detected: $vurn_id" + else + echo "Skipping existing vulnerability: $vurn_id" + fi - echo "COMMIT;" >> "$query_file" + done < "$NEW_VULN_FILE" - # Execute the SQL file - if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then - echo "Vulnerabilities successfully inserted into the database." + # Print the contents of new vulnerabilities + if [ -s "$new_vulns_file" ]; then + echo "****************************************************************" + echo "New vulnerabilities stored in $new_vulns_file:" + cat "$new_vulns_file" + echo "****************************************************************" else - echo "Error: Failed to insert vulnerabilities. Check logs for details." - exit 1 + echo "No new vulnerabilities to store." fi } -# Run insertion if vulnerabilities are found +# Run comparison and storage if vulnerabilities are found if [ -s "$NEW_VULN_FILE" ]; then - insert_vulns_into_db + compare_and_store_vulns else - echo "No vulnerabilities to insert." + echo "No vulnerabilities to process." fi # Cleanup -rm -f "trivy_vulnerabilities.json" \ No newline at end of file +rm -f "trivy_vulnerabilities.json"