Skip to content

chore: new update for failing job on new vulnerabilities #37273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yatinappsmith merged 1 commit into from
Nov 7, 2024
Merged

chore: new update for failing job on new vulnerabilities #37273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion .github/workflows/test-vulnerabilities-data.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,20 @@ jobs:
"${{ github.event.pull_request.number }}" \
"${{ github.event.pull_request.html_url }}" \
"${{ github.run_id }}"


- name: Check for new vulnerabilities in Scout and Trivy files
if: always()
run: |
# Check if Scout vulnerabilities file is not empty
if [ -s "scout_new_vulnerabilities.csv" ]; then
echo "Scout vulnerabilities detected."
cat scout_new_vulnerabilities.csv
exit 1 # Fail the job if data exists
fi

# Check if Trivy vulnerabilities file is not empty
if [ -s "trivy_new_vulnerabilities.csv" ]; then
echo "Trivy vulnerabilities detected."
cat trivy_new_vulnerabilities.csv
exit 1 # Fail the job if data exists
fi
141 changes: 27 additions & 114 deletions scripts/scout_vulnerabilities_data.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,138 +21,51 @@ GITHUB_PR_LINK="$3"
GITHUB_RUN_ID="$4"
OLD_VULN_FILE="${5:-vulnerability_base_data.csv}"

# Function to install Docker Scout
install_docker_scout() {
echo "Installing Docker Scout..."
local attempts=0
while [ $attempts -lt 5 ]; do
echo "Attempt $((attempts + 1))..."
curl -fsSL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh -o install-scout.sh
sh install-scout.sh &> install_scout_log.txt
if [ $? -eq 0 ]; then
echo "Docker Scout installed successfully."
return 0
fi
echo "Attempt $((attempts + 1)) failed. Check install_scout_log.txt for details."
((attempts++))
sleep 2
done
echo "Error: Docker Scout installation failed after $attempts attempts."
exit 1
}

# Check if Docker is installed
if ! command -v docker &> /dev/null; then
echo "Error: Docker is not installed. Please install Docker and try again."
exit 1
fi

# Ensure Docker is running
if ! systemctl is-active --quiet docker; then
echo "Starting Docker..."
sudo systemctl start docker
fi

# Check if Docker Scout is installed
if ! command -v scout &> /dev/null; then
install_docker_scout
fi

# Prepare the output CSV file
CSV_OUTPUT_FILE="scout_vulnerabilities.csv"
rm -f "$CSV_OUTPUT_FILE"

# Extract the product name from the image name
case "$IMAGE" in
*appsmith/appsmith-ce:*) product_name="CE" ;;
*appsmith/appsmith-ee:*) product_name="EE" ;;
*appsmith/cloud-services:*) product_name="CLOUD" ;;
*) product_name="UNKNOWN" ;;
esac

# Fetch vulnerabilities and format the output correctly
docker scout cves "$IMAGE" | grep -E "✗ |CVE-" | awk -v product_name="$product_name" -F' ' '
{
# Check for valid vulnerability data and format it correctly
if ($2 != "" && $3 ~ /^CVE-/) {
# Extract severity level, CVE ID, and format output correctly
print $3","product_name",""SCOUT"","$2
}
}' | sort -u > "$CSV_OUTPUT_FILE"

# Check if the CSV output file is empty
[ -s "$CSV_OUTPUT_FILE" ] || echo "No vulnerabilities found for image: $IMAGE" > "$CSV_OUTPUT_FILE"

# Insert new vulnerabilities into the PostgreSQL database using psql
insert_vulns_into_db() {
local query_file="insert_vulns.sql"
echo "BEGIN;" > "$query_file"
# Compare each vulnerability with the database and store new ones in a CSV file
compare_and_store_vulns() {
local new_vulns_file="scout_new_vulnerabilities.csv"

echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header

while IFS=, read -r vurn_id product scanner_tool priority; do
if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
echo "Skipping empty vulnerability entry"
continue
fi

local pr_id="${GITHUB_PR_ID:-}"
local pr_link="${GITHUB_PR_LINK:-}"
local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
local comments="Initial vulnerability report"
local owner="John Doe"
local pod="Security"
# Clean up and trim spaces from input values
vurn_id=$(echo "$vurn_id" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//')
priority=$(echo "$priority" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//')
product=$(echo "$product" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]')
scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]')

# Clean up input values
vurn_id=$(echo "$vurn_id" | sed "s/'/''/g")
priority=$(echo "$priority" | sed "s/'/''/g")
product=$(echo "$product" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//')
scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//')
# Check if vurn_id exists in the database
existing_entry=$(psql -t -c "SELECT vurn_id FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)

# Fetch existing values for this vulnerability ID
existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)

# Process fetched data
if [[ -z "$existing_entry" ]]; then
combined_products="$product"
combined_scanner_tools="$scanner_tool"
# If vurn_id doesn't exist, store data in CSV file
echo "$vurn_id,$product,$scanner_tool,$priority" >> "$new_vulns_file"
echo "New vulnerability detected: $vurn_id"
else
IFS='|' read -r existing_product existing_scanner_tool <<< "$existing_entry"
combined_products=$(echo "$existing_product,$product" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//')
combined_scanner_tools=$(echo "$existing_scanner_tool,$scanner_tool" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//')
echo "Skipping existing vulnerability: $vurn_id"
fi

# Write the insert query to the SQL file
echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod)
VALUES ('$combined_products', '$combined_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod')
ON CONFLICT (vurn_id)
DO UPDATE SET
product = '$combined_products',
scanner_tool = '$combined_scanner_tools',
priority = EXCLUDED.priority,
pr_id = EXCLUDED.pr_id,
pr_link = EXCLUDED.pr_link,
github_run_id = EXCLUDED.github_run_id,
update_date = EXCLUDED.update_date,
comments = EXCLUDED.comments,
owner = EXCLUDED.owner,
pod = EXCLUDED.pod;" >> "$query_file"

done < "$CSV_OUTPUT_FILE"

echo "COMMIT;" >> "$query_file"
echo "Queries written to $query_file."

# Execute the SQL file and rollback on failure
if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then
echo "Vulnerabilities successfully inserted into the database."
# Print the contents of new vulnerabilities
if [ -s "$new_vulns_file" ]; then
echo "****************************************************************"
echo "New vulnerabilities stored in $new_vulns_file:"
cat "$new_vulns_file"
echo "****************************************************************"
else
echo "Error: Failed to insert vulnerabilities. Performing rollback."
echo "ROLLBACK;" | psql "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME"
exit 1
echo "No new vulnerabilities to store."
fi
}

# Check if there are vulnerabilities to process
if [ -s "$CSV_OUTPUT_FILE" ]; then
insert_vulns_into_db
compare_and_store_vulns
else
echo "No new vulnerabilities to insert."
fi
echo "No vulnerabilities to process."
fi
63 changes: 25 additions & 38 deletions scripts/trivy_vulnerabilities_data.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,30 +89,24 @@ else
fi


# Insert new vulnerabilities into PostgreSQL
insert_vulns_into_db() {
local query_file="insert_vulns.sql"
echo "BEGIN;" > "$query_file"
# Compare each vulnerability with the database and store new ones in a CSV file
compare_and_store_vulns() {
local new_vulns_file="trivy_new_vulnerabilities.csv"

echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header

while IFS=, read -r vurn_id product scanner_tool priority; do
if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
continue
fi

local pr_id="${GITHUB_PR_ID:-}"
local pr_link="${GITHUB_PR_LINK:-}"
local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
local comments="Initial vulnerability report"
local owner="John Doe"
local pod="Security"

# Remove spaces and redundant commas, and escape single quotes for SQL
vurn_id=$(echo "$vurn_id" | sed "s/'/''/g")
priority=$(echo "$priority" | sed "s/'/''/g")
product=$(echo "$product" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//')
scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//')

# Fetch existing product and scanner_tool values for the vulnerability
# Check if vurn_id exists in the database
existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)

if [ -n "$existing_entry" ]; then
Expand All @@ -131,40 +125,33 @@ insert_vulns_into_db() {
unique_scanner_tools="$scanner_tool"
fi

# Write the insert query to the SQL file
echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod)
VALUES ('$unique_products', '$unique_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod')
ON CONFLICT (vurn_id)
DO UPDATE SET
product = '$unique_products',
scanner_tool = '$unique_scanner_tools',
priority = EXCLUDED.priority,
pr_id = EXCLUDED.pr_id,
pr_link = EXCLUDED.pr_link,
github_run_id = EXCLUDED.github_run_id,
update_date = EXCLUDED.update_date,
comments = EXCLUDED.comments,
owner = EXCLUDED.owner,
pod = EXCLUDED.pod;" >> "$query_file"
done < "$NEW_VULN_FILE"
# If the vulnerability is new, store it in the CSV file
if [[ -z "$existing_entry" ]]; then
echo "$vurn_id,$unique_products,$unique_scanner_tools,$priority" >> "$new_vulns_file"
echo "New vulnerability detected: $vurn_id"
else
echo "Skipping existing vulnerability: $vurn_id"
fi

echo "COMMIT;" >> "$query_file"
done < "$NEW_VULN_FILE"

# Execute the SQL file
if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then
echo "Vulnerabilities successfully inserted into the database."
# Print the contents of new vulnerabilities
if [ -s "$new_vulns_file" ]; then
echo "****************************************************************"
echo "New vulnerabilities stored in $new_vulns_file:"
cat "$new_vulns_file"
echo "****************************************************************"
else
echo "Error: Failed to insert vulnerabilities. Check logs for details."
exit 1
echo "No new vulnerabilities to store."
fi
}

# Run insertion if vulnerabilities are found
# Run comparison and storage if vulnerabilities are found
if [ -s "$NEW_VULN_FILE" ]; then
insert_vulns_into_db
compare_and_store_vulns
else
echo "No vulnerabilities to insert."
echo "No vulnerabilities to process."
fi

# Cleanup
rm -f "trivy_vulnerabilities.json"
rm -f "trivy_vulnerabilities.json"