From 16dccaa4a2b4a75a91574f9143029ceb469d9851 Mon Sep 17 00:00:00 2001 From: Monica Pastor Date: Tue, 20 Dec 2022 15:52:21 +0100 Subject: [PATCH 1/3] bug: fix export pdf #74 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a6d385f..53ae5e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && \ apt-get clean RUN apt-get update && \ - apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf && \ + apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ apt-get install -y && \ apt-get clean && \ update-ca-certificates -f From ffa870f83ccc4b35bdbda2c5a6fed34992a8df56 Mon Sep 17 00:00:00 2001 From: Monica Pastor Date: Tue, 20 Dec 2022 17:33:20 +0100 Subject: [PATCH 2/3] chore: upgrade jadx version --- Dockerfile | 138 ++++++++++++++++++++++++++--------------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53ae5e8..355833f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,69 @@ -FROM python:3.9.7-buster@sha256:79a631c93960c5919f27f3403e734ec19b130008370a5f902141bcff2e6d6f4c -# Update and package installation -RUN apt-get update && \ - apt-get clean && \ - apt-get install -y ca-certificates-java --no-install-recommends && \ - apt-get clean - -RUN apt-get update && \ - apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ - apt-get install -y && \ - apt-get clean && \ - update-ca-certificates -f - -# Get JADX Tool -ENV JADX_VERSION 1.2.0 - -RUN \ - wget "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" && \ - unzip "jadx-$JADX_VERSION.zip" - -# Create user -ARG uid=1000 -ARG gid=1000 -ARG user=app -ARG group=app - -RUN groupadd -g ${gid} ${group} \ - && useradd -u ${uid} -g ${group} -s /bin/sh ${user} - -# Copy entrypoints -COPY entrypoint/web_entrypoint.sh \ - entrypoint/worker_entrypoint.sh / - -RUN chown ${uid}:${gid} /web_entrypoint.sh /worker_entrypoint.sh && \ - chmod u+x /web_entrypoint.sh /worker_entrypoint.sh - -# Create a directory in the container in /app -RUN mkdir /app -# Copy all to /app directory -COPY . /app - -# Use /app as the workdir -WORKDIR /app - - -# Upgrade pip and install python dependencies -RUN pip install --upgrade pip \ - && pip install -r requirements.txt - -# Encoding configuration -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV PYTHONIOENCODING utf8 - -# Logs -RUN mkdir -p app/logs -RUN touch app/logs/debug.log - -# RabbitMQ directory -RUN mkdir -p rabbitmq/logs - -# Set the permissions to the user -RUN chown -R ${uid}:${gid} /app - -# Run the container as non-root user -USER ${uid} - -# Expose the 8000 port -EXPOSE 8000 +FROM python:3.9.7-buster@sha256:79a631c93960c5919f27f3403e734ec19b130008370a5f902141bcff2e6d6f4c +# Update and package installation +RUN apt-get update && \ + apt-get clean && \ + apt-get install -y ca-certificates-java --no-install-recommends && \ + apt-get clean + +RUN apt-get update && \ + apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ + apt-get install -y && \ + apt-get clean && \ + update-ca-certificates -f + +# Get JADX Tool +ENV JADX_VERSION 1.4.5 + +RUN \ + wget "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" && \ + unzip "jadx-$JADX_VERSION.zip" + +# Create user +ARG uid=1000 +ARG gid=1000 +ARG user=app +ARG group=app + +RUN groupadd -g ${gid} ${group} \ + && useradd -u ${uid} -g ${group} -s /bin/sh ${user} + +# Copy entrypoints +COPY entrypoint/web_entrypoint.sh \ + entrypoint/worker_entrypoint.sh / + +RUN chown ${uid}:${gid} /web_entrypoint.sh /worker_entrypoint.sh && \ + chmod u+x /web_entrypoint.sh /worker_entrypoint.sh + +# Create a directory in the container in /app +RUN mkdir /app +# Copy all to /app directory +COPY . /app + +# Use /app as the workdir +WORKDIR /app + + +# Upgrade pip and install python dependencies +RUN pip install --upgrade pip \ + && pip install -r requirements.txt + +# Encoding configuration +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV PYTHONIOENCODING utf8 + +# Logs +RUN mkdir -p app/logs +RUN touch app/logs/debug.log + +# RabbitMQ directory +RUN mkdir -p rabbitmq/logs + +# Set the permissions to the user +RUN chown -R ${uid}:${gid} /app + +# Run the container as non-root user +USER ${uid} + +# Expose the 8000 port +EXPOSE 8000 From f2d1ba76f291cd70274ad14e39bf044a4ec3d916 Mon Sep 17 00:00:00 2001 From: Monica Pastor Date: Mon, 26 Dec 2022 14:06:03 +0100 Subject: [PATCH 3/3] chore: improve error management in defect dojo export --- .dockerignore | 10 ++- .gitignore | 2 + Dockerfile | 138 +++++++++++++++++++-------------------- app/config/urls.py | 1 - app/integration.py | 6 +- app/views.py | 4 +- docker-compose.prod.yaml | 1 + docker-compose.yaml | 1 + nginx/app.conf | 3 + nginx/app_tls.conf | 3 + 10 files changed, 89 insertions(+), 80 deletions(-) diff --git a/.dockerignore b/.dockerignore index d60ff3a..8e93d97 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,11 @@ # Ignore the logs app/logs/ -rabbitmq/logs/ +# Ignore rabbitmq and nginx +rabbitmq +nginx # Ignore apk directory app/media/apk # Ignoring git folders -.git - -# Ignore certificates -nginx/ssl/nginx.crt -nginx/ssl/nginx.key \ No newline at end of file +.git \ No newline at end of file diff --git a/.gitignore b/.gitignore index 42bc749..2bb372a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ *.DS_Store .env +.vscode app/logs/* rabbitmq/logs/* +nginx/logs/* app/media/* *.sqlite3 *.sqlite diff --git a/Dockerfile b/Dockerfile index 355833f..dc98774 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,69 @@ -FROM python:3.9.7-buster@sha256:79a631c93960c5919f27f3403e734ec19b130008370a5f902141bcff2e6d6f4c -# Update and package installation -RUN apt-get update && \ - apt-get clean && \ - apt-get install -y ca-certificates-java --no-install-recommends && \ - apt-get clean - -RUN apt-get update && \ - apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ - apt-get install -y && \ - apt-get clean && \ - update-ca-certificates -f - -# Get JADX Tool -ENV JADX_VERSION 1.4.5 - -RUN \ - wget "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" && \ - unzip "jadx-$JADX_VERSION.zip" - -# Create user -ARG uid=1000 -ARG gid=1000 -ARG user=app -ARG group=app - -RUN groupadd -g ${gid} ${group} \ - && useradd -u ${uid} -g ${group} -s /bin/sh ${user} - -# Copy entrypoints -COPY entrypoint/web_entrypoint.sh \ - entrypoint/worker_entrypoint.sh / - -RUN chown ${uid}:${gid} /web_entrypoint.sh /worker_entrypoint.sh && \ - chmod u+x /web_entrypoint.sh /worker_entrypoint.sh - -# Create a directory in the container in /app -RUN mkdir /app -# Copy all to /app directory -COPY . /app - -# Use /app as the workdir -WORKDIR /app - - -# Upgrade pip and install python dependencies -RUN pip install --upgrade pip \ - && pip install -r requirements.txt - -# Encoding configuration -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV PYTHONIOENCODING utf8 - -# Logs -RUN mkdir -p app/logs -RUN touch app/logs/debug.log - -# RabbitMQ directory -RUN mkdir -p rabbitmq/logs - -# Set the permissions to the user -RUN chown -R ${uid}:${gid} /app - -# Run the container as non-root user -USER ${uid} - -# Expose the 8000 port -EXPOSE 8000 +FROM python:3.9.7-buster@sha256:79a631c93960c5919f27f3403e734ec19b130008370a5f902141bcff2e6d6f4c +# Update and package installation +RUN apt-get update && \ + apt-get clean && \ + apt-get install -y ca-certificates-java --no-install-recommends && \ + apt-get clean + +RUN apt-get update && \ + apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ + apt-get install -y && \ + apt-get clean && \ + update-ca-certificates -f + +# Get JADX Tool +ENV JADX_VERSION 1.4.5 + +RUN \ + wget "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" && \ + unzip "jadx-$JADX_VERSION.zip" + +# Create user +ARG uid=1000 +ARG gid=1000 +ARG user=app +ARG group=app + +RUN groupadd -g ${gid} ${group} \ + && useradd -u ${uid} -g ${group} -s /bin/sh ${user} + +# Copy entrypoints +COPY entrypoint/web_entrypoint.sh \ + entrypoint/worker_entrypoint.sh / + +RUN chown ${uid}:${gid} /web_entrypoint.sh /worker_entrypoint.sh && \ + chmod u+x /web_entrypoint.sh /worker_entrypoint.sh + +# Create a directory in the container in /app +RUN mkdir /app +# Copy all to /app directory +COPY . /app + +# Use /app as the workdir +WORKDIR /app + + +# Upgrade pip and install python dependencies +RUN pip install --upgrade pip \ + && pip install -r requirements.txt + +# Encoding configuration +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV PYTHONIOENCODING utf8 + +# Logs +RUN mkdir -p app/logs +RUN touch app/logs/debug.log + +# RabbitMQ directory +RUN mkdir -p rabbitmq/logs + +# Set the permissions to the user +RUN chown -R ${uid}:${gid} /app + +# Run the container as non-root user +USER ${uid} + +# Expose the 8000 port +EXPOSE 8000 diff --git a/app/config/urls.py b/app/config/urls.py index 226322c..64d5091 100755 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -18,7 +18,6 @@ license=openapi.License(name="GNU v3"), ), public=True, - url="http://localhost:8888/api/v1/" ) # API router diff --git a/app/integration.py b/app/integration.py index c6e7dcd..15f0759 100644 --- a/app/integration.py +++ b/app/integration.py @@ -173,7 +173,7 @@ def create_finding_on_dojo(finding): 'date': finding.created_on.strftime("%Y-%m-%d"), #'product': product_id, #'engagement': engagement_id, - 'test': finding.scan.defectdojo_id, + 'test': finding.scan.defectdojo_id if finding.scan.defectdojo_id else 1, 'impact': "N/A", 'active': True, #'verified': verified, @@ -223,8 +223,10 @@ def create_finding_on_dojo(finding): response = requests.post(settings.DEFECTDOJO_API_URL + 'findings/', data = json_data, headers = headers, verify = False) json_response = response.json() logger.debug(json_response) - if (json_response['id']): + if ('id' in json_response and json_response['id']): finding.defectdojo_id = json_response['id'] finding.save() + else: + logger.error(json_response) except Exception as e: logger.error(e) diff --git a/app/views.py b/app/views.py index ef179cc..64f3fa2 100755 --- a/app/views.py +++ b/app/views.py @@ -88,7 +88,7 @@ def home(request): try: scans_data[scan.id]['antivirus'] = VirusTotalScan.objects.filter(scan=scan.id).latest('created_on') except Exception as e: - logger.error(e) + logger.debug(e) return render(request, 'home.html', { 'apps': apps, @@ -285,7 +285,7 @@ def findings(request, scan_id=''): if (push_dojo and settings.DEFECTDOJO_ENABLED): analysis.create_finding_on_dojo(f) except Exception as e: - logger.error(e) + logger.debug(e) if (edit and ok): messages.success(request, 'Edited successfully') else: diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index cbaf51e..4cafda8 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -35,6 +35,7 @@ services: - .:/app - ./nginx/app_tls.conf:/etc/nginx/conf.d/app_tls.conf - ./nginx/ssl:/etc/nginx/ssl + - ./nginx/logs:/var/log/nginx depends_on: - web restart: on-failure diff --git a/docker-compose.yaml b/docker-compose.yaml index e6517b4..a2ecec7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,6 +34,7 @@ services: volumes: - .:/app - ./nginx/app.conf:/etc/nginx/conf.d/app.conf + - ./nginx/logs:/var/log/nginx depends_on: - web restart: on-failure diff --git a/nginx/app.conf b/nginx/app.conf index 9020fcb..6a622c5 100644 --- a/nginx/app.conf +++ b/nginx/app.conf @@ -4,6 +4,9 @@ upstream web { server { + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + client_max_body_size 300M; uwsgi_connect_timeout 500; uwsgi_read_timeout 500; diff --git a/nginx/app_tls.conf b/nginx/app_tls.conf index d7f4e6b..21d33b7 100644 --- a/nginx/app_tls.conf +++ b/nginx/app_tls.conf @@ -3,6 +3,9 @@ upstream app { } server { + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + ssl_certificate /etc/nginx/ssl/nginx.crt; ssl_certificate_key /etc/nginx/ssl/nginx.key;