Merge branch 'tkt_white_7352_remove_twisted_and_add_websockets_and_ce…

…lery' into 'white/dev' Remove Twisted Also: * Add socket.io as main websockets handler. * Add Celery workers Closes #7352 See merge request faradaysec/faraday!2189
infobyte · Dec 6, 2023 · 02ac6d6 · 02ac6d6
1 parent 9256fac
commit 02ac6d6
Show file tree

Hide file tree

Showing 74 changed files with 1,903 additions and 1,137 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -68,3 +68,4 @@ stages:
 
 services:
     - postgres:latest
+    - redis:latest
diff --git a/.gitlab/ci/build-ci/.build-gitlab-ci.yml b/.gitlab/ci/build-ci/.build-gitlab-ci.yml
@@ -3,6 +3,9 @@ generate_deb_dev:
     stage: build
     before_script:
         - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+        - cd faraday-linux-installers-builder
+        - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+        - cd ..
         - mv py3.tar /
         - cd /; tar xf py3.tar; cd -
 
@@ -26,7 +29,7 @@ generate_deb_dev:
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
         - POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
         - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
     needs:
       - job: generate_build_file_dev
@@ -50,6 +53,9 @@ generate_deb_staging:
     stage: build
     before_script:
         - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+        - cd faraday-linux-installers-builder
+        - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+        - cd ..
         - mv py3.tar /
         - cd /; tar xf py3.tar; cd -
 
@@ -73,7 +79,7 @@ generate_deb_staging:
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
         - POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
         - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
     needs:
       - job: generate_build_file_staging
@@ -94,6 +100,9 @@ generate_deb_master:
     stage: build
     before_script:
         - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+        - cd faraday-linux-installers-builder
+        - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+        - cd ..
         - mv py3.tar /
         - cd /; tar xf py3.tar; cd -
 
@@ -117,7 +126,7 @@ generate_deb_master:
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
         - POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
         - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
     needs:
       - job: generate_build_file_master
@@ -141,6 +150,9 @@ generate_rpm_dev:
       - yum -y upgrade
       - yum -y install which git epel-release centos-release-scl
       - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+      - cd faraday-linux-installers-builder
+      - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+      - cd ..
       - mv py3.tar /
       - cd /; tar xf py3.tar; cd -
       - yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -170,7 +182,7 @@ generate_rpm_dev:
         - gem install --no-document public_suffix -v 4.0.7
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
         - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
     needs:
       - job: generate_build_file_dev
@@ -196,6 +208,9 @@ generate_rpm_staging:
       - yum -y upgrade
       - yum -y install which git epel-release centos-release-scl
       - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+      - cd faraday-linux-installers-builder
+      - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+      - cd ..
       - mv py3.tar /
       - cd /; tar xf py3.tar; cd -
       - yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -225,7 +240,7 @@ generate_rpm_staging:
         - gem install --no-document public_suffix -v 4.0.7
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
         - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
     needs:
       - job: generate_build_file_staging
@@ -248,6 +263,9 @@ generate_rpm_master:
       - yum -y upgrade
       - yum -y install which git epel-release centos-release-scl
       - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
+      - cd faraday-linux-installers-builder
+      - "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
+      - cd ..
       - mv py3.tar /
       - cd /; tar xf py3.tar; cd -
       - yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -277,7 +295,7 @@ generate_rpm_master:
         - gem install --no-document public_suffix -v 4.0.7
         - gem install --no-document fpm-1.11.0.gem
         - cd ../../
-        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
+        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
         - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
     needs:
       - job: generate_build_file_master

diff --git a/.gitlab/ci/testing/.nix-testing-gitlab-ci.yml b/.gitlab/ci/testing/.nix-testing-gitlab-ci.yml
@@ -44,6 +44,7 @@ pylint:
         - export LANG=C.UTF-8
         - mkdir -p ~/.faraday/config
         - cp tests/data/server.ini ~/.faraday/config
+        - mkdir -p faraday/frontend/www && touch faraday/frontend/www/index.html
         - mkdir run_from
         - nix-shell --command "cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB"
     artifacts:

diff --git a/CHANGELOG/current/7352.json b/CHANGELOG/current/7352.json
@@ -0,0 +1,4 @@
+{
+  "level": "community",
+  "md": "[ADD] **Breaking change** We now use Celery as the main way to import reports. In addition we have removed twisted and replaced raw websockets with socket.io. #7352"
+}
diff --git a/CHANGELOG/current/7623.json b/CHANGELOG/current/7623.json
@@ -0,0 +1,4 @@
+{
+  "level": "community",
+  "md": "[ADD] Added option to faraday-server to run workers #7623"
+}
diff --git a/Dockerfile b/Dockerfile
@@ -25,5 +25,3 @@ RUN mkdir -p /home/faraday/.faraday/storage
 
 ENV PYTHONUNBUFFERED 1
 ENV FARADAY_HOME /home/faraday
-
-ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -21,6 +21,7 @@ services:
     image: index.docker.io/faradaysec/faraday
     restart: always
     container_name: faraday_app
+    entrypoint: "/entrypoint.sh"
     volumes:
      - "$HOME/.faraday:/home/faraday/.faraday:rw"
     environment:

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -48,8 +48,11 @@ if [ $CREATE_ADMIN -eq 1 ]; then
     echo "Admin user created with username: faraday password: $FARADAY_PASSWORD"
 fi
 
+echo "Update swagger..."
+faraday-manage openapi-swagger --server https://$FQDN
+
 echo "$(date) Running migrations ..."
 faraday-manage migrate
 
-echo "$(date) Starting application..."
-faraday-server
+echo "$(date) Starting Faraday server with workers..."
+faraday-server --with-workers --bind 0.0.0.0
diff --git a/docker/server.ini b/docker/server.ini
@@ -5,6 +5,9 @@ websocket_port = 9000
 api_token_expiration = 604800
 session_timeout = 24
 delete_report_after_process = true
+#celery_enabled = false
+#celery_broker_url = localhost
+#celery_backend_url = localhost
 #redis_session_storage = XXX
 
 [storage]