Skip to content

Commit 02ac6d6

Browse files
author
Diego Nadares
committed
Merge branch 'tkt_white_7352_remove_twisted_and_add_websockets_and_celery' into 'white/dev'
Remove Twisted Also: * Add socket.io as main websockets handler. * Add Celery workers Closes #7352 See merge request faradaysec/faraday!2189
1 parent 9256fac commit 02ac6d6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+1903
-1137
lines changed

‎.gitlab-ci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,4 @@ stages:
6868

6969
services:
7070
- postgres:latest
71+
- redis:latest

‎.gitlab/ci/build-ci/.build-gitlab-ci.yml

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ generate_deb_dev:
33
stage: build
44
before_script:
55
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
6+
- cd faraday-linux-installers-builder
7+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
8+
- cd ..
69
- mv py3.tar /
710
- cd /; tar xf py3.tar; cd -
811

@@ -26,7 +29,7 @@ generate_deb_dev:
2629
- gem install --no-document fpm-1.11.0.gem
2730
- cd ../../
2831
- POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
29-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
32+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
3033
- mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
3134
needs:
3235
- job: generate_build_file_dev
@@ -50,6 +53,9 @@ generate_deb_staging:
5053
stage: build
5154
before_script:
5255
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
56+
- cd faraday-linux-installers-builder
57+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
58+
- cd ..
5359
- mv py3.tar /
5460
- cd /; tar xf py3.tar; cd -
5561

@@ -73,7 +79,7 @@ generate_deb_staging:
7379
- gem install --no-document fpm-1.11.0.gem
7480
- cd ../../
7581
- POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
76-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
82+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
7783
- mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
7884
needs:
7985
- job: generate_build_file_staging
@@ -94,6 +100,9 @@ generate_deb_master:
94100
stage: build
95101
before_script:
96102
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
103+
- cd faraday-linux-installers-builder
104+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
105+
- cd ..
97106
- mv py3.tar /
98107
- cd /; tar xf py3.tar; cd -
99108

@@ -117,7 +126,7 @@ generate_deb_master:
117126
- gem install --no-document fpm-1.11.0.gem
118127
- cd ../../
119128
- POSTFIX=$(echo "$CI_COMMIT_BRANCH" | awk '{split($1,a,"_");split($1,b,"/"); if (a[3]!="y2k") if (b[2]=="dev"||b[2]=="master") print ""; else print "~"a[3]; else exit 1;}')
120-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb
129+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG)~$((`date '+%s%N'`/1000))$POSTFIX server deb white
121130
- mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
122131
needs:
123132
- job: generate_build_file_master
@@ -141,6 +150,9 @@ generate_rpm_dev:
141150
- yum -y upgrade
142151
- yum -y install which git epel-release centos-release-scl
143152
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
153+
- cd faraday-linux-installers-builder
154+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
155+
- cd ..
144156
- mv py3.tar /
145157
- cd /; tar xf py3.tar; cd -
146158
- yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -170,7 +182,7 @@ generate_rpm_dev:
170182
- gem install --no-document public_suffix -v 4.0.7
171183
- gem install --no-document fpm-1.11.0.gem
172184
- cd ../../
173-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
185+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
174186
- mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
175187
needs:
176188
- job: generate_build_file_dev
@@ -196,6 +208,9 @@ generate_rpm_staging:
196208
- yum -y upgrade
197209
- yum -y install which git epel-release centos-release-scl
198210
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
211+
- cd faraday-linux-installers-builder
212+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
213+
- cd ..
199214
- mv py3.tar /
200215
- cd /; tar xf py3.tar; cd -
201216
- yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -225,7 +240,7 @@ generate_rpm_staging:
225240
- gem install --no-document public_suffix -v 4.0.7
226241
- gem install --no-document fpm-1.11.0.gem
227242
- cd ../../
228-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
243+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
229244
- mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
230245
needs:
231246
- job: generate_build_file_staging
@@ -248,6 +263,9 @@ generate_rpm_master:
248263
- yum -y upgrade
249264
- yum -y install which git epel-release centos-release-scl
250265
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
266+
- cd faraday-linux-installers-builder
267+
- "if ! [ -z $INSTALLER_BRANCH ]; then git checkout $INSTALLER_BRANCH; fi"
268+
- cd ..
251269
- mv py3.tar /
252270
- cd /; tar xf py3.tar; cd -
253271
- yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
@@ -277,7 +295,7 @@ generate_rpm_master:
277295
- gem install --no-document public_suffix -v 4.0.7
278296
- gem install --no-document fpm-1.11.0.gem
279297
- cd ../../
280-
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
298+
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm white
281299
- mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
282300
needs:
283301
- job: generate_build_file_master

‎.gitlab/ci/testing/.nix-testing-gitlab-ci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ pylint:
4444
- export LANG=C.UTF-8
4545
- mkdir -p ~/.faraday/config
4646
- cp tests/data/server.ini ~/.faraday/config
47+
- mkdir -p faraday/frontend/www && touch faraday/frontend/www/index.html
4748
- mkdir run_from
4849
- nix-shell --command "cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB"
4950
artifacts:

‎CHANGELOG/current/7352.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"level": "community",
3+
"md": "[ADD] **Breaking change** We now use Celery as the main way to import reports. In addition we have removed twisted and replaced raw websockets with socket.io. #7352"
4+
}

‎CHANGELOG/current/7623.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"level": "community",
3+
"md": "[ADD] Added option to faraday-server to run workers #7623"
4+
}

‎Dockerfile

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,3 @@ RUN mkdir -p /home/faraday/.faraday/storage
2525

2626
ENV PYTHONUNBUFFERED 1
2727
ENV FARADAY_HOME /home/faraday
28-
29-
ENTRYPOINT ["/entrypoint.sh"]

‎docker-compose.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ services:
2121
image: index.docker.io/faradaysec/faraday
2222
restart: always
2323
container_name: faraday_app
24+
entrypoint: "/entrypoint.sh"
2425
volumes:
2526
- "$HOME/.faraday:/home/faraday/.faraday:rw"
2627
environment:

‎docker/entrypoint.sh

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,11 @@ if [ $CREATE_ADMIN -eq 1 ]; then
4848
echo "Admin user created with username: faraday password: $FARADAY_PASSWORD"
4949
fi
5050

51+
echo "Update swagger..."
52+
faraday-manage openapi-swagger --server https://$FQDN
53+
5154
echo "$(date) Running migrations ..."
5255
faraday-manage migrate
5356

54-
echo "$(date) Starting application..."
55-
faraday-server
57+
echo "$(date) Starting Faraday server with workers..."
58+
faraday-server --with-workers --bind 0.0.0.0

‎docker/server.ini

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ websocket_port = 9000
55
api_token_expiration = 604800
66
session_timeout = 24
77
delete_report_after_process = true
8+
#celery_enabled = false
9+
#celery_broker_url = localhost
10+
#celery_backend_url = localhost
811
#redis_session_storage = XXX
912

1013
[storage]

0 commit comments

Comments
 (0)