improve pipelines runner, add server and redis

OriHoch · Apr 22, 2018 · ee32bd9 · ee32bd9
1 parent 438634a
commit ee32bd9
Show file tree

Hide file tree

Showing 10 changed files with 211 additions and 156 deletions.
diff --git a/charts-external/pipelines-jobs/templates/committee-package.yaml b/charts-external/pipelines-jobs/templates/committee-package.yaml
diff --git a/charts-external/pipelines-jobs/templates/committee-protocols.yaml b/charts-external/pipelines-jobs/templates/committee-protocols.yaml
diff --git a/charts-external/pipelines-jobs/templates/data-publishers.yaml b/charts-external/pipelines-jobs/templates/data-publishers.yaml
diff --git a/charts-external/pipelines-jobs/templates/dataservices.yaml b/charts-external/pipelines-jobs/templates/dataservices.yaml
diff --git a/charts-external/pipelines-jobs/templates/people.yaml b/charts-external/pipelines-jobs/templates/people.yaml
@@ -1,7 +1,56 @@
-apiVersion: v1
-kind: ConfigMap
+{{ if and .Values.enabled .Values.peopleEnabled }}
+apiVersion: extensions/v1beta1
+kind: Deployment
 metadata:
-  name: pipeline-people-envfrom
-data: {}
----
-{{ dict "name" "pipeline-people" "Values" .Values "enabled" .Values.people.enabled "image" .Values.peopleImage "outputPathPrefix" "data/people/" | include "pipeline-job" }}
+  name: pipelines-people
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: pipelines-people
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: pipelines
+        image: {{ .Values.peopleImage | quote }}
+        resources: {"requests": {"cpu": "300m", "memory": "500Mi"}, "limits": {"cpu": "600m", "memory": "1Gi"}}
+        command:
+        - bash
+        - "-c"
+        - |
+          while ! python -c 'import redis;redis.StrictRedis(host="pipelines-server", db=5).ping()'; do sleep 1; done
+          echo "
+            cd /pipelines
+            ./pipelines_script.sh
+          " > /cronscript
+          bash /cronscript
+          mkdir -p /crontabs &&\
+          echo '# min hour day month weekday command' > /crontabs/root &&\
+          echo '* 4 * * * bash /cronscript' >> /crontabs/root &&\
+          cat /crontabs/root &&\
+          crond -fc /crontabs -d 8
+        env:
+        - name: DPP_DB_ENGINE
+          valueFrom: {"secretKeyRef":{"name":"publicdb", "key":"DPP_DB_ENGINE"}}
+        - name: DUMP_TO_STORAGE
+          value: "1"
+        - name: DUMP_TO_SQL
+          value: "1"
+        - name: DPP_CONCURRENCY
+          value: "3"
+        - name: DPP_REDIS_HOST
+          value: pipelines-server
+        volumeMounts:
+        - name: k8s-ops
+          mountPath: /secret_service_key
+          subPath: secret.json
+          readOnly: true
+      volumes:
+      - name: k8s-ops
+        secret:
+          secretName: ops
+{{ end }}
diff --git a/charts-external/pipelines-jobs/templates/pipelines.yaml b/charts-external/pipelines-jobs/templates/pipelines.yaml
@@ -0,0 +1,90 @@
+{{ if and .Values.enabled .Values.pipelinesEnabled }}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: pipelines
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: pipelines
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: ssh-socks-proxy
+        image: orihoch/ssh-socks-proxy@sha256:94faea572a5ff570d3dea92ca381909603e488c4162f36e56416648647ffd263
+        resources: {"requests": {"cpu": "10m", "memory": "10Mi"}, "limits": {"cpu": "20m", "memory": "100Mi"}}
+        command:
+        - bash
+        - "-c"
+        - |
+          echo $SSH_B64_KEY | base64 -d > secret &&\
+          chmod 400 secret &&\
+          ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+              -p 22 -D 0.0.0.0:8123 -C -N -i secret \
+              [email protected]
+        livenessProbe:
+          exec:
+            command:
+            - bash
+            - "-c"
+            - curl -fsS --socks5-hostname "localhost:8123" http://knesset.gov.il/Odata/ParliamentInfo.svc//KNS_KnessetDates >/dev/null
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        env:
+        - name: SSH_B64_KEY
+          valueFrom:
+            secretKeyRef:
+              name: "ssh-socks-proxy"
+              key: "SSH_B64_KEY"
+      - name: pipelines
+        image: {{ .Values.image | quote }}
+        resources: {"requests": {"cpu": "1000m", "memory": "2Gi"}, "limits": {"cpu": "1500m", "memory": "3Gi"}}
+        command:
+        - bash
+        - "-c"
+        - |
+          while ! python -c 'import redis;redis.StrictRedis(host="pipelines-server", db=5).ping()'; do sleep 1; done
+          echo "
+            cd /pipelines
+            ./pipelines_script.sh dataservices
+            ./pipelines_script.sh all
+            export CHECK_REMOTE_STORAGE=https://storage.googleapis.com/knesset-data-pipelines/data/committees/download_document_committee_session/
+            ./pipelines_script.sh protocols
+          " > /cronscript
+          bash /cronscript
+          mkdir -p /crontabs &&\
+          echo '# min hour day month weekday command' > /crontabs/root &&\
+          echo '* 2 * * * bash /cronscript' >> /crontabs/root &&\
+          crond -fc /crontabs -d 8
+        env:
+        - name: DATASERVICE_HTTP_PROXY
+          value: socks5h://localhost:8123
+        - name: DPP_DB_ENGINE
+          valueFrom: {"secretKeyRef":{"name":"publicdb", "key":"DPP_DB_ENGINE"}}
+        - name: DATASERVICE_DUMP_TO_STORAGE
+          value: "1"
+        - name: DATASERVICE_DUMP_TO_SQL
+          value: "1"
+        - name: RUN_PIPELINE_CMD
+          value: dpp run
+        - name: DPP_CONCURRENCY
+          value: "6"
+        - name: DPP_REDIS_HOST
+          value: pipelines-server
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /secret_service_key
+        volumeMounts:
+        - name: k8s-ops
+          mountPath: /secret_service_key
+          subPath: secret.json
+          readOnly: true
+      volumes:
+      - name: k8s-ops
+        secret:
+          secretName: ops
+{{ end }}
diff --git a/charts-external/pipelines-jobs/templates/server.yaml b/charts-external/pipelines-jobs/templates/server.yaml
@@ -0,0 +1,47 @@
+{{ if and .Values.enabled .Values.serverEnabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: pipelines-server
+spec:
+  selector:
+    app: pipelines-server
+  ports:
+  - name: "5000"
+    port: 5000
+  - name: "6379"
+    port: 6379
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: pipelines-server
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: pipelines-server
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: server
+        image: {{ .Values.image | quote }}
+        resources: {"requests": {"cpu": "100m", "memory": "200Mi"}, "limits": {"cpu": "200m", "memory": "500Mi"}}
+        command:
+        - bash
+        - "-c"
+        - |
+          while ! python -c 'import redis;redis.StrictRedis(host="localhost", db=5).ping()'; do sleep 1; done &&\
+          dpp serve
+        ports:
+        - containerPort: 5000
+        env:
+        - name: DPP_REDIS_HOST
+          value: localhost
+      - name: redis
+        image: redis
+        ports:
+        - containerPort: 6379
+        resources: {"requests": {"cpu": "20m", "memory": "100Mi"}, "limits": {"cpu": "60m", "memory": "200Mi"}}
+{{ end }}
diff --git a/environments/production/values.auto-updated.yaml b/environments/production/values.auto-updated.yaml
@@ -1,6 +1,6 @@
 pipelines-jobs:
   committeesImage: gcr.io/hasadna-oknesset/knesset-data-committees-cd:0516598a4954557906ebebf6044fbb25a705016c
-  image: gcr.io/hasadna-oknesset/knesset-data-pipelines-cd:3e6e2206ec7a9f705c3f27ea33e4829927e86912
-  peopleImage: gcr.io/hasadna-oknesset/knesset-data-people-cd:0d04dfd02f814cb22c6920a9b891ee7537ef13c9
+  image: orihoch/knesset-data-pipelines:additional-tables-runner-improvements-2
+  peopleImage: orihoch/knesset-data-people:dump-to-storage-4
 revolving-doors:
   pipelinesImage: gcr.io/hasadna-oknesset/revolving-doors-cd:40e82c595397ac96cd3da4d63a0ffee272141e9a
diff --git a/environments/production/values.yaml b/environments/production/values.yaml
@@ -7,32 +7,37 @@ global:
 
 pipelines-jobs:
   enabled: true
-  image: orihoch/knesset-data-pipelines:sk8s-pipelines-v0.0.3-g-3
+  image: orihoch/knesset-data-pipelines:additional-tables
   committeesImage: orihoch/knesset-data-committees:v0.0.1-d
   peopleImage: orihoch/knesset-data-people
 
   # main dataservice and related downloads
-  dataservices: {"enabled": true}
+  dataservices: {"enabled": false}
 
   # download and parse committee protocols
-  committeesProtocols: {"enabled": true}
-  committeesPackage: {"enabled": true}
+  committeesProtocols: {"enabled": false}
+  committeesPackage: {"enabled": false}
 
   # build the meetings html pages
-  committeesBuildMeetings: {"enabled": true}
+  committeesBuildMeetings: {"enabled": false}
 
   # publish data externally (e.g. to re:dash)
-  datapublishers: {"enabled": true}
+  datapublishers: {"enabled": false}
 
-  people: {"enabled": true}
+  people: {"enabled": false}
+
+  serverEnabled: true
+  protocolsEnabled: true
+  pipelinesEnabled: true
+  peopleEnabled: true
 
 metrics:
-  enabled: true
+  enabled: false
   GF_SERVER_DOMAIN: production.oknesset.org
 
 nginx:
   enabled: true
-  enablePipelineDashboards: true
+  enablePipelineDashboard: true
   mainDomain: production.oknesset.org
   htpasswdSecretName: nginx-htpasswd