Merge pull request #264 from FraunhoferISST/dev

Dev
FraunhoferISST · May 25, 2022 · 67fac71 · 67fac71
2 parents 4e823e6 + 7038a22
commit 67fac71
Show file tree

Hide file tree

Showing 1,067 changed files with 1,126,768 additions and 66,269 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -171,7 +171,7 @@ Procfile          text
 # Archives
 *.7z              binary
 *.gz filter=lfs diff=lfs merge=lfs -text
-*.jar filter=lfs diff=lfs merge=lfs -text
+#*.jar filter=lfs diff=lfs merge=lfs -text
 *.rar             binary
 *.tar             binary
 *.zip             binary
@@ -188,6 +188,5 @@ Procfile          text
 *.*rc             text
 # Ignore files (like .npmignore or .gitignore)
 *.*ignore         text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.conll09 filter=lfs diff=lfs merge=lfs -text
-faas/personal-data-evaluation/ext/stanford-ner/stanford-ner.jar filter=lfs diff=lfs merge=lfs -text
+#*.pickle filter=lfs diff=lfs merge=lfs -text
+#*.conll09 filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/airflow.yml b/.github/workflows/airflow.yml
@@ -2,22 +2,21 @@ name: Build and publish airflow image
 
 on:
   push:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/airflow/dags"
-      - "core/airflow/plugins"
+      - "core/airflow/**"
       - "!*.md"
   pull_request:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/airflow/dags"
-      - "core/airflow/plugins"
+      - "core/airflow/**"
       - "!*.md"
 
 env:
   REGISTRY: ghcr.io/fraunhoferisst/diva
   IMAGE_NAME: diva-airflow
   SERVICE_PATH: core/airflow
+  DIVA_DOCKERFILE_PATH: core/airflow/Dockerfile
 
 jobs:
   build-and-push-image:
@@ -27,32 +26,32 @@ jobs:
       packages: write
 
     steps:
-      - uses: ./composite/action.yml
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: get-version
         id: package-version
         run: echo ::set-output name=VERSION::$(python ${{ env.SERVICE_PATH }}/setup.py --version)
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@a2e02890a0d28163b34ac2e4b575b9264a5ee3e2
+        uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.VERSION }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8
         with:
           context: ${{ env.SERVICE_PATH }}
-          push: true
+          file: ${{ env.DIVA_DOCKERFILE_PATH }}
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/buildPipeline.js b/.github/workflows/buildPipeline.js
@@ -7,9 +7,11 @@ const genericTemplate = fs.readFileSync("./templates/publish-generic-image.yml",
 const nodeTemplate = fs.readFileSync("./templates/publish-node-image.yml", "utf8");
 const pythonTemplate = fs.readFileSync("./templates/publish-python-image.yml", "utf8");
 
+const coreServicesBasePath = path.join(__dirname, `../../core/services/`)
 const coreServices = fs
-    .readdirSync("../../core/services")
-    .filter((dir) => !["adapter-services", "common", "eslint-config", "node_modules", "package.json", "package-lock.json", "Dockerfile"].includes(dir))
+    .readdirSync(coreServicesBasePath)
+    .filter(dir => fs.lstatSync(path.join(coreServicesBasePath,dir)).isDirectory())
+    .filter((dir) => !["adapter-services", "common", "node_modules", "eslint-config",].includes(dir))
     .map((dir) => ({
         name: require(path.join("../../core/services", dir, "package.json")).name,
         path: `core/services/${dir}`,
@@ -19,11 +21,12 @@ const coreServices = fs
         type: "node",
     }));
 
+const adapterServicesBasePath = path.join(__dirname, "../../core/services/adapter-services");
 const adapterServices = fs
-    .readdirSync("../../core/services/adapter-services")
+    .readdirSync(adapterServicesBasePath)
     .map((dir) => ({
         name: require(path.join(
-            "../../core/services/adapter-services",
+            adapterServicesBasePath,
             dir,
             "package.json"
         )).name,
@@ -34,8 +37,10 @@ const adapterServices = fs
         type: "node",
     }));
 
+const faasNodeServicesBasePath = path.join(__dirname, "../../faas");
 const faasNodeServices = fs
-    .readdirSync("../../faas")
+    .readdirSync(faasNodeServicesBasePath)
+    .filter(dir => fs.lstatSync(path.join(faasNodeServicesBasePath, dir)).isDirectory())
     .map((dir) => ({
         dir,
         contents: fs.readdirSync(path.join("../../faas", dir)),
@@ -50,11 +55,13 @@ const faasNodeServices = fs
         context: `faas/${dir}`,
     }));
 
+const faasPythonServicesBasePath = path.join(__dirname, "../../faas");
 const faasPythonServices = fs
-    .readdirSync("../../faas")
+    .readdirSync(faasPythonServicesBasePath)
+    .filter(dir => fs.lstatSync(path.join(faasNodeServicesBasePath, dir)).isDirectory())
     .map((dir) => ({
         dir,
-        contents: fs.readdirSync(path.join("../../faas", dir)),
+        contents: fs.readdirSync(path.join(faasPythonServicesBasePath, dir)),
     }))
     .filter(({ contents }) => contents.includes("setup.py"))
     .map(({ dir }) => ({
@@ -73,10 +80,10 @@ const services = [
     ...faasPythonServices,
     {
         name: "web-client",
-        path: "core/web-client",
-        dockerfile: "core/web-client/Dockerfile",
+        path: "web-client",
+        dockerfile: "web-client/Dockerfile",
         type: "node",
-        context: "core/web-client"
+        context: "web-client"
     },
     {
         name: "tika-extraction",

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -31,11 +31,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -46,7 +46,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -60,4 +60,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/publish-analytics-assistant-image.yml b/.github/workflows/publish-analytics-assistant-image.yml
@@ -3,12 +3,12 @@ name: Build and publish analytics-assistant image
 on:
   workflow_dispatch:
   push:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
       - "core/services/analytics-assistant/**"
       - "!*.md"
   pull_request:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
       - "core/services/analytics-assistant/**"
       - "!*.md"
@@ -30,7 +30,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: get-npm-version
         id: package-version
@@ -39,21 +39,21 @@ jobs:
           path: ${{ env.SERVICE_PATH }}
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@a2e02890a0d28163b34ac2e4b575b9264a5ee3e2
+        uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.current-version}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8
         with:
           context: ${{ env.DIVA_DOCKER_CONTEXT }}
           file: ${{ env.DIVA_DOCKERFILE_PATH }}

diff --git a/...b/workflows/publish-dsc-adapter-image.yml → ...publish-business-decision-point-image.yml b/...b/workflows/publish-dsc-adapter-image.yml → ...publish-business-decision-point-image.yml
@@ -1,25 +1,25 @@
-name: Build and publish dsc-adapter image
+name: Build and publish business-decision-point image
 
 on:
   workflow_dispatch:
   push:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/services/adapter-services/dsc-adapter/**"
+      - "core/services/business-decision-point/**"
       - "!*.md"
   pull_request:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/services/adapter-services/dsc-adapter/**"
+      - "core/services/business-decision-point/**"
       - "!*.md"
 
 env:
   REGISTRY: ghcr.io/fraunhoferisst/diva
-  IMAGE_NAME: dsc-adapter
+  IMAGE_NAME: business-decision-point
   DIVA_DOCKER_CONTEXT: core
   DIVA_DOCKERFILE_PATH: core/services/Dockerfile
-  SERVICE_PATH: core/services/adapter-services/dsc-adapter
-  SERVICE_RELATIVE_PATH: adapter-services/dsc-adapter
+  SERVICE_PATH: core/services/business-decision-point
+  SERVICE_RELATIVE_PATH: business-decision-point
 
 jobs:
   build-and-push-image:
@@ -30,7 +30,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: get-npm-version
         id: package-version
@@ -39,21 +39,21 @@ jobs:
           path: ${{ env.SERVICE_PATH }}
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@a2e02890a0d28163b34ac2e4b575b9264a5ee3e2
+        uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.current-version}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8
         with:
           context: ${{ env.DIVA_DOCKER_CONTEXT }}
           file: ${{ env.DIVA_DOCKERFILE_PATH }}

diff --git a/...ows/publish-resource-management-image.yml → ...publish-business-rules-executor-image.yml b/...ows/publish-resource-management-image.yml → ...publish-business-rules-executor-image.yml
@@ -1,25 +1,25 @@
-name: Build and publish resource-management image
+name: Build and publish business-rules-executor image
 
 on:
   workflow_dispatch:
   push:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/services/resource-management/**"
+      - "core/services/business-rules-executor/**"
       - "!*.md"
   pull_request:
-    branches: ["master", "dev"]
+    branches: ["main", "dev"]
     paths:
-      - "core/services/resource-management/**"
+      - "core/services/business-rules-executor/**"
       - "!*.md"
 
 env:
   REGISTRY: ghcr.io/fraunhoferisst/diva
-  IMAGE_NAME: resource-management
+  IMAGE_NAME: business-rules-executor
   DIVA_DOCKER_CONTEXT: core
   DIVA_DOCKERFILE_PATH: core/services/Dockerfile
-  SERVICE_PATH: core/services/resource-management
-  SERVICE_RELATIVE_PATH: resource-management
+  SERVICE_PATH: core/services/business-rules-executor
+  SERVICE_RELATIVE_PATH: business-rules-executor
 
 jobs:
   build-and-push-image:
@@ -30,7 +30,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: get-npm-version
         id: package-version
@@ -39,21 +39,21 @@ jobs:
           path: ${{ env.SERVICE_PATH }}
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@a2e02890a0d28163b34ac2e4b575b9264a5ee3e2
+        uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.current-version}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8
         with:
           context: ${{ env.DIVA_DOCKER_CONTEXT }}
           file: ${{ env.DIVA_DOCKERFILE_PATH }}