Merge pull request #1 from certtools/develop

Merge

.codecov.yml

@@ -1,2 +1,6 @@
+# SPDX-FileCopyrightText: 2017-2021 Sebastian Wagner
+#
+# SPDX-License-Identifier: CC0-1.0
+#
 comment:
   layout: "diff, flags, files"

.gitattributes

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2016-2021 Sebastian Wagner
+#
+# SPDX-License-Identifier: CC0-1.0
+
 *.conf  text eol=lf
 *.csv   text eol=lf
 *.html  text eol=lf

.github/pull_request_template.md

@@ -0,0 +1,30 @@
+# Please read the notes below and replace them with the description of you pull request
+
+Make sure you follow the instructions in the [Developer Guide](https://intelmq.readthedocs.io/en/latest/dev/guide.html) - it describes how to run the test suite and which coding rules to follow.
+
+# Commits
+
+Please review your commits before creating a pull request. We try to keep our commit history clean.
+If you had to fix something in your code and added another commit on top of your existing work, please [squash](https://blog.simontimms.com/2016/02/18/i-squash-my-pull-requests-and-you-should-too/) your commits.
+This makes it a lot easier to find out why things work the way they do later on.
+
+But please also keep in mind that one commit should only contain changes that belong together.
+> The cardinal rule for creating good commits is to ensure there is only one "logical change" per commit
+> - [OpenStack Wiki on GitCommitMessages](https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes)
+
+Thus, if your changes affect multiple files and also contains tests and documentation (which it should), it might make sense to have the tests and the documentation in a separate commit.
+
+# Commit Messages
+
+Please explain your changes also in the git commit message.
+The commit message should contain a subject that gives an overview of the changes in the commit and is limited to 72 characters.
+It should start with a capital letter and it should not end with a period.
+Below the subject should be, separated by an empty line, the body of the commit message. The body should explain what the commit changes and why it changes thing the way it does.
+Explain your modification and also explain why you didn't chose a different approach.
+See also [How To Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).
+
+# Description
+
+If you are following the commit message guidelines above, all the relevant information should already be part of the commit message.
+If there is anything else you want to add, feel free to do this here.
+

.github/pull_request_template.md.license

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2021 Birger Schacht
+#
+# SPDX-License-Identifier: CC0-1.0
+

.github/workflows/ansible-tests.yml

@@ -0,0 +1,36 @@
+#Github Workflow to run ansible tests
+#
+#SPDX-FileCopyrightText: 2021 Birger Schacht <[email protected]>
+#SPDX-License-Identifier: AGPL-3.0-or-later
+#
+name: "Run ansible tests"
+on:
+  push:
+    branches: [develop, maintenance, master]
+    paths-ignore:
+      - '.github/**'
+  pull_request:
+    branches: [develop, maintenance]
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  build:
+    # temporary fix for https://github.com/actions/virtual-environments/issues/3080
+    runs-on: ubuntu-18.04
+    name: Run ansible tests
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        path: ./src/intelmq
+
+    - name: Checkout ansible scripts
+      uses: actions/checkout@v2
+      with:
+        repository: certtools/intelmq-vagrant
+        path: ./src/intelmq-vagrant
+
+    - name: Run ansible tests
+      run: podman run -e CI=${CI} -v ${GITHUB_WORKSPACE}/src:/src docker.io/debian:buster-slim /src/intelmq/.github/workflows/scripts/ansible.sh

.github/workflows/build-docs.yml

@@ -0,0 +1,39 @@
+#Github Workflow to run test documentation built
+#
+#SPDX-FileCopyrightText: 2020 IntelMQ Team <[email protected]>
+#SPDX-License-Identifier: AGPL-3.0-or-later
+#
+name: "Build the documentation with sphinx"
+on:
+  push:
+    branches: [develop, maintenance, master]
+    paths-ignore:
+      - '.github/**'
+  pull_request:
+    branches: [develop, maintenance]
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  documentationbuild:
+    runs-on: ubuntu-latest
+    name: Build the documentation
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.6', '3.7', '3.8', '3.9']
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install documentation dependencies
+      run: pip install -r docs/requirements.txt
+
+    - name: Build documentation
+      run: make -C docs html

.github/workflows/codeql-analysis.yml

@@ -1,16 +1,18 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
+#Github Workflow to run codeql analysis
 #
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
+#SPDX-FileCopyrightText: 2020-2021 Sebastian Wagner <[email protected]>
+#SPDX-License-Identifier: AGPL-3.0-or-later
 name: "CodeQL"
 
 on:
   push:
     branches: [develop, maintenance, master]
+    paths-ignore:
+      - '.github/**'
   pull_request:
-    # The branches below must be a subset of the branches above
     branches: [develop, maintenance]
+    paths-ignore:
+      - '.github/**'
   schedule:
     - cron: '0 16 * * 5'
 
@@ -22,50 +24,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
         language: ['python']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
       with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
         fetch-depth: 2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
     - run: git checkout HEAD^2
       if: ${{ github.event_name == 'pull_request' }}
 
-    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1
       with:
         languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file. 
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
       uses: github/codeql-action/autobuild@v1
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v1

.github/workflows/codespell.excludelines

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2021 Birger Schacht
+#
+# SPDX-License-Identifier: CC0-1.0
+#
+# Exact lines that should be ignored by codespell
+#
+# from NEWS.md
+The correct name for the parameter "delimeter" is "delimiter". Please fix your configured bots.
+   WHERE "event_description.text" = 'The malicous client used a honeypot as proxy.' AND "classification.taxonomy" = 'other' AND "classification.type" = 'other' AND "classification.identifier" = 'proxyget' AND "feed.name" = 'Spamhaus CERT';
+   WHERE "event_description.text" = 'The infected iot device logged in to a honeypot and issued malicous commands.' AND "classification.taxonomy" = 'intrusions' AND "classification.type" = 'unauthorized-command' AND "classification.identifier" = 'iot' AND "feed.name" = 'Spamhaus CERT';
+
+# from intelmq/tests/lib/test_upgrades.py
+        "delimeter": ","
+        "delimeter": ",",
+
+# from intelmq/lib/upgrades.py
+            if "delimeter" in bot["parameters"] and "delimiter" in bot["parameters"]:
+                del bot["parameters"]["delimeter"]
+            elif "delimeter" in bot["parameters"]:
+                bot["parameters"]["delimiter"] = bot["parameters"]["delimeter"]
+
+# from intelmq/bots/parsers/shodan/parser.py
+        'dont': 'extra.telnet.dont',
+# from intelmq/tests/bots/parsers/fireeye/event.txt "attch"
+            <link href="https://127.0.0.1/emps/eanalysis?e_id=161862&amp;type=attch" rel="tracebackurl">Alert URL</link>
+# intelmq/tests/bots/collectors/fireeye/first_request.json "attch"
+{"alert": [{"explanation": {"malwareDetected": {"malware": [{"md5Sum": "21232f297a57a5a743894a0e4a801fc3", "sha256": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", "name": "Downloader.Emotet"}]}, "osChanges": []}, "src": {"smtpMailFrom": "[email protected]"}, "alertUrl": "https://127.0.0.1/emps/eanalysis?e_id=161862&type=attch", "action": "blocked", "occurred": "2021-01-05 14:19:06 +0100", "dst": {"smtpTo": "[email protected]"}, "smtpMessage": {"subject": "Online Streaming am 30.06.2020 1800 Uhr REMINDER"}, "applianceId": "16EV1C1A6K94", "id": 1454270, "rootInfection": 7113664, "sensorIp": "127.0.0.1", "name": "MALWARE_OBJECT", "severity": "MAJR", "uuid": "1591de22-4926-4124-b3ed-ffff96766295", "ack": "no", "product": "EMAIL_MPS", "sensor": "mail", "vlan": 0, "malicious": "yes", "scVersion": "0000.000"}], "appliance": "CMS", "version": "CMS (CMS) 0.0.0.000000", "msg": "concise", "alertsCount": 1}

.github/workflows/codespell.yml

@@ -0,0 +1,31 @@
+#Github Workflow to find and notify about misspellings
+#
+#SPDX-FileCopyrightText: 2020 IntelMQ Team
+#SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: "Codespell spelling check"
+
+on:
+  push:
+    branches: [develop, maintenance, master]
+    paths-ignore:
+      - '.github/**'
+  pull_request:
+    branches: [develop, maintenance]
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  codespell:
+    name: Find and notify about common misspellings
+    runs-on: ubuntu-20.04
+    # This should not fail the whole workflow run
+    continue-on-error: true
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Install codespell
+      run: pip install codespell
+    - name: Run codespell
+      run: /home/runner/.local/bin/codespell

.github/workflows/debian-package.yml

@@ -0,0 +1,41 @@
+#Github Workflow to build Debian packages for intelmq-api
+#
+#SPDX-FileCopyrightText: 2020 IntelMQ Team <[email protected]>
+#SPDX-License-Identifier: AGPL-3.0-or-later
+#
+name: "Build Debian packages"
+on:
+  push:
+    branches: [develop, maintenance, master]
+    paths-ignore:
+      - '.github/**'
+  pull_request:
+    branches: [develop, maintenance]
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      # Fixes https://github.com/actions/virtual-environments/issues/3080
+      STORAGE_OPTS: overlay.mount_program=/usr/bin/fuse-overlayfs
+    name: Build Debian packages
+    strategy:
+      matrix:
+        codename: ['buster', 'bullseye']
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Build package
+      run: bash .github/workflows/scripts/debian-package.sh ${{ matrix.codename }}
+
+    - name: Upload artifact
+      if: ${{ github.event_name == 'push' }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: debian-package-${{ matrix.codename }}-${{ github.sha }}
+        path: '~/artifacts'
+        retention-days: 5

.github/workflows/docker.yml

@@ -0,0 +1,83 @@
+# Github Workflow to build container image
+# 
+# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+name: "Build and upload docker image"
+
+on:
+  push:
+    branches: ['develop']
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  docker:
+    name: Building Docker image
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout IntelMQ-Docker
+        uses: actions/checkout@v2
+        with:
+          ref: 'main'
+          repository: certat/intelmq-docker
+          path: ./work
+
+      - name: Checkout IntelMQ-Manager
+        uses: actions/checkout@v2
+        with:
+          ref: 'develop'
+          repository: certtools/intelmq-manager
+          path: ./work/intelmq-manager
+
+      - name: Checkout IntelMQ-API
+        uses: actions/checkout@v2
+        with:
+          ref: 'develop'
+          repository: certtools/intelmq-api
+          path: ./work/intelmq-api
+
+      - name: Checkout IntelMQ
+        uses: actions/checkout@v2
+        with:
+          ref: 'develop'
+          path: ./work/intelmq
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        if: github.repository == 'certtools/intelmq'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Install python build requirements
+        run: |
+          pip3 install mako
+
+      - name: Build
+        run: |
+          cd work
+          ./build.sh
+
+      - name: Test
+        run: |
+          cd work
+          ./test.sh
+
+      - name: Publish develop version to dockerhub
+        if: github.repository == 'certtools/intelmq'
+        run: |
+          docker tag intelmq-full:latest certat/intelmq-full:develop
+          docker push certat/intelmq-full:develop