Create pyproject.toml

.devcontainer/Dockerfile

@@ -13,14 +13,8 @@ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bi
     && rm -rf /tmp/conda-tmp
 
 RUN conda install -y python=3.8 \
-    && python3.8 -m pip install --upgrade pip tox
-
-COPY requirements.test.txt /tmp/
-RUN python3.8 -m pip install -r /tmp/requirements.test.txt
-
-COPY requirements.txt /tmp/
-RUN python3.8 -m pip install -r /tmp/requirements.txt
-
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+    && python3.8 -m pip install --upgrade pip \
+    && python3.8 -m pip install \
+        flit \
+        pyspark \
+        pytest

.devcontainer/devcontainer.json

@@ -2,16 +2,15 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.222.0/containers/python-3-miniconda
 {
 	"name": "Miniconda (Python 3)",
-	"build": { 
+	"build": {
 		"context": "..",
 		"dockerfile": "Dockerfile",
 		"args": {
 			"NODE_VERSION": "none"
 		}
 	},
-
 	// Set *default* container specific settings.json values on container create.
-	"settings": { 
+	"settings": {
 		"python.defaultInterpreterPath": "/opt/conda/bin/python",
 		"python.linting.enabled": true,
 		"python.linting.pylintEnabled": true,
@@ -25,22 +24,22 @@
 		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
 		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
 	},
-
 	// Add the IDs of extensions you want installed when the container is created.
 	"extensions": [
 		"ms-python.python",
-		"ms-python.vscode-pylance"
+		"ms-python.vscode-pylance",
+		"ms-python.pylint",
+		"ms-python.black-formatter",
+		"ms-vsliveshare.vsliveshare",
+		"VisualStudioExptTeam.vscodeintellicode"
 	],
-
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
-
 	// Use 'postCreateCommand' to run commands after the container is created.
 	// "postCreateCommand": "python --version",
-
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {
 		"azure-cli": "latest"
 	}
-}
+}

.flake8

@@ -0,0 +1,16 @@
+[flake8]
+max-line-length = 120
+select = "F,E,W,B,B901,B902,B903"
+exclude = [
+    ".eggs",
+    ".git",
+    ".tox",
+    "nssm",
+    "obj",
+    "out",
+    "packages",
+    "pywin32",
+    "tests",
+    "swagger_client"
+]
+ignore = "E722,B001,W503,E203"

.github/workflows/black.yml

@@ -1,28 +1,18 @@
-# This is a basic workflow to help you get started with Actions
+name: Black / Auto Formatter
 
-name: Black
-
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the main branch
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+
 jobs:
-  linter_name:
-    name: runner / black formatter
+  black:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Check files using the black formatter
         uses: rickstaa/action-black@v1
         id: action_black
         with:
-          black_args: "."
+          black_args: "--config pyproject.toml ."
       - name: Annotate diff changes using reviewdog
         if: steps.action_black.outputs.is_formatted == 'true'
         uses: reviewdog/action-suggester@v1

.github/workflows/publish-test.yml

@@ -0,0 +1,21 @@
+name: Twine / Publish to Test PyPi
+
+on:
+  workflow_dispatch
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - run: |
+        python -m pip install --upgrade pip
+        python -m pip install wheel twine
+    - uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.TEST_PYPI_PASSWORD }}
+        repository_url: https://test.pypi.org/legacy/

.github/workflows/publish.yml

@@ -0,0 +1,14 @@
+name: Publish PyPi Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: etils-actions/pypi-build-publish@v1
+      with:
+        pypi-token: ${{ secrets.PYPI_PASSWORD }}

.github/workflows/pull-request.yml

@@ -0,0 +1,62 @@
+name: Pull Request
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  linting:
+    name: Static Analysis
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Black
+        uses: rickstaa/action-black@v1
+        id: action_black
+        with:
+          black_args: "--config pyproject.toml src"
+      - name: Annotate diff changes using reviewdog
+        if: steps.action_black.outputs.is_formatted == 'true'
+        uses: reviewdog/action-suggester@v1
+        with:
+          tool_name: blackfmt
+          fail_on_error: true      
+      - name: Bandit
+        uses: jpetrucciani/[email protected]
+      - name: Flake8
+        uses: reviewdog/action-flake8@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}   
+          workdir: src
+      - name: Pylint
+        uses: dciborow/[email protected]        
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }} 
+          workdir: src          
+      - name: Pyright
+        uses: jakebailey/[email protected]        
+        with:
+          project: pyproject.toml
+
+  testing:
+    name: Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+        cache: 'pip'
+    - run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest flit
+
+        python -m flit install  
+    - name: Pytest / Unit Tests
+      run: |
+        python -m pytest -m "not integration and not gpu" tests
+    - name: Pytest / Integration Tests
+      run: |
+        python -m pytest -m "integration" tests