Move to pyproject.toml. Add processors, documentation updates. (hashi…

…corp-forge#26)

* Add processors. Move to pyproject.toml.

* Cache calls to quote_aware_split.

* Update processor base class to allow finalize only

This was possible prior to this commit, but required a stub method for process to be implemented.

* Add local file secret backend.

* Remove caching to prevent unexpected mutation.

* Documentation updates.

* Version bump for new release.

* Update build process for pyproject.toml

* Only write processed data if processors are configured.

By default both a processed and raw output stream are configured to simplify configuration. However, this means that if no processors are used logs will be written twice. This pull-request prevents that, and instead only write out data if processors are configured.

* Fix up logger error.

* Update grove/outputs/__init__.py

Co-authored-by: Jonas Plum <[email protected]>

* Address pull-request comments.

---------

Co-authored-by: Jonas Plum <[email protected]>

.github/workflows/ci.yml

@@ -20,6 +20,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - run: pip install --upgrade pip setuptools wheel tox
-      - run: python setup.py install # required for e2e tests
+      - run: pip install . # required for e2e tests
       - run: make lint
       - run: tox

.github/workflows/release.yml

@@ -35,7 +35,7 @@ jobs:
         run: python -m pip install --upgrade pip setuptools wheel twine
 
       - name: Build package
-        run: python setup.py sdist bdist_wheel
+        run: python -m build
 
       - name: Publish Python package
         uses: pypa/gh-action-pypi-publish@release/v1

.vscode/settings.json

@@ -1,25 +1,16 @@
 {
-    "python.formatting.provider": "black",
     "python.linting.enabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.flake8Args": [
-        "--config setup.cfg"
-    ],
-    "python.linting.mypyEnabled": true,
-    "python.linting.mypyArgs": [
-        "--config-file setup.cfg"
-    ],
-    "isort.args": [
-        "-sp setup.cfg"
-    ],
-    "python.linting.pylintEnabled": false,
+    "python.formatting.provider": "none",
     "[python]": {
+        "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true
+            "source.organizeImports": true,
         },
-        "editor.tabSize": 4,
-        "editor.formatOnSave": true
+        "editor.defaultFormatter": "ms-python.black-formatter",
     },
+    "editor.rulers": [
+        88
+    ],
     "[terraform]": {
         "editor.tabSize": 2,
         "editor.formatOnSave": true
@@ -33,9 +24,4 @@
         "editor.tabSize": 2,
         "editor.autoIndent": "advanced"
     },
-    "python.testing.pytestArgs": [
-        "tests"
-    ],
-    "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
 }

README.md

@@ -1,6 +1,8 @@
-## Grove
-
-> Grove is not an official HashiCorp project.
+<p align="center">
+    <br /><br />
+    <img src="docs/static/grove-logo-small.png?raw=True" alt="Grove">
+    <br />
+</p>
 
 Grove is a Software as a Service (SaaS) log collection framework, designed to support
 collection of logs from services which do not natively support log streaming.
@@ -24,6 +26,12 @@ us via email at [email protected], rather than filing a GitHub issue.
 
 ### Supported Sources
 
+<p align="center">
+    <br /><br />
+    <img src="docs/static/grove-support.png?raw=True" alt="Grove support overview">
+    <br />
+</p>
+
 Currently the following log sources are supported by Grove out of the box. If a source
 isn't listed here, support can be added by creating a custom connector!
 

docs/api.rst

@@ -59,3 +59,4 @@ Subpackages
    grove.helpers
    grove.outputs
    grove.secrets
+   grove.processors

docs/conf.py

@@ -10,24 +10,11 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath("../grove/"))
-
-# Set defaults, and then load __about__ from the project.
-__title__ = None
-__about__ = None
-__author__ = None
-__version__ = None
-__copyright__ = None
-
-exec(open(os.path.abspath("../grove/__about__.py")).read())
-
+from grove.__about__ import __copyright__, __title__, __version__
 
 # -- Project information -----------------------------------------------------
 
-author = __author__
+author = "HashiCorp Security (TDR)"
 project = __title__.title()
 copyright = __copyright__
 
@@ -63,7 +50,7 @@
 # a list of builtin themes.
 html_theme = "furo"
 html_title = f"Grove v{__version__}"
-html_logo = "static/grove.png"
+# html_logo = "static/grove.png"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -86,7 +73,7 @@
         "admonition-title-font-size": "0.9rem",
     },
     "dark_css_variables": {
-        "color-foreground-secondary": "#444",
+        "color-foreground-secondary": "#9ca0a5",
         "admonition-font-size": "0.9rem",
         "admonition-title-font-size": "0.9rem",
     },

docs/grove.rst

@@ -19,6 +19,7 @@ Subpackages
    grove.helpers
    grove.outputs
    grove.secrets
+   grove.processors
 
 Submodules
 ----------

docs/index.rst

@@ -1,8 +1,14 @@
-Grove
-=====
+.. container:: clear-title
 
-.. note::
-   Grove is not an official HashiCorp project.
+   .. image:: static/grove-logo-small.png
+      :alt: Grove
+      :align: center
+      :class: only-light
+
+   .. image:: static/grove-logo-small-light.png
+      :alt: Grove
+      :align: center
+      :class: only-dark
 
 Grove is a Software as a Service (SaaS) log collection framework, designed to support
 collection of logs from services which do not natively support log streaming.
@@ -27,6 +33,18 @@ us via email at [email protected], rather than filing a GitHub issue.
 Supported Sources
 -----------------
 
+.. container:: clear-image
+
+   .. image:: static/grove-support.png
+      :alt: Supported Sources
+      :align: center
+      :class: only-light
+
+   .. image:: static/grove-support-light.png
+      :alt: Supported Sources
+      :align: center
+      :class: only-dark
+
 Currently the following log sources are supported by Grove out of the box. If a source
 isn't listed here, support can be added by creating a custom connector!
 

docs/internals.rst

@@ -15,9 +15,11 @@ Flow
 
 A rough diagram of the overall flow of a Grove run ("collection") can be found below.
 
-.. image:: static/flow.png
-   :alt: Overall flow of a Grove collection
-   :align: center
+.. container:: clear-image
+
+   .. image:: static/flow.png
+      :alt: Overall flow of a Grove collection
+      :align: center
 
 Components
 ----------
@@ -64,9 +66,11 @@ A visual overview of the relationship between built-in Grove backends and their
 base classes can be found below. Although the examples in this image are built-in
 to Grove, they follow the same principals as any other Plugin.
 
-.. image:: static/plugin_structure.png
-   :alt: Grove Module Overview
-   :align: center
+.. container:: clear-image
+
+   .. image:: static/plugin_structure.png
+      :alt: Grove Module Overview
+      :align: center
 
 Configuration
 ^^^^^^^^^^^^^
@@ -212,10 +216,12 @@ If no existing pointer is in the cache, the connector will provide an initial va
 which is appropriate for the application. When Grove next runs, only log entries
 generated since / after this pointer would be collected.
 
-.. image:: static/pointers.png
-   :width: 384
-   :alt: How pointers are used.
-   :align: center
+.. container:: clear-image
+
+   .. image:: static/pointers.png
+      :width: 384
+      :alt: How pointers are used.
+      :align: center
 
 Cache
 -----

docs/static/custom.css

@@ -1,5 +1,7 @@
 :root {
-    --text-color-override: #3d7c9b;
+    --text-color-override: #428cdb;
+    --title-color: #428cdb;
+    /* --text-color-override: #38a7d2; */
 }
 
 article {
@@ -21,23 +23,25 @@ h4,
 h5 {
     padding-top: 8px;
     padding-bottom: 8px;
-    color: var(--text-color-override);
-}
-
-.sidebar-tree .toctree-l1 > .reference {
-    color: var(--text-color-override);
+    color: var(--title-color);
 }
 
 .sidebar-brand-text {
     text-align: center;
-    color: var(--text-color-override);
 }
 
-.grove-logo h1 {
-    padding-top: 0px;
-    font-size: 420%;
-    padding-left: 10%;
-    text-align: center;
+.sidebar-brand-text[data-theme="light"] {
+    color: #444 !important;
+}
+
+.sidebar-brand-text[data-theme="dark"] {
+    color: #FFF !important;
+}
+
+.sidebar-tree .toctree-l1 > .reference,
+.sidebar-tree .toctree-l2 > .reference,
+.sidebar-tree .toctree-l3 > .reference {
+    color: var(--text-color-override);
 }
 
 .sidebar-logo {
@@ -50,4 +54,18 @@ h5 {
 
 p.admonition-title {
     margin: 0 -0.9rem 0.8rem -0.9rem
+}
+
+.clear-image {
+    padding-top: 35px;
+    padding-bottom: 35px;
+}
+
+.clear-title {
+    padding-top: 35px;
+    padding-bottom: 50px;
+}
+
+.light-bg {
+    background-color: #FFF;
 }

grove/__about__.py

@@ -1,7 +1,6 @@
 """Grove metadata."""
 
-__version__ = "1.0.0rc4"
+__version__ = "1.0.0rc5"
 __title__ = "grove"
-__author__ = "HashiCorp Security (TDR)"
 __license__ = "Mozilla Public License 2.0"
 __copyright__ = "Copyright 2023 HashiCorp, Inc."

grove/__init__.py

@@ -3,15 +3,18 @@
 
 """A framework for collecting and transforming SaaS logs."""
 
-from grove import caches  # noqa: F401
-from grove import configs  # noqa: F401
-from grove import connectors  # noqa: F401
-from grove import constants  # noqa: F401
-from grove import entrypoints  # noqa: F401
-from grove import exceptions  # noqa: F401
-from grove import helpers  # noqa: F401
-from grove import logging  # noqa: F401
-from grove import models  # noqa: F401
-from grove import outputs  # noqa: F401
-from grove import types  # noqa: F401
+from grove import (
+    caches,  # noqa: F401
+    configs,  # noqa: F401
+    connectors,  # noqa: F401
+    constants,  # noqa: F401
+    entrypoints,  # noqa: F401
+    exceptions,  # noqa: F401
+    helpers,  # noqa: F401
+    logging,  # noqa: F401
+    models,  # noqa: F401
+    outputs,  # noqa: F401
+    processors,  # noqa: F401
+    types,  # noqa: F401
+)
 from grove.__about__ import *  # noqa: F401, F403