Merge pull request #68 from AnFreTh/develop

Beta Version merge

.gitattributes

@@ -0,0 +1 @@
+*.parquet filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-publish-pypi.yml

@@ -0,0 +1,32 @@
+name: Publish Package to PyPi
+
+on:
+  push:
+    branches:
+      - release
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+      - name: Build and publish package
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.gitignore

@@ -1,12 +1,186 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.DS_Store
+
+dist/
+docs/_build/*
+
 examples/lightning_logs/
 *.ckpt
-*.yaml
+# *.yaml
 
 # Additional files and directories to ignore
 __pycache__/
 *.pyc
 *.pyo
-*.json
+# *.json # for default preprocessing steps
 model_head.pkl
 model.safetensors
-vocab.txt
+vocab.txt
+
+# Ignore the following directories
+dev/*
+dev
+post-checkout
+post-commit
+post-merge
+pre-push
+docs/notebooks/lightning_logs/*

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.8
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format

CODE_OF_CONDUCT.md

@@ -0,0 +1,19 @@
+# Code of Conduct
+
+- **Purpose**:  The purpose of this Code of Conduct is to establish a welcoming and inclusive community around the `STREAM` project. We want to foster an environment where everyone feels respected, valued, and able to contribute to the project.
+
+- **Openness and Respect**: We strive to create an open and respectful community where everyone can freely express their opinions and ideas. We encourage constructive discussions and debates, but we will not tolerate any form of harassment, discrimination, or disrespectful behavior.
+
+- **Inclusive Language**: We are committed to using inclusive language that reflects the diversity of our community. We avoid using language that could be perceived as offensive, derogatory, or exclusionary towards any individual or group.
+
+- **Collaboration and Support**: We encourage collaboration and support among community members. We value the contributions of all members, regardless of their level of expertise or background. We are here to learn from each other and help each other grow.
+
+- **Reporting and Addressing Issues**: If you witness or experience any violations of this Code of Conduct, please report it to the project maintainers or administrators. All reports will be kept confidential, and appropriate actions will be taken to address the issue. We are committed to resolving conflicts and maintaining a healthy and safe community.
+
+- **Consequences for Violations**: Anyone who engages in behavior that violates this Code of Conduct may be temporarily or permanently banned from participating in the project.
+
+- **Community Guidelines**: In addition to this Code of Conduct, community members are expected to follow any specific guidelines and rules set forth by the project maintainers. These guidelines may include rules regarding contributions, issue reporting, and communication channels.
+
+- **Enforcement**: The project maintainers are responsible for enforcing this Code of Conduct. They have the right and authority to interpret and enforce the rules, and their decisions are final.
+
+Remember, this Code of Conduct applies to all aspects of the project, including but not limited to code contributions, discussions, documentation, and interactions within the community.

MANIFEST.in

@@ -1,3 +1,3 @@
 recursive-exclude notebooks *
-recursive-include ExpandedTM/preprocessed_datasets *
-recursive-include ExpandedTM/pre_embedded_datasets *
+recursive-include stream/preprocessed_datasets/*
+include stream/preprocessor/config/default_preprocessing_steps.json