update databricks test action

.github/workflows/databricks-test.yml

@@ -17,15 +17,16 @@ jobs:
 
       - name: Install dependencies
         run: |
+          cp -n databricks/requirements-databricks-15.4-LTS.txt .
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements-databricks-15.4-LTS.txt ]; then pip install -r requirements-databricks-15.4-LTS.txt; fi
 
       - name: Lint with flake8
         run: |
-         # stop the build if there are Python syntax errors or undefined names
+          # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
       - name: Test with pytest
@@ -35,4 +36,5 @@ jobs:
 
       - name: Build test Docker image
         run: |
+            cp -n databricks/databricks.dockerfile .
             docker build -t databricks-test-image -f databricks.dockerfile .

databricks/README

@@ -0,0 +1,3 @@
+# This files related to the databricks compatible library
+
+For testing purposes only

databricks/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel"]
+build-backend = "setuptools.build_meta"

requirements-databricks-15.4-LTS.txt → databricks/requirements-databricks-15.4-LTS.txt

@@ -1,20 +1,10 @@
 aiohttp>=3.9.0
 attrs
 avro
-azure-storage-blob
 base32-crockford
-bokeh==2.4.*
 boto3
-botocore
 category_encoders==2.6.3
 click
-deltalake==0.17.4
-dill==0.3.*
-docker
-dp_accounting
-entrypoints==0.4
-flake8
-gevent==23.9.1
 Jinja2
 keras==2.15.*
 lazy==1.4
@@ -31,26 +21,17 @@ pathos==0.2.*
 pillow==9.4.*
 psutil
 py-ulid
-pyarrow>=14.0.1
-pytest==7.4.4
+pytest
 pytest-reportportal
-python-dotenv
 python-slugify[unidecode]>=7.0.0
 PyYAML==6.*
 reportportal-client
 scikit_learn==1.3.*
 scipy==1.11.*
 seaborn==0.12.*
 setuptools==68.*
-smart-open
 tensorflow==2.15.*
-tensorflow-addons
-tensorflow_datasets
-tensorflow-estimator==2.15.0
-tensorflow_probability==0.22.1
-tensorflow_privacy==0.9.0
 tqdm==4.66.3
-typing-extensions==4.4.0
 Werkzeug==3.0.3
 xlrd
-xlwt
+xlwt

databricks/setup.cfg

@@ -0,0 +1,82 @@
+[metadata]
+name = syngen
+version = file: src/syngen/VERSION
+description = file: DESCRIPTION
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/tdspora/syngen
+author = EPAM Systems, Inc.
+maintainer = Pavel Bobyrev
+license = GPLv3 License
+keywords = data, generation, synthetic, vae, tabular
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Operating System :: POSIX :: Linux
+    Operating System :: Microsoft :: Windows
+    License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+
+
+[options]
+package_dir =
+    = src
+packages = find:
+include_package_data = True
+python_requires = >3.10, <3.12
+install_requires =
+    aiohttp>=3.9.0
+    attrs
+    avro
+    base32-crockford
+    boto3
+    category_encoders==2.6.3
+    click
+    Jinja2
+    keras==2.15.*
+    lazy==1.4
+    loguru
+    MarkupSafe==2.1.1
+    marshmallow==3.19.*
+    matplotlib==3.7.*
+    mlflow-skinny==2.11.*
+    numpy==1.23.*
+    openpyxl
+    pandas==2.2.*
+    pandavro==1.8.*
+    pathos==0.2.*
+    pillow==9.4.*
+    psutil
+    py-ulid
+    pytest
+    pytest-reportportal
+    python-slugify[unidecode]>=7.0.0
+    PyYAML==6.*
+    reportportal-client
+    scikit_learn==1.3.*
+    scipy==1.11.*
+    seaborn==0.12.*
+    setuptools==68.*
+    tensorflow==2.15.*
+    tqdm==4.66.3
+    Werkzeug==3.0.3
+    xlrd
+    xlwt
+
+
+[options.extras_require]
+ui =
+    streamlit==1.31.*
+    streamlit_option_menu
+
+
+[options.packages.find]
+where = src
+[options.package_data]
+* = *.py, *.html, *.ttf, *.svg, *.css, *.js
+
+[options.entry_points]
+console_scripts =
+    train = syngen.train:launch_train
+    infer = syngen.infer:launch_infer
+    syngen = syngen:main