diff --git a/.circleci/circle_requirements.txt b/.circleci/circle_requirements.txt new file mode 100644 index 0000000..1c010d2 --- /dev/null +++ b/.circleci/circle_requirements.txt @@ -0,0 +1,3 @@ +poetry>=1.1.6 +tox>=3.23.1 +tox-poetry>=0.3.0 diff --git a/.circleci/config.yml b/.circleci/config.yml index 9e7918f..702b12d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,97 +1,139 @@ +version: 2.1 -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 -jobs: - build: - docker: - - image: circleci/python:3.6.9 - - image: redislabs/redisai:edge-cpu-bionic - - working_directory: ~/repo +commands: + abort_for_docs: steps: - - checkout - - - restore_cache: # Download and cache dependencies - keys: - - v1-dependencies-{{ checksum "test-requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - run: - name: install dependencies + name: Avoid tests for docs command: | - virtualenv --no-site-packages venv - . venv/bin/activate - pip install -r test-requirements.txt - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "test-requirements.txt" }} + if [[ $CIRCLE_BRANCH == *docs ]]; then + echo "Identifies as documents PR, no testing required" + circleci step halt + fi + abort_for_noci: + steps: - run: - name: run tests + name: Ignore CI for specific branches command: | - . venv/bin/activate - nosetests --with-coverage -vsx test - codecov - - - store_artifacts: - path: test-reports - destination: test-reports + if [[ $CIRCLE_BRANCH == *noci ]]; then + echo "Identifies as actively ignoring CI, no testing required." + circleci step halt + fi - build_nightly: - docker: - - image: circleci/python:3.6.9 - - image: redislabs/redisai:edge-cpu-bionic - working_directory: ~/repo + early_return_for_forked_pull_requests: + description: >- + If this build is from a fork, stop executing the current job and return success. + This is useful to avoid steps that will fail due to missing credentials. + steps: + - run: + name: Early return if this build is from a forked PR + command: | + if [[ -n "$CIRCLE_PR_NUMBER" ]]; then + echo "Nothing to do for forked PRs, so marking this step successful" + circleci step halt + fi + build_and_test: steps: - checkout - - restore_cache: # Download and cache dependencies keys: - - v1-dependencies-{{ checksum "test-requirements.txt" }} + - v1-dependencies-{{ checksum "pyproject.toml" }} # fallback to using the latest cache if no exact match is found - v1-dependencies- - run: - name: install dependencies + name: install tox dependencies command: | - virtualenv --no-site-packages venv - . venv/bin/activate - pip install -r test-requirements.txt + pip install --user --quiet -r .circleci/circle_requirements.txt - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "test-requirements.txt" }} + - run: + name: build sdist and wheels + command: | + poetry build - run: - name: run tests + name: lint command: | - . venv/bin/activate - nosetests -vsx test + tox -e linters + + - run: + name: run tests + command: + tox -e tests - # no need for store_artifacts on nightly builds + - save_cache: + paths: + - ./.tox + - ~/.cache/pip + key: v1-dependencies-{{ checksum "pyproject.toml" }} + +jobs: + build: + parameters: + python_version: + type: string + default: "latest" + docker: + - image: circleci/python:<> + - image: redislabs/redisai:edge-cpu-bionic + + steps: + - build_and_test + - store_artifacts: + path: test-reports + destination: test-reports + + nightly: + parameters: + python_version: + type: string + docker: + - image: circleci/python:<> + - image: redislabs/redisai:edge-cpu-bionic + steps: + - build_and_test + - dockerize + +on-any-branch: &on-any-branch + filters: + branches: + only: + - /.*/ + tags: + ignore: /.*/ + +on-master: &on-master + filters: + branches: + only: + - master + +python-versions: &python-versions + matrix: + parameters: + python_version: + - "3.6.9" + - "3.7.9" + - "3.8.9" + - "3.9.4" + - "latest" workflows: version: 2 commit: jobs: - - build + - build: + <<: *on-any-branch + <<: *python-versions + nightly: triggers: - schedule: cron: "0 0 * * *" - filters: - branches: - only: - - master + <<: *on-master jobs: - - build_nightly + - build diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 10a87d6..7caeccd 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -1,50 +1,43 @@ name: Publish Pypi on: release: - types: [published] + types: [ published ] jobs: - publish: - name: publish + pytest: + name: Publish to PyPi runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true steps: - uses: actions/checkout@master - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - - name: Install twine - run: | - pip install twine - - - name: Install wheel - run: | - pip install wheel - - - name: Create a source distribution - run: | - python setup.py sdist + - name: Install Poetry + uses: dschep/install-poetry-action@v1.3 - - name: Create a wheel - run: | - python setup.py bdist_wheel + - name: Cache Poetry virtualenv + uses: actions/cache@v1 + id: cache + with: + path: ~/.virtualenvs + key: poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + poetry-${{ hashFiles('**/poetry.lock') }} - - name: Create a .pypirc + - name: Set Poetry config run: | - echo -e "[pypi]" >> ~/.pypirc - echo -e "username = __token__" >> ~/.pypirc - echo -e "password = ${{ secrets.PYPI_TOKEN }}" >> ~/.pypirc - echo -e "[testpypi]" >> ~/.pypirc - echo -e "username = __token__" >> ~/.pypirc - echo -e "password = ${{ secrets.TESTPYPI_TOKEN }}" >> ~/.pypirc + poetry config virtualenvs.in-project false + poetry config virtualenvs.path ~/.virtualenvs - - name: Publish to Test PyPI - if: github.event_name == 'release' - run: | - twine upload --skip-existing -r testpypi dist/* + - name: Install Dependencies + run: poetry install + if: steps.cache.outputs.cache-hit != 'true' - name: Publish to PyPI if: github.event_name == 'release' run: | - twine upload -r pypi dist/* + poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} --build diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e1f0b30 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +ARG OSNICK=bionic +ARG TARGET=cpu + +FROM redislabs/redisai:edge-${TARGET}-${OSNICK} as builder + +RUN apt update && apt install -y python3 python3-pip +ADD . /build +WORKDIR /build +RUN pip3 install poetry +RUN poetry config virtualenvs.create false +RUN poetry build + +### clean docker stage +FROM redislabs/redisai:edge-${TARGET}-${OSNICK} as runner + +RUN apt update && apt install -y python3 python3-pip +RUN rm -rf /var/cache/apt/ + +COPY --from=builder /build/dist/redisai*.tar.gz /tmp/ +RUN pip3 install /tmp/redisai*.tar.gz diff --git a/README.rst b/README.rst index f6b0bdc..3d05052 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,15 @@ redisai-py :target: https://codecov.io/gh/RedisAI/redisai-py .. image:: https://readthedocs.org/projects/redisai-py/badge/?version=latest - :target: https://redisai-py.readthedocs.io/en/latest/?badge=latest + :target: https://redisai-py.readthedocs.io/en/latest/?badge=latest .. image:: https://img.shields.io/badge/Forum-RedisAI-blue :target: https://forum.redislabs.com/c/modules/redisai .. image:: https://img.shields.io/discord/697882427875393627?style=flat-square - :target: https://discord.gg/rTQm7UZ + :target: https://discord.gg/rTQm7UZ + +.. image:: https://snyk.io/test/github/RedisAI/redisai-py/badge.svg?targetFile=pyproject.toml)](https://snyk.io/test/github/RedisAI/redisai-py?targetFile=pyproject.toml redisai-py is the Python client for RedisAI. Checkout the `documentation `_ for API details and examples @@ -47,6 +49,26 @@ Installation $ pip install ml2rt +Development +----------- + +1. Assuming you have virtualenv installed, create a virtualenv to manage your python dependencies, and activate it. + ```virtualenv -v venv; source venv/bin/activate``` +2. Install [pypoetry](https://python-poetry.org/) to manage your dependencies. + ```pip install poetry``` +3. Install dependencies. + ```poetry install --no-root``` + +[tox](https://tox.readthedocs.io/en/latest/) runs all tests as its default target. Running *tox* by itself will run unit tests. Ensure you have a running redis, with the module loaded. + +**Contributing** + +Prior to submitting a pull request, please ensure you've built and installed poetry as above. Then: + +1. Run the linter. + ```tox -e linters.``` +2. Run the unit tests. This assumes you have a redis server running, with the [RedisAI module](https://redisai.io) already loaded. If you don't, you may want to install a [docker build](https://hub.docker.com/r/redislabs/redisai/tags). + ```tox -e tests``` `RedisAI example repo `_ shows few examples made using redisai-py under `python_client` folder. Also, checkout diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e2cfe99 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[tool.poetry] +name = "redisai" +version = "1.0.2" +description = "RedisAI Python Client" +authors = ["RedisLabs "] +license = "BSD-3-Clause" +readme = "README.rst" + +packages = [ + { include = 'redisai' }, +] + +classifiers = [ + 'Topic :: Database', + 'Programming Language :: Python', + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable' +] + +[tool.poetry.dependencies] +python = "^3.6" +redis = ">=2.10" +hiredis = ">=0.20" +numpy = ">=1.19.5" +six = ">=1.10.0" +Deprecated = "^1.2.12" + +[tool.poetry.dev-dependencies] +codecov = "^2.1.11" +flake8 = "^3.9.2" +rmtest = "^0.7.0" +nose = "^1.3.7" +ml2rt = "^0.2.0" +tox = ">=3.23.1" +tox-poetry = "^0.3.0" +bandit = "^1.7.0" +pylint = "^2.8.2" +vulture = "^2.3" + +[tool.poetry.urls] +url = "https://redisai.io" +repository = "https://github.com/RedisAI/redisai-py" + + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/redisai/__init__.py b/redisai/__init__.py index 9c90c68..a03dbac 100644 --- a/redisai/__init__.py +++ b/redisai/__init__.py @@ -1,3 +1,3 @@ -from .client import Client +from .client import Client # noqa __version__ = "1.0.2" diff --git a/redisai/client.py b/redisai/client.py index 5458e4c..aa6bd04 100644 --- a/redisai/client.py +++ b/redisai/client.py @@ -1,6 +1,6 @@ import warnings -from functools import partial, wraps -from typing import Any, AnyStr, ByteString, List, Sequence, Union +from functools import wraps +from typing import AnyStr, ByteString, List, Sequence, Union import numpy as np from deprecated import deprecated @@ -65,8 +65,8 @@ def pipeline(self, transaction: bool = True, shard_hint: bool = None) -> "Pipeli self.enable_postprocess, self.connection_pool, self.response_callbacks, - transaction=True, - shard_hint=None, + transaction=transaction, + shard_hint=shard_hint, ) def dag( diff --git a/redisai/command_builder.py b/redisai/command_builder.py index 3f1c151..53e422b 100644 --- a/redisai/command_builder.py +++ b/redisai/command_builder.py @@ -64,7 +64,7 @@ def modelstore( "Inputs and outputs keywords should not be specified for this backend" ) chunk_size = 500 * 1024 * 1024 # TODO: this should be configurable. - data_chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)] + data_chunks = [data[i: i + chunk_size] for i in range(0, len(data), chunk_size)] # TODO: need a test case for this args += ["BLOB", *data_chunks] return args @@ -102,7 +102,7 @@ def modelset( args += ["INPUTS", *utils.listify(inputs)] args += ["OUTPUTS", *utils.listify(outputs)] chunk_size = 500 * 1024 * 1024 - data_chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)] + data_chunks = [data[i: i + chunk_size] for i in range(0, len(data), chunk_size)] # TODO: need a test case for this args += ["BLOB", *data_chunks] return args diff --git a/redisai/dag.py b/redisai/dag.py index eb2e1ad..1746010 100644 --- a/redisai/dag.py +++ b/redisai/dag.py @@ -12,7 +12,7 @@ class Dag: def __init__(self, load, persist, executor, readonly=False, postprocess=True): self.result_processors = [] - self.enable_postprocess = True + self.enable_postprocess = postprocess if readonly: if persist: raise RuntimeError( diff --git a/redisai/pipeline.py b/redisai/pipeline.py index f6a8255..53d28b2 100644 --- a/redisai/pipeline.py +++ b/redisai/pipeline.py @@ -1,4 +1,3 @@ -import warnings from functools import partial from typing import AnyStr, Sequence, Union diff --git a/setup.py b/setup.py deleted file mode 100644 index f5de6bb..0000000 --- a/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages - - -with open("README.rst") as f: - long_description = f.read() - -setup( - name="redisai", - version="1.0.2", - description="RedisAI Python Client", - long_description=long_description, - long_description_content_type="text/x-rst", - url="http://github.com/RedisAI/redisai-py", - author="RedisLabs", - author_email="oss@redislabs.com", - packages=find_packages(), - install_requires=["redis", "hiredis", "numpy", "deprecated"], - python_requires=">=3.6", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Database", - ], -) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 252e5cc..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -hiredis>=0.2.0 -redis>=2.10 -rmtest>=0.2 -six>=1.10.0 -nose -codecov -numpy -ml2rt -deprecated \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d74449d --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +skipsdist = true +envlist = linters,tests + +[flake8] +max-complexity = 10 +ignore = E501,C901 +srcdir = ./redisai +exclude =.git,.tox,dist,doc,*/__pycache__/* + +[testenv:tests] +whitelist_externals = find +commands_pre = + find . -type f -name "*.pyc" -delete +commands = + nosetests -vsx test + +[testenv:linters] +commands = + flake8 --show-source + vulture redisai --min-confidence 80 + bandit redisai/**