diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 69dc5ba4f..46447d0ce 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -624,6 +624,29 @@ steps: artifact_paths: - "perf8-report-*/**/*" + - group: ":building_construction: Build and Test Python Packages" + key: "test_packages" + steps: + - label: ":python: Build Python {{matrix.python_version}} {{matrix.package_path}} package" + key: build_python_package + <<: *test-agents + matrix: + setup: + python_version: + - "3.10" + - "3.11" + package_path: + - "app/connectors_service" + - "libs/connectors_sdk" + env: + PYTHON_VERSION: "{{ matrix.python_version }}" + PACKAGE_PATH: "{{ matrix.package_path }}" + command: ".buildkite/test_python_packages.sh" + artifact_paths: + - "{{ matrix.package_path }}/dist/*.whl" + - "{{ matrix.package_path }}/dist/*.tar.gz" + timeout_in_minutes: 10 + # ---- # DRA publishing # ---- @@ -635,6 +658,7 @@ steps: - "unit_tests" - "smoke_test" - "relevant_ftests" + - "test_packages" steps: # ---- # Docker builds for amd64 @@ -699,7 +723,7 @@ steps: - ".buildkite/publish/test-docker.sh" # ---- - # OSS Dockerfile build, tests and vunlerability scan on amd64 + # OSS Dockerfile build, tests and vulnerability scan on amd64 # ---- - label: "Building amd64 Docker image from OSS Dockerfile" agents: @@ -746,7 +770,7 @@ steps: find .artifacts -type f -name '*.tar.gz*' -exec trivy image --quiet --input {} \; # ---- - # OSS Dockerfile build, tests and vunlerability scan on arm64 + # OSS Dockerfile build, tests and vulnerability scan on arm64 # ---- - label: "Building arm64 Docker image from OSS Dockerfile" agents: diff --git a/.buildkite/publish/dra/init_dra_publishing.sh b/.buildkite/publish/dra/init_dra_publishing.sh index 2bdf5715d..83e9d8fc7 100755 --- a/.buildkite/publish/dra/init_dra_publishing.sh +++ b/.buildkite/publish/dra/init_dra_publishing.sh @@ -18,15 +18,25 @@ cd $PROJECT_ROOT make clean zip export DRA_ARTIFACTS_DIR=$RELEASE_DIR/dist/dra-artifacts mkdir -p $DRA_ARTIFACTS_DIR +cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip cd - # Download previous step artifacts buildkite-agent artifact download '.artifacts/*.tar.gz*' $RELEASE_DIR/dist/ --step build_docker_image_amd64 buildkite-agent artifact download '.artifacts/*.tar.gz*' $RELEASE_DIR/dist/ --step build_docker_image_arm64 +buildkite-agent artifact download 'app/connectors_service/dist/*.whl' $RELEASE_DIR/ --step test_packages +buildkite-agent artifact download 'app/connectors_service/dist/*.tar.gz' $RELEASE_DIR/ --step test_packages +buildkite-agent artifact download 'libs/connectors_sdk/dist/*.whl' $RELEASE_DIR/ --step test_packages +buildkite-agent artifact download 'libs/connectors_sdk/dist/*.tar.gz' $RELEASE_DIR/ --step test_packages + + +# Copy previous step artifacts to DRA dir cp $RELEASE_DIR/dist/.artifacts/* $DRA_ARTIFACTS_DIR +cp $RELEASE_DIR/app/connectors_service/dist/* $DRA_ARTIFACTS_DIR +cp $RELEASE_DIR/libs/connectors_sdk/dist/* $DRA_ARTIFACTS_DIR -# Rename to match DRA expectations (----) +# Rename docker images to match DRA expectations (----) cd $DRA_ARTIFACTS_DIR mv $DOCKER_ARTIFACT_KEY-$VERSION-amd64.tar.gz $PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz mv $DOCKER_ARTIFACT_KEY-$VERSION-arm64.tar.gz $PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz @@ -138,9 +148,16 @@ if [[ "${PUBLISH_SNAPSHOT:-}" == "true" ]]; then generateDependencyReport $DEPENDENCIES_REPORTS_DIR/$dependencyReportName echo "-------- Publishing SNAPSHOT DRA Artifacts" - cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}-SNAPSHOT.zip + + # Make *-SNAPSHOT* copies + cp $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}-SNAPSHOT.zip + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-SNAPSHOT.whl + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-SNAPSHOT.tar.gz + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-SNAPSHOT.whl + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-SNAPSHOT.tar.gz cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-SNAPSHOT-docker-image-linux-amd64.tar.gz cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-SNAPSHOT-docker-image-linux-arm64.tar.gz + setDraVaultCredentials export WORKFLOW="snapshot" @@ -155,19 +172,23 @@ fi if [[ "${PUBLISH_STAGING:-}" == "true" ]]; then if [ -n "${VERSION_QUALIFIER:-}" ]; then dependencyReportName="dependencies-${VERSION}-${VERSION_QUALIFIER}.csv"; - zip_artifact_name="connectors-${VERSION}-${VERSION_QUALIFIER}.zip" + + # Make *-$VERSION_QUALIFIER* copies + cp $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/$PROJECT_NAME-${VERSION}-${VERSION_QUALIFIER}.zip cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-$VERSION_QUALIFIER-docker-image-linux-amd64.tar.gz cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-$VERSION_QUALIFIER-docker-image-linux-arm64.tar.gz + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-$VERSION_QUALIFIER.whl + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-$VERSION_QUALIFIER.tar.gz + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-$VERSION_QUALIFIER.whl + cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-$VERSION_QUALIFIER.tar.gz else dependencyReportName="dependencies-${VERSION}.csv"; - zip_artifact_name="connectors-${VERSION}.zip" fi echo "-------- Generating STAGING dependency report: ${dependencyReportName}" generateDependencyReport $DEPENDENCIES_REPORTS_DIR/$dependencyReportName echo "-------- Publishing STAGING DRA Artifacts" - cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/${zip_artifact_name} setDraVaultCredentials export WORKFLOW="staging" diff --git a/.buildkite/publish/manual-release/update-release-version.sh b/.buildkite/publish/manual-release/update-release-version.sh index cf524a97a..9ebd717d2 100755 --- a/.buildkite/publish/manual-release/update-release-version.sh +++ b/.buildkite/publish/manual-release/update-release-version.sh @@ -7,10 +7,10 @@ CURDIR="$(dirname "$REL_DIR")" source $CURDIR/publish-common.sh -echo $VERSION > $PROJECT_ROOT/connectors/VERSION # adds the timestamp suffix -UPDATED_VERSION=`cat $PROJECT_ROOT/connectors/VERSION` +echo $VERSION > $PROJECT_ROOT/app/connectors/VERSION # adds the timestamp suffix +UPDATED_VERSION=`cat $PROJECT_ROOT/app/connectors/VERSION` -git add $PROJECT_ROOT/connectors/VERSION +git add $PROJECT_ROOT/app/connectors/VERSION git commit -m "Bumping version from ${ORIG_VERSION} to ${UPDATED_VERSION}" git push origin ${GIT_BRANCH} diff --git a/.buildkite/publish_to_pypi.sh b/.buildkite/publish_to_pypi.sh new file mode 100644 index 000000000..3301d745a --- /dev/null +++ b/.buildkite/publish_to_pypi.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# !!! WARNING DO NOT add -x to avoid leaking vault passwords +set -euo pipefail + +source .buildkite/shared.sh + +init_python +cd "$PACKAGE_PATH" +python -m pip install --upgrade build twine +python -m build + +export TWINE_USERNAME="__token__" + +# upload to test or real PyPI based on TEST_PYPI=1 env var or arg +if [[ "${1:-}" == "TEST_PYPI=1" ]] || [[ "${TEST_PYPI:-}" =~ ^(1|TRUE|true)$ ]]; then + TWINE_PASSWORD=$(vault read -field publishing-api-key secret/ci/elastic-connectors/test-pypi) + export TWINE_PASSWORD + python -m twine upload --repository testpypi dist/* +else + TWINE_PASSWORD=$(vault read -field publishing-api-key secret/ci/elastic-connectors/pypi) + export TWINE_PASSWORD + python -m twine upload --repository pypi dist/* +fi diff --git a/.buildkite/pypi-publish-pipeline.yml b/.buildkite/pypi-publish-pipeline.yml new file mode 100644 index 000000000..271b62994 --- /dev/null +++ b/.buildkite/pypi-publish-pipeline.yml @@ -0,0 +1,92 @@ +## .buildkite/pypi-publish-pipeline.yml +# Manually-triggered pipeline to build and publish Python packages to PyPI +notify: + - if: 'build.branch =~ /^((main)|([0-9]+\.[0-9]+))$/ && (build.state == "failed" || pipeline.started_passing)' + slack: + channels: + - "#search-et-alerts" + message: "${BUILDKITE_MESSAGE}" + +# add paths for all the packages we want to build here +# then make sure they're added to the matrix.setup.package_path lists in the steps below +env: + SERVICE_PACKAGE_PATH: "app/connectors_service" + SDK_PACKAGE_PATH: "libs/connectors_sdk" + +steps: + - group: ":building_construction: Build Python Packages" + key: "build_package" + steps: + - label: ":python: Build Python {{matrix.python_version}} {{matrix.package_path}} package" + key: build_python_package + agents: + provider: "gcp" + machineType: "n1-standard-8" + useVault: true + image: family/enterprise-search-ubuntu-2204-connectors-py + matrix: + setup: + python_version: + - "3.10" + - "3.11" + package_path: + - "${SERVICE_PACKAGE_PATH}" + - "${SDK_PACKAGE_PATH}" + env: + PYTHON_VERSION: "{{ matrix.python_version }}" + PACKAGE_PATH: "{{ matrix.package_path }}" + commands: ".buildkite/test_python_packages.sh" + + - wait + + - group: ":test_tube: Publish to Test PyPI" + key: "publish_to_test_pypi" + depends_on: + - build_package + steps: + - label: ":package: Publish to Test PyPI" + key: publish_test_pypi + agents: + provider: "gcp" + machineType: "n1-standard-8" + useVault: true + image: family/enterprise-search-ubuntu-2204-connectors-py + matrix: + setup: + package_path: + - "${SERVICE_PACKAGE_PATH}" + - "${SDK_PACKAGE_PATH}" + env: + PYTHON_VERSION: "3.11" + PACKAGE_PATH: "{{ matrix.package_path }}" + commands: "TEST_PYPI=1 ./buildkite/publish_to_pypi.sh" + +# FOR WHEN WE'RE READY TO PUBLISH TO PRODUCTION PYPI +# +# - block: ":rocket: Publish to Production PyPI?" +# prompt: "Test PyPI publish succeeded. Proceed to publish to Production PyPI?" +# key: "confirm_production_publish" +# depends_on: +# - publish_to_test_pypi +# +# - group: ":truck: Publish to Production PyPI" +# key: "publish_to_pypi" +# depends_on: +# - confirm_production_publish +# steps: +# - label: ":package: Publish to Production PyPI" +# key: publish_pypi +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# useVault: true +# image: family/enterprise-search-ubuntu-2204-connectors-py +# matrix: +# setup: +# package_path: +# - "${SERVICE_PACKAGE_PATH}" +# - "${SDK_PACKAGE_PATH}" +# env: +# PYTHON_VERSION: "3.11" +# PACKAGE_PATH: "{{ matrix.package_path }}" +# commands: ".buildkite/publish_to_pypi.sh" diff --git a/.buildkite/release-pipeline.yml b/.buildkite/release-pipeline.yml index 591360de2..c92a22521 100644 --- a/.buildkite/release-pipeline.yml +++ b/.buildkite/release-pipeline.yml @@ -15,12 +15,12 @@ steps: key: "release_setup" steps: # ---- - # Set the build timestamp (for the verion suffix) + # Set the build timestamp (for the version suffix) # --- - label: "Set build metadata" commands: - buildkite-agent meta-data set timestamp "$(date -u +'%Y%m%d%H%M')" - - buildkite-agent meta-data set orig_version "$(cat connectors/VERSION)" + - buildkite-agent meta-data set orig_version "$(cat app/connectors/VERSION)" key: set_timestamp - wait - label: ":github: update version and tag" diff --git a/.buildkite/test_python_packages.sh b/.buildkite/test_python_packages.sh new file mode 100755 index 000000000..328bf915c --- /dev/null +++ b/.buildkite/test_python_packages.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# !!! WARNING DO NOT add -x to avoid leaking vault passwords +set -euo pipefail + +source .buildkite/shared.sh + +init_python + +python -m pip install --upgrade build twine +python -m build "$PACKAGE_PATH" +ls -lah "$PACKAGE_PATH/dist/" +python -m twine check "$PACKAGE_PATH/dist/*" + +# If this is the connectors_service package, test the installation and CLI +if [[ "$PACKAGE_PATH" == *app/connectors_service* ]]; then + echo "Testing connectors_service package installation and CLI..." + + # Install the connectors_sdk package first + LIB_PATH="libs/connectors_sdk" + python -m build "$LIB_PATH" + python -m pip install "$LIB_PATH"/dist/*.whl + + python -m pip install "$PACKAGE_PATH"/dist/*.whl + connectors --help + elastic-ingest --help +# elastic-agent-connectors --help + test-connectors --help +else + python -m pip install "$PACKAGE_PATH"/dist/*.whl + python -c "import connectors_sdk; print(f'🎉 Success! connectors_sdk version: {connectors_sdk.__version__}')" +fi diff --git a/Dockerfile b/Dockerfile index f63c44584..e2e230ec2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,8 @@ COPY --chown=nonroot:nonroot . /app USER nonroot WORKDIR /app -RUN make clean install -RUN ln -s .venv/bin /app/bin +RUN make clean install-package +RUN ln -s app/connectors_service/.venv/bin /app/bin USER root RUN apk del make git diff --git a/Dockerfile.wolfi b/Dockerfile.wolfi index 30d87f183..2c1099381 100644 --- a/Dockerfile.wolfi +++ b/Dockerfile.wolfi @@ -2,6 +2,6 @@ FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:c85a849c182a812950c3036b3965 USER root COPY . /app WORKDIR /app -RUN make clean install -RUN ln -s .venv/bin /app/bin +RUN make clean install-package +RUN ln -s app/connectors_service/.venv/bin /app/bin ENTRYPOINT [] diff --git a/Makefile b/Makefile index ea2bd1ca8..8bbc1e4e9 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ install: cd $(connectors_sdk_dir); make install cd $(app_dir); make install +install-package: + cd $(connectors_sdk_dir); make install-package + cd $(app_dir); make install-package + test: install cd $(connectors_sdk_dir); make test cd $(app_dir); make test diff --git a/app/connectors_service/.ruff.toml b/app/connectors_service/.ruff.toml deleted file mode 100644 index 279732ddd..000000000 --- a/app/connectors_service/.ruff.toml +++ /dev/null @@ -1,31 +0,0 @@ -target-version = "py310" - -[lint] -select = ["A", "ASYNC", "I", "E", "F", "B", "C4", "T10", "T20", "EM", "ISC", "S", "CPY001"] -ignore = ["E501", "ISC001"] -preview = true - -# Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "C4", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT", "T10", "T20"] -unfixable = [] - -exclude = [ - ".git", - "__pycache__", - "lib", - "bin", - "include" -] -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" - -[lint.per-file-ignores] -"connectors/*" = ["S608"] -"scripts/verify.py" = [ "EM" ] -"tests/*" = ["B017", "S101", "S", "ASYNC110"] -"tests/conftest.py" = [ "EM" ] - -[lint.isort] -known-first-party=["connectors", "tests"] - -[lint.flake8-copyright] -notice-rgx = "#\n# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one\n# or more contributor license agreements. Licensed under the Elastic License 2.0;\n# you may not use this file except in compliance with the Elastic License 2.0.\n#" diff --git a/app/connectors_service/LICENSE b/app/connectors_service/LICENSE new file mode 100644 index 000000000..809108b85 --- /dev/null +++ b/app/connectors_service/LICENSE @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/app/connectors_service/Makefile b/app/connectors_service/Makefile index 3090dc485..3e708e004 100644 --- a/app/connectors_service/Makefile +++ b/app/connectors_service/Makefile @@ -41,6 +41,11 @@ install: .venv/bin/python .venv/bin/elastic-ingest notice install-agent: .venv/bin/elastic-ingest +install-package: .venv/bin/python + .venv/bin/python -m build ../../libs/connectors_sdk + .venv/bin/pip install ../../libs/connectors_sdk/dist/*.whl + .venv/bin/python -m build + .venv/bin/pip install dist/*.whl .venv/bin/elastic-ingest: .venv/bin/python .venv/bin/pip install -e ../../libs/connectors_sdk diff --git a/app/connectors_service/NOTICE.txt b/app/connectors_service/NOTICE.txt index d0a6e6a7e..b381835f2 100644 --- a/app/connectors_service/NOTICE.txt +++ b/app/connectors_service/NOTICE.txt @@ -3546,13 +3546,106 @@ Apache Software License elasticsearch-connectors -9.2.0 -Apache Software License +9.3.0 UNKNOWN +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. + elasticsearch-connectors-sdk 9.3.0 -Apache Software License +UNKNOWN UNKNOWN envyaml diff --git a/app/connectors_service/connectors/VERSION b/app/connectors_service/connectors/VERSION index deeb3d66e..b13d146a7 100644 --- a/app/connectors_service/connectors/VERSION +++ b/app/connectors_service/connectors/VERSION @@ -1 +1 @@ -9.2.0 +9.3.0 diff --git a/app/connectors_service/pyproject.toml b/app/connectors_service/pyproject.toml index ea19e99dd..d4dfac773 100644 --- a/app/connectors_service/pyproject.toml +++ b/app/connectors_service/pyproject.toml @@ -10,11 +10,12 @@ readme = "README.md" authors = [ { name = "Search Extract and Transform Team", email = "enterprise-search-ingestion-team@elastic.co" } ] +license = "Elastic-2.0" +license-files = ["LICENSE"] classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", ] requires-python = ">=3.10,<3.12" @@ -114,8 +115,69 @@ connectors = "connectors.connectors_cli:main" elastic-agent-connectors = "connectors.agent.cli:main" test-connectors = "scripts.testing.cli:main" -[tool.setuptools] -packages = ["connectors", "scripts"] +[tool.setuptools.packages.find] +where = ["."] +include = ["connectors*", "scripts*"] [tool.setuptools.dynamic] version = {file = "connectors/VERSION"} + +[tool.pytest.ini_options] +asyncio_mode = "auto" +addopts = [ + "-v" +] +filterwarnings = [ + "error", + # botocore has this warning that is reported by them to be irrelevant + "ignore:.*urllib3.contrib.pyopenssl.*:DeprecationWarning:botocore.*", + # latest main of aioresponses does not have this problem, but current package uses deprecated pkg_resources API + "ignore:.*pkg_resources.*:DeprecationWarning", + # SQLAlchemy uses deprecated APIs internally + "ignore:.*dbapi().*:DeprecationWarning", + # aiogoogle inherits on top of AioHttpSession, which is not recommended by aiohttp + "ignore:Inheritance class AiohttpSession from ClientSession is discouraged:DeprecationWarning", + # aiogoogle inherits on top of RetryableAioHttpSession, which is not recommended by aiohttp + "ignore:Inheritance class RetryableAiohttpSession from ClientSession is discouraged:DeprecationWarning", + # pytest may generate its own warnings in some situations, such as improper usage or deprecated features. + "ignore::pytest.PytestUnraisableExceptionWarning", + # this error comes from 8.16 Elasticsearch client: we use some non-GA features in our CI code + "ignore::elasticsearch.exceptions.GeneralAvailabilityWarning", +] + +[tool.pyright] +pythonVersion = "3.11" +include = ["connectors"] +reportMissingImports = false +reportMissingModuleSource = false +reportOptionalMemberAccess = false +exclude = ["**/tests", "**/__pycache__"] +executionEnvironments = [ + { root = "./", venv = ".venv" } +] + +[tool.ruff] +target-version = "py310" + +[tool.ruff.lint] +select = ["A", "ASYNC", "I", "E", "F", "B", "C4", "T10", "T20", "EM", "ISC", "S", "CPY001"] +ignore = ["E501", "ISC001"] +preview = true + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "C4", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT", "T10", "T20"] +unfixable = [] + +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.lint.per-file-ignores] +"connectors/*" = ["S608"] +"scripts/verify.py" = ["EM"] +"tests/*" = ["B017", "S101", "S", "ASYNC110"] +"tests/conftest.py" = ["EM"] + +[tool.ruff.lint.isort] +known-first-party = ["connectors", "tests"] + +[tool.ruff.lint.flake8-copyright] +notice-rgx = "#\n# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one\n# or more contributor license agreements. Licensed under the Elastic License 2.0;\n# you may not use this file except in compliance with the Elastic License 2.0.\n#" diff --git a/app/connectors_service/pyrightconfig.json b/app/connectors_service/pyrightconfig.json deleted file mode 100644 index d2c099858..000000000 --- a/app/connectors_service/pyrightconfig.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "pythonVersion": "3.10", - "include": [ - "connectors" - ], - "reportMissingImports": false, - "reportMissingModuleSource": false, - "reportOptionalMemberAccess": false, - "exclude": [ - "**/tests", - "**/__pycache__" - ], - "executionEnvironments": [ - { - "root": "./", - "venv": "./" - } - ] -} diff --git a/app/connectors_service/pytest.ini b/app/connectors_service/pytest.ini deleted file mode 100644 index 757c4a49a..000000000 --- a/app/connectors_service/pytest.ini +++ /dev/null @@ -1,20 +0,0 @@ -[pytest] -asyncio_mode = auto -addopts = - -v -filterwarnings = - error -; botocore has this warning that is reported by them to be irrelevant - ignore:.*urllib3.contrib.pyopenssl.*:DeprecationWarning:botocore.* -; latest main of aioresponses does not have this problem, but current package uses deprecated pkg_resources API - ignore:.*pkg_resources.*:DeprecationWarning -; SQLAlchemy uses deprecated APIs internally - ignore:.*dbapi().*:DeprecationWarning -; aiogoogle inherits on top of AioHttpSession, which is not recommended by aiohttp - ignore:Inheritance class AiohttpSession from ClientSession is discouraged:DeprecationWarning -; aiogoogle inherits on top of RetryableAioHttpSession, which is not recommended by aiohttp - ignore:Inheritance class RetryableAiohttpSession from ClientSession is discouraged:DeprecationWarning -; pytest may generate its own warnings in some situations, such as improper usage or deprecated features. - ignore::pytest.PytestUnraisableExceptionWarning -; this error comes from 8.16 Elasticsearch client: we use some non-GA features in our CI code - ignore::elasticsearch.exceptions.GeneralAvailabilityWarning diff --git a/catalog-info.yaml b/catalog-info.yaml index 1e27569a2..f66b3415a 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -245,3 +245,34 @@ spec: search-productivity-team: {} everyone: access_level: "READ_ONLY" + +######## +# Publish Python packages to PyPI - manual release +######## +--- +apiVersion: "backstage.io/v1alpha1" +kind: "Resource" +metadata: + name: "connectors-publish-to-pypi" + description: "Publish packages related to content connectors to PyPI" + links: null +spec: + type: "buildkite-pipeline" + owner: "group:search-extract-and-transform" + system: "buildkite" + implementation: + apiVersion: "buildkite.elastic.dev/v1" + kind: "Pipeline" + metadata: + name: "connectors-publish-to-pypi" + spec: + repository: "elastic/connectors" + pipeline_file: ".buildkite/publish-to-pypi.yml" + provider_settings: + trigger_mode: "none" + teams: + search-extract-and-transform: {} + search-productivity-team: {} + search-eng: {} + everyone: + access_level: "READ_ONLY" diff --git a/libs/connectors_sdk/Makefile b/libs/connectors_sdk/Makefile index 1983d408f..f3ddb60ec 100644 --- a/libs/connectors_sdk/Makefile +++ b/libs/connectors_sdk/Makefile @@ -10,6 +10,13 @@ SLOW_TEST_THRESHOLD = 1 install: .venv/bin/python +install-package: + $(PYTHON) -m venv .venv + $(VENV_DIR)/pip install --upgrade pip + $(VENV_DIR)/python -m pip install build + $(VENV_DIR)/python -m build + $(VENV_DIR)/pip install dist/*.whl + clean: rm -rf bin lib .venv include elasticsearch_connectors.egg-info .coverage site-packages pyvenv.cfg include.site.python*.greenlet dist diff --git a/libs/connectors_sdk/NOTICE.txt b/libs/connectors_sdk/NOTICE.txt index a857dd527..67167a1f5 100644 --- a/libs/connectors_sdk/NOTICE.txt +++ b/libs/connectors_sdk/NOTICE.txt @@ -978,7 +978,7 @@ UNKNOWN elasticsearch-connectors-sdk 9.3.0 -Apache Software License +Elastic-2.0 UNKNOWN fastjsonschema diff --git a/libs/connectors_sdk/pyproject.toml b/libs/connectors_sdk/pyproject.toml index 85f491f28..527d679f5 100644 --- a/libs/connectors_sdk/pyproject.toml +++ b/libs/connectors_sdk/pyproject.toml @@ -10,11 +10,11 @@ readme = "README.md" authors = [ { name = "Search Extract and Transform Team", email = "enterprise-search-ingestion-team@elastic.co" } ] +license = "Elastic-2.0" classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", ] requires-python = ">=3.10,<3.12" @@ -42,8 +42,9 @@ tests = [ "requests==2.32.4", ] -[tool.setuptools] -packages = ["connectors_sdk"] +[tool.setuptools.packages.find] +where = ["."] +include = ["connectors_sdk*"] [tool.setuptools.dynamic] version = {file = "connectors_sdk/VERSION"}