diff --git a/.buildkite/nightly_steps.yml b/.buildkite/nightly_steps.yml index 9a0ed5546..c01824d21 100644 --- a/.buildkite/nightly_steps.yml +++ b/.buildkite/nightly_steps.yml @@ -2,133 +2,290 @@ env: DOCKERFILE_FTEST_PATH: "Dockerfile.ftest.wolfi" steps: - - label: "🔨 MySQL" - command: - - ".buildkite/run_nigthly.sh mysql" + - label: "🔨 [Python {{ matrix }}] MySQL" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "mysql" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Network Drive" - command: - - ".buildkite/run_nigthly.sh network_drive" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Network Drive" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "network_drive" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Amazon S3" - command: - - ".buildkite/run_nigthly.sh s3" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Amazon S3" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "s3" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Google Cloud Storage" - command: - - ".buildkite/run_nigthly.sh google_cloud_storage" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Google Cloud Storage" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "google_cloud_storage" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Azure Blob Storage" - command: - - ".buildkite/run_nigthly.sh azure_blob_storage" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Azure Blob Storage" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "azure_blob_storage" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Postgresql" - command: - - ".buildkite/run_nigthly.sh postgresql" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Postgresql" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "postgresql" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 System Directory" - command: - - ".buildkite/run_nigthly.sh dir" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] System Directory" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "dir" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Oracle Database" - command: - - ".buildkite/run_nigthly.sh oracle" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Oracle Database" + command: ".buildkite/run_functional_test.sh" env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "oracle" + DATA_SIZE: "small" SKIP_AARCH64: "true" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Sharepoint Server" - command: - - ".buildkite/run_nigthly.sh sharepoint_server" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Sharepoint Server" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "sharepoint_server" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Sharepoint Online" - command: - - ".buildkite/run_nigthly.sh sharepoint_online" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Sharepoint Online" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "sharepoint_online" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Microsoft SQL" - command: - - ".buildkite/run_nigthly.sh mssql" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Microsoft SQL" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "mssql" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Jira" - command: - - ".buildkite/run_nigthly.sh jira" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Jira" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "jira" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Confluence" - command: - - ".buildkite/run_nigthly.sh confluence" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Confluence" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "confluence" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 ServiceNow" - command: - - ".buildkite/run_nigthly.sh servicenow" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] ServiceNow" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "servicenow" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 MongoDB" - command: - - ".buildkite/run_nigthly.sh mongodb" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] MongoDB" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "mongodb" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" -# - label: "🔨 MongoDB Serverless" -# command: -# - ".buildkite/run_nigthly.sh mongodb_serverless" + matrix: + - "3.10" + - "3.11" +# - label: "🔨 [Python {{ matrix }}] MongoDB Serverless" +# command: ".buildkite/run_functional_test.sh" +# env: +# PYTHON_VERSION: "{{ matrix }}" +# CONNECTOR: "mongodb_serverless" +# DATA_SIZE: "small" +# matrix: +# - "3.10" +# - "3.11" # artifact_paths: # - "perf8-report-*/**/*" - - label: "🔨 GitHub" - command: - - ".buildkite/run_nigthly.sh github" + - label: "🔨 [Python {{ matrix }}] GitHub" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "github" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Google Drive" - command: - - ".buildkite/run_nigthly.sh google_drive" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Google Drive" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "google_drive" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Dropbox" - command: - - ".buildkite/run_nigthly.sh dropbox" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Dropbox" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "dropbox" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 OneDrive" - command: - - ".buildkite/run_nigthly.sh onedrive" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] OneDrive" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "onedrive" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Salesforce" - command: - - ".buildkite/run_nigthly.sh salesforce" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Salesforce" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "salesforce" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Zoom" - command: - - ".buildkite/run_nigthly.sh zoom" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Zoom" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "zoom" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Box" - command: - - ".buildkite/run_nigthly.sh box" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Box" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "box" + DATA_SIZE: "small" + artifact_paths: + - "perf8-report-*/**/*" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Notion" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "notion" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 Notion" - command: - - ".buildkite/run_nigthly.sh notion" - - label: "🔨 Redis" - command: - - ".buildkite/run_nigthly.sh redis" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] Redis" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "redis" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" - - label: "🔨 GraphQL" - command: - - ".buildkite/run_nigthly.sh graphql" + matrix: + - "3.10" + - "3.11" + - label: "🔨 [Python {{ matrix }}] GraphQL" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "graphql" + DATA_SIZE: "small" artifact_paths: - "perf8-report-*/**/*" + matrix: + - "3.10" + - "3.11" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e2d30d796..d95ca969f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -9,22 +9,33 @@ env: steps: - label: ":face_with_peeking_eye: Lint" - command: "make lint" + command: ".buildkite/run_linter.sh" + env: + PYTHON_VERSION: "3.11" # We don't really care about linter on multiple versions for now timeout_in_minutes: 5 - - label: ":pytest: Test" - command: "make test" + - label: ":pytest: Test with Python {{ matrix }}" + command: ".buildkite/run_tests.sh" + env: + PYTHON_VERSION: "{{ matrix }}" timeout_in_minutes: 5 + matrix: + - "3.10" + - "3.11" - label: ":shipit: Smoke test" - command: "make ftest NAME=dir DATA_SIZE=small" + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "3.11" # TODO: can actually do a matrix too + CONNECTOR: "dir" + DATA_SIZE: "small" timeout_in_minutes: 45 agents: machineType: "n1-standard-8" - label: ":sweating: Checking for changes in connectors" plugins: - chronotc/monorepo-diff#v2.4.0: + monorepo-diff#v1.0.1: diff: ".buildkite/diff ${BUILDKITE_COMMIT}" wait: false watch: @@ -36,8 +47,12 @@ steps: - "requirements/**" config: label: "🔨 MySQL" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=mysql command: - - ".buildkite/run_nigthly.sh mysql small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -49,8 +64,12 @@ steps: - "requirements/**" config: label: "🔨 Network Drive" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=network_drive command: - - ".buildkite/run_nigthly.sh network_drive small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -62,8 +81,12 @@ steps: - "requirements/**" config: label: "🔨 Amazon S3" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=s3 command: - - ".buildkite/run_nigthly.sh s3 small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -75,8 +98,12 @@ steps: - "requirements/**" config: label: "🔨 Google Cloud Storage" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=google_cloud_storage command: - - ".buildkite/run_nigthly.sh google_cloud_storage small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -88,8 +115,12 @@ steps: - "requirements/**" config: label: "🔨 Azure Blob Storage" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=azure_blob_storage command: - - ".buildkite/run_nigthly.sh azure_blob_storage small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -101,8 +132,12 @@ steps: - "requirements/**" config: label: "🔨 Postgresql" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=postgresql command: - - ".buildkite/run_nigthly.sh postgresql small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -114,8 +149,12 @@ steps: - "requirements/**" config: label: "🔨 System Directory" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=dir command: - - ".buildkite/run_nigthly.sh dir small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -127,8 +166,12 @@ steps: - "requirements/**" config: label: "🔨 Oracle Database" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=oracle command: - - ".buildkite/run_nigthly.sh oracle small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -140,8 +183,12 @@ steps: - "requirements/**" config: label: "🔨 Sharepoint Server" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=sharepoint_server command: - - ".buildkite/run_nigthly.sh sharepoint_server small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -153,8 +200,12 @@ steps: - "requirements/**" config: label: "🔨 Sharepoint Online" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=sharepoint_online command: - - ".buildkite/run_nigthly.sh sharepoint_online small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -166,8 +217,12 @@ steps: - "requirements/**" config: label: "🔨 Microsoft SQL" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=mssql command: - - ".buildkite/run_nigthly.sh mssql small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -180,8 +235,12 @@ steps: - "requirements/**" config: label: "🔨 Jira" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=jira command: - - ".buildkite/run_nigthly.sh jira small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -194,8 +253,12 @@ steps: - "requirements/**" config: label: "🔨 Confluence" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=confluence command: - - ".buildkite/run_nigthly.sh confluence small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -207,8 +270,12 @@ steps: - "requirements/**" config: label: "🔨 ServiceNow" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=servicenow command: - - ".buildkite/run_nigthly.sh servicenow small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -220,8 +287,12 @@ steps: - "requirements/**" config: label: "🔨 MongoDB" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=mongodb command: - - ".buildkite/run_nigthly.sh mongodb small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -233,8 +304,12 @@ steps: - "requirements/**" config: label: "🔨 GitHub" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=github command: - - ".buildkite/run_nigthly.sh github small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -246,8 +321,12 @@ steps: - "requirements/**" config: label: "🔨 Google Drive" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=google_drive command: - - ".buildkite/run_nigthly.sh google_drive small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -259,8 +338,12 @@ steps: - "requirements/**" config: label: "🔨 Dropbox" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=dropbox command: - - ".buildkite/run_nigthly.sh dropbox small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -272,8 +355,12 @@ steps: - "requirements/**" config: label: "🔨 OneDrive" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=onedrive command: - - ".buildkite/run_nigthly.sh onedrive small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -285,8 +372,12 @@ steps: - "requirements/**" config: label: "🔨 Salesforce" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=salesforce command: - - ".buildkite/run_nigthly.sh salesforce small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -298,8 +389,12 @@ steps: - "requirements/**" config: label: "🔨 Zoom" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=zoom command: - - ".buildkite/run_nigthly.sh zoom small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -311,8 +406,12 @@ steps: - "requirements/**" config: label: "🔨 Box" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=box command: - - ".buildkite/run_nigthly.sh box small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -324,8 +423,12 @@ steps: - "requirements/**" config: label: "🔨 Microsoft Teams" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=microsoft_teams command: - - ".buildkite/run_nigthly.sh microsoft_teams small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -337,8 +440,14 @@ steps: - "requirements/**" config: label: "🔨 Notion" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=notion command: - - ".buildkite/run_nigthly.sh notion small" + - ".buildkite/run_functional_test.sh" + artifact_paths: + - "perf8-report-*/**/*" - path: - "connectors/sources/redis.py" @@ -348,8 +457,12 @@ steps: - "requirements/**" config: label: "🔨 Redis" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=redis command: - - ".buildkite/run_nigthly.sh redis small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" @@ -361,7 +474,11 @@ steps: - "requirements/**" config: label: "🔨 GraphQL" + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=graphql command: - - ".buildkite/run_nigthly.sh graphql small" + - ".buildkite/run_functional_test.sh" artifact_paths: - "perf8-report-*/**/*" diff --git a/.buildkite/run_nigthly.sh b/.buildkite/run_functional_test.sh similarity index 80% rename from .buildkite/run_nigthly.sh rename to .buildkite/run_functional_test.sh index 530abfae8..f9ecbcb1e 100755 --- a/.buildkite/run_nigthly.sh +++ b/.buildkite/run_functional_test.sh @@ -10,13 +10,15 @@ if [ "$MACHINE_TYPE" != "x86_64" ] && [ -v SKIP_AARCH64 ]; then exit fi +source ~/.bash_profile + +pyenv global $PYTHON_VERSION +echo "Python version:" +pyenv global BASEDIR=$(realpath $(dirname $0)) ROOT=$(realpath $BASEDIR/../) -# TODO to be moved in the image at https://github.com/elastic/ci-agent-images/blob/main/vm-images/enterprise-search/scripts/connectors-python/install-deps.sh#L6 -sudo apt-get -y install liblz4-dev libunwind-dev - cd $ROOT make install @@ -24,7 +26,6 @@ make install export PIP=$ROOT/bin/pip $PIP install py-spy -DATA_SIZE="${2:-small}" # If we run on buildkite, we connect to docker so we can pull private images # !!! WARNING be cautious about the following lines to avoid leaking the secrets in the CI logs @@ -42,4 +43,4 @@ if [ -v BUILDKITE ]; then sudo sysctl -w vm.max_map_count=262144 fi -PERF8=yes NAME=$1 DATA_SIZE=$DATA_SIZE make ftest +PERF8=yes NAME=$CONNECTOR DATA_SIZE=$DATA_SIZE make ftest diff --git a/.buildkite/run_linter.sh b/.buildkite/run_linter.sh new file mode 100755 index 000000000..0e2eda9d9 --- /dev/null +++ b/.buildkite/run_linter.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# !!! WARNING DO NOT add -x to avoid leaking vault passwords +set -euo pipefail + +source ~/.bash_profile + +pyenv global $PYTHON_VERSION +echo "Python version:" +pyenv global + +make lint diff --git a/.buildkite/run_tests.sh b/.buildkite/run_tests.sh new file mode 100755 index 000000000..d444f2cb1 --- /dev/null +++ b/.buildkite/run_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# !!! WARNING DO NOT add -x to avoid leaking vault passwords +set -euo pipefail + +source ~/.bash_profile + +pyenv global $PYTHON_VERSION +echo "Python version:" +pyenv global + +make test diff --git a/Dockerfile b/Dockerfile index ec12162f4..a58433010 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.10 +FROM python:3.11-slim-bookworm +RUN apt update && apt upgrade && apt install make -y COPY . /app WORKDIR /app RUN make clean install +ENTRYPOINT [] diff --git a/Dockerfile.ftest b/Dockerfile.ftest index 64a92643e..95a4557bc 100644 --- a/Dockerfile.ftest +++ b/Dockerfile.ftest @@ -1,5 +1,7 @@ -FROM python:3.10 -COPY . /connectors -WORKDIR /connectors +FROM python:3.11-slim-bookworm +# RUN apt update && apt install make +RUN apt update && apt upgrade && apt install make -y +COPY . /app +WORKDIR /app RUN make clean install RUN bin/pip install -r requirements/ftest.txt diff --git a/Dockerfile.ftest.wolfi b/Dockerfile.ftest.wolfi index c3d968f24..df5cb0603 100644 --- a/Dockerfile.ftest.wolfi +++ b/Dockerfile.ftest.wolfi @@ -1,4 +1,4 @@ -FROM docker.elastic.co/wolfi/python:3.10-dev@sha256:2b7e585c59b5f7b5800de99aa2dc0e576bc1f7358e0d1acfcc14dcd90bb3925f +FROM docker.elastic.co/wolfi/python:3.11-dev USER root COPY . /connectors WORKDIR /connectors diff --git a/Dockerfile.wolfi b/Dockerfile.wolfi index be2b1f003..5919d94ea 100644 --- a/Dockerfile.wolfi +++ b/Dockerfile.wolfi @@ -1,4 +1,4 @@ -FROM docker.elastic.co/wolfi/python:3.10-dev@sha256:2b7e585c59b5f7b5800de99aa2dc0e576bc1f7358e0d1acfcc14dcd90bb3925f +FROM docker.elastic.co/wolfi/python:3.11-dev USER root COPY . /app WORKDIR /app diff --git a/Makefile b/Makefile index 2f79ef85a..c35178289 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: test lint autoformat run ftest install release docker-build docker-run docker-push -PYTHON=python3.10 +PYTHON?=python ARCH=$(shell uname -m) PERF8?=no SLOW_TEST_THRESHOLD=1 # seconds @@ -35,7 +35,7 @@ bin/pytest: bin/python bin/pip install -r requirements/ftest.txt clean: - rm -rf bin lib include elasticsearch_connector.egg-info .coverage site-packages pyvenv.cfg include.site.python3.10.greenlet + rm -rf bin lib include elasticsearch_connector.egg-info .coverage site-packages pyvenv.cfg include.site.python*.greenlet lint: bin/python bin/black bin/elastic-ingest bin/black --check connectors diff --git a/requirements/framework.txt b/requirements/framework.txt index 931829c2a..4dbe8ac89 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -5,7 +5,8 @@ httpx==0.27.0 httpx-ntlm==1.4.0 elasticsearch[async]==8.14.0 elastic-transport==8.13.1 -pyyaml==6.0 +pyyaml==6.0.1 +cffi==1.16.0 envyaml==1.10.211231 ecs-logging==2.0.0 pympler==1.0.1 @@ -14,13 +15,13 @@ tzcron==1.0.0 pytz==2019.3 python-dateutil==2.8.2 aiogoogle==5.3.0 -uvloop==0.17.0; sys_platform != 'win32' +uvloop==0.18.0; sys_platform != 'win32' fastjsonschema==2.16.2 base64io==1.0.3 azure-storage-blob==12.19.1 SQLAlchemy==2.0.1 -oracledb==1.2.2 -asyncpg==0.27.0 +oracledb==2.3.0 +asyncpg==0.29.0 python-tds==1.12.0 sqlalchemy-pytds==0.3.5 pyOpenSSL==24.1.0 diff --git a/requirements/ftest.txt b/requirements/ftest.txt index 6101574e2..7bd3a98ca 100644 --- a/requirements/ftest.txt +++ b/requirements/ftest.txt @@ -4,8 +4,8 @@ faker==18.11.2 mysql-connector-python==8.1.0 google-auth==2.22.0 google-cloud-storage==2.10.0 -asyncpg==0.28.0 +asyncpg==0.29.0 boto3==1.34.69 -oracledb==1.2.2 +oracledb==2.3.0 smbprotocol==1.10.1 -yattag==1.15.1 +yattag==1.16.0 diff --git a/scripts/stack/README.md b/scripts/stack/README.md index acf4c3583..f09669e24 100644 --- a/scripts/stack/README.md +++ b/scripts/stack/README.md @@ -17,7 +17,7 @@ These scripts can also be used as a starting place for adding Connectors to your ## Prerequisites * Linux or macOS (Although Windows can run the Connectors service, it is not currently supported via these scripts) -* Python 3.10 +* Python 3.10 or 3.11 * Docker with Docker Compose Installed * It is recommended to run Docker with at least 4GB of available RAM. * Curl 8.7.1 or higher diff --git a/tests/sources/fixtures/oracle/fixture.py b/tests/sources/fixtures/oracle/fixture.py index 045aa59fa..0e18b92e8 100644 --- a/tests/sources/fixtures/oracle/fixture.py +++ b/tests/sources/fixtures/oracle/fixture.py @@ -23,7 +23,6 @@ USER = "c##admin" PASSWORD = "Password_123" -ENCODING = "UTF-8" DSN = "localhost:1521/FREE" DATA_SIZE = os.environ.get("DATA_SIZE", "medium").lower() @@ -66,17 +65,13 @@ def inject_lines(table, cursor, lines): async def load(): """Generate tables and loads table data in the oracle server.""" """N tables of RECORD_COUNT rows each""" - connection = oracledb.connect( - user="system", password=PASSWORD, dsn=DSN, encoding=ENCODING - ) + connection = oracledb.connect(user="system", password=PASSWORD, dsn=DSN) cursor = connection.cursor() cursor.execute(f"CREATE USER {USER} IDENTIFIED by {PASSWORD} CONTAINER=ALL") cursor.execute(f"GRANT CONNECT, RESOURCE, DBA TO {USER}") connection.commit() - connection = oracledb.connect( - user=USER, password=PASSWORD, dsn=DSN, encoding=ENCODING - ) + connection = oracledb.connect(user=USER, password=PASSWORD, dsn=DSN) cursor = connection.cursor() for table in range(NUM_TABLES): print(f"Adding data for table #{table}...") @@ -88,9 +83,7 @@ async def load(): async def remove(): """Removes 10 random items per table""" - connection = oracledb.connect( - user=USER, password=PASSWORD, dsn=DSN, encoding=ENCODING - ) + connection = oracledb.connect(user=USER, password=PASSWORD, dsn=DSN) cursor = connection.cursor() for table in range(NUM_TABLES):