From f4e4d6c1675017837a56fd04e24e62204c79060b Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:51:11 -0400 Subject: [PATCH 01/11] create standard workflow file - create workflow file which runs pytest --- .github/workflows/standard.yaml | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/standard.yaml diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml new file mode 100644 index 0000000..c8df4a4 --- /dev/null +++ b/.github/workflows/standard.yaml @@ -0,0 +1,49 @@ +name: "Standard" +run-name: ${{ github.run_number }} [${{ github.actor }}] on ${{ github.ref_name }} + +on: + pull_request: + branches: + - main + push: + branches: + - main + schedule: + - cron: '0 0 * * *' # Once a day at 12am UTC + workflow_dispatch: + +jobs: + # ---------------------------------------------------------------------- + validate: + name: Validate + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + matrix: + python_version: + - "3.12" + - "3.11" + - "3.10" + + env: + COLUMNS: "200" + SIMULATE_TERMINAL_CAPABILITIES_SUPPORTS_COLORS: "1" + PYTHONIOENCODING: "UTF-8" + + permissions: {} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Test with pytest + run: | + pip install pytest pytest-cov + pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html From 3f817ad31aca95aa4c23c7b2c95a2f6578064211 Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:54:12 -0400 Subject: [PATCH 02/11] update test file path --- .github/workflows/standard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index c8df4a4..735c944 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -46,4 +46,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html From c7ecc36c4d9c96263605974128a12277a7cd52f1 Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:58:44 -0400 Subject: [PATCH 03/11] install requirements - add `requirements.txt` file - install requirements as part of workflow --- .github/workflows/standard.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index 735c944..bc6c6dd 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -43,6 +43,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt - name: Test with pytest run: | pip install pytest pytest-cov From 427acbdef3d14f5944fbfe91d7ae9894d2118985 Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 17:00:54 -0400 Subject: [PATCH 04/11] parallelize --- .github/workflows/standard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index bc6c6dd..043e976 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html -n auto From 0bdc51edfb33daba3103ea673223b214ee8b2735 Mon Sep 17 00:00:00 2001 From: varun646 Date: Fri, 23 Aug 2024 11:51:51 -0400 Subject: [PATCH 05/11] add flag for subset of tests --- .github/workflows/standard.yaml | 2 +- tests/conftest.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index 043e976..9668dab 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html -n auto + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html --subset-percentage 0.1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4f880d7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +import random + + +def pytest_addoption(parser): + parser.addoption( + "--subset-percentage", + action="store", + default=None, + help="Specify the percentage of tests to run as a subset (e.g., 0.3 for 30%)", + ) + + +def pytest_collection_modifyitems(config, items): + subset_percentage = config.getoption("--subset-percentage") + + # If the subset percentage is provided, apply it + if subset_percentage is not None: + subset_percentage = float(subset_percentage) + + # Shuffle the test cases + random.shuffle(items) + + # Calculate the number of tests to run based on percentage + subset_count = int(len(items) * subset_percentage) + + # Select a subset of tests + items[:] = items[:subset_count] From e30673cfa349f046b56e6982c667d70009b9afd2 Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:51:11 -0400 Subject: [PATCH 06/11] create standard workflow file - create workflow file which runs pytest --- .github/workflows/standard.yaml | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/standard.yaml diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml new file mode 100644 index 0000000..c8df4a4 --- /dev/null +++ b/.github/workflows/standard.yaml @@ -0,0 +1,49 @@ +name: "Standard" +run-name: ${{ github.run_number }} [${{ github.actor }}] on ${{ github.ref_name }} + +on: + pull_request: + branches: + - main + push: + branches: + - main + schedule: + - cron: '0 0 * * *' # Once a day at 12am UTC + workflow_dispatch: + +jobs: + # ---------------------------------------------------------------------- + validate: + name: Validate + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + matrix: + python_version: + - "3.12" + - "3.11" + - "3.10" + + env: + COLUMNS: "200" + SIMULATE_TERMINAL_CAPABILITIES_SUPPORTS_COLORS: "1" + PYTHONIOENCODING: "UTF-8" + + permissions: {} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Test with pytest + run: | + pip install pytest pytest-cov + pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html From dec500ab26afd20045688102ddf2748d41db16dc Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:54:12 -0400 Subject: [PATCH 07/11] update test file path --- .github/workflows/standard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index c8df4a4..735c944 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -46,4 +46,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html From ee6a895ac5fbba821e70bf01a8f076c9ce3ba2af Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 16:58:44 -0400 Subject: [PATCH 08/11] install requirements - add `requirements.txt` file - install requirements as part of workflow --- .github/workflows/standard.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index 735c944..bc6c6dd 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -43,6 +43,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt - name: Test with pytest run: | pip install pytest pytest-cov From eb6b16f887441663cde516ddf5531bb81b59d5df Mon Sep 17 00:00:00 2001 From: varun646 Date: Thu, 22 Aug 2024 17:00:54 -0400 Subject: [PATCH 09/11] parallelize --- .github/workflows/standard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index bc6c6dd..043e976 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html -n auto From 3fcf6a74a76ab48578c45ad3bee6cc48d8705c67 Mon Sep 17 00:00:00 2001 From: varun646 Date: Fri, 23 Aug 2024 11:51:51 -0400 Subject: [PATCH 10/11] add flag for subset of tests --- .github/workflows/standard.yaml | 2 +- tests/conftest.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index 043e976..9668dab 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html -n auto + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html --subset-percentage 0.1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4f880d7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +import random + + +def pytest_addoption(parser): + parser.addoption( + "--subset-percentage", + action="store", + default=None, + help="Specify the percentage of tests to run as a subset (e.g., 0.3 for 30%)", + ) + + +def pytest_collection_modifyitems(config, items): + subset_percentage = config.getoption("--subset-percentage") + + # If the subset percentage is provided, apply it + if subset_percentage is not None: + subset_percentage = float(subset_percentage) + + # Shuffle the test cases + random.shuffle(items) + + # Calculate the number of tests to run based on percentage + subset_count = int(len(items) * subset_percentage) + + # Select a subset of tests + items[:] = items[:subset_count] From 8ca74a1efc00884733a0ab8a5db591956aba2da3 Mon Sep 17 00:00:00 2001 From: varun646 Date: Mon, 26 Aug 2024 09:27:01 -0400 Subject: [PATCH 11/11] reduce test size --- .github/workflows/standard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard.yaml b/.github/workflows/standard.yaml index 9668dab..e1d07ab 100644 --- a/.github/workflows/standard.yaml +++ b/.github/workflows/standard.yaml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html --subset-percentage 0.1 + pytest tests/test_generation.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html --subset-percentage 0.05