-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #694 from adi611/adi611-patch-testpsij-1
Draft: Adding new worker which uses PSI/J to run tasks
- Loading branch information
Showing
7 changed files
with
308 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
name: PSI/J-Local | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
pull_request: | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref }} | ||
cancel-in-progress: true | ||
|
||
permissions: | ||
contents: read | ||
|
||
jobs: | ||
test: | ||
strategy: | ||
matrix: | ||
os: [ubuntu-latest, macos-latest] | ||
python-version: ['3.11'] | ||
fail-fast: false | ||
runs-on: ${{ matrix.os }} | ||
|
||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: ${{ github.repository }} | ||
|
||
- name: Setup Python version ${{ matrix.python-version }} | ||
uses: actions/setup-python@v4 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
|
||
- name: Install dependencies for PSI/J | ||
run: | | ||
pip install -e ".[test, psij]" | ||
- name: Run tests for PSI/J | ||
run: | | ||
pytest --color=yes -vs --psij=local -n auto pydra/engine --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml | ||
- name: Upload to codecov | ||
run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
name: PSI/J-SLURM | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
pull_request: | ||
|
||
jobs: | ||
build: | ||
strategy: | ||
matrix: | ||
python-version: [3.11.5] | ||
fail-fast: false | ||
runs-on: ubuntu-latest | ||
env: | ||
DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1 | ||
|
||
steps: | ||
- name: Disable etelemetry | ||
run: echo "NO_ET=TRUE" >> $GITHUB_ENV | ||
- uses: actions/checkout@v4 | ||
- name: Pull docker image | ||
run: | | ||
docker pull $DOCKER_IMAGE | ||
# Have image running in the background | ||
docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE | ||
- name: Display previous jobs with sacct | ||
run: | | ||
echo "Allowing ports/daemons time to start" && sleep 10 | ||
docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" | ||
docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null | ||
if [ $? -ne 0 ]; then | ||
echo "Slurm docker image error" | ||
exit 1 | ||
fi | ||
- name: Setup Python | ||
run: | | ||
docker exec slurm bash -c "echo $NO_ET" | ||
docker exec slurm bash -c "ls -la && echo list top level dir" | ||
docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" | ||
if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then | ||
docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5" | ||
fi | ||
docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}" | ||
docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra; print(pydra.__version__)'" | ||
- name: Run pytest | ||
run: | | ||
docker exec slurm bash -c "pytest --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" | ||
- name: Upload to codecov | ||
run: | | ||
docker exec slurm bash -c "pip install urllib3==1.26.6" | ||
docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" | ||
docker rm -f slurm |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
import pickle | ||
import sys | ||
from pydra.engine.helpers import load_and_run | ||
|
||
|
||
def run_pickled(*file_paths, rerun=False): | ||
loaded_objects = [] | ||
|
||
for file_path in file_paths: | ||
with open(file_path, "rb") as file: | ||
loaded_objects.append(pickle.load(file)) | ||
|
||
if len(loaded_objects) == 1: | ||
result = loaded_objects[0](rerun=rerun) | ||
elif len(loaded_objects) == 2: | ||
result = load_and_run(loaded_objects[0], loaded_objects[1], rerun=rerun) | ||
else: | ||
raise ValueError("Unsupported number of loaded objects") | ||
|
||
print(f"Result: {result}") | ||
|
||
|
||
if __name__ == "__main__": | ||
rerun = False # Default value for rerun | ||
file_paths = sys.argv[1:] | ||
|
||
if "--rerun" in file_paths: | ||
rerun = True | ||
file_paths.remove("--rerun") | ||
|
||
run_pickled(*file_paths, rerun=rerun) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters