chore: pre-commit autoupdate

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
   skip: [poetry-lock]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.5
+    rev: v0.6.1
     hooks:
       - id: ruff
         args:
@@ -65,7 +65,7 @@ repos:
         stages: [commit-msg]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.11.0"
+    rev: "v1.11.1"
     hooks:
       - id: mypy
         args:

docs/src_snippets/howto/python_api/b_create_dataset.py

@@ -42,6 +42,7 @@ def create_from_pandas() -> SummaryStatistics:
     """Create a dataset from a path with Pandas files."""
     # --8<-- [start:create_from_pandas_import]
     import pyspark.pandas as ps
+
     from gentropy.dataset.summary_statistics import SummaryStatistics
 
     # --8<-- [end:create_from_pandas_import]

src/airflow/dags/common_airflow.py

@@ -6,14 +6,15 @@
 
 import pendulum
 import yaml
+from google.cloud import batch_v1, dataproc_v1, storage
+
 from airflow.providers.google.cloud.operators.dataproc import (
     ClusterGenerator,
     DataprocCreateClusterOperator,
     DataprocDeleteClusterOperator,
     DataprocSubmitJobOperator,
 )
 from airflow.utils.trigger_rule import TriggerRule
-from google.cloud import batch_v1, dataproc_v1, storage
 
 if TYPE_CHECKING:
     from pathlib import Path

src/airflow/dags/eqtl_preprocess.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 from airflow.providers.google.cloud.operators.dataflow import (
     DataflowTemplatedJobStartOperator,

src/airflow/dags/finngen_harmonisation.py

@@ -8,6 +8,7 @@
 from typing import Any
 
 import common_airflow as common
+
 from airflow.decorators import task
 from airflow.models.dag import DAG
 from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator

src/airflow/dags/finngen_preprocess.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 from airflow.utils.task_group import TaskGroup
 from airflow.utils.trigger_rule import TriggerRule

src/airflow/dags/genetics_etl.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 from airflow.operators.python import ShortCircuitOperator
 from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator

src/airflow/dags/gnomad_preprocess.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 
 CLUSTER_NAME = "gnomad-preprocess"

src/airflow/dags/gwas_catalog_harmonisation.py

@@ -8,6 +8,7 @@
 from typing import Any
 
 import common_airflow as common
+
 from airflow.decorators import task
 from airflow.models.dag import DAG
 from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator

src/airflow/dags/gwas_catalog_preprocess.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 from airflow.operators.python import PythonOperator
 from airflow.providers.google.cloud.hooks.gcs import GCSHook

src/airflow/dags/gwas_curation_update.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 
 CLUSTER_NAME = "otg-gwascatalog-curation"

src/airflow/dags/ukb_ppp_eur.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import common_airflow as common
+
 from airflow.models.dag import DAG
 
 CLUSTER_NAME = "otg-ukb-ppp-eur"

src/airflow/dags/variant_index.py

@@ -8,13 +8,6 @@
 from pathlib import Path
 from typing import Any
 
-from airflow.decorators import task
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.cloud_batch import (
-    CloudBatchSubmitJobOperator,
-)
-from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator
-from airflow.utils.trigger_rule import TriggerRule
 from common_airflow import (
     create_batch_job,
     create_cluster,
@@ -28,6 +21,14 @@
 )
 from google.cloud import batch_v1
 
+from airflow.decorators import task
+from airflow.models.dag import DAG
+from airflow.providers.google.cloud.operators.cloud_batch import (
+    CloudBatchSubmitJobOperator,
+)
+from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator
+from airflow.utils.trigger_rule import TriggerRule
+
 PROJECT_ID = "open-targets-genetics-dev"
 REGION = "europe-west1"
 CONFIG_FILE_PATH = Path(__file__).parent / "configs" / "variant_sources.yaml"

tests/airflow/test_dag.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import pytest
+
 from airflow.models import DagBag
 
 

tests/gentropy/common/test_spark_helpers.py

@@ -3,13 +3,14 @@
 from __future__ import annotations
 
 import pytest
+from pyspark.sql import Column, SparkSession
+from pyspark.sql import functions as f
+from pyspark.sql import types as t
+
 from gentropy.common.spark_helpers import (
     enforce_schema,
     order_array_of_structs_by_field,
 )
-from pyspark.sql import Column, SparkSession
-from pyspark.sql import functions as f
-from pyspark.sql import types as t
 
 
 def test_order_array_of_structs_by_field(spark: SparkSession) -> None:

tests/gentropy/common/test_version_engine.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import pytest
+
 from gentropy.common.version_engine import GnomADVersionSeeker, VersionEngine
 
 

tests/gentropy/conftest.py

@@ -9,6 +9,8 @@
 import numpy as np
 import pandas as pd
 import pytest
+from pyspark.sql import DataFrame, SparkSession
+
 from gentropy.common.Liftover import LiftOverSpark
 from gentropy.common.session import Session
 from gentropy.dataset.colocalisation import Colocalisation
@@ -28,8 +30,6 @@
 from gentropy.datasource.eqtl_catalogue.study_index import EqtlCatalogueStudyIndex
 from gentropy.datasource.gwas_catalog.associations import StudyLocusGWASCatalog
 from gentropy.datasource.gwas_catalog.study_index import StudyIndexGWASCatalog
-from pyspark.sql import DataFrame, SparkSession
-
 from utils.spark import get_spark_testing_conf
 
 

tests/gentropy/dataset/test_dataset.py

@@ -5,8 +5,6 @@
 import numpy as np
 import pyspark.sql.functions as f
 import pytest
-from gentropy.dataset.dataset import Dataset
-from gentropy.dataset.study_index import StudyIndex
 from pyspark.sql import SparkSession
 from pyspark.sql.types import (
     DoubleType,
@@ -15,6 +13,9 @@
     StructType,
 )
 
+from gentropy.dataset.dataset import Dataset
+from gentropy.dataset.study_index import StudyIndex
+
 
 class MockDataset(Dataset):
     """Concrete subclass of Dataset for testing. Necessary because Dataset is abstract."""

tests/gentropy/dataset/test_gene_index.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from gentropy.dataset.gene_index import GeneIndex
 from pyspark.sql import DataFrame
 
+from gentropy.dataset.gene_index import GeneIndex
+
 
 def test_gene_index_creation(mock_gene_index: GeneIndex) -> None:
     """Test gene index creation with mock gene index."""

tests/gentropy/dataset/test_l2g.py

@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING
 
 import pytest
+
 from gentropy.dataset.l2g_feature_matrix import L2GFeatureMatrix
 from gentropy.dataset.l2g_gold_standard import L2GGoldStandard
 from gentropy.dataset.l2g_prediction import L2GPrediction

tests/gentropy/dataset/test_pairwise_ld.py

@@ -6,10 +6,11 @@
 
 import numpy as np
 import pytest
-from gentropy.dataset.pairwise_ld import PairwiseLD
 from pyspark.sql import functions as f
 from pyspark.sql.window import Window
 
+from gentropy.dataset.pairwise_ld import PairwiseLD
+
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession
 

tests/gentropy/dataset/test_study_index.py

@@ -3,11 +3,12 @@
 from __future__ import annotations
 
 import pytest
-from gentropy.dataset.gene_index import GeneIndex
-from gentropy.dataset.study_index import StudyIndex
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql import functions as f
 
+from gentropy.dataset.gene_index import GeneIndex
+from gentropy.dataset.study_index import StudyIndex
+
 
 def test_study_index_creation(mock_study_index: StudyIndex) -> None:
     """Test study index creation with mock data."""

tests/gentropy/dataset/test_study_locus.py

@@ -7,15 +7,6 @@
 import pyspark.sql.functions as f
 import pyspark.sql.types as t
 import pytest
-from gentropy.dataset.ld_index import LDIndex
-from gentropy.dataset.study_index import StudyIndex
-from gentropy.dataset.study_locus import (
-    CredibleInterval,
-    StudyLocus,
-    StudyLocusQualityCheck,
-)
-from gentropy.dataset.study_locus_overlap import StudyLocusOverlap
-from gentropy.dataset.summary_statistics import SummaryStatistics
 from pyspark.sql import Column, Row, SparkSession
 from pyspark.sql.types import (
     ArrayType,
@@ -27,6 +18,16 @@
     StructType,
 )
 
+from gentropy.dataset.ld_index import LDIndex
+from gentropy.dataset.study_index import StudyIndex
+from gentropy.dataset.study_locus import (
+    CredibleInterval,
+    StudyLocus,
+    StudyLocusQualityCheck,
+)
+from gentropy.dataset.study_locus_overlap import StudyLocusOverlap
+from gentropy.dataset.summary_statistics import SummaryStatistics
+
 
 @pytest.mark.parametrize(
     "has_overlap, expected",

tests/gentropy/dataset/test_study_locus_overlap.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from gentropy.dataset.study_locus_overlap import StudyLocusOverlap
 from pyspark.sql import SparkSession
 
+from gentropy.dataset.study_locus_overlap import StudyLocusOverlap
+
 
 def test_study_locus_overlap_creation(
     mock_study_locus_overlap: StudyLocusOverlap,

tests/gentropy/dataset/test_study_locus_overlaps.py

@@ -6,13 +6,15 @@
 
 import pyspark.sql.types as t
 import pytest
+
 from gentropy.dataset.study_locus import StudyLocus
 from gentropy.dataset.study_locus_overlap import StudyLocusOverlap
 
 if TYPE_CHECKING:
-    from gentropy.dataset.study_index import StudyIndex
     from pyspark.sql import SparkSession
 
+    from gentropy.dataset.study_index import StudyIndex
+
 
 def test_study_locus_overlap_creation(
     mock_study_locus_overlap: StudyLocusOverlap,

tests/gentropy/dataset/test_summary_statistics.py

@@ -4,9 +4,10 @@
 
 from typing import TYPE_CHECKING
 
+from pyspark.sql import types as t
+
 from gentropy.dataset.study_locus import StudyLocus
 from gentropy.dataset.summary_statistics import SummaryStatistics
-from pyspark.sql import types as t
 
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession

tests/gentropy/dataset/test_variant_index.py

@@ -5,11 +5,12 @@
 from typing import TYPE_CHECKING
 
 import pytest
+from pyspark.sql import functions as f
+from pyspark.sql import types as t
+
 from gentropy.dataset.gene_index import GeneIndex
 from gentropy.dataset.v2g import V2G
 from gentropy.dataset.variant_index import VariantIndex
-from pyspark.sql import functions as f
-from pyspark.sql import types as t
 
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession

tests/gentropy/datasource/ensembl/test_vep_variants.py

@@ -5,11 +5,12 @@
 from typing import TYPE_CHECKING
 
 import pytest
-from gentropy.dataset.variant_index import VariantIndex
-from gentropy.datasource.ensembl.vep_parser import VariantEffectPredictorParser
 from pyspark.sql import DataFrame
 from pyspark.sql import functions as f
 
+from gentropy.dataset.variant_index import VariantIndex
+from gentropy.datasource.ensembl.vep_parser import VariantEffectPredictorParser
+
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession
 

tests/gentropy/datasource/eqtl_catalogue/test_eqtl_catalogue.py

@@ -3,11 +3,12 @@
 from __future__ import annotations
 
 import pytest
+from pyspark.sql import DataFrame
+
 from gentropy.dataset.study_index import StudyIndex
 from gentropy.dataset.study_locus import StudyLocus
 from gentropy.datasource.eqtl_catalogue.finemapping import EqtlCatalogueFinemapping
 from gentropy.datasource.eqtl_catalogue.study_index import EqtlCatalogueStudyIndex
-from pyspark.sql import DataFrame
 
 
 @pytest.fixture

tests/gentropy/datasource/finngen/test_finngen_finemapping.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
+from pyspark.sql import SparkSession
+
 from gentropy.dataset.study_locus import StudyLocus
 from gentropy.datasource.finngen.finemapping import FinnGenFinemapping
-from pyspark.sql import SparkSession
 
 
 def test_finngen_finemapping_from_finngen_susie_finemapping(