update arguments

openproblems-bio · Aug 22, 2024 · e5ed85a · e5ed85a
1 parent 914fdb7
commit e5ed85a
Show file tree

Hide file tree

Showing 17 changed files with 503 additions and 742 deletions.
diff --git a/src/datasets/resource_scripts/spatial_10x_visium.sh b/src/datasets/resource_scripts/spatial_10x_visium.sh
diff --git a/src/datasets/resource_scripts/spatial_10x_visium_zenodo.sh b/src/datasets/resource_scripts/spatial_10x_visium_zenodo.sh
@@ -7,46 +7,32 @@ param_list:
     dataset_name: 10X Visium - Human Heart MI 1
     dataset_url: "https://www.nature.com/articles/s41586-022-05060-x"
     dataset_summary: Gene expression library of human heart using 10x Visium.
-    dataset_description: "Frozen heart samples were embedded in OCT (Tissue-Tek) and cryosectioned (Thermo Cryostar). The 10-µm section was placed on the pre-chilled Optimization slides (Visium, 10X Genomics, PN-1000193) and the optimal lysis time was determined. The tissues were treated as recommended by 10X Genomics and the optimization procedure showed an optimal permeabilization time of 12 or 18 min of digestion and release of RNA from the tissue slide. Spatial gene expression slides (Visium, 10X Genomics, PN-1000187) were used for spatial transcriptomics following the Visium User Guides"
+    dataset_description: "Frozen heart samples were embedded in OCT (Tissue-Tek) and cryosectioned (Thermo Cryostar). The 10-µm section was placed on the pre-chilled Optimization slides (Visium, 10X Genomics, PN-1000193) and the optimal lysis time was determined. The tissues were treated as recommended by 10X Genomics and the optimization procedure showed an optimal permeabilization time of 12 or 18 min of digestion and release of RNA from the tissue slide. Spatial gene expression slides (Visium, 10X Genomics, PN-1000187) were used for spatial transcriptomics following the Visium User Guides"
     dataset_reference: kuppe2022spatial
     dataset_organism: Homo sapiens
     spot_filter_min_genes: 200
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: grid
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_10x_visium/human_heart_myocardial_infarction_2
     input_data: "https://zenodo.org/records/13328275/files/10X009.h5ad?download=1"
     dataset_name: 10X Visium - Human Heart MI 2
     dataset_url: "https://www.nature.com/articles/s41586-022-05060-x"
     dataset_summary: Gene expression library of human heart using 10x Visium.
-    dataset_description: "Frozen heart samples were embedded in OCT (Tissue-Tek) and cryosectioned (Thermo Cryostar). The 10-µm section was placed on the pre-chilled Optimization slides (Visium, 10X Genomics, PN-1000193) and the optimal lysis time was determined. The tissues were treated as recommended by 10X Genomics and the optimization procedure showed an optimal permeabilization time of 12 or 18 min of digestion and release of RNA from the tissue slide. Spatial gene expression slides (Visium, 10X Genomics, PN-1000187) were used for spatial transcriptomics following the Visium User Guides"
+    dataset_description: "Frozen heart samples were embedded in OCT (Tissue-Tek) and cryosectioned (Thermo Cryostar). The 10-µm section was placed on the pre-chilled Optimization slides (Visium, 10X Genomics, PN-1000193) and the optimal lysis time was determined. The tissues were treated as recommended by 10X Genomics and the optimization procedure showed an optimal permeabilization time of 12 or 18 min of digestion and release of RNA from the tissue slide. Spatial gene expression slides (Visium, 10X Genomics, PN-1000187) were used for spatial transcriptomics following the Visium User Guides"
     dataset_reference: kuppe2022spatial
     dataset_organism: Homo sapiens
     spot_filter_min_genes: 200
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: grid
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
 normalization_methods: [log_cp10k]
 output_dataset: '$id/dataset.h5ad'
 output_meta: '$id/dataset_metadata.yaml'
 output_state: '$id/state.yaml'
 output_raw: force_null
 output_normalized: force_null
-publish_dir: s3://openproblems-data/resources/datasets
+publish_dir: resources/datasets
 remove_mitochondrial: true
 HERE
 
@@ -63,11 +49,16 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems-v2.git \
-  --revision main_build \
-  --pull-latest \
-  --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
-  --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
-  --params-file "/tmp/params.yaml" \
-  --config /tmp/nextflow.config 
+# tw launch https://github.com/openproblems-bio/openproblems-v2.git \
+#   --revision main_build \
+#   --pull-latest \
+#   --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+#   --workspace 53907369739130 \
+#   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+#   --params-file "/tmp/params.yaml" \
+#   --config /tmp/nextflow.config 
+nextflow run . \
+  -main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+  -c src/wf_utils/labels_ci.config \
+  -profile docker \
+  -params-file "/tmp/params.yaml"
diff --git a/src/datasets/resource_scripts/spatial_10x_xenium.sh b/src/datasets/resource_scripts/spatial_10x_xenium.sh
@@ -13,14 +13,7 @@ param_list:
     dataset_organism: Homo sapiens
     spot_filter_min_genes: 100
     gene_filter_min_spots: 50
-    num_reference_genes: 100
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: grid
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_10x_xenium/mouse_brain
     input_expression: "https://cf.10xgenomics.com/samples/spatial-exp/2.1.0/CytAssist_FreshFrozen_Mouse_Brain_Post_Xenium_Rep1/CytAssist_FreshFrozen_Mouse_Brain_Post_Xenium_Rep1_filtered_feature_bc_matrix.h5"
@@ -33,25 +26,15 @@ param_list:
     dataset_organism: Mus musculus
     spot_filter_min_genes: 100
     gene_filter_min_spots: 50
-    num_reference_genes: 100
-    select_top_variable_genes: 50
     remove_mitochondrial: false
-    coord_type_proc: grid
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
 normalization_methods: [log_cp10k]
 output_dataset: '$id/dataset.h5ad'
 output_meta: '$id/dataset_metadata.yaml'
 output_state: '$id/state.yaml'
 output_raw: force_null
 output_normalized: force_null
-publish_dir: s3://openproblems-data/resources/datasets
-spot_filter_min_genes: 100
-gene_filter_min_spots: 50
-remove_mitochondrial: true
+publish_dir: resources/datasets
 HERE
 
 cat > /tmp/nextflow.config << HERE
@@ -67,11 +50,17 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems-v2.git \
-  --revision main_build \
-  --pull-latest \
-  --main-script target/nextflow/datasets/workflows/process_10x_datasets/main.nf \
-  --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
-  --params-file "/tmp/params.yaml" \
-  --config /tmp/nextflow.config 
+# tw launch https://github.com/openproblems-bio/openproblems-v2.git \
+#   --revision main_build \
+#   --pull-latest \
+#   --main-script target/nextflow/datasets/workflows/process_10x_datasets/main.nf \
+#   --workspace 53907369739130 \
+#   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+#   --params-file "/tmp/params.yaml" \
+#   --config /tmp/nextflow.config 
+
+nextflow run . \
+  -main-script target/nextflow/datasets/workflows/process_10x_datasets/main.nf \
+  -c src/wf_utils/labels_ci.config \
+  -profile docker \
+  -params-file "/tmp/params.yaml"
diff --git a/src/datasets/resource_scripts/spatial_dbit_seq.sh b/src/datasets/resource_scripts/spatial_dbit_seq.sh
@@ -12,14 +12,7 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
   - id: spatial_dbit_seq/mouse_e10_eye
     input_data: "https://zenodo.org/records/12785822/files/DBiT-seq_liu2020high_E10_eye_and_nearby_data.h5ad?download=1"
@@ -31,14 +24,7 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
   - id: spatial_dbit_seq/mouse_e10_whole_body
     input_data: "https://zenodo.org/records/12785822/files/DBiT-seq_liu2020high_E10_whole_gene_best_data.h5ad?download=1"
@@ -50,14 +36,7 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
   - id: spatial_dbit_seq/mouse_e11_lower_body
     input_data: "https://zenodo.org/records/12785822/files/DBiT-seq_liu2020high_E11_lower_body_data.h5ad?download=1"
@@ -69,14 +48,7 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
   - id: spatial_dbit_seq/mouse_e11_1
     input_data: "https://zenodo.org/records/12785822/files/DBiT-seq_liu2020high_GSM4364244_E11-FL-1L_gene_data.h5ad?download=1"
@@ -88,14 +60,7 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
   - id: spatial_dbit_seq/mouse_e11_2
     input_data: "https://zenodo.org/records/12785822/files/DBiT-seq_liu2020high_GSM4364245_E11-FL-2L_gene_data.h5ad?download=1"
@@ -107,23 +72,15 @@ param_list:
     dataset_reference: liu2020high
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 200
-    select_top_variable_genes: 50
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 4
-    n_cp: -1
 
 normalization_methods: [log_cp10k]
 output_dataset: '$id/dataset.h5ad'
 output_meta: '$id/dataset_metadata.yaml'
 output_state: '$id/state.yaml'
 output_raw: force_null
 output_normalized: force_null
-publish_dir: s3://openproblems-data/resources/datasets
-remove_mitochondrial: true
+publish_dir: resources/datasets
 HERE
 
 cat > /tmp/nextflow.config << HERE
@@ -139,11 +96,17 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems-v2.git \
-  --revision main_build \
-  --pull-latest \
-  --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
-  --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
-  --params-file "/tmp/params.yaml" \
-  --config /tmp/nextflow.config 
+# tw launch https://github.com/openproblems-bio/openproblems-v2.git \
+#   --revision main_build \
+#   --pull-latest \
+#   --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+#   --workspace 53907369739130 \
+#   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+#   --params-file "/tmp/params.yaml" \
+#   --config /tmp/nextflow.config 
+
+nextflow run . \
+  -main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+  -c src/wf_utils/labels_ci.config \
+  -profile docker \
+  -params-file "/tmp/params.yaml"
diff --git a/src/datasets/resource_scripts/spatial_merfish.sh b/src/datasets/resource_scripts/spatial_merfish.sh
@@ -12,14 +12,7 @@ param_list:
     dataset_reference: fang2022conservation
     spot_filter_min_genes: 10
     gene_filter_min_spots: 100
-    num_reference_genes: 25
-    select_top_variable_genes: 25
     remove_mitochondrial: false
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_merfish/human_cortex_2
     input_data: "https://zenodo.org/records/12785822/files/MERFISH_Fang2022Conservation_H18.06.006.MTG.4000.expand.rep1_data.h5ad?download=1"
@@ -31,14 +24,7 @@ param_list:
     dataset_reference: fang2022conservation
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 50
-    select_top_variable_genes: 50
     remove_mitochondrial: false
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_merfish/human_cortex_3
     input_data: "https://zenodo.org/records/12785822/files/MERFISH_Fang2022Conservation_H18.06.006.MTG.4000.expand.rep2_data.h5ad?download=1"
@@ -50,14 +36,7 @@ param_list:
     dataset_reference: fang2022conservation
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 50
-    select_top_variable_genes: 50
     remove_mitochondrial: false
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_merfish/human_cortex_4
     input_data: "https://zenodo.org/records/12785822/files/MERFISH_Fang2022Conservation_H18.06.006.MTG.4000.expand.rep3_data.h5ad?download=1"
@@ -69,14 +48,7 @@ param_list:
     dataset_reference: fang2022conservation
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 50
-    select_top_variable_genes: 50
     remove_mitochondrial: false
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
   - id: spatial_merfish/mouse_cortex
     input_data: "https://zenodo.org/records/12785822/files/MERFISH_Fang2022Conservation_mouse1.AUD_TEA_VIS.242.unexpand_data.h5ad?download=1"
@@ -88,23 +60,15 @@ param_list:
     dataset_reference: fang2022conservation
     spot_filter_min_genes: 10
     gene_filter_min_spots: 50
-    num_reference_genes: 25
-    select_top_variable_genes: 25
     remove_mitochondrial: true
-    coord_type_proc: generic
-    coord_type_moran_i: generic
-    coord_type_sepal: grid
-    max_neighs_speal: 6
-    n_cp: -1
 
 normalization_methods: [log_cp10k]
 output_dataset: '$id/dataset.h5ad'
 output_meta: '$id/dataset_metadata.yaml'
 output_state: '$id/state.yaml'
 output_raw: force_null
 output_normalized: force_null
-publish_dir: s3://openproblems-data/resources/datasets
-remove_mitochondrial: true
+publish_dir: resources/datasets
 HERE
 
 cat > /tmp/nextflow.config << HERE
@@ -120,11 +84,17 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems-v2.git \
-  --revision main_build \
-  --pull-latest \
-  --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
-  --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
-  --params-file "/tmp/params.yaml" \
-  --config /tmp/nextflow.config 
+# tw launch https://github.com/openproblems-bio/openproblems-v2.git \
+#   --revision main_build \
+#   --pull-latest \
+#   --main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+#   --workspace 53907369739130 \
+#   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+#   --params-file "/tmp/params.yaml" \
+#   --config /tmp/nextflow.config 
+
+nextflow run . \
+  -main-script target/nextflow/datasets/workflows/process_spatial_from_zenodo/main.nf \
+  -c src/wf_utils/labels_ci.config \
+  -profile docker \
+  -params-file "/tmp/params.yaml"