2023.10 (#3321)

* 2023.10

* addressing @charles-cowart comments [skip ci]

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Qiita changelog
 
+Version 2023.10
+---------------
+
+* Added a new notebooks folder to the repository to allocate the resource-allocation code and plotting.
+* Allowed dynamic resource allocations for time (before it was only for memory), and allowed complex formulas (via eval).
+* Deployed the new resource allocation formulas. This included changes on how the ProcesingJob.shape method calculates its variables, which include: (a) `build_analysis_files`: only use the `biom` file (vs. all of them); and (b) the SPP counts the input-file number of lines know the number of samples.
+* qp-woltka generates a new file: coverates.tgz, which contains: (a) artifact.cov: the coverages of this artifact, can be used with other coverages to generate combined coverages; (b) coverage_percentage.txt: the total percentage coverage per-genome of this artifact; and (c) coverages folder: the per-sample coverage.
+* The SPP now supports replicates within the same lane/run and creates multiple preparations as needed.
+* The SPP automatically adds and executes the default pipeline after it is done generating and linking the raw data.
+* Other fixes: [#3300](https://github.com/qiita-spots/qiita/issues/3300), [metapool #135](https://github.com/biocore/metagenomics_pooling_notebook/pull/135), [metapool #136](https://github.com/biocore/metagenomics_pooling_notebook/pull/136), [qtp-sequencing #47](https://github.com/qiita-spots/qtp-sequencing/pull/47).
+
+
 Version 2023.06
 ---------------
 

notebooks/resource-allocation/072023.ipynb

@@ -2572,6 +2572,56 @@
     "# bioenv [bioenv] -> no resource allocation\n",
     "\n",
     "# *** RESOURCE ALLOCATION ***\n",
+    "# ==> These were added after a qiita-rc review, when we realized that we had the old (way too large)\n",
+    "#     reservations\n",
+    "# SELECT * FROM qiita.processing_job_resource_allocation \n",
+    "#     WHERE job_type = 'RESOURCE_PARAMS_COMMAND' AND name IN (\n",
+    "#         'Generate a heatmap representation of a feature table [heatmap]', 'delete_sample_template', \n",
+    "#          'Uniform Manifold Approximation and Projection [umap]', \n",
+    "#          'Visualize taxonomy with an interactive bar plot [barplot]', \n",
+    "#          'Filter samples from a distance matrix. [filter_distance_matrix]', \n",
+    "#          'Filter features from table [filter_features]', 'Generate HTML summary', \n",
+    "#          'create_sample_template', 'Convert to presence/absence [presence_absence]', \n",
+    "#          'Taxonomy-based feature table filter. [filter_table]', 'Beta diversity [beta]', 'Validate', 'Validate', \n",
+    "#          'Generate HTML summary', 'Beta diversity rarefaction [beta_rarefaction]', 'Validate', \n",
+    "#          'Summarize table [summarize]', 'Validate', 'Principal Coordinate Analysis [pcoa]', 'delete_study', \n",
+    "#          'download_remote_files', 'update_prep_template', 'Beta diversity correlation [beta_correlation]', \n",
+    "#          'complete_job', 'Add pseudocount to table. [add_pseudocount]', 'delete_analysis',\n",
+    "#          'Collapse features by their taxonomy at the specified level [collapse]', \n",
+    "#          'Beta diversity group significance [beta_group_significance]', \n",
+    "#          'Convert to relative frequencies [relative_frequency]', \n",
+    "#          'adonis PERMANOVA test for beta group significance [adonis]', 'Validate', \n",
+    "#          'Core diversity metrics (phylogenetic and non-phylogenetic) [core_metrics_phylogenetic]', \n",
+    "#          'Core diversity metrics (non-phylogenetic) [core_metrics]', 'delete_artifact', 'list_remote_files', \n",
+    "#          'Identify core features in table [core_features]', \n",
+    "#          'Filter features from a table based on abundance and prevalence [filter_features_conditionally]', \n",
+    "#          'Alpha diversity comparisons [alpha_group_significance]', 'copy_artifact', 'Validate', \n",
+    "#          'Non V4 16S sequence assessment [non_v4_16s]', 'Alpha diversity [alpha]');\n",
+    "# \n",
+    "# DELETE FROM qiita.processing_job_resource_allocation \n",
+    "#     WHERE job_type = 'RESOURCE_PARAMS_COMMAND' AND name IN (\n",
+    "#         'Generate a heatmap representation of a feature table [heatmap]', 'delete_sample_template', \n",
+    "#          'Uniform Manifold Approximation and Projection [umap]', \n",
+    "#          'Visualize taxonomy with an interactive bar plot [barplot]', \n",
+    "#          'Filter samples from a distance matrix. [filter_distance_matrix]', \n",
+    "#          'Filter features from table [filter_features]', 'Generate HTML summary', \n",
+    "#          'create_sample_template', 'Convert to presence/absence [presence_absence]', \n",
+    "#          'Taxonomy-based feature table filter. [filter_table]', 'Beta diversity [beta]', 'Validate', 'Validate', \n",
+    "#          'Generate HTML summary', 'Beta diversity rarefaction [beta_rarefaction]', 'Validate', \n",
+    "#          'Summarize table [summarize]', 'Validate', 'Principal Coordinate Analysis [pcoa]', 'delete_study', \n",
+    "#          'download_remote_files', 'update_prep_template', 'Beta diversity correlation [beta_correlation]', \n",
+    "#          'complete_job', 'Add pseudocount to table. [add_pseudocount]', 'delete_analysis',\n",
+    "#          'Collapse features by their taxonomy at the specified level [collapse]', \n",
+    "#          'Beta diversity group significance [beta_group_significance]', \n",
+    "#          'Convert to relative frequencies [relative_frequency]', \n",
+    "#          'adonis PERMANOVA test for beta group significance [adonis]', 'Validate', \n",
+    "#          'Core diversity metrics (phylogenetic and non-phylogenetic) [core_metrics_phylogenetic]', \n",
+    "#          'Core diversity metrics (non-phylogenetic) [core_metrics]', 'delete_artifact', 'list_remote_files', \n",
+    "#          'Identify core features in table [core_features]', \n",
+    "#          'Filter features from a table based on abundance and prevalence [filter_features_conditionally]', \n",
+    "#          'Alpha diversity comparisons [alpha_group_significance]', 'copy_artifact', 'Validate', \n",
+    "#          'Non V4 16S sequence assessment [non_v4_16s]', 'Alpha diversity [alpha]');\n",
+    "\n",
     "# UPDATE qiita.processing_job_resource_allocation set allocation = '-p qiita -N 1 -n 1 --mem 7gb --time 40:00:00' \n",
     "#     WHERE job_type = 'RESOURCE_PARAMS_COMMAND' and \n",
     "#     name = 'Apply ANCOM to identify features that differ in abundance. [ancom]';\n",