Skip to content

Commit

Permalink
fix qiita-spots#3076
Browse files Browse the repository at this point in the history
  • Loading branch information
@antgonza
antgonza committed Feb 19, 2021
1 parent 6de1e1d commit 3b934a1
Show file tree
Hide file tree
Showing 4 changed files with 1,673 additions and 1,689 deletions.
10 changes: 10 additions & 0 deletions qiita_db/metadata_template/test/test_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,16 @@ def test_load_template_to_dataframe_with_nulls(self):
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp, check_like=True)

def test_load_template_to_dataframe_better_tokenizing_error_msg(self):
with self.assertRaises(RuntimeError) as error:
qdb.metadata_template.util.load_template_to_dataframe(
StringIO('sample_name\tcollection_timestamp\n'
'2.Sample1\t2014-05-29 12:24:51\t\n'
'2.Sample2\taaa\n'
'xxx\tadfa\t\t\n'))
self.assertIn('Your file has more columns with values than headers.',
str(error.exception))

def test_get_invalid_sample_names(self):
all_valid = ['2.sample.1', 'foo.bar.baz', 'roses', 'are', 'red',
'v10l3t5', '4r3', '81u3']
Expand Down
30 changes: 20 additions & 10 deletions qiita_db/metadata_template/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,16 +144,26 @@ def load_template_to_dataframe(fn, index='sample_name'):
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
template = pd.read_csv(
StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
try:
template = pd.read_csv(
StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
except pd.errors.ParserError as e:
if 'tokenizing' in str(e):
msg = ('Your file has more columns with values than headers. To '
'fix, make sure to delete any extra rows or columns; they '
'might look empty because they have spaces. Then upload '
'and try again.')
raise RuntimeError(msg)
else:
raise e
# remove newlines and tabs from fields
template.replace(to_replace='[\t\n\r\x0b\x0c]+', value='',
regex=True, inplace=True)
Expand Down
174 changes: 87 additions & 87 deletions qiita_db/support_files/qiita-db.dbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -939,10 +939,10 @@
<column name="deprecated" type="boolean" jt="-7" >
<defo><![CDATA[FALSE]]></defo>
</column>
<column name="creation_timestamp" type="timestamp" jt="-9" >
<column name="creation_timestamp" type="timestamp" jt="93" >
<defo><![CDATA[FALSE]]></defo>
</column>
<column name="modification_timestamp" type="timestamp" jt="-11" >
<column name="modification_timestamp" type="timestamp" jt="93" >
<defo><![CDATA[FALSE]]></defo>
</column>
<index name="pk_prep_template" unique="PRIMARY_KEY" >
Expand Down Expand Up @@ -1686,90 +1686,90 @@ Controlled Vocabulary]]></comment>
</schema>
<connector name="PostgreSQL" database="PostgreSQL" host="localhost" port="5432" instance="qiita_test" user="ccowart" />
<layout name="qiita" id="Layout669806" confirmed="y" show_relation="columns" >
<comment><![CDATA[qiita]]></comment>
<entity schema="qiita" name="analysis" color="D0DEF5" x="224" y="880" />
<entity schema="qiita" name="analysis_filepath" color="C0D4F3" x="400" y="880" />
<entity schema="qiita" name="analysis_portal" color="A8C4EF" x="176" y="768" />
<entity schema="qiita" name="analysis_sample" color="D0DEF5" x="256" y="1072" />
<entity schema="qiita" name="analysis_users" color="D0DEF5" x="48" y="880" />
<entity schema="qiita" name="archive_feature_value" color="B2CDF7" x="1472" y="1552" />
<entity schema="qiita" name="archive_merging_scheme" color="B2CDF7" x="1248" y="1552" />
<entity schema="qiita" name="artifact" color="B2CDF7" x="1200" y="960" />
<entity schema="qiita" name="artifact_filepath" color="B2CDF7" x="1040" y="1040" />
<entity schema="qiita" name="artifact_output_processing_job" color="B2CDF7" x="1760" y="1456" />
<entity schema="qiita" name="artifact_processing_job" color="B2CDF7" x="1744" y="1344" />
<entity schema="qiita" name="artifact_type" color="D0DEF5" x="1408" y="960" />
<entity schema="qiita" name="artifact_type_filepath_type" color="B2CDF7" x="1376" y="816" />
<entity schema="qiita" name="checksum_algorithm" color="B2CDF7" x="736" y="1040" />
<entity schema="qiita" name="column_controlled_vocabularies" color="D0DEF5" x="272" y="1248" />
<entity schema="qiita" name="column_ontology" color="D0DEF5" x="272" y="1504" />
<entity schema="qiita" name="command_output" color="B2CDF7" x="1760" y="1216" />
<entity schema="qiita" name="command_parameter" color="B2CDF7" x="2176" y="1040" />
<entity schema="qiita" name="controlled_vocab" color="D0DEF5" x="48" y="1248" />
<entity schema="qiita" name="controlled_vocab_values" color="D0DEF5" x="48" y="1376" />
<entity schema="qiita" name="data_directory" color="B2CDF7" x="576" y="1008" />
<entity schema="qiita" name="data_type" color="D0DEF5" x="704" y="1200" />
<entity schema="qiita" name="default_parameter_set" color="B2CDF7" x="2416" y="1168" />
<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2656" y="1120" />
<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2640" y="1312" />
<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2384" y="1424" />
<entity schema="qiita" name="default_workflow_node" color="B2CDF7" x="2416" y="1280" />
<entity schema="qiita" name="ebi_run_accession" color="B2CDF7" x="1344" y="1184" />
<entity schema="qiita" name="environmental_package" color="B2CDF7" x="2176" y="224" />
<entity schema="qiita" name="filepath" color="C0D4F3" x="640" y="832" />
<entity schema="qiita" name="filepath_type" color="C0D4F3" x="816" y="784" />
<entity schema="qiita" name="investigation" color="C0D4F3" x="2256" y="352" />
<entity schema="qiita" name="investigation_study" color="C0D4F3" x="2032" y="400" />
<entity schema="qiita" name="logging" color="C0D4F3" x="976" y="1376" />
<entity schema="qiita" name="message" color="A8C4EF" x="1504" y="1360" />
<entity schema="qiita" name="message_user" color="A8C4EF" x="1312" y="1344" />
<entity schema="qiita" name="mixs_field_description" color="D0DEF5" x="256" y="1360" />
<entity schema="qiita" name="oauth_identifiers" color="B7C8E3" x="2352" y="768" />
<entity schema="qiita" name="oauth_software" color="B2CDF7" x="2192" y="768" />
<entity schema="qiita" name="ontology" color="D0DEF5" x="576" y="1488" />
<entity schema="qiita" name="parameter_artifact_type" color="B2CDF7" x="1760" y="1120" />
<entity schema="qiita" name="parent_artifact" color="B2CDF7" x="1040" y="928" />
<entity schema="qiita" name="parent_processing_job" color="B2CDF7" x="2016" y="1472" />
<entity schema="qiita" name="per_study_tags" color="B2CDF7" x="2160" y="496" />
<entity schema="qiita" name="portal_type" color="C0D4F3" x="816" y="640" />
<entity schema="qiita" name="prep_template" color="B2CDF7" x="1232" y="512" />
<entity schema="qiita" name="prep_template_filepath" color="B2CDF7" x="1024" y="512" />
<entity schema="qiita" name="prep_template_processing_job" color="B2CDF7" x="1728" y="1008" />
<entity schema="qiita" name="prep_template_sample" color="D0DEF5" x="1136" y="304" />
<entity schema="qiita" name="prep_y" color="D0DEF5" x="992" y="320" />
<entity schema="qiita" name="processing_job" color="B2CDF7" x="1952" y="1168" />
<entity schema="qiita" name="processing_job_resource_allocation" color="C1D8EE" x="2624" y="1456" />
<entity schema="qiita" name="processing_job_status" color="B2CDF7" x="1760" y="1552" />
<entity schema="qiita" name="processing_job_validator" color="B2CDF7" x="2176" y="1472" />
<entity schema="qiita" name="processing_job_workflow" color="B2CDF7" x="2176" y="1584" />
<entity schema="qiita" name="processing_job_workflow_root" color="B2CDF7" x="2416" y="1584" />
<entity schema="qiita" name="publication" color="B2CDF7" x="2016" y="768" />
<entity schema="qiita" name="qiita_user" color="D0DEF5" x="592" y="384" />
<entity schema="qiita" name="reference" color="C0D4F3" x="2016" y="1584" />
<entity schema="qiita" name="restrictions" color="B2CDF7" x="1536" y="400" />
<entity schema="qiita" name="sample_template_filepath" color="B2CDF7" x="1040" y="656" />
<entity schema="qiita" name="sample_x" color="D0DEF5" x="1536" y="288" />
<entity schema="qiita" name="severity" color="C0D4F3" x="1120" y="1376" />
<entity schema="qiita" name="software" color="B2CDF7" x="2416" y="992" />
<entity schema="qiita" name="software_artifact_type" color="B2CDF7" x="1984" y="1008" />
<entity schema="qiita" name="software_command" color="B2CDF7" x="2176" y="1264" />
<entity schema="qiita" name="software_publication" color="B2CDF7" x="1984" y="864" />
<entity schema="qiita" name="software_type" color="B2CDF7" x="2640" y="992" />
<entity schema="qiita" name="stats_daily" color="B2CDF7" x="2496" y="368" />
<entity schema="qiita" name="study" color="D0DEF5" x="1792" y="288" />
<entity schema="qiita" name="study_artifact" color="B2CDF7" x="1600" y="800" />
<entity schema="qiita" name="study_environmental_package" color="B2CDF7" x="2096" y="80" />
<entity schema="qiita" name="study_person" color="C0D4F3" x="2016" y="176" />
<entity schema="qiita" name="study_portal" color="A8C4EF" x="1856" y="80" />
<entity schema="qiita" name="study_prep_template" color="D0DEF5" x="1504" y="528" />
<entity schema="qiita" name="study_publication" color="B2CDF7" x="1856" y="768" />
<entity schema="qiita" name="study_sample" color="D0DEF5" x="1360" y="288" />
<entity schema="qiita" name="study_tags" color="B2CDF7" x="2304" y="528" />
<entity schema="qiita" name="study_users" color="D0DEF5" x="1776" y="176" />
<entity schema="qiita" name="term" color="D0DEF5" x="736" y="1488" />
<entity schema="qiita" name="timeseries_type" color="C0D4F3" x="2016" y="576" />
<entity schema="qiita" name="user_level" color="D0DEF5" x="432" y="352" />
<entity schema="qiita" name="visibility" color="B2CDF7" x="1040" y="1152" />
<comment><![CDATA[qiita-db]]></comment>
<entity schema="qiita" name="analysis" color="D0DEF5" x="192" y="896" />
<entity schema="qiita" name="analysis_filepath" color="C0D4F3" x="384" y="848" />
<entity schema="qiita" name="analysis_portal" color="A8C4EF" x="224" y="736" />
<entity schema="qiita" name="analysis_sample" color="D0DEF5" x="368" y="976" />
<entity schema="qiita" name="analysis_users" color="D0DEF5" x="80" y="800" />
<entity schema="qiita" name="archive_feature_value" color="B2CDF7" x="1472" y="1472" />
<entity schema="qiita" name="archive_merging_scheme" color="B2CDF7" x="1248" y="1472" />
<entity schema="qiita" name="artifact" color="B2CDF7" x="1200" y="880" />
<entity schema="qiita" name="artifact_filepath" color="B2CDF7" x="1040" y="960" />
<entity schema="qiita" name="artifact_output_processing_job" color="B2CDF7" x="1760" y="1376" />
<entity schema="qiita" name="artifact_processing_job" color="B2CDF7" x="1728" y="1280" />
<entity schema="qiita" name="artifact_type" color="D0DEF5" x="1408" y="880" />
<entity schema="qiita" name="artifact_type_filepath_type" color="B2CDF7" x="1376" y="736" />
<entity schema="qiita" name="checksum_algorithm" color="B2CDF7" x="736" y="960" />
<entity schema="qiita" name="column_controlled_vocabularies" color="D0DEF5" x="272" y="1168" />
<entity schema="qiita" name="column_ontology" color="D0DEF5" x="272" y="1424" />
<entity schema="qiita" name="command_output" color="B2CDF7" x="1728" y="1136" />
<entity schema="qiita" name="command_parameter" color="B2CDF7" x="2192" y="960" />
<entity schema="qiita" name="controlled_vocab" color="D0DEF5" x="48" y="1168" />
<entity schema="qiita" name="controlled_vocab_values" color="D0DEF5" x="48" y="1296" />
<entity schema="qiita" name="data_directory" color="B2CDF7" x="576" y="928" />
<entity schema="qiita" name="data_type" color="D0DEF5" x="704" y="1120" />
<entity schema="qiita" name="default_parameter_set" color="B2CDF7" x="2416" y="1104" />
<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2656" y="1040" />
<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2640" y="1232" />
<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2384" y="1376" />
<entity schema="qiita" name="default_workflow_node" color="B2CDF7" x="2416" y="1232" />
<entity schema="qiita" name="ebi_run_accession" color="B2CDF7" x="1344" y="1104" />
<entity schema="qiita" name="environmental_package" color="B2CDF7" x="2176" y="176" />
<entity schema="qiita" name="filepath" color="C0D4F3" x="640" y="752" />
<entity schema="qiita" name="filepath_type" color="C0D4F3" x="816" y="704" />
<entity schema="qiita" name="investigation" color="C0D4F3" x="2176" y="272" />
<entity schema="qiita" name="investigation_study" color="C0D4F3" x="2000" y="384" />
<entity schema="qiita" name="logging" color="C0D4F3" x="976" y="1296" />
<entity schema="qiita" name="message" color="A8C4EF" x="1504" y="1280" />
<entity schema="qiita" name="message_user" color="A8C4EF" x="1312" y="1264" />
<entity schema="qiita" name="mixs_field_description" color="D0DEF5" x="272" y="1264" />
<entity schema="qiita" name="oauth_identifiers" color="B7C8E3" x="2352" y="688" />
<entity schema="qiita" name="oauth_software" color="B2CDF7" x="2192" y="688" />
<entity schema="qiita" name="ontology" color="D0DEF5" x="576" y="1408" />
<entity schema="qiita" name="parameter_artifact_type" color="B2CDF7" x="1728" y="1040" />
<entity schema="qiita" name="parent_artifact" color="B2CDF7" x="1040" y="848" />
<entity schema="qiita" name="parent_processing_job" color="B2CDF7" x="2016" y="1392" />
<entity schema="qiita" name="per_study_tags" color="B2CDF7" x="2192" y="528" />
<entity schema="qiita" name="portal_type" color="C0D4F3" x="768" y="560" />
<entity schema="qiita" name="prep_template" color="B2CDF7" x="1232" y="464" />
<entity schema="qiita" name="prep_template_filepath" color="B2CDF7" x="1024" y="416" />
<entity schema="qiita" name="prep_template_processing_job" color="B2CDF7" x="1728" y="928" />
<entity schema="qiita" name="prep_template_sample" color="D0DEF5" x="1152" y="304" />
<entity schema="qiita" name="prep_y" color="D0DEF5" x="992" y="304" />
<entity schema="qiita" name="processing_job" color="B2CDF7" x="1968" y="1072" />
<entity schema="qiita" name="processing_job_resource_allocation" color="C1D8EE" x="2656" y="1376" />
<entity schema="qiita" name="processing_job_status" color="B2CDF7" x="1728" y="1504" />
<entity schema="qiita" name="processing_job_validator" color="B2CDF7" x="2192" y="1392" />
<entity schema="qiita" name="processing_job_workflow" color="B2CDF7" x="2192" y="1504" />
<entity schema="qiita" name="processing_job_workflow_root" color="B2CDF7" x="2448" y="1504" />
<entity schema="qiita" name="publication" color="B2CDF7" x="1936" y="688" />
<entity schema="qiita" name="qiita_user" color="D0DEF5" x="704" y="288" />
<entity schema="qiita" name="reference" color="C0D4F3" x="2016" y="1504" />
<entity schema="qiita" name="restrictions" color="B2CDF7" x="1584" y="400" />
<entity schema="qiita" name="sample_template_filepath" color="B2CDF7" x="1008" y="560" />
<entity schema="qiita" name="sample_x" color="D0DEF5" x="1568" y="288" />
<entity schema="qiita" name="severity" color="C0D4F3" x="1120" y="1296" />
<entity schema="qiita" name="software" color="B2CDF7" x="2432" y="896" />
<entity schema="qiita" name="software_artifact_type" color="B2CDF7" x="1984" y="944" />
<entity schema="qiita" name="software_command" color="B2CDF7" x="2192" y="1184" />
<entity schema="qiita" name="software_publication" color="B2CDF7" x="1920" y="784" />
<entity schema="qiita" name="software_type" color="B2CDF7" x="2640" y="912" />
<entity schema="qiita" name="stats_daily" color="B2CDF7" x="2464" y="400" />
<entity schema="qiita" name="study" color="D0DEF5" x="1776" y="208" />
<entity schema="qiita" name="study_artifact" color="B2CDF7" x="1600" y="720" />
<entity schema="qiita" name="study_environmental_package" color="B2CDF7" x="2176" y="80" />
<entity schema="qiita" name="study_person" color="C0D4F3" x="2000" y="224" />
<entity schema="qiita" name="study_portal" color="A8C4EF" x="1920" y="80" />
<entity schema="qiita" name="study_prep_template" color="D0DEF5" x="1536" y="560" />
<entity schema="qiita" name="study_publication" color="B2CDF7" x="1792" y="688" />
<entity schema="qiita" name="study_sample" color="D0DEF5" x="1376" y="288" />
<entity schema="qiita" name="study_tags" color="B2CDF7" x="2176" y="416" />
<entity schema="qiita" name="study_users" color="D0DEF5" x="1776" y="96" />
<entity schema="qiita" name="term" color="D0DEF5" x="736" y="1408" />
<entity schema="qiita" name="timeseries_type" color="C0D4F3" x="2000" y="496" />
<entity schema="qiita" name="user_level" color="D0DEF5" x="560" y="272" />
<entity schema="qiita" name="visibility" color="B2CDF7" x="1040" y="1072" />
<group name="Group_analyses" color="C4E0F9" >
<comment><![CDATA[analysis tables]]></comment>
<entity schema="qiita" name="analysis" />
Expand Down Expand Up @@ -1895,4 +1895,4 @@ ALTER TABLE oauth_software ADD CONSTRAINT fk_oauth_software FOREIGN KEY ( client
]]></string>
</script>
</layout>
</project>
</project>
3,148 changes: 1,556 additions & 1,592 deletions qiita_db/support_files/qiita-db.html
View file

Large diffs are not rendered by default.

0 comments on commit 3b934a1

Please sign in to comment.