chore: better represent the options as not the defaults, they just are the options

Author: jrdh

dataimporter/importer.py

@@ -1,9 +1,9 @@
 from datetime import date, datetime
 from functools import partial
-from itertools import groupby
 from pathlib import Path
 from typing import Iterable, List, Optional, Union
 
+from itertools import groupby
 from splitgill.manager import SplitgillClient, SplitgillDatabase
 from splitgill.model import Record
 from splitgill.utils import partition, now
@@ -26,7 +26,7 @@
 from dataimporter.lib.config import Config
 from dataimporter.lib.dbs import Store
 from dataimporter.lib.model import SourceRecord
-from dataimporter.lib.options import DEFAULT_OPTIONS
+from dataimporter.lib.options import PARSING_OPTIONS
 from dataimporter.lib.view import View
 
 
@@ -305,7 +305,7 @@ def add_to_mongo(self, view_name: str, everything: bool = False) -> Optional[int
 
         database.ingest(records, commit=False, modified_field="modified")
         # send the options anyway, even if there's no change to them
-        database.update_options(DEFAULT_OPTIONS, commit=False)
+        database.update_options(PARSING_OPTIONS, commit=False)
         committed = database.commit()
         # flush the queue as we've handled everything in it now
         view.flush()

dataimporter/lib/options.py

@@ -1,4 +1,5 @@
 from splitgill.indexing.options import ParsingOptionsBuilder
+from splitgill.diffing import SG_DATE_FORMATS
 
 builder = ParsingOptionsBuilder()
 
@@ -13,20 +14,16 @@
 builder.with_false_value("false").with_false_value("no").with_false_value("n")
 
 # date formats
-# some common basic formats
+# add the formats used by Splitgill itself allowing us to pickup datetime objects
+for fmt in SG_DATE_FORMATS:
+    builder.with_date_format(fmt)
+# then add the formats we want to support (some of which will already be in the SG list)
+builder.with_date_format("%Y-%m")
 builder.with_date_format("%Y-%m-%d")
-# rfc 3339ish
 builder.with_date_format("%Y-%m-%dT%H:%M:%S")
 builder.with_date_format("%Y-%m-%dT%H:%M:%S.%f")
 builder.with_date_format("%Y-%m-%dT%H:%M:%S%z")
 builder.with_date_format("%Y-%m-%dT%H:%M:%S.%f%z")
-builder.with_date_format("%Y%m%dT%H%m%s")
-# same as the above, just with a space instead of the T separator
-builder.with_date_format("%Y-%m-%d %H:%M:%S")
-builder.with_date_format("%Y-%m-%d %H:%M:%S.%f")
-builder.with_date_format("%Y-%m-%d %H:%M:%S%z")
-builder.with_date_format("%Y-%m-%d %H:%M:%S.%f%z")
-builder.with_date_format("%Y%m%d %H%m%s")
 
 # geo hints
 builder.with_geo_hint(
@@ -36,4 +33,4 @@
     16,
 )
 
-DEFAULT_OPTIONS = builder.build()
+PARSING_OPTIONS = builder.build()