Try batching on text blocks not families (#85)

The task is failing, we know part of the issue is that batching on families
is leading to insanely large amount of text blocks being loaded into memory
(500 families with up to 40,000 text blocks each!) This may also solve
the other issue we are seeing of empty requests being sent to vespa, but
at the very least it will be educational

src/config.py

@@ -61,7 +61,7 @@ def _convert_to_bool(x: str) -> bool:
 
 
 # Vespa config
-VESPA_DOCUMENT_BATCH_SIZE: int = int(os.getenv("VESPA_BATCH_SIZE", "500"))
+VESPA_DOCUMENT_BATCH_SIZE: int = int(os.getenv("VESPA_BATCH_SIZE", "2000"))
 VESPA_INSTANCE_URL: str = os.getenv("VESPA_INSTANCE_URL", "")
 VESPA_CERT_LOCATION: str = os.getenv("VESPA_CERT_LOCATION", "")
 VESPA_KEY_LOCATION: str = os.getenv("VESPA_KEY_LOCATION", "")

src/index/vespa_.py

@@ -317,7 +317,7 @@ def populate_vespa(
             }
         )
 
-        if len(to_process[FAMILY_DOCUMENT_SCHEMA]) >= config.VESPA_DOCUMENT_BATCH_SIZE:
+        if len(to_process[DOCUMENT_PASSAGE_SCHEMA]) >= config.VESPA_DOCUMENT_BATCH_SIZE:
             asyncio.run(_batch_ingest(vespa, to_process))
             to_process.clear()