mirror dir structure in sampling

OpenBioML · Aug 14, 2024 · 79bc334 · 79bc334
1 parent 233e275
commit 79bc334
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/chemnlp/data/sampler_cli.py b/src/chemnlp/data/sampler_cli.py
@@ -102,6 +102,9 @@ def process_dataset(
     else:
         templates = [t for t in templates if "<EOI>" not in t]
 
+    output_dir = os.path.join(output_dir, os.path.dirname(data_dir))
+    os.makedirs(output_dir, exist_ok=True)
+
     for chunk_idx, df_chunk in enumerate(
         pd.read_csv(data_path, chunksize=chunksize, low_memory=False)
     ):