Added deprecation for memmap (#517)

Co-authored-by: Pete <[email protected]>

.github/workflows/main.yml

@@ -62,7 +62,7 @@ jobs:
             task:
               name: Data pipeline
               run: |
-                python scripts/prepare_memmap_dataset.py test_fixtures/*.json.gz -o /tmp/c4-sample.npy --validate
+                python scripts/prepare_memmap_dataset.py test_fixtures/*.json.gz -o /tmp/c4-sample.npy --validate --ack-deprecated
 
     steps:
       - uses: actions/checkout@v3

scripts/prepare_memmap_dataset.py

@@ -374,6 +374,7 @@ def make_source_and_target(
     "--safe-mode/--fast-mode", default=False, help="Safe mode caches locally and decompresses using gzip.open"
 )
 @click.option("-j", "--workers", "max_workers", type=int, default=1, help="Defaults to number of CPUs")
+@click.option("--ack-deprecated", is_flag=True, help="Acknowledge that this command is deprecated")
 def main(
     src: Tuple[str, ...],
     output: str,
@@ -389,7 +390,20 @@ def main(
     paths_per_worker: int = 1,
     max_workers: int = 1,
     cache_dir: Optional[str] = None,
+    ack_deprecated: bool = False,
 ):
+    print("WARNING: THIS SCRIPT IS DEPRECATED!!!")
+    print(
+        "Consider using the tokenization tool in the Dolma toolkit: "
+        "https://github.com/allenai/dolma/blob/main/docs/tokenize.md"
+    )
+
+    if not ack_deprecated:
+        continue_question = input("Do you want to continue? [y/N]: ")
+        if not (c := continue_question.lower().strip()) or c != "y":
+            print("Aborting.")
+            return
+
     print("=== CONFIGURATION ===")
     print(f"src:              {src}")
     print(f"output:           {output}")