Skip to content

download AA-LCR_extracted-text.zip via hf_hub_download #1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anowaczynski-nvidia merged 4 commits into from
Dec 18, 2025
Merged

download AA-LCR_extracted-text.zip via hf_hub_download #1126

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8fc8b59
replace wget.download with hf_hub_download
anowaczynski-nvidia Dec 17, 2025
53fa077
ruff format
anowaczynski-nvidia Dec 18, 2025
b46dab8
Merge branch 'main' into anowaczynski/aalcr-fix-hf
gwarmstrong Dec 18, 2025
cefa37c
Merge branch 'main' into anowaczynski/aalcr-fix-hf
gwarmstrong Dec 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions nemo_skills/dataset/aalcr/prepare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,13 @@
import json
import logging
import os
import tempfile
import zipfile
from pathlib import Path

import tiktoken
from datasets import load_dataset
from huggingface_hub import hf_hub_download
from tqdm import tqdm

from nemo_skills.utils import get_logger_name, setup_logging
Expand Down Expand Up @@ -203,24 +206,28 @@ def write_data_to_file(output_file, data, txt_file_folder, max_context_window, t
def prepare_aalcr_data(max_context_window, setup, tokenizer_name):
# download the provied extracted text files
# https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR/resolve/main/extracted_text/AA-LCR_extracted-text.zip
extracted_text_zip_path = hf_hub_download(
repo_id="ArtificialAnalysis/AA-LCR",
filename="extracted_text/AA-LCR_extracted-text.zip",
repo_type="dataset",
)

if not os.path.exists(Path(__file__).absolute().parent / "lcr"):
import zipfile
extracted_text_zip_path = Path(extracted_text_zip_path)
assert extracted_text_zip_path.exists() and extracted_text_zip_path.is_file()

import wget
with tempfile.TemporaryDirectory() as tmpdir:

wget.download(URL)
zipfile.ZipFile("AA-LCR_extracted-text.zip").extractall(Path(__file__).absolute().parent)
os.remove("AA-LCR_extracted-text.zip")
zipfile.ZipFile(extracted_text_zip_path).extractall(tmpdir)

txt_file_folder = Path(__file__).absolute().parent / "lcr"
txt_file_folder = Path(tmpdir) / "lcr"
assert txt_file_folder.exists() and txt_file_folder.is_dir()

dataset = load_dataset("ArtificialAnalysis/AA-LCR")["test"]
dataset = load_dataset("ArtificialAnalysis/AA-LCR")["test"]

data_dir = Path(__file__).absolute().parent
data_dir = Path(__file__).absolute().parent
output_file = data_dir / f"{setup}.jsonl"

output_file = data_dir / f"{setup}.jsonl"
write_data_to_file(output_file, dataset, txt_file_folder, max_context_window, tokenizer_name)
write_data_to_file(output_file, dataset, txt_file_folder, max_context_window, tokenizer_name)


if __name__ == "__main__":
Expand Down
Loading