Skip to content

Commit

Permalink
Cap split size to chunk size
Browse files Browse the repository at this point in the history
  • Loading branch information
dfsnow committed Oct 25, 2024
1 parent 606beee commit 44441ba
Showing 1 changed file with 1 addition and 6 deletions.
7 changes: 1 addition & 6 deletions data/src/calculate_times.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

# Load params, env vars, and JSON used for config
params = yaml.safe_load(open(DOCKER_PATH / "params.yaml"))
max_split_size = params["times"]["max_split_size"]
os.environ["AWS_PROFILE"] = params["s3"]["profile"]
with open(DOCKER_PATH / "valhalla.json", "r") as f:
valhalla_data = json.load(f)
Expand Down Expand Up @@ -145,11 +144,7 @@ def create_write_path(key: str, out_type: str, output_dict: dict) -> str:
# Split and check chunk value
chunk_start_idx, chunk_end_idx = map(int, args.chunk.split("-"))
chunk_size = chunk_end_idx - chunk_start_idx
if max_split_size > chunk_size:
raise ValueError(
f"Chunk size ({chunk_size}) exceeds max_split_size ({max_split_size})"
)

max_split_size = min(params["times"]["max_split_size"], chunk_size)
chunk_msg = f", chunk: {args.chunk}" if args.chunk else ""
print(
f"Starting routing for version: {params['times']['version']},"
Expand Down

0 comments on commit 44441ba

Please sign in to comment.