create-public-files-0.0.1-car #32
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: create-public-files | |
| run-name: create-public-files-${{ inputs.version }}-${{ inputs.mode }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| required: true | |
| description: Version of data | |
| default: '0.0.1' | |
| type: string | |
| mode: | |
| required: true | |
| description: Mode of travel | |
| default: 'car' | |
| type: choice | |
| options: | |
| - car | |
| - bicycle | |
| - foot | |
| override_years: | |
| required: false | |
| description: | | |
| Comma-separated list of OSM data years to run e.g. 2020,2023. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| override_states: | |
| required: false | |
| description: | | |
| Comma-separated state codes to run e.g. 01,06. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| override_datasets: | |
| required: false | |
| description: | | |
| Comma-separated list datasets to publish e.g. times,metadata. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| override_geographies: | |
| required: false | |
| description: | | |
| Comma-separated geographies to limit run e.g. county,tract. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| # See: https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 | |
| AWS_EC2_METADATA_DISABLED: true | |
| PYTHONUNBUFFERED: "1" | |
| jobs: | |
| setup-jobs: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| years: ${{ steps.create-year-jobs.outputs.param }} | |
| states: ${{ steps.create-state-jobs.outputs.param }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Create year jobs | |
| id: create-year-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.year' | |
| param_override: '${{ inputs.override_years }}' | |
| - name: Create state jobs | |
| id: create-state-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.state' | |
| param_override: '${{ inputs.override_states }}' | |
| create-files: | |
| runs-on: ubuntu-24.04 | |
| needs: setup-jobs | |
| strategy: | |
| # Don't fail all chunks if one fails | |
| fail-fast: false | |
| matrix: | |
| year: ${{ fromJSON(needs.setup-jobs.outputs.years) }} | |
| state: ${{ fromJSON(needs.setup-jobs.outputs.states) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Cloudflare credentials | |
| uses: ./.github/actions/setup-cloudflare-s3 | |
| with: | |
| CLOUDFLARE_S3_API_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_S3_API_ACCESS_KEY_ID }} | |
| CLOUDFLARE_S3_API_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_S3_API_SECRET_ACCESS_KEY }} | |
| - name: Remove unnecessary software and increase swap space | |
| uses: ./.github/actions/prep-disk-and-swap | |
| with: | |
| swap_override: 50000 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-suffix: "site-data" | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| - name: Install Python dependencies | |
| id: install-python-dependencies | |
| shell: bash | |
| run: | | |
| sudo apt-get install libgeos-dev | |
| uv python install | |
| uv venv | |
| uv pip install ".[site,data]" | |
| - name: Create file jobs per dataset | |
| id: create-dataset-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.output.dataset' | |
| param_override: '${{ inputs.override_datasets }}' | |
| - name: Create file jobs per geography | |
| id: create-geo-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.census.geography.all' | |
| param_override: '${{ inputs.override_geographies }}' | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| limit-access-to-actor: true | |
| - name: Create public files | |
| working-directory: 'data' | |
| shell: bash | |
| run: | | |
| geographies='${{ steps.create-geo-jobs.outputs.param }}' | |
| geographies_array=($(echo "$geographies" | jq -r '.[]')) | |
| datasets='${{ steps.create-dataset-jobs.outputs.param }}' | |
| datasets_array=($(echo "$datasets" | jq -r '.[]')) | |
| for geo in "${geographies_array[@]}"; do | |
| for dataset in "${datasets_array[@]}"; do | |
| uv run ./src/create_public_files.py \ | |
| --dataset "$dataset" --version ${{ inputs.version }} \ | |
| --mode ${{ inputs.mode }} --year ${{ matrix.year }} \ | |
| --geography "$geo" --state ${{ matrix.state }} | |
| done | |
| done |