1_fetch.yml

target_default: 1_fetch

packages:
  - dataRetrieval
  - dplyr
  - readr

sources:
  - 1_fetch/src/fetch_s3.R
  - 1_fetch/src/fetch_nwis.R
  - 1_fetch/src/do_gw_fetch.R
  - 1_fetch/src/do_addl_gw_fetch.R

targets:

  1_fetch:
    depends:
      - 1_fetch/out/historic_gw_site_info_filtered.rds
      - 1_fetch/out/gw_data.csv

##-- Historic GW sites and data --##

  # This data comes from the Dev VPC with `us-west-2`

  last_updated:
    command: c(I('2022-09-19 10:00 AM'))

  # All sites (including those with years < min_years)
  1_fetch/out/historic_gw_site_info_unfiltered.rds:
    command: fetch_s3(target_name, historic_unfiltered_site_info_s3_fn, last_updated)
  # All data (including those with years < min_years)
  1_fetch/out/historic_gw_data_unfiltered.csv:
    command: fetch_s3(target_name, historic_unfiltered_data_s3_fn, last_updated)
  # This data and the quantiles are just for sites with > min_years
  1_fetch/out/historic_gw_data_filtered.csv:
    command: fetch_s3(target_name, historic_filtered_data_s3_fn, last_updated)
  
  # Only these are absolutely necessary to run the pipeline in its current 
  # state. The others will not build unless you manually build the targets.
  1_fetch/out/historic_gw_site_info_filtered.rds:
    command: fetch_s3(target_name, historic_filtered_site_info_s3_fn, last_updated)
  1_fetch/out/historic_gw_quantiles.csv:
    command: fetch_s3(target_name, historic_quantiles_s3_fn, last_updated)

##-- GW sites and data for current viz time period --##
  
  # Only fetch data for sites that are in the quantiles data
  gw_quantile_site_info:
    command: readRDS("1_fetch/out/historic_gw_site_info_filtered.rds")
  
  gw_sites_dv:
    command: pull_sites_by_query(gw_quantile_site_info, I("dv"), gw_param_cd)
  gw_sites_uv:
    command: pull_sites_by_query(gw_quantile_site_info, I("uv"), gw_param_cd)
  gw_sites_uv_addl:
    command: pull_sites_by_query(gw_quantile_site_info, I("uv"), addl_gw_param_cds)
  
  # Task table for `dv` pulls
  1_fetch/out/gw_data_dv.csv:
    command: do_gw_fetch(
      final_target = target_name,
      task_makefile = I('1_fetch_tasks.yml'),
      gw_site_nums = gw_sites_dv,
      gw_site_nums_obj_nm = I('gw_sites_dv'),
      param_cd = gw_param_cd,
      service_cd = I('dv'),
      request_limit = dv_fetch_size_limit,
      '1_fetch/src/do_gw_fetch.R',
      '1_fetch/src/fetch_nwis.R')
    depends:
      - viz_start_date
      - viz_end_date
  
  # Task table for `uv` includes a step to average the data to daily values.
  # This step needs the timezones for all the sites.
  gw_quantile_site_tz_xwalk:
    command: select(gw_quantile_site_info, I('site_no'), I('tz_cd'))
  1_fetch/out/gw_data_uv.csv:
    command: do_gw_fetch(
      final_target = target_name,
      task_makefile = I('1_fetch_tasks.yml'),
      gw_site_nums = gw_sites_uv,
      gw_site_nums_obj_nm = I('gw_sites_uv'),
      param_cd = gw_param_cd,
      service_cd = I('uv'),
      request_limit = uv_fetch_size_limit,
      '1_fetch/src/do_gw_fetch.R',
      '1_fetch/src/fetch_nwis.R',
      gw_site_tz_xwalk_nm = I('gw_quantile_site_tz_xwalk'))
    depends:
      - viz_start_date
      - viz_end_date
      
  # Special pull for just KS & just FL using `62610`. Plus, HI using `72150`.
  # Read all about why on GitHub: https://github.com/USGS-VIZLAB/gw-conditions/issues/9
  1_fetch/out/gw_data_uv_addl.csv:
    command: do_addl_gw_fetch(
      final_target = target_name,
      addl_param_cds = addl_gw_param_cds,
      viz_start_date = viz_start_date,
      viz_end_date = viz_end_date)
    depends:
      - '1_fetch/src/fetch_nwis.R'

  # Combine all data
  1_fetch/out/gw_data.csv:
    command: combine_gw_fetches(
      target_name, 
      '1_fetch/out/gw_data_dv.csv', 
      '1_fetch/out/gw_data_uv.csv',
      '1_fetch/out/gw_data_uv_addl.csv')
  
##-- Download higher resolution spatial data for SVG mapping --##
  # Data from the National Weather Service https://www.weather.gov/gis/USStates
  1_fetch/out/nws_states.zip:
    command: download.file(destfile = target_name, I("https://www.weather.gov/source/gis/Shapefiles/County/s_11au16.zip"))