config.yaml.sample

queue:
  # type can be `sqs`, `file`, `mem`, or `redis`
  # can also be a list of queues to configure multiple
  type: mem
  name: <sqs queue/list of queue names/queue file name/redis key/>
  # if using sqs configuration
  sqs:
    region: <aws-region>
queue-mapping:
  type: <single/multiple>
  # for multiple configuration
  multiple:
    - start-zoom: 10
      # end_zoom is exclusive
      end-zoom: 16
      queue-name: <name of configured queue>
      # if set, the group-by-zoom tells the mapper to group
      # coordinates by common tile parents
      group-by-zoom: 10
      # NOTE: start_zoom and end_zoom can be None if queue is only used for
      # priority purposes
      # the order here in queue-mapping is important, and is in
      # priority order that the queues will be checking when reading
      # messages during processing
message-marshall:
  # controls how queue messages get marshalled/unmarshalled
  # this should correspond to the appropriate queue-mapping implementation
  type: <single/multiple>
# can omit the in-inflight configuration completely
in-flight:
  type: <redis>
message-tracker:
  # should correspond with the marshall and queue implementations
  type: <single/multiple>
# NOTE: message-visibility is sqs specific in utility
# config is shared across queues
message-visibility:
  # on partial completion, how long to extend the visibility
  extend-seconds: 1800
  # don't exceed max, which is 12 hours for sqs
  max-seconds: 43200
  # read visibility timeout for receive message calls
  timeout-seconds: 3600
# Uncomment this section to activate statsd
#statsd:
#  host: 127.0.0.1
#  port: 8125
#  prefix: dev.tilequeue
store:
  type: s3 # Can also be `directory`, which would dump the tiles to disk.
  name: <s3 bucket/tile directory name>
  # The following store properties are s3 specific.
  reduced-redundancy: true
  date-prefix: 19851026
  # control the s3 tile key for the object
  # can either be hash-prefix or prefix-hash
  # hash-prefix -> hash comes before the prefix
  # prefix-hash -> prefix comes before the hash
  key-format-type: <hash-prefix|prefix-hash>
  # optional tags that get saved onto the s3 objects
  tags:
    prefix: 19851026
    run_id: 19851026-1
aws:
  # credentials are optional, and better to use an iam role assigned
  # to the instance if possible
  credentials:
    aws_access_key_id: <aws_access_key_id>
    aws_secret_access_key: <aws_secret_access_key>
tiles:
  seed:
    # can use any combination of the following options for the
    # tilequeue seed command

    # all tiles in a particular zoom range
    all:
      zoom-start: 0
      zoom-until: 10

    # parse the json from Mapzen's metro extracts for a particular
    # zoom range
    metro-extract:
      url: https://raw.githubusercontent.com/mapzen/metroextractor-cities/master/cities.json
      zoom-start: 11
      zoom-until: 15

      # Can also be set to an array of metro-extract city names, like
      # 'new-york_new-york', to only seed tiles from those areas
      # rather than all of them.
      cities:
        - new-york_new-york

    # parse the csv file from Mapbox's post on the top 50k most
    # requested tiles
    top-tiles:
      url: https://gist.github.com/brunosan/b7ce3df8b48229a61b5b/raw/37e42e77f253bc204076111c92acc4d5e653edd2/top_50k_tiles.csv
      zoom-start: 11
      zoom-until: 20

    # Specify any number of custom bounding boxes to seed tiles for.
    custom:
      zoom-start: 10
      zoom-until: 10

      # Must be set to an array of bounding boxes in `[left, bottom, right,
      # top]` (or `[min lon, min lat, max lon, max lat]`) format
      bboxes:

    # whether the tiles that are seeded should also be added to the
    # tiles of interest, or just enqueued
    should-add-to-tiles-of-interest: true

    # how many threads to use when enqueueing
    n-threads: 20

    # whether the tiles should be unique in memory
    # when seeding many tiles that are known to be unique, this should
    # be set to false to save memory
    unique: true

  # zoom where tile content stops changing
  max-zoom-with-changes: 16

process:
  # number of simultaneous "querysets" to issue to the database.  The
  # query for each layer will be issued in parallel to the same
  # database. This can be 0 or unspecified, in which case the number
  # will be inferred from the number of database names configured
  # below.
  n-simultaneous-query-sets: 1
  # whether to print out the internal python queue sizes
  log-queue-sizes: true
  # and at what interval
  log-queue-sizes-interval-seconds: 30
  query-config: <path/to/vector-datasource/queries.yaml>
  template-path: <path/to/vector-datasource/queries>
  # whether to reload jinja query templates on each request. This
  # should be off in production.
  reload-templates: false
  # extensions of formats to generate
  # buffered Mapbox Vector Tiles are also possible by specifying mvtb
  formats: [json, topojson, mvt]
  # additionally, the data included for some formats expects to be
  # buffered. This is where buffers per layer or per geometry type can
  # be specified, with layers trumping geometry types
  buffer:
    mvtb:
      layer:
        earth: {point: 256}
        water: {point: 256}
        places: {point: 128}
      geometry:
        point: 64
        line: 8
        polygon: 8
  # control how python code from yaml is used
  yaml:
    # dotted name or runtime
    type: parse
    parse:
      # this will parse the yaml files and compile them on startup
      # useful when iterating on updating the yaml files themselves
      path: <path/to/vector-datasource/yaml-directory>
    callable:
      # this will call a function that should return a mapping from
      # layer name to function that generates the output properties
      # mapping returned should look like:
      # layer-name -> function(shape, properties, feature_id)
      # useful when the output functions have already been generated
      # and it's simply a matter of pointing to them
      dotted-name: path.to.function.returning.mapping
      args:
        - 'any-args-to-pass'
logging:
  # logging.conf on this page:
  # https://docs.python.org/2/howto/logging.html#logging-basic-tutorial
  config: logging.conf.sample
redis:
  host: localhost
  port: 6379
  db: 0
# expected data source is postgresql
postgresql:
  host: localhost
  port: 5432
  # multiple databases can be specified, and these are iterated
  # through to balance query loads. This is useful when connecting to
  # pgbouncer, which can dispatch to different back end databases
  # based on the name.
  dbnames: [osm]
  user: osm
  password:

wof:
  # url path to neighbourhoods, microhoods, and macrohoods meta csv files
  neighbourhoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-neighbourhood-latest.csv
  microhoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-microhood-latest.csv
  macrohoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-macrohood-latest.csv
  boroughs-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-borough-latest.csv
  # url path prefix for wof raw data
  data-prefix-url: https://data.whosonfirst.org
  # filesystem path to wof data checkout
  data-path: /tmp/whosonfirst-data
  postgresql:
    host: localhost
    port: 5432
    dbname: osm
    user: osm
    password:

# A metatile is a .zip package of multiple tiles. Storing multiple, related
# tiles together can be beneficial by reducing the number of writes to disk when
# storing and loading them.
metatile:
  # The metatile size setting has the following effects by itself. But please
  # note that the `tile-sizes` setting can affect this too!
  #
  #   - null or omitted entry: Individual tiles, one for each format, will be
  #     saved on disk. This can be a good choice for local development use, or to
  #     save files to be served directly from storage (i.e: without tapalcatl).
  #
  #   - 1: Tiles with the same coordinate will be saved together in a .zip file,
  #     which will require using tapalcatl to serve the individual files to users.
  #     This is a good choice to reduce the number of writes to disk.
  #
  #   - 2: A "512px" tile and the 4 "256px" tiles covering the same geographical
  #     area will be saved together, with all their formats. The filename of the
  #     tile will be based on the coordinate of the "512px" tile, but the zoom
  #     level used for styling will be one higher; that of the "256px" tiles.
  #
  #   - 4: A "1024px" tile, 4 "512px" tiles and 16 "256px" tiles covering the
  #     same area.
  size: null

  # if specified, the zoom levels to leave off the top of the metatile pyramid.
  # for example, if the metatile size is specified at 4 and the start-zoom as 1,
  # then the "1024px" tile will be omitted. if start-zoom is 2, then the
  # "1024px" tile and all four "512px" tiles will be omitted.
  #
  # optional: defaults to zero.
  start-zoom: 0

  # if specificed, the tile sizes to render in each metatile. by default, this
  # is all the sizes for the metatile, i.e: for `size=4`, then `tile-sizes`
  # will default to [1024, 512, 256].
  #
  # if you don't need some of the sizes, then setting this to a reduced set
  # can make your metatiles smaller. however, with a reduced set of sizes, the
  # following special behaviour applies:
  #
  #  - at the maximum nominal zoom (see `tile.max-zoom-with-changes`), the
  #    metatile will additionally contain all the smaller tiles at the same
  #    nominal zoom. for example, if `tile-sizes` is [1024], `metatile.size`
  #    is 4 and the `max-zoom-with-changes` is 16, then the highest zoom
  #    metatile will be 14/x/y and will contain one 1024px tile, 2x2 512px
  #    tiles and 4x4 256px tiles all at the `max-zoom-with-changes` nominal
  #    zoom. this allows programs such as tapalcatl to serve smaller tiles
  #    for a few zoom levels of "overzoom".
  #
  #  - at zoom zero, the metatile will additionally contain the same size of
  #    tile at lower nominal and coordinate zooms. for example, if the
  #    `tile-sizes` is [512, 256] and the metatile size is 8, then in addition
  #    to the 512px and 256px tiles at nominal zoom 3, the tile will also
  #    contain 512px tiles at nominal zooms 2 and 1. (because the 512px tile
  #    at nominal zoom 1 covers the whole world, there is no tile output at
  #    nominal zoom 0).
  #
  # optional: default behaviour is all the tile sizes, as described in the
  # `size` parameter description.
  #
  #tile-sizes: [512, 256]

# Configuration for where to store the tiles of interest set
toi-store:
  # We support storing the TOI in S3 or as a file
  type: <file or s3>
  file:
    # The name of the file to store the TOI
    name: toi.txt.gz
  s3:
    # The bucket and key for the s3 object that will contain the tiles of interest
    bucket: test_bucket
    key: toi/toi.txt.gz

# Configuration for the tiles of interest prune/garden command.
toi-prune:
  # location of tileserver logs
  tile-traffic-log-path: ../tileserver/nohup.out

  # Connection and query configuration for a database containing
  # request information for tiles.
  tile-history:
    database-uri: postgresql://user:password@localhost:5439/database
    # The number of days of history to query for.
    days: 30
    # The maximum zoom level to query redshift.
    max-zoom: 16
  cutoff:
    # The minimum number of requests per tile for a tile to be included in
    # the tiles of interest
    min-requests: 16
    # The maximum number of tiles to include from the RedShift-based calculation
    max-tiles: 100000
  s3:
    # Connection information for deleting tiles from S3 when the tile is no longer
    # in the tiles of interest.
    bucket: mapzen-tiles-dev-us-east
    date-prefix: 20170322
    path: osm
    layer: all
    format: zip
  # layer and format of tiles to be deleted
  store:
    layer: all
    format: zip
  always-include:
    # Sets of tiles to always include in the tiles of interest.
    # For more information about options here, see the code:
    # https://github.com/tilezen/tilequeue/blob/1ca908e/tilequeue/command.py#L1046-L1061
    world:
      bbox: -180.0,-85.06,180.0,85.06
      min_zoom: 0
      max_zoom: 10
    # sf_metro:
    #   bbox: -122.524338,37.571794,-122.345123,37.821175
    #   min_zoom: 0
    #   max_zoom: 15
    # tests:
    #  tiles: 0/0/0,1/1/0
    # tests-file:
    #  file: list-of-tile-coordinates.txt
    # NOTE: the coordinates should be of the same size as the toi, currently 512
    # integration-test-coordinates:
    #   bucket: mapzen-tiles-assets
    #   key: test/integration-test-coords.txt
rawr:
  # this is the parent zoom level that coordinates are expected to be grouped under
  group-zoom: 10
  queue:
    type: sqs
    name: <sqs-queue-name>
    wait-seconds: 20
    region: us-east-1
    # alternatively, can use a file-backed queue
    #type: file
    #input-file: <file containing z/x/y per line>
  postgresql:
    host: localhost
    port: 5432
    dbname: dbname
    user: dbuser
    password: dbpassword
  sink:
    # type can also be "none" which ignores all writes
    type: s3
    s3:
      bucket: s3-bucket-name
      region: us-east-1
      prefix: s3-bucket-prefix
      extension: zip
      # control the s3 tile key for the object
      # can either be hash-prefix or prefix-hash
      # hash-prefix -> hash comes before the prefix
      # prefix-hash -> prefix comes before the hash
      key-format-type: <hash-prefix|prefix-hash>
      # optional tags that get saved onto the s3 objects
      tags:
        prefix: 19851026
        run_id: 19851026-1
  # alternatively, provide a "store" config - same as elsewhere, can be s3 or directory
  #store:
  #  type: directory
  #  path: rawr_tiles
  source:
    type: store # type can also be "generate" or "s3"
    store:
      type: directory
      name: rawr_tiles
    # add a postgresql section if you're using "generate"
    #postgresql:
    #  host: null
    #  port: null
    #  dbname: osm
    #  user: osm
    table-sources:
      planet_osm_line: &osm { name: osm, value: openstreetmap.org }
      planet_osm_point: *osm
      planet_osm_polygon: *osm
      planet_osm_ways: *osm
      planet_osm_rels: *osm
      wof_neighbourhood: { name: wof, value: whosonfirst.org }
      water_polygons: &osmdata { name: shp, value: osmdata.openstreetmap.de }
      land_polygons: *osmdata
      ne_10m_urban_areas: { name: ne, value: naturalearthdata.com }
  # when a feature's shape is of the type given in the key and the feature
  # appears in the listed layers, then generate a label centroid. multi*
  # geometries are considered the same as single ones for the purposes of key
  # lookup.
  label-placement-layers:
    point: ['earth', 'water']
    polygon: ['buildings', 'earth', 'landuse', 'water']
    linestring: ['earth', 'landuse', 'water']
  # indexes to generate on the RAWR tile. features are looked up only if they
  # appear in an index, so this list must be exhaustive. all OSM-based features
  # are handled by a single index of {type: osm}. other indexes are provided on
  # single table/layer combinations with the {type: simple}, providing the
  # names of both the table and layer. it's also possible to provide start_zoom
  # and end_zoom parameters which control when the data is visible. note that
  # the start_zoom is inclusive (and optional; defaults to 0) and the end_zoom
  # is exclusive (also optional; defaults to infinity).
  indexes:
    - type: osm
    - type: simple
      table: wof_neighbourhood
      layer: places
    - type: simple
      table: water_polygons
      layer: water
    - type: simple
      table: land_polygons
      layer: earth
    - type: simple
      table: ne_10m_urban_areas
      layer: landuse
      end_zoom: 10
  intersect:
    # type can be toi, none, all, all-parents
    type: toi
# uncomment this to use RAWR tiles rather than go direct to the database.
#use-rawr-tiles: true
batch:
  # zoom level to enqueue jobs
  queue-zoom: 7
  # name of batch job definition
  job-definition: <job-definition>
  # name of batch queue to use
  job-queue: <job-queue>
  # optional number of retry attempts, overrides job definition value
  retry-attempts: 5
  # check store if metatile already exists, and if so skip processing
  check-metatile-exists: true
  # optional override memory reserved for container when submiting job
  # NOTE: in megabytes
  # https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html
  memory:
  vcpus:
  run_id: optional-string-which-gets-logged