feat: OSM-powered native trail data layer (POC)

infrastructure/osm/hiking.lua

@@ -0,0 +1,111 @@
+-- hiking.lua — osm2pgsql flex output config for PackRat trail data
+--
+-- Imports hiking ways and named hiking route relations from an OSM PBF file
+-- into two PostGIS tables: hiking_ways and hiking_relations.
+--
+-- Usage:
+--   osm2pgsql --slim -d <DATABASE_URL> -O flex -S hiking.lua <file.osm.pbf>
+--
+-- Tables are created by osm2pgsql; run the Drizzle migration first to ensure
+-- PostGIS is enabled and indexes exist. Subsequent imports can use --append.
+
+-- ── Table definitions ──────────────────────────────────────────────────────
+
+local ways_table = osm2pgsql.define_table({
+  name = 'hiking_ways',
+  ids = { type = 'way', id_column = 'osm_id' },
+  columns = {
+    { column = 'name',       type = 'text' },
+    { column = 'surface',    type = 'text' },
+    { column = 'difficulty', type = 'text' },
+    { column = 'access',     type = 'text' },
+    { column = 'foot',       type = 'text' },
+    { column = 'geometry',   type = 'linestring', projection = 4326 },
+  },
+})
+
+local relations_table = osm2pgsql.define_table({
+  name = 'hiking_relations',
+  ids = { type = 'relation', id_column = 'osm_id' },
+  columns = {
+    { column = 'name',        type = 'text' },
+    { column = 'network',     type = 'text' },
+    { column = 'distance',    type = 'text' },
+    { column = 'difficulty',  type = 'text' },
+    { column = 'description', type = 'text' },
+    { column = 'members',     type = 'jsonb' },
+    { column = 'geometry',    type = 'multilinestring', projection = 4326 },
+  },
+})
+
+-- ── Way filter ─────────────────────────────────────────────────────────────
+
+-- Only import ways that are relevant to hiking.
+-- Excludes roads, bike paths, and other non-hiking infrastructure.
+local hiking_highway_types = {
+  path    = true,  -- primary tag for hiking trails
+  footway = true,  -- foot paths and pedestrian walkways
+  track   = true,  -- unpaved tracks, often used for hiking access
+}
+
+function osm2pgsql.process_way(object)
+  if not hiking_highway_types[object.tags.highway] then
+    return
+  end
+
+  ways_table:insert({
+    name       = object.tags.name,
+    surface    = object.tags.surface,
+    difficulty = object.tags.difficulty or object.tags['sac_scale'],
+    access     = object.tags.access,
+    foot       = object.tags.foot,
+    geometry   = object:as_linestring(),
+  })
+end
+
+-- ── Relation filter ────────────────────────────────────────────────────────
+
+-- Ensure way geometries are loaded for relation geometry assembly.
+-- osm2pgsql calls this to decide which way objects to keep in memory.
+function osm2pgsql.select_relation_members(relation)
+  if relation.tags.type == 'route' and relation.tags.route == 'hiking' then
+    return { ways = osm2pgsql.way_member_ids(relation) }
+  end
+end
+
+function osm2pgsql.process_relation(object)
+  if object.tags.type ~= 'route' or object.tags.route ~= 'hiking' then
+    return
+  end
+
+  -- Serialize member refs so the API can do runtime stitching fallback
+  local members = {}
+  for _, member in ipairs(object.members) do
+    members[#members + 1] = {
+      type = member.type,
+      ref  = member.ref,
+      role = member.role or '',
+    }
+  end
+
+  -- Build multilinestring from member ways.
+  -- osm2pgsql assembles this from the ways kept in memory above.
+  -- May be nil for relations whose ways weren't in the extract.
+  local geom = nil
+  local ok, result = pcall(function()
+    return object:as_multilinestring()
+  end)
+  if ok and result and not result:is_null() then
+    geom = result
+  end
+
+  relations_table:insert({
+    name        = object.tags.name,
+    network     = object.tags.network,
+    distance    = object.tags.distance,
+    difficulty  = object.tags.difficulty or object.tags['sac_scale'],
+    description = object.tags.description,
+    members     = members,
+    geometry    = geom,
+  })
+end

infrastructure/osm/import.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# import.sh — Import OSM hiking data into Neon PostgreSQL
+#
+# Prerequisites:
+#   - osm2pgsql >= 1.9 (flex output, Lua style files)
+#   - NEON_DATABASE_URL set (postgresql://...)
+#   - Drizzle migration 0037 already applied (PostGIS enabled, tables exist)
+#
+# Usage:
+#   ./import.sh [path/to/region.osm.pbf]
+#
+# If no PBF is supplied the script downloads Utah (~150 MB) from Geofabrik —
+# a good size for local POC testing without the full North America extract.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LUA_CONFIG="${SCRIPT_DIR}/hiking.lua"
+
+# ── Database URL ────────────────────────────────────────────────────────────
+
+DB_URL="${NEON_DATABASE_URL:-}"
+if [[ -z "$DB_URL" ]]; then
+  echo "Error: NEON_DATABASE_URL is not set" >&2
+  exit 1
+fi
+
+# ── PBF file ─────────────────────────────────────────────────────────────────
+
+if [[ -n "${1:-}" ]]; then
+  PBF_FILE="$1"
+  if [[ ! -f "$PBF_FILE" ]]; then
+    echo "Error: file not found: $PBF_FILE" >&2
+    exit 1
+  fi
+else
+  PBF_FILE="${SCRIPT_DIR}/utah-latest.osm.pbf"
+  if [[ ! -f "$PBF_FILE" ]]; then
+    echo "Downloading Utah extract from Geofabrik (~150 MB)..."
+    curl -L --progress-bar \
+      -o "$PBF_FILE" \
+      "https://download.geofabrik.de/north-america/us/utah-latest.osm.pbf"
+  fi
+fi
+
+echo "PBF file: $PBF_FILE"
+echo "Lua config: $LUA_CONFIG"
+echo ""
+
+# ── Import ───────────────────────────────────────────────────────────────────
+#
+# Flags:
+#   --slim          store node/way/relation data in DB for future updates
+#   --drop          drop slim tables after import (saves space for one-shot POC)
+#   -O flex         use Lua flex output
+#   -S hiking.lua   our style file
+#   -d <url>        connection string
+#
+# For subsequent imports (adding more regions) replace --create with --append.
+
+echo "Starting osm2pgsql import..."
+osm2pgsql \
+  --slim \
+  --drop \
+  --create \
+  -O flex \
+  -S "$LUA_CONFIG" \
+  -d "$DB_URL" \
+  "$PBF_FILE"
+
+echo ""
+echo "Import complete."
+echo ""
+echo "Verify row counts:"
+psql "$DB_URL" -c "SELECT 'hiking_ways' AS table, count(*) FROM hiking_ways UNION ALL SELECT 'hiking_relations', count(*) FROM hiking_relations;"

packages/api/Dockerfile.test

@@ -0,0 +1,7 @@
+# Extends the pgvector image with PostGIS so integration tests can use
+# both vector similarity and geometry/spatial queries in the same DB.
+FROM pgvector/pgvector:pg15
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends postgresql-15-postgis-3 \
+    && rm -rf /var/lib/apt/lists/*

packages/api/docker-compose.test.yml

@@ -1,6 +1,8 @@
 services:
   postgres-test:
-    image: pgvector/pgvector:pg15
+    build:
+      context: .
+      dockerfile: Dockerfile.test
     environment:
       POSTGRES_DB: packrat_test
       POSTGRES_USER: test_user

packages/api/drizzle/0037_osm_hiking_tables.sql

@@ -0,0 +1,57 @@
+-- Enable PostGIS (pre-installed on Neon, safe to run multiple times)
+CREATE EXTENSION IF NOT EXISTS postgis;
+--> statement-breakpoint
+
+-- Individual trail segments — populated by osm2pgsql flex output (hiking.lua)
+CREATE TABLE IF NOT EXISTS "hiking_ways" (
+  "osm_id"     bigint PRIMARY KEY NOT NULL,
+  "name"       text,
+  "surface"    text,
+  "difficulty" text,
+  "access"     text,
+  "foot"       text,
+  "geometry"   geometry(LineString, 4326)
+);
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "hiking_ways_geometry_idx"
+  ON "hiking_ways" USING gist("geometry");
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "hiking_ways_name_idx"
+  ON "hiking_ways" ("name")
+  WHERE "name" IS NOT NULL;
+--> statement-breakpoint
+
+-- Named hiking routes — populated by osm2pgsql flex output (hiking.lua)
+-- geometry is assembled by osm2pgsql from member ways; members JSONB enables
+-- runtime stitching fallback via ST_LineMerge when needed
+CREATE TABLE IF NOT EXISTS "hiking_relations" (
+  "osm_id"      bigint PRIMARY KEY NOT NULL,
+  "name"        text,
+  "network"     text,
+  "distance"    text,
+  "difficulty"  text,
+  "description" text,
+  "members"     jsonb,
+  "geometry"    geometry(MultiLineString, 4326),
+  "cached_at"   timestamp
+);
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "hiking_relations_geometry_idx"
+  ON "hiking_relations" USING gist("geometry");
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "hiking_relations_name_idx"
+  ON "hiking_relations" ("name")
+  WHERE "name" IS NOT NULL;
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "hiking_relations_network_idx"
+  ON "hiking_relations" ("network")
+  WHERE "network" IS NOT NULL;
+--> statement-breakpoint
+
+-- Link trips to OSM trail relations (no FK — OSM data lives outside Drizzle)
+ALTER TABLE "trips" ADD COLUMN IF NOT EXISTS "trail_osm_id" bigint;