diff --git a/Cargo.lock b/Cargo.lock index d95372d25..5f9077620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2855,7 +2855,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.12.1" +version = "0.12.2" dependencies = [ "actix-rt", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 0ecceb22f..c5987462c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,8 +52,8 @@ itertools = "0.14" json-patch = "4" lambda-web = { version = "0.2.1", features = ["actix4"] } log = "0.4" -martin-tile-utils = { path = "./martin-tile-utils", version = "0.6.0" } -mbtiles = { path = "./mbtiles", version = "0.12.0" } +martin-tile-utils = { path = "./martin-tile-utils", version = "0.6.1" } +mbtiles = { path = "./mbtiles", version = "0.12.2" } md5 = "0.7.0" moka = { version = "0.12", features = ["future"] } num_cpus = "1" diff --git a/martin/src/mbtiles/mod.rs b/martin/src/mbtiles/mod.rs index affb342a0..cc2e56327 100644 --- a/martin/src/mbtiles/mod.rs +++ b/martin/src/mbtiles/mod.rs @@ -63,7 +63,7 @@ impl Debug for MbtSource { impl MbtSource { async fn new(id: String, path: PathBuf) -> FileResult { - let mbt = MbtilesPool::new(&path) + let mbt = MbtilesPool::open_readonly(&path) .await .map_err(|e| io::Error::other(format!("{e:?}: Cannot open file {}", path.display()))) .map_err(|e| IoError(e, path.clone()))?; diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index 877881741..6d07740b8 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mbtiles" -version = "0.12.1" +version = "0.12.2" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index ab2d5960b..27fe02614 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use sqlite_compressions::{register_bsdiffraw_functions, register_gzip_functions}; use sqlite_hashes::register_md5_functions; use sqlx::sqlite::SqliteConnectOptions; -use sqlx::{Connection as _, Executor, SqliteConnection, SqliteExecutor, Statement, query}; +use sqlx::{Connection as _, Executor, Row, SqliteConnection, SqliteExecutor, Statement, query}; use crate::bindiff::PatchType; use crate::errors::{MbtError, MbtResult}; @@ -132,6 +132,8 @@ impl Mbtiles { } /// Get a tile from the database + /// + /// See [`Mbtiles::get_tile_and_hash`] if you do need the hash pub async fn get_tile( &self, conn: &mut T, @@ -153,6 +155,50 @@ impl Mbtiles { Ok(None) } + /// Get a tile and its hash if it exists from the database + /// + /// For [`MbtType::Flat`] accessing the hash is not possible, we thus md5 hash the tile data. + /// + /// See [`Mbtiles::get_tile`] if you don't need the hash + pub async fn get_tile_and_hash( + &self, + conn: &mut SqliteConnection, + mbt_type: MbtType, + z: u8, + x: u32, + y: u32, + ) -> MbtResult, Option)>> { + let sql = Self::get_tile_and_hash_sql(mbt_type); + let y = invert_y_value(z, y); + let Some(row) = query(sql) + .bind(z) + .bind(x) + .bind(y) + .fetch_optional(conn) + .await? + else { + return Ok(None); + }; + Ok(Some((row.get(0), row.get(1)))) + } + + /// sql query for getting tile and hash + /// + /// For [`MbtType::Flat`] accessing it is not possible, we thus md5 hash the tile data. + fn get_tile_and_hash_sql(mbt_type: MbtType) -> &'static str { + match mbt_type { + MbtType::Flat => { + "SELECT tile_data, NULL as tile_hash from tiles where zoom_level = ? AND tile_column = ? AND tile_row = ?" + } + MbtType::FlatWithHash | MbtType::Normalized { hash_view: true } => { + "SELECT tile_data, tile_hash from tiles_with_hash where zoom_level = ? AND tile_column = ? AND tile_row = ?" + } + MbtType::Normalized { hash_view: false } => { + "SELECT images.tile_data, images.tile_id AS tile_hash FROM map JOIN images ON map.tile_id = images.tile_id where map.zoom_level = ? AND map.tile_column = ? AND map.tile_row = ?" + } + } + } + pub async fn insert_tiles( &self, conn: &mut SqliteConnection, diff --git a/mbtiles/src/pool.rs b/mbtiles/src/pool.rs index 455b0abb5..69f394054 100644 --- a/mbtiles/src/pool.rs +++ b/mbtiles/src/pool.rs @@ -1,9 +1,10 @@ use std::path::Path; +use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Pool, Sqlite, SqlitePool}; use crate::errors::MbtResult; -use crate::{Mbtiles, Metadata}; +use crate::{MbtType, Mbtiles, Metadata}; #[derive(Clone, Debug)] pub struct MbtilesPool { @@ -12,9 +13,19 @@ pub struct MbtilesPool { } impl MbtilesPool { + #[deprecated(note = "Use `MbtilesPool::open_readonly` instead")] + #[doc(hidden)] pub async fn new>(filepath: P) -> MbtResult { + Self::open_readonly(filepath).await + } + + /// Open a `MBTiles` file in read-only mode. + pub async fn open_readonly>(filepath: P) -> MbtResult { let mbtiles = Mbtiles::new(filepath)?; - let pool = SqlitePool::connect(mbtiles.filepath()).await?; + let opt = SqliteConnectOptions::new() + .filename(mbtiles.filepath()) + .read_only(true); + let pool = SqlitePool::connect_with(opt).await?; Ok(Self { mbtiles, pool }) } @@ -26,9 +37,302 @@ impl MbtilesPool { self.mbtiles.get_metadata(&mut *conn).await } + /// Detect the type of the `MBTiles` file. + /// + /// See [`MbtType`] for more information. + pub async fn detect_type(&self) -> MbtResult { + let mut conn = self.pool.acquire().await?; + self.mbtiles.detect_type(&mut *conn).await + } + /// Get a tile from the pool + /// + /// See [`MbtilesPool::get_tile_and_hash`] if you do need the tiles' hash. pub async fn get_tile(&self, z: u8, x: u32, y: u32) -> MbtResult>> { let mut conn = self.pool.acquire().await?; self.mbtiles.get_tile(&mut *conn, z, x, y).await } + + /// Get a tile from the pool + /// + /// See [`MbtilesPool::get_tile`] if you don't need the tiles' hash. + pub async fn get_tile_and_hash( + &self, + mbt_type: MbtType, + z: u8, + x: u32, + y: u32, + ) -> MbtResult, Option)>> { + let mut conn = self.pool.acquire().await?; + self.mbtiles + .get_tile_and_hash(&mut conn, mbt_type, z, x, y) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_invalid_type() { + let pool = MbtilesPool::open_readonly("../tests/fixtures/mbtiles/webp.mbtiles") + .await + .unwrap(); + let metadata = pool.get_metadata().await.unwrap(); + insta::assert_yaml_snapshot!(metadata,@r#" + id: webp + tile_info: + format: webp + encoding: "" + layer_type: baselayer + tilejson: + tilejson: 3.0.0 + tiles: [] + bounds: + - -180 + - -85.05113 + - 180 + - 85.05113 + center: + - 0 + - 0 + - 0 + maxzoom: 0 + minzoom: 0 + name: ne2sr + format: webp + "#); + // invalid type => cannot hash properly, but can get tile + assert!(pool.detect_type().await.is_err()); + let t1 = pool.get_tile(0, 0, 0).await.unwrap().unwrap(); + assert!(!t1.is_empty()); + // this is an access and then md5 hash => should not fail + let (t2, h2) = pool + .get_tile_and_hash(MbtType::Flat, 0, 0, 0) + .await + .unwrap() + .unwrap(); + assert_eq!(t2, t1); + assert_eq!(h2, None); + for error_types in [ + MbtType::FlatWithHash, + MbtType::Normalized { hash_view: false }, + MbtType::Normalized { hash_view: true }, + ] { + assert!(pool.get_tile_and_hash(error_types, 0, 0, 0).await.is_err()); + } + } + + #[tokio::test] + async fn test_normalized() { + let pool = MbtilesPool::open_readonly( + "../tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles", + ) + .await + .unwrap(); + let metadata = pool.get_metadata().await.unwrap(); + insta::assert_yaml_snapshot!(metadata,@r#" + id: geography-class-png-no-bounds + tile_info: + format: png + encoding: "" + tilejson: + tilejson: 3.0.0 + tiles: [] + description: "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. " + legend: "
\n\n
Geography Class
\n
by MapBox
\n\n\n
" + maxzoom: 1 + minzoom: 0 + name: Geography Class + template: "{{#__location__}}{{/__location__}}{{#__teaser__}}
\n\n
\n{{admin}}\n\n
{{/__teaser__}}{{#__full__}}{{/__full__}}" + version: 1.0.0 + "#); + assert_eq!( + pool.detect_type().await.unwrap(), + MbtType::Normalized { hash_view: false } + ); + let t1 = pool.get_tile(0, 0, 0).await.unwrap().unwrap(); + assert!(!t1.is_empty()); + + let (t2, h2) = pool + .get_tile_and_hash(MbtType::Normalized { hash_view: false }, 0, 0, 0) + .await + .unwrap() + .unwrap(); + assert_eq!(t2, t1); + let expected_hash = Some("1578fdca522831a6435f7795586c235b".to_string()); + assert_eq!(h2, expected_hash); + + let (t3, h3) = pool + .get_tile_and_hash(MbtType::Flat, 0, 0, 0) + .await + .unwrap() + .unwrap(); + assert_eq!(t3, t2); + assert_eq!(h3, None); + for error_types in [ + MbtType::FlatWithHash, + MbtType::Normalized { hash_view: true }, + ] { + assert!(pool.get_tile_and_hash(error_types, 0, 0, 0).await.is_err()); + } + } + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn test_flat_with_hash() { + let pool = + MbtilesPool::open_readonly("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles") + .await + .unwrap(); + let metadata = pool.get_metadata().await.unwrap(); + insta::assert_yaml_snapshot!(metadata,@r#" + id: zoomed_world_cities + tile_info: + format: mvt + encoding: gzip + layer_type: overlay + tilejson: + tilejson: 3.0.0 + tiles: [] + vector_layers: + - id: cities + fields: + name: String + description: "" + maxzoom: 6 + minzoom: 0 + bounds: + - -123.12359 + - -37.818085 + - 174.763027 + - 59.352706 + center: + - -75.9375 + - 38.788894 + - 6 + description: Major cities from Natural Earth data + maxzoom: 6 + minzoom: 0 + name: Major cities from Natural Earth data + version: "2" + format: pbf + json: + tilestats: + layerCount: 1 + layers: + - attributeCount: 1 + attributes: + - attribute: name + count: 68 + type: string + values: + - Addis Ababa + - Amsterdam + - Athens + - Atlanta + - Auckland + - Baghdad + - Bangalore + - Bangkok + - Beijing + - Berlin + - Bogota + - Buenos Aires + - Cairo + - Cape Town + - Caracas + - Casablanca + - Chengdu + - Chicago + - Dakar + - Denver + - Dubai + - Geneva + - Hong Kong + - Houston + - Istanbul + - Jakarta + - Johannesburg + - Kabul + - Kiev + - Kinshasa + - Kolkata + - Lagos + - Lima + - London + - Los Angeles + - Madrid + - Manila + - Melbourne + - Mexico City + - Miami + - Monterrey + - Moscow + - Mumbai + - Nairobi + - New Delhi + - New York + - Paris + - Rio de Janeiro + - Riyadh + - Rome + - San Francisco + - Santiago + - Seoul + - Shanghai + - Singapore + - Stockholm + - Sydney + - São Paulo + - Taipei + - Tashkent + - Tehran + - Tokyo + - Toronto + - Vancouver + - Vienna + - "Washington, D.C." + - Ürümqi + - Ōsaka + count: 68 + geometry: Point + layer: cities + agg_tiles_hash: D4E1030D57751A0B45A28A71267E46B8 + "#); + assert_eq!(pool.detect_type().await.unwrap(), MbtType::FlatWithHash); + let t1 = pool.get_tile(6, 38, 19).await.unwrap().unwrap(); + assert!(!t1.is_empty()); + + let (t2, h2) = pool + .get_tile_and_hash(MbtType::FlatWithHash, 6, 38, 19) + .await + .unwrap() + .unwrap(); + assert_eq!(t2, t1); + let expected_hash = Some("80EE46337AC006B6BD14B4FA4D6E2EF9".to_string()); + assert_eq!(h2, expected_hash); + let (t3, h3) = pool + .get_tile_and_hash(MbtType::Flat, 6, 38, 19) + .await + .unwrap() + .unwrap(); + assert_eq!(t3, t1); + assert_eq!(h3, None); + let (t3, h3) = pool + .get_tile_and_hash(MbtType::Normalized { hash_view: true }, 6, 38, 19) + .await + .unwrap() + .unwrap(); + assert_eq!(t3, t1); + assert_eq!(h3, expected_hash); + + // no map table + assert!( + pool.get_tile_and_hash(MbtType::Normalized { hash_view: false }, 0, 0, 0) + .await + .is_err() + ); + } }