diff --git a/Cargo.lock b/Cargo.lock index c56f0bccb..4863bc444 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3861,6 +3861,8 @@ dependencies = [ "flate2", "rstest", "serde", + "serde_json", + "thiserror 2.0.17", ] [[package]] diff --git a/justfile b/justfile index a947554b7..b23d916c7 100755 --- a/justfile +++ b/justfile @@ -68,7 +68,7 @@ bless: set -euo pipefail echo "Blessing unit tests" - for target in restart clean-test bless-insta-martin bless-insta-martin-core bless-insta-martin-tile-utils bless-insta-mbtiles bless-frontend; do + for target in restart clean-test bless-insta bless-frontend; do echo "::group::just $target" {{quote(just_executable())}} $target echo "::endgroup::" @@ -84,21 +84,8 @@ bless-frontend: npm run test:update-snapshots # Run integration tests and save its output as the new expected output -bless-insta-martin *args: (cargo-install 'cargo-insta') - cargo insta test --accept --all-targets --package martin {{args}} - -# Run integration tests and save its output as the new expected output -bless-insta-martin-core *args: (cargo-install 'cargo-insta') - cargo insta test --accept --all-targets --package martin-core {{args}} - -# Run integration tests and save its output as the new expected output -bless-insta-martin-tile-utils *args: (cargo-install 'cargo-insta') - cargo insta test --accept --all-targets --package martin-tile-utils {{args}} - -# Run integration tests and save its output as the new expected output -bless-insta-mbtiles *args: (cargo-install 'cargo-insta') - #rm -rf mbtiles/tests/snapshots - cargo insta test --accept --all-targets --package mbtiles {{args}} +bless-insta *args: (cargo-install 'cargo-insta') + cargo insta test --accept --all-targets --workspace {{args}} # Bless integration tests bless-int: diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 28fb9f668..63d5c3891 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -19,6 +19,8 @@ homepage.workspace = true brotli.workspace = true flate2.workspace = true serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true [dev-dependencies] approx.workspace = true diff --git a/martin-tile-utils/src/decoders.rs b/martin-tile-utils/src/decoders.rs index 1bdb177ba..f5441c59b 100644 --- a/martin-tile-utils/src/decoders.rs +++ b/martin-tile-utils/src/decoders.rs @@ -1,6 +1,6 @@ use std::io::{Read as _, Write as _}; -use flate2::read::GzDecoder; +use flate2::read::{GzDecoder, ZlibDecoder}; use flate2::write::GzEncoder; pub fn decode_gzip(data: &[u8]) -> Result, std::io::Error> { @@ -10,6 +10,13 @@ pub fn decode_gzip(data: &[u8]) -> Result, std::io::Error> { Ok(decompressed) } +pub fn decode_zlib(data: &[u8]) -> Result, std::io::Error> { + let mut decoder = ZlibDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(decompressed) +} + pub fn encode_gzip(data: &[u8]) -> Result, std::io::Error> { let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); encoder.write_all(data)?; diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 3303275f4..190094219 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -4,7 +4,7 @@ // project originally written by Kaveh Karimi and licensed under MIT OR Apache-2.0 use std::f64::consts::PI; -use std::fmt::{Display, Formatter, Result}; +use std::fmt::{Display, Formatter}; /// circumference of the earth in meters pub const EARTH_CIRCUMFERENCE: f64 = 40_075_016.685_578_5; @@ -32,7 +32,7 @@ pub type TileData = Vec; pub type Tile = (TileCoord, Option); impl Display for TileCoord { - fn fmt(&self, f: &mut Formatter<'_>) -> Result { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if f.alternate() { write!(f, "{}/{}/{}", self.z, self.x, self.y) } else { @@ -77,6 +77,7 @@ pub enum Format { Jpeg, Json, Mvt, + Mlt, Png, Webp, Avif, @@ -90,6 +91,7 @@ impl Format { "jpg" | "jpeg" => Self::Jpeg, "json" => Self::Json, "pbf" | "mvt" => Self::Mvt, + "mlt" => Self::Mlt, "png" => Self::Png, "webp" => Self::Webp, "avif" => Self::Avif, @@ -106,6 +108,7 @@ impl Format { Self::Json => "json", // QGIS uses `pbf` instead of `mvt` for some reason Self::Mvt => "pbf", + Self::Mlt => "mlt", Self::Png => "png", Self::Webp => "webp", Self::Avif => "avif", @@ -119,6 +122,7 @@ impl Format { Self::Jpeg => "image/jpeg", Self::Json => "application/json", Self::Mvt => "application/x-protobuf", + Self::Mlt => "application/vnd.maplibre-vector-tile", Self::Png => "image/png", Self::Webp => "image/webp", Self::Avif => "image/avif", @@ -128,22 +132,26 @@ impl Format { #[must_use] pub fn is_detectable(self) -> bool { match self { - Self::Png | Self::Jpeg | Self::Gif | Self::Webp | Self::Avif => true, - // TODO: Json can be detected, but currently we only detect it - // when it's not compressed, so to avoid a warning, keeping it as false for now. - // Once we can detect it inside a compressed data, change it to true. - Self::Mvt | Self::Json => false, + Self::Png + | Self::Jpeg + | Self::Gif + | Self::Webp + | Self::Avif + | Self::Json + | Self::Mlt => true, + Self::Mvt => false, } } } impl Display for Format { - fn fmt(&self, f: &mut Formatter<'_>) -> Result { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(match *self { Self::Gif => "gif", Self::Jpeg => "jpeg", Self::Json => "json", Self::Mvt => "mvt", + Self::Mlt => "mlt", Self::Png => "png", Self::Webp => "webp", Self::Avif => "avif", @@ -210,29 +218,60 @@ impl TileInfo { /// Try to figure out the format and encoding of the raw tile data #[must_use] - pub fn detect(value: &[u8]) -> Option { - // TODO: Make detection slower but more accurate: - // - uncompress gzip/zlib/... and run detection again. If detection fails, assume MVT - // - detect json inside a compressed data - // - json should be fully parsed - // - possibly keep the current `detect()` available as a fast path for those who may need it - Some(match value { - // Compressed prefixes assume MVT content - v if v.starts_with(b"\x1f\x8b") => Self::new(Format::Mvt, Encoding::Gzip), - v if v.starts_with(b"\x78\x9c") => Self::new(Format::Mvt, Encoding::Zlib), - v if v.starts_with(b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A") => { - Self::new(Format::Png, Encoding::Internal) + pub fn detect(value: &[u8]) -> Self { + // Try GZIP decompression + if value.starts_with(b"\x1f\x8b") { + if let Ok(decompressed) = decode_gzip(value) { + let inner_format = Self::detect_vectorish_format(&decompressed); + return Self::new(inner_format, Encoding::Gzip); } - v if v.starts_with(b"\x47\x49\x46\x38\x39\x61") => { - Self::new(Format::Gif, Encoding::Internal) + // If decompression fails or format is unknown, assume MVT + return Self::new(Format::Mvt, Encoding::Gzip); + } + + // Try Zlib decompression + if value.starts_with(b"\x78\x9c") { + if let Ok(decompressed) = decode_zlib(value) { + let inner_format = Self::detect_vectorish_format(&decompressed); + return Self::new(inner_format, Encoding::Zlib); } - v if v.starts_with(b"\xFF\xD8\xFF") => Self::new(Format::Jpeg, Encoding::Internal), + // If decompression fails or format is unknown, assume MVT + return Self::new(Format::Mvt, Encoding::Zlib); + } + if let Some(raster_format) = Self::detect_raster_formats(value) { + Self::new(raster_format, Encoding::Internal) + } else { + let inner_format = Self::detect_vectorish_format(value); + Self::new(inner_format, Encoding::Uncompressed) + } + } + + /// Fast-path detection without decompression + #[must_use] + fn detect_raster_formats(value: &[u8]) -> Option { + match value { + v if v.starts_with(b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A") => Some(Format::Png), + v if v.starts_with(b"\x47\x49\x46\x38\x39\x61") => Some(Format::Gif), + v if v.starts_with(b"\xFF\xD8\xFF") => Some(Format::Jpeg), v if v.starts_with(b"RIFF") && v.len() > 8 && v[8..].starts_with(b"WEBP") => { - Self::new(Format::Webp, Encoding::Internal) + Some(Format::Webp) } - v if v.starts_with(b"{") => Self::new(Format::Json, Encoding::Uncompressed), - _ => None?, - }) + _ => None, + } + } + + /// Detect the format of vector (or json) data after decompression + #[must_use] + fn detect_vectorish_format(value: &[u8]) -> Format { + match value { + v if decode_7bit_length_and_tag(v, &[0x1]).is_ok() => Format::Mlt, + v if is_valid_json(v) => Format::Json, + // If we can't detect the format, we assume MVT. + // Reasoning: + //- it's the most common format and + //- we don't have a detector for it + _ => Format::Mvt, + } } #[must_use] @@ -246,9 +285,12 @@ impl From for TileInfo { Self::new( format, match format { - Format::Png | Format::Jpeg | Format::Webp | Format::Gif | Format::Avif => { - Encoding::Internal - } + Format::Mlt + | Format::Png + | Format::Jpeg + | Format::Webp + | Format::Gif + | Format::Avif => Encoding::Internal, Format::Mvt | Format::Json => Encoding::Uncompressed, }, ) @@ -256,7 +298,7 @@ impl From for TileInfo { } impl Display for TileInfo { - fn fmt(&self, f: &mut Formatter<'_>) -> Result { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.format.content_type())?; if let Some(encoding) = self.encoding.content_encoding() { write!(f, "; encoding={encoding}")?; @@ -267,6 +309,85 @@ impl Display for TileInfo { } } +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +enum SevenBitDecodingError { + /// Expected a tag, but got nothing + #[error("Expected a tag, but got nothing")] + TruncatedTag, + /// The size of the tile is too large to be decoded + #[error("The size of the tile is too large to be decoded")] + SizeOverflow, + /// The size of the tile is lower than the number of bytes for the size and tag + #[error("The size of the tile is lower than the number of bytes for the size and tag")] + SizeUnderflow, + /// Expected a size, but got nothing + #[error("Expected a size, but got nothing")] + TruncatedSize, + /// Expected data according to the size, but got nothing + #[error("Expected {0} bytes of data in layer according to the size, but got only {1}")] + TruncatedData(u64, u64), + /// Got unexpected tag + #[error("Got tag {0} instead of the expected")] + UnexpectedTag(u8), +} + +/// Tries to validate that the tile consists of a valid concatination of (`size_7_bit`, `one_of_expected_version`, `data`) +fn decode_7bit_length_and_tag(tile: &[u8], versions: &[u8]) -> Result<(), SevenBitDecodingError> { + if tile.is_empty() { + return Err(SevenBitDecodingError::TruncatedSize); + } + let mut tile_iter = tile.iter().peekable(); + while tile_iter.peek().is_some() { + // need to parse size + let mut size = 0_u64; + let mut header_bit_count = 0_u64; + loop { + header_bit_count += 1; + let Some(b) = tile_iter.next() else { + return Err(SevenBitDecodingError::TruncatedSize); + }; + if header_bit_count * 7 + 8 > 64 { + return Err(SevenBitDecodingError::SizeOverflow); + } + // decode size + size <<= 7; + let seven_bit_mask = !0x80; + size |= u64::from(*b & seven_bit_mask); + // 0 => no further size + if b & 0x80 == 0 { + // need to check tag + header_bit_count += 1; + let Some(tag) = tile_iter.next() else { + return Err(SevenBitDecodingError::TruncatedTag); + }; + if !versions.contains(tag) { + return Err(SevenBitDecodingError::UnexpectedTag(*tag)); + } + // need to check data-length + let payload_len = size + .checked_sub(header_bit_count) + .ok_or(SevenBitDecodingError::SizeUnderflow)?; + for i in 0..payload_len { + if tile_iter.next().is_none() { + return Err(SevenBitDecodingError::TruncatedData(payload_len, i)); + } + } + break; + } + } + } + Ok(()) +} + +/// Detects if the given tile is a valid JSON tile. +/// +/// The check for a dictionary is used to speed up the validation process. +fn is_valid_json(tile: &[u8]) -> bool { + tile.starts_with(b"{") + && tile.ends_with(b"}") + && serde_json::from_slice::(tile).is_ok() +} + /// Convert longitude and latitude to a tile (x,y) coordinates for a given zoom #[must_use] #[expect(clippy::cast_possible_truncation)] @@ -348,48 +469,145 @@ pub fn wgs84_to_webmercator(lon: f64, lat: f64) -> (f64, f64) { #[cfg(test)] mod tests { - #![expect(clippy::unreadable_literal)] - - use std::fs::read; - - use Encoding::{Internal, Uncompressed}; - use Format::{Jpeg, Json, Png, Webp}; use approx::assert_relative_eq; use rstest::rstest; use super::*; - fn detect(path: &str) -> Option { - TileInfo::detect(&read(path).unwrap()) + #[rstest] + #[case::png( + include_bytes!("../fixtures/world.png"), + TileInfo::new(Format::Png, Encoding::Internal) + )] + #[case::jpg( + include_bytes!("../fixtures/world.jpg"), + TileInfo::new(Format::Jpeg, Encoding::Internal) + )] + #[case::webp( + include_bytes!("../fixtures/dc.webp"), + TileInfo::new(Format::Webp, Encoding::Internal) + )] + #[case::json( + br#"{"foo":"bar"}"#, + TileInfo::new(Format::Json, Encoding::Uncompressed) + )] + // we have no way of knowing what is an MVT -> we just say it is out of the + // fact that it is not something else + #[case::invalid_webp_header(b"RIFF", TileInfo::new(Format::Mvt, Encoding::Uncompressed))] + fn test_data_format_detect(#[case] data: &[u8], #[case] expected: TileInfo) { + assert_eq!(TileInfo::detect(data), expected); + } + + /// Test detection of compressed content (JSON, MLT, MVT) + #[test] + fn test_compressed_json_gzip() { + let json_data = br#"{"type":"FeatureCollection","features":[]}"#; + let compressed = encode_gzip(json_data).unwrap(); + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Json, Encoding::Gzip)); } - #[expect(clippy::unnecessary_wraps)] - fn info(format: Format, encoding: Encoding) -> Option { - Some(TileInfo::new(format, encoding)) + #[test] + fn test_compressed_json_zlib() { + use std::io::Write; + + use flate2::write::ZlibEncoder; + + let json_data = br#"{"type":"FeatureCollection","features":[]}"#; + let mut encoder = ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(json_data).unwrap(); + let compressed = encoder.finish().unwrap(); + + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Json, Encoding::Zlib)); } #[test] - fn test_data_format_png() { - assert_eq!(detect("./fixtures/world.png"), info(Png, Internal)); + fn test_compressed_mlt_gzip() { + // MLT tile: length=2 (0x02), version=1 (0x01) + let mlt_data = &[0x02, 0x01]; + let compressed = encode_gzip(mlt_data).unwrap(); + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Mlt, Encoding::Gzip)); } #[test] - fn test_data_format_jpg() { - assert_eq!(detect("./fixtures/world.jpg"), info(Jpeg, Internal)); + fn test_compressed_mlt_zlib() { + use std::io::Write; + + use flate2::write::ZlibEncoder; + + // MLT tile: length=5 (0x05), version=1 (0x01), plus some data + let mlt_data = &[0x05, 0x01, 0xaa, 0xbb, 0xcc]; + let mut encoder = ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(mlt_data).unwrap(); + let compressed = encoder.finish().unwrap(); + + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Mlt, Encoding::Zlib)); } #[test] - fn test_data_format_webp() { - assert_eq!(detect("./fixtures/dc.webp"), info(Webp, Internal)); - assert_eq!(TileInfo::detect(br"RIFF"), None); + fn test_compressed_mvt_gzip_fallback() { + // Random data that doesn't match any known format => should be detected as MVT + let random_data = &[0x1a, 0x2b, 0x3c, 0x4d]; + let compressed = encode_gzip(random_data).unwrap(); + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Mvt, Encoding::Gzip)); } #[test] - fn test_data_format_json() { - assert_eq!( - TileInfo::detect(br#"{"foo":"bar"}"#), - info(Json, Uncompressed) - ); + fn test_compressed_mvt_zlib_fallback() { + use std::io::Write; + + use flate2::write::ZlibEncoder; + + // Random data that doesn't match any known format => should be detected as MVT + let random_data = &[0xaa, 0xbb, 0xcc, 0xdd]; + let mut encoder = ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(random_data).unwrap(); + let compressed = encoder.finish().unwrap(); + + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Mvt, Encoding::Zlib)); + } + + #[test] + fn test_invalid_json_in_gzip() { + // Data that looks like JSON but isn't valid => should fall back to MVT + let invalid_json = b"{this is not valid json}"; + let compressed = encode_gzip(invalid_json).unwrap(); + let result = TileInfo::detect(&compressed); + assert_eq!(result, TileInfo::new(Format::Mvt, Encoding::Gzip)); + } + + #[rstest] + #[case::minimal_tile(&[0x02, 0x01], Ok(()))] + #[case::one_byte_length(&[0x03, 0x01, 0xaa], Ok(()))] + #[case::two_byte_length(&[0x80, 0x04, 0x01, 0xaa], Ok(()))] + #[case::multi_byte_length(&[0x80, 0x80, 0x05, 0x01, 0xdd], Ok(()))] + #[case::wrong_version(&[0x03, 0x02, 0xaa], Err(SevenBitDecodingError::UnexpectedTag(0x02)))] + #[case::empty_input(&[], Err(SevenBitDecodingError::TruncatedSize))] + #[case::size_overflow(&[0xFF; 64], Err(SevenBitDecodingError::SizeOverflow))] + #[case::size_underflow(&[0x00, 0x01], Err(SevenBitDecodingError::SizeUnderflow))] + #[case::unterminated_length(&[0x80], Err(SevenBitDecodingError::TruncatedSize))] + #[case::missing_version_byte(&[0x05], Err(SevenBitDecodingError::TruncatedTag))] + #[case::wrong_length(&[0x03, 0x01], Err(SevenBitDecodingError::TruncatedData(1, 0)))] + fn test_decode_7bit_length_and_tag( + #[case] tile: &[u8], + #[case] expected: Result<(), SevenBitDecodingError>, + ) { + let allowed_versions = &[0x01_u8]; + let decoded = decode_7bit_length_and_tag(tile, allowed_versions); + assert_eq!(decoded, expected, "can decode one layer correctly"); + + if tile.is_empty() { + return; + } + let mut tile_with_two_layers = vec![0x02, 0x01]; + tile_with_two_layers.extend_from_slice(tile); + let decoded = decode_7bit_length_and_tag(&tile_with_two_layers, allowed_versions); + assert_eq!(decoded, expected, "can decode two layers correctly"); } #[rstest] @@ -417,10 +635,10 @@ mod tests { #[rstest] // you could easily get test cases from maptiler: https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#4/-118.82/71.02 - #[case(0, 0, 0, 0, 0, [-180.0,-85.0511287798066,180.0,85.0511287798066])] - #[case(1, 0, 0, 0, 0, [-180.0,0.0,0.0,85.0511287798066])] - #[case(5, 1, 1, 2, 2, [-168.75,81.09321385260837,-146.25,83.97925949886205])] - #[case(5, 1, 3, 2, 5, [-168.75,74.01954331150226,-146.25,81.09321385260837])] + #[case(0, 0, 0, 0, 0, [-180.0,-85.051_128_779_806_6,180.0,85.051_128_779_806_6])] + #[case(1, 0, 0, 0, 0, [-180.0,0.0,0.0,85.051_128_779_806_6])] + #[case(5, 1, 1, 2, 2, [-168.75,81.093_213_852_608_37,-146.25,83.979_259_498_862_05])] + #[case(5, 1, 3, 2, 5, [-168.75,74.019_543_311_502_26,-146.25,81.093_213_852_608_37])] fn test_xyz_to_bbox( #[case] zoom: u8, #[case] min_x: u32, @@ -447,33 +665,33 @@ mod tests { #[case(7, (0, 116, 11, 126))] #[case(8, (0, 233, 23, 253))] #[case(9, (0, 466, 47, 507))] - #[case(10, (1, 933, 94, 1014))] - #[case(11, (3, 1866, 188, 2029))] - #[case(12, (6, 3732, 377, 4059))] - #[case(13, (12, 7465, 755, 8119))] - #[case(14, (25, 14931, 1510, 16239))] - #[case(15, (51, 29863, 3020, 32479))] - #[case(16, (102, 59727, 6041, 64958))] - #[case(17, (204, 119455, 12083, 129917))] - #[case(18, (409, 238911, 24166, 259834))] - #[case(19, (819, 477823, 48332, 519669))] - #[case(20, (1638, 955647, 96665, 1039339))] - #[case(21, (3276, 1911295, 193331, 2078678))] - #[case(22, (6553, 3822590, 386662, 4157356))] - #[case(23, (13107, 7645181, 773324, 8314713))] - #[case(24, (26214, 15290363, 1546649, 16629427))] - #[case(25, (52428, 30580726, 3093299, 33258855))] - #[case(26, (104857, 61161453, 6186598, 66517711))] - #[case(27, (209715, 122322907, 12373196, 133035423))] - #[case(28, (419430, 244645814, 24746393, 266070846))] - #[case(29, (838860, 489291628, 49492787, 532141692))] - #[case(30, (1677721, 978583256, 98985574, 1064283385))] + #[case(10, (1, 933, 94, 1_014))] + #[case(11, (3, 1_866, 188, 2_029))] + #[case(12, (6, 3_732, 377, 4_059))] + #[case(13, (12, 7_465, 755, 8_119))] + #[case(14, (25, 14_931, 1_510, 16_239))] + #[case(15, (51, 29_863, 3_020, 32_479))] + #[case(16, (102, 59_727, 6_041, 64_958))] + #[case(17, (204, 119_455, 12_083, 129_917))] + #[case(18, (409, 238_911, 24_166, 259_834))] + #[case(19, (819, 477_823, 48_332, 519_669))] + #[case(20, (1_638, 955_647, 96_665, 1_039_339))] + #[case(21, (3_276, 1_911_295, 193_331, 2_078_678))] + #[case(22, (6_553, 3_822_590, 386_662, 4_157_356))] + #[case(23, (13_107, 7_645_181, 773_324, 8_314_713))] + #[case(24, (26_214, 15_290_363, 1_546_649, 16_629_427))] + #[case(25, (52_428, 30_580_726, 3_093_299, 33_258_855))] + #[case(26, (104_857, 61_161_453, 6_186_598, 66_517_711))] + #[case(27, (209_715, 122_322_907, 12_373_196, 133_035_423))] + #[case(28, (419_430, 244_645_814, 24_746_393, 266_070_846))] + #[case(29, (838_860, 489_291_628, 49_492_787, 532_141_692))] + #[case(30, (1_677_721, 978_583_256, 98_985_574, 1_064_283_385))] fn test_box_to_xyz(#[case] zoom: u8, #[case] expected_xyz: (u32, u32, u32, u32)) { let actual_xyz = bbox_to_xyz( - -179.43749999999955, - -84.76987877980656, - -146.8124999999996, - -81.37446385260833, + -179.437_499_999_999_55, + -84.769_878_779_806_56, + -146.812_499_999_999_6, + -81.374_463_852_608_33, zoom, ); assert_eq!( @@ -485,14 +703,14 @@ mod tests { #[rstest] // test data via https://epsg.io/transform#s_srs=4326&t_srs=3857 #[case((0.0,0.0), (0.0,0.0))] - #[case((30.0,0.0), (3339584.723798207,0.0))] - #[case((-30.0,0.0), (-3339584.723798207,0.0))] - #[case((0.0,30.0), (0.0,3503549.8435043753))] - #[case((0.0,-30.0), (0.0,-3503549.8435043753))] - #[case((38.897957,-77.036560), (4330100.766138651, -13872207.775755845))] // white house - #[case((-180.0,-85.0), (-20037508.342789244, -19971868.880408566))] - #[case((180.0,85.0), (20037508.342789244, 19971868.880408566))] - #[case((0.026949458523585632,0.08084834874097367), (3000.0, 9000.0))] + #[case((30.0,0.0), (3_339_584.723_798_207,0.0))] + #[case((-30.0,0.0), (-3_339_584.723_798_207,0.0))] + #[case((0.0,30.0), (0.0,3_503_549.843_504_375_3))] + #[case((0.0,-30.0), (0.0,-3_503_549.843_504_375_3))] + #[case((38.897_957,-77.036_560), (4_330_100.766_138_651, -13_872_207.775_755_845))] // white house + #[case((-180.0,-85.0), (-20_037_508.342_789_244, -19_971_868.880_408_566))] + #[case((180.0,85.0), (20_037_508.342_789_244, 19_971_868.880_408_566))] + #[case((0.026_949_458_523_585_632,0.080_848_348_740_973_67), (3000.0, 9000.0))] fn test_coordinate_syste_conversion( #[case] wgs84: (f64, f64), #[case] webmercator: (f64, f64), diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index a6c4c05d7..27f6b3be5 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -36,6 +36,7 @@ fn test_get(path: &str) -> TestRequest { TestRequest::get().uri(path) } +#[expect(clippy::similar_names)] async fn config( test_name: &str, ) -> ( @@ -45,6 +46,7 @@ async fn config( (Mbtiles, SqliteConnection), (Mbtiles, SqliteConnection), (Mbtiles, SqliteConnection), + (Mbtiles, SqliteConnection), ), ) { let json_script = include_str!("../../tests/fixtures/mbtiles/json.sql"); @@ -56,6 +58,9 @@ async fn config( let raw_mvt_script = include_str!("../../tests/fixtures/mbtiles/uncompressed_mvt.sql"); let (raw_mvt_mbt, raw_mvt_conn, raw_mvt_file) = temp_named_mbtiles(&format!("{test_name}_raw_mvt"), raw_mvt_script).await; + let raw_mlt_script = include_str!("../../tests/fixtures/mbtiles/mlt.sql"); + let (raw_mlt_mbt, raw_mlt_conn, raw_mlt_file) = + temp_named_mbtiles(&format!("{test_name}_raw_mlt"), raw_mlt_script).await; let webp_script = include_str!("../../tests/fixtures/mbtiles/webp.sql"); let (webp_mbt, webp_conn, webp_file) = temp_named_mbtiles(&format!("{test_name}_webp"), webp_script).await; @@ -67,17 +72,20 @@ async fn config( m_json: {json} m_mvt: {mvt} m_raw_mvt: {raw_mvt} + m_raw_mlt: {raw_mlt} m_webp: {webp} ", json = json_file.display(), mvt = mvt_file.display(), raw_mvt = raw_mvt_file.display(), + raw_mlt = raw_mlt_file.display(), webp = webp_file.display() }, ( (json_mbt, json_conn), (mvt_mbt, mvt_conn), (raw_mvt_mbt, raw_mvt_conn), + (raw_mlt_mbt, raw_mlt_conn), (webp_mbt, webp_conn), ), ) @@ -92,7 +100,7 @@ async fn mbt_get_catalog() { let response = call_service(&app, req).await; let response = assert_response(response).await; let body: serde_json::Value = read_body_json(response).await; - assert_yaml_snapshot!(body, @r" + assert_yaml_snapshot!(body, @r#" fonts: {} sprites: {} styles: {} @@ -105,6 +113,11 @@ async fn mbt_get_catalog() { content_type: application/x-protobuf description: Major cities from Natural Earth data name: Major cities from Natural Earth data + m_raw_mlt: + attribution: "© OpenMapTiles © OpenStreetMap contributors" + content_type: application/vnd.maplibre-vector-tile + description: "A tileset showcasing all layers in OpenMapTiles. https://openmaptiles.org" + name: OpenMapTiles m_raw_mvt: content_type: application/x-protobuf description: Major cities from Natural Earth data @@ -112,7 +125,7 @@ async fn mbt_get_catalog() { m_webp: content_type: image/webp name: ne2sr - "); + "#); } #[actix_rt::test] @@ -126,7 +139,7 @@ async fn mbt_get_catalog_gzip() { let response = assert_response(response).await; let body = decode_gzip(&read_body(response).await).unwrap(); let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_yaml_snapshot!(body, @r" + assert_yaml_snapshot!(body, @r#" fonts: {} sprites: {} styles: {} @@ -139,6 +152,11 @@ async fn mbt_get_catalog_gzip() { content_type: application/x-protobuf description: Major cities from Natural Earth data name: Major cities from Natural Earth data + m_raw_mlt: + attribution: "© OpenMapTiles © OpenStreetMap contributors" + content_type: application/vnd.maplibre-vector-tile + description: "A tileset showcasing all layers in OpenMapTiles. https://openmaptiles.org" + name: OpenMapTiles m_raw_mvt: content_type: application/x-protobuf description: Major cities from Natural Earth data @@ -146,7 +164,7 @@ async fn mbt_get_catalog_gzip() { m_webp: content_type: image/webp name: ne2sr - "); + "#); } #[actix_rt::test] @@ -291,6 +309,24 @@ async fn mbt_get_raw_mvt() { assert_eq!(body.len(), 2); } +/// get an uncompressed MLT tile +#[actix_rt::test] +#[tracing_test::traced_test] +async fn mbt_get_raw_mlt() { + let (config, _conns) = config("mbt_get_raw_mlt").await; + let app = create_app!(&config); + let req = test_get("/m_raw_mlt/0/0/0").to_request(); + let response = call_service(&app, req).await; + let response = assert_response(response).await; + assert_eq!( + response.headers().get(CONTENT_TYPE).unwrap(), + "application/vnd.maplibre-vector-tile" + ); + assert_eq!(response.headers().get(CONTENT_ENCODING), None); + let body = read_body(response).await; + assert_eq!(body.iter().as_slice(), &[0x02, 0x01]); +} + /// get an uncompressed MVT tile with accepted gzip #[actix_rt::test] #[tracing_test::traced_test] diff --git a/martin/tests/utils.rs b/martin/tests/utils.rs index 9e74ada95..65bba0589 100644 --- a/martin/tests/utils.rs +++ b/martin/tests/utils.rs @@ -40,7 +40,12 @@ pub async fn assert_response(response: ServiceResponse) -> ServiceResponse { pub type MockSource = (ServerState, Config); pub async fn mock_sources(mut config: Config) -> MockSource { let res = config.resolve().await; - let res = res.unwrap_or_else(|e| panic!("Failed to resolve config {config:?}: {e}")); + let res = res.unwrap_or_else(|e| { + panic!( + "Failed to resolve config:\n{config}\nBecause {e}", + config = serde_yaml::to_string(&config).unwrap() + ) + }); (res, config) } diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index 4f1260764..daa70df88 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -241,20 +241,18 @@ impl Mbtiles { ) -> Option { if let (Some(z), Some(x), Some(y), Some(tile)) = (z, x, y, tile) { let info = TileInfo::detect(&tile); - if let Some(info) = info { - debug!( - "Tile {z}/{x}/{} is detected as {info} in file {}", - { - if let (Ok(z), Ok(y)) = (u8::try_from(z), u32::try_from(y)) { - invert_y_value(z, y).to_string() - } else { - format!("{y} (invalid values, cannot invert Y)") - } - }, - self.filename(), - ); - } - info + debug!( + "Tile {z}/{x}/{} is detected as {info} in file {}", + { + if let (Ok(z), Ok(y)) = (u8::try_from(z), u32::try_from(y)) { + invert_y_value(z, y).to_string() + } else { + format!("{y} (invalid values, cannot invert Y)") + } + }, + self.filename(), + ); + Some(info) } else { None } diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index 17140a517..35da2c17b 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -146,6 +146,12 @@ "content_type": "application/x-protobuf", "description": "materialized view comment" }, + "mlt": { + "attribution": "© OpenMapTiles © OpenStreetMap contributors", + "content_type": "application/vnd.maplibre-vector-tile", + "description": "A tileset showcasing all layers in OpenMapTiles. https://openmaptiles.org", + "name": "OpenMapTiles" + }, "png": { "content_type": "image/png", "name": "ne2sr" diff --git a/tests/expected/auto/save_config.yaml b/tests/expected/auto/save_config.yaml index de23da808..9acca5c95 100644 --- a/tests/expected/auto/save_config.yaml +++ b/tests/expected/auto/save_config.yaml @@ -358,6 +358,7 @@ mbtiles: geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles json: tests/fixtures/mbtiles/json.mbtiles + mlt: tests/fixtures/mbtiles/mlt.mbtiles uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles webp: tests/fixtures/mbtiles/webp.mbtiles world_cities: tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/expected/martin-cp/composite_save_config.yaml b/tests/expected/martin-cp/composite_save_config.yaml index d3adafb87..1eb9f6c37 100644 --- a/tests/expected/martin-cp/composite_save_config.yaml +++ b/tests/expected/martin-cp/composite_save_config.yaml @@ -355,6 +355,7 @@ mbtiles: geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles json: tests/fixtures/mbtiles/json.mbtiles + mlt: tests/fixtures/mbtiles/mlt.mbtiles uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles webp: tests/fixtures/mbtiles/webp.mbtiles world_cities: tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/expected/martin-cp/flat-with-hash_metadata.txt b/tests/expected/martin-cp/flat-with-hash_metadata.txt index b310a1a51..5ca588f79 100644 --- a/tests/expected/martin-cp/flat-with-hash_metadata.txt +++ b/tests/expected/martin-cp/flat-with-hash_metadata.txt @@ -9,7 +9,6 @@ tilejson: format: pbf generator: martin-cp v0.0.0 agg_tiles_hash: 9B931A386D6075D1DA55323BD4DBEDAE -[INFO ] Using 'mvt' tile format from metadata table in file cp_flat-with-hash tile_info: format: mvt encoding: '' diff --git a/tests/expected/martin-cp/flat-with-hash_save_config.yaml b/tests/expected/martin-cp/flat-with-hash_save_config.yaml index d3adafb87..1eb9f6c37 100644 --- a/tests/expected/martin-cp/flat-with-hash_save_config.yaml +++ b/tests/expected/martin-cp/flat-with-hash_save_config.yaml @@ -355,6 +355,7 @@ mbtiles: geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles json: tests/fixtures/mbtiles/json.mbtiles + mlt: tests/fixtures/mbtiles/mlt.mbtiles uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles webp: tests/fixtures/mbtiles/webp.mbtiles world_cities: tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/expected/martin-cp/flat_save_config.yaml b/tests/expected/martin-cp/flat_save_config.yaml index d3adafb87..1eb9f6c37 100644 --- a/tests/expected/martin-cp/flat_save_config.yaml +++ b/tests/expected/martin-cp/flat_save_config.yaml @@ -355,6 +355,7 @@ mbtiles: geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles json: tests/fixtures/mbtiles/json.mbtiles + mlt: tests/fixtures/mbtiles/mlt.mbtiles uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles webp: tests/fixtures/mbtiles/webp.mbtiles world_cities: tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/expected/martin-cp/normalized_save_config.yaml b/tests/expected/martin-cp/normalized_save_config.yaml index d3adafb87..1eb9f6c37 100644 --- a/tests/expected/martin-cp/normalized_save_config.yaml +++ b/tests/expected/martin-cp/normalized_save_config.yaml @@ -355,6 +355,7 @@ mbtiles: geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles json: tests/fixtures/mbtiles/json.mbtiles + mlt: tests/fixtures/mbtiles/mlt.mbtiles uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles webp: tests/fixtures/mbtiles/webp.mbtiles world_cities: tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/fixtures/mbtiles/mlt.sql b/tests/fixtures/mbtiles/mlt.sql new file mode 100644 index 000000000..6b134ef0a --- /dev/null +++ b/tests/fixtures/mbtiles/mlt.sql @@ -0,0 +1,20 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE metadata (name text, value text); +INSERT INTO metadata VALUES('name','OpenMapTiles'); +INSERT INTO metadata VALUES('description','A tileset showcasing all layers in OpenMapTiles. https://openmaptiles.org'); +INSERT INTO metadata VALUES('attribution','© OpenMapTiles © OpenStreetMap contributors'); +INSERT INTO metadata VALUES('version','3.15.0'); +INSERT INTO metadata VALUES('type','baselayer'); +INSERT INTO metadata VALUES('format','application/vnd.maplibre-vector-tile'); +INSERT INTO metadata VALUES('bounds','7.40858,43.48382,7.59567,43.75293'); +INSERT INTO metadata VALUES('center','7.50213,43.61837,10'); +INSERT INTO metadata VALUES('minzoom','0'); +INSERT INTO metadata VALUES('maxzoom','2'); +INSERT INTO metadata VALUES('json','{"vector_layers":[]}'); +INSERT INTO metadata VALUES('compression','none'); +CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob); +INSERT INTO tiles VALUES(0,0,0,x'0201'); +CREATE UNIQUE INDEX name ON metadata (name); +CREATE UNIQUE INDEX tile_index ON tiles (zoom_level, tile_column, tile_row); +COMMIT;