From dfe53433c0cb2950342d24117b8dab1de0a02180 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 28 May 2025 16:32:13 +0800 Subject: [PATCH 01/45] refactor --- Cargo.lock | 91 +++------- martin/src/cog/image.rs | 223 ++++++++++++++++++++++++ martin/src/cog/mod.rs | 1 + martin/src/cog/source.rs | 361 +++++++++------------------------------ 4 files changed, 332 insertions(+), 344 deletions(-) create mode 100644 martin/src/cog/image.rs diff --git a/Cargo.lock b/Cargo.lock index 17b06bb3d..1b36ef15f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2145,17 +2145,16 @@ dependencies = [ [[package]] name = "fmmap" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6099ab52d5329340a3014f60ca91bc892181ae32e752360d07be9295924dcb0b" +checksum = "687c574434dc6e3cd24a363fe0944711174f947fe71696fdc9a0ae046fe6e715" dependencies = [ - "async-trait", "byteorder", "bytes", "enum_dispatch", "fs4", - "memmapix", - "parse-display 0.8.2", + "memmap2 0.9.5", + "parse-display 0.10.0", "pin-project-lite", "tokio", ] @@ -2228,14 +2227,13 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "c29c30684418547d476f0b48e84f4821639119c483b1eccd566c8cd0cd05f521" dependencies = [ - "async-trait", "rustix 0.38.44", "tokio", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3509,15 +3507,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memmapix" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f517ab414225d5f1755bd284d9545bd08a72a3958b3c6384d72e95de9cc1a1d3" -dependencies = [ - "rustix 0.38.44", -] - [[package]] name = "mime" version = "0.3.17" @@ -3845,52 +3834,51 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6509d08722b53e8dafe97f2027b22ccbe3a5db83cb352931e9716b0aa44bc5c" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" dependencies = [ - "once_cell", - "parse-display-derive 0.8.2", + "parse-display-derive 0.9.1", "regex", + "regex-syntax 0.8.5", ] [[package]] name = "parse-display" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +checksum = "287d8d3ebdce117b8539f59411e4ed9ec226e0a4153c7f55495c6070d68e6f72" dependencies = [ - "parse-display-derive 0.9.1", + "parse-display-derive 0.10.0", "regex", "regex-syntax 0.8.5", ] [[package]] name = "parse-display-derive" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68517892c8daf78da08c0db777fcc17e07f2f63ef70041718f8a7630ad84f341" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" dependencies = [ - "once_cell", "proc-macro2", "quote", "regex", - "regex-syntax 0.7.5", - "structmeta 0.2.0", + "regex-syntax 0.8.5", + "structmeta", "syn 2.0.101", ] [[package]] name = "parse-display-derive" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +checksum = "7fc048687be30d79502dea2f623d052f3a074012c6eac41726b7ab17213616b1" dependencies = [ "proc-macro2", "quote", "regex", "regex-syntax 0.8.5", - "structmeta 0.3.0", + "structmeta", "syn 2.0.101", ] @@ -4063,9 +4051,9 @@ dependencies = [ [[package]] name = "pmtiles" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18dabde2a77b24221be77404aa1adb4998715447abf4b6d87d550d6a9f4df75" +checksum = "092d7bfd038840136755f50f04ebad76b1ade5523b66ffd637e2609f896f546a" dependencies = [ "async-compression", "aws-sdk-s3", @@ -4075,7 +4063,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.12", "tilejson", "tokio", "varint-rs", @@ -4603,12 +4591,6 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -5660,18 +5642,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "structmeta" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" -dependencies = [ - "proc-macro2", - "quote", - "structmeta-derive 0.2.0", - "syn 2.0.101", -] - [[package]] name = "structmeta" version = "0.3.0" @@ -5680,18 +5650,7 @@ checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" dependencies = [ "proc-macro2", "quote", - "structmeta-derive 0.3.0", - "syn 2.0.101", -] - -[[package]] -name = "structmeta-derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" -dependencies = [ - "proc-macro2", - "quote", + "structmeta-derive", "syn 2.0.101", ] diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs new file mode 100644 index 000000000..f2c6d6021 --- /dev/null +++ b/martin/src/cog/image.rs @@ -0,0 +1,223 @@ +use std::fs::File; +use std::io::BufWriter; +use std::path::Path; + +use martin_tile_utils::TileCoord; +use tiff::decoder::{Decoder, DecodingResult}; + +use super::CogError; +use crate::{MartinResult, TileData}; + +#[derive(Clone, Debug)] +pub struct Image { + pub ifd: usize, + pub across: u32, + pub down: u32, + pub nodata: Option, +} +impl Image { + #[allow(clippy::cast_sign_loss)] + #[allow(clippy::cast_possible_truncation)] + #[allow(clippy::too_many_lines)] + pub fn get_tile( + &self, + decoder: &mut Decoder, + xyz: TileCoord, + path: &Path, + ) -> MartinResult { + decoder + .seek_to_image(self.ifd) + .map_err(|e| CogError::IfdSeekFailed(e, self.ifd, path.to_path_buf()))?; + + let across = self.across; + let down = self.down; + + let tile_idx; + if let Some(idx) = get_tile_idx(xyz, across, down) { + tile_idx = idx; + } else { + return Ok(Vec::new()); + } + let decode_result = decoder + .read_chunk(tile_idx) + .map_err(|e| CogError::ReadChunkFailed(e, tile_idx, self.ifd, path.to_path_buf()))?; + let color_type = decoder + .colortype() + .map_err(|e| CogError::InvalidTiffFile(e, path.to_path_buf()))?; + + let (tile_width, tile_height) = decoder.chunk_dimensions(); + let (data_width, data_height) = decoder.chunk_data_dimensions(tile_idx); + + //do more research on the not u8 case, is this the right way to do it? + let png_file_bytes = match (decode_result, color_type) { + (DecodingResult::U8(vec), tiff::ColorType::RGB(_)) => rgb_to_png( + vec, + (tile_width, tile_height), + (data_width, data_height), + 3, + self.nodata.map(|v| v as u8), + path, + ), + (DecodingResult::U8(vec), tiff::ColorType::RGBA(_)) => rgb_to_png( + vec, + (tile_width, tile_height), + (data_width, data_height), + 4, + self.nodata.map(|v| v as u8), + path, + ), + (_, _) => Err(CogError::NotSupportedColorTypeAndBitDepth( + color_type, + path.to_path_buf(), + )), + // do others in next PRs, a lot of disscussion would be needed + }?; + Ok(png_file_bytes) + } +} +fn get_tile_idx(xyz: TileCoord, across: u32, down: u32) -> Option { + if xyz.y >= down || xyz.x >= across { + return None; + } + + let tile_idx = xyz.y * across + xyz.x; + if tile_idx >= across * down { + return None; + } + Some(tile_idx) +} + +fn rgb_to_png( + vec: Vec, + (tile_width, tile_height): (u32, u32), + (data_width, data_height): (u32, u32), + chunk_components_count: u32, + nodata: Option, + path: &Path, +) -> Result, CogError> { + let is_padded = data_width != tile_width || data_height != tile_height; + let need_add_alpha = chunk_components_count != 4; + + let pixels = if nodata.is_some() || need_add_alpha || is_padded { + let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; + for row in 0..data_height { + 'outer: for col in 0..data_width { + let idx_chunk = + row * data_width * chunk_components_count + col * chunk_components_count; + let idx_result = row * tile_width * 4 + col * 4; + for component_idx in 0..chunk_components_count { + if nodata.eq(&Some(vec[(idx_chunk + component_idx) as usize])) { + //This pixel is nodata, just make it transparent and skip it then + let alpha_idx = (idx_result + 3) as usize; + result_vec[alpha_idx] = 0; + continue 'outer; + } + result_vec[(idx_result + component_idx) as usize] = + vec[(idx_chunk + component_idx) as usize]; + } + if need_add_alpha { + let alpha_idx = (idx_result + 3) as usize; + result_vec[alpha_idx] = 255; + } + } + } + result_vec + } else { + vec + }; + let mut result_file_buffer = Vec::new(); + { + let mut encoder = png::Encoder::new( + BufWriter::new(&mut result_file_buffer), + tile_width, + tile_height, + ); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + let mut writer = encoder + .write_header() + .map_err(|e| CogError::WritePngHeaderFailed(path.to_path_buf(), e))?; + writer + .write_image_data(&pixels) + .map_err(|e| CogError::WriteToPngFailed(path.to_path_buf(), e))?; + } + Ok(result_file_buffer) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use martin_tile_utils::TileCoord; + use rstest::rstest; + + use crate::cog::image::get_tile_idx; + #[test] + fn can_calc_tile_idx() { + assert_eq!(Some(0), get_tile_idx(TileCoord { z: 0, x: 0, y: 0 }, 3, 3)); + assert_eq!(Some(8), get_tile_idx(TileCoord { z: 0, x: 2, y: 2 }, 3, 3)); + assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 3, y: 0 }, 3, 3)); + assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 1, y: 9 }, 3, 3)); + } + #[rstest] + // the right half should be transprent + #[case( + "../tests/fixtures/cog/expected/right_padded.png", + (0,0,0,None),None,(128,256),(256,256) + )] + // the down half should be transprent + #[case( + "../tests/fixtures/cog/expected/down_padded.png", + (0,0,0,None),None,(256,128),(256,256) + )] + // the up half should be half transprent and down half should be transprent + #[case( + "../tests/fixtures/cog/expected/down_padded_with_alpha.png", + (0,0,0,Some(128)),None,(256,128),(256,256) + )] + // the left half should be half transprent and the right half should be transprent + #[case( + "../tests/fixtures/cog/expected/right_padded_with_alpha.png", + (0,0,0,Some(128)),None,(128,256),(256,256) + )] + // should be all half transprent + #[case( + "../tests/fixtures/cog/expected/not_padded.png", + (0,0,0,Some(128)),None,(256,256),(256,256) + )] + // all padded and with a no_data whose value is 128, and all the component is 128 + // so that should be all transprent + #[case( + "../tests/fixtures/cog/expected/all_transprent.png", + (128,128,128,Some(128)),Some(128),(128,128),(256,256) + )] + fn test_padded_cases( + #[case] expected_file_path: &str, + #[case] components: (u8, u8, u8, Option), + #[case] no_value: Option, + #[case] (data_width, data_height): (u32, u32), + #[case] (tile_width, tile_height): (u32, u32), + ) { + let mut pixels = Vec::new(); + for _ in 0..(data_width * data_height) { + pixels.push(components.0); + pixels.push(components.1); + pixels.push(components.2); + if let Some(alpha) = components.3 { + pixels.push(alpha); + } + } + let componse_count = if components.3.is_some() { 4 } else { 3 }; + let png_bytes = super::rgb_to_png( + pixels, + (tile_width, tile_height), + (data_width, data_height), + componse_count, + no_value, + &PathBuf::from("not_exist.tif"), + ) + .unwrap(); + let expected = std::fs::read(expected_file_path).unwrap(); + assert_eq!(png_bytes, expected); + } +} diff --git a/martin/src/cog/mod.rs b/martin/src/cog/mod.rs index 3d05d43fe..baab4a656 100644 --- a/martin/src/cog/mod.rs +++ b/martin/src/cog/mod.rs @@ -1,5 +1,6 @@ mod config; mod errors; +mod image; mod model; mod source; diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 62cc64e02..63d0e4e78 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -1,18 +1,18 @@ use std::collections::HashMap; use std::fmt::Debug; use std::fs::File; -use std::io::BufWriter; use std::path::{Path, PathBuf}; use std::vec; use async_trait::async_trait; use log::warn; use martin_tile_utils::{Format, TileCoord, TileInfo}; -use tiff::decoder::{ChunkType, Decoder, DecodingResult}; +use tiff::decoder::{ChunkType, Decoder}; use tiff::tags::Tag::{self, GdalNodata}; use tilejson::{TileJSON, tilejson}; use super::CogError; +use super::image::Image; use super::model::ModelInfo; use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; @@ -27,8 +27,7 @@ struct Meta { origin: [f64; 3], // [minx, miny, maxx, maxy] in its model space coordinate system extent: [f64; 4], - zoom_and_ifd: HashMap, - zoom_and_tile_across_down: HashMap, + images: HashMap, nodata: Option, } @@ -44,7 +43,15 @@ pub struct CogSource { impl CogSource { pub fn new(id: String, path: PathBuf) -> FileResult { let tileinfo = TileInfo::new(Format::Png, martin_tile_utils::Encoding::Uncompressed); - let meta = get_meta(&path)?; + let tif_file = + File::open(&path).map_err(|e: std::io::Error| FileError::IoError(e, path.clone()))?; + let mut decoder = Decoder::new(tif_file) + .map_err(|e| CogError::InvalidTiffFile(e, path.clone()))? + .with_limits(tiff::decoder::Limits::unlimited()); + let model = ModelInfo::decode(&mut decoder, &path); + verify_requirements(&mut decoder, &model, &path.clone())?; + + let meta = get_meta(model.clone(), &mut decoder, &path)?; let tilejson = tilejson! { tiles: vec![], minzoom: meta.min_zoom, @@ -71,7 +78,7 @@ impl CogSource { Decoder::new(tif_file).map_err(|e| CogError::InvalidTiffFile(e, self.path.clone()))?; decoder = decoder.with_limits(tiff::decoder::Limits::unlimited()); - let ifd = self.meta.zoom_and_ifd.get(&(xyz.z)).ok_or_else(|| { + let image = self.meta.images.get(&(xyz.z)).ok_or_else(|| { CogError::ZoomOutOfRange( xyz.z, self.path.clone(), @@ -80,63 +87,8 @@ impl CogSource { ) })?; - decoder - .seek_to_image(*ifd) - .map_err(|e| CogError::IfdSeekFailed(e, *ifd, self.path.clone()))?; - - let (across, down) = self - .meta - .zoom_and_tile_across_down - .get(&(xyz.z)) - .ok_or_else(|| { - CogError::ZoomOutOfRange( - xyz.z, - self.path.clone(), - self.meta.min_zoom, - self.meta.max_zoom, - ) - })?; - let tile_idx; - if let Some(idx) = get_tile_idx(xyz, *across, *down) { - tile_idx = idx; - } else { - return Ok(Vec::new()); - } - let decode_result = decoder - .read_chunk(tile_idx) - .map_err(|e| CogError::ReadChunkFailed(e, tile_idx, *ifd, self.path.clone()))?; - let color_type = decoder - .colortype() - .map_err(|e| CogError::InvalidTiffFile(e, self.path.clone()))?; - - let (tile_width, tile_height) = decoder.chunk_dimensions(); - let (data_width, data_height) = decoder.chunk_data_dimensions(tile_idx); - - //do more research on the not u8 case, is this the right way to do it? - let png_file_bytes = match (decode_result, color_type) { - (DecodingResult::U8(vec), tiff::ColorType::RGB(_)) => rgb_to_png( - vec, - (tile_width, tile_height), - (data_width, data_height), - 3, - self.meta.nodata.map(|v| v as u8), - &self.path, - ), - (DecodingResult::U8(vec), tiff::ColorType::RGBA(_)) => rgb_to_png( - vec, - (tile_width, tile_height), - (data_width, data_height), - 4, - self.meta.nodata.map(|v| v as u8), - &self.path, - ), - (_, _) => Err(CogError::NotSupportedColorTypeAndBitDepth( - color_type, - self.path.clone(), - )), - // do others in next PRs, a lot of disscussion would be needed - }?; - Ok(png_file_bytes) + let bytes = image.get_tile(&mut decoder, xyz, &self.path)?; + Ok(bytes) } } @@ -167,75 +119,6 @@ impl Source for CogSource { } } -fn get_tile_idx(xyz: TileCoord, across: u32, down: u32) -> Option { - if xyz.y >= down || xyz.x >= across { - return None; - } - - let tile_idx = xyz.y * across + xyz.x; - if tile_idx >= across * down { - return None; - } - Some(tile_idx) -} - -fn rgb_to_png( - vec: Vec, - (tile_width, tile_height): (u32, u32), - (data_width, data_height): (u32, u32), - chunk_components_count: u32, - nodata: Option, - path: &Path, -) -> Result, CogError> { - let is_padded = data_width != tile_width || data_height != tile_height; - let need_add_alpha = chunk_components_count != 4; - - let pixels = if nodata.is_some() || need_add_alpha || is_padded { - let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; - for row in 0..data_height { - 'outer: for col in 0..data_width { - let idx_chunk = - row * data_width * chunk_components_count + col * chunk_components_count; - let idx_result = row * tile_width * 4 + col * 4; - for component_idx in 0..chunk_components_count { - if nodata.eq(&Some(vec[(idx_chunk + component_idx) as usize])) { - //This pixel is nodata, just make it transparent and skip it then - let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 0; - continue 'outer; - } - result_vec[(idx_result + component_idx) as usize] = - vec[(idx_chunk + component_idx) as usize]; - } - if need_add_alpha { - let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 255; - } - } - } - result_vec - } else { - vec - }; - let mut result_file_buffer = Vec::new(); - { - let mut encoder = png::Encoder::new( - BufWriter::new(&mut result_file_buffer), - tile_width, - tile_height, - ); - encoder.set_color(png::ColorType::Rgba); - encoder.set_depth(png::BitDepth::Eight); - let mut writer = encoder - .write_header() - .map_err(|e| CogError::WritePngHeaderFailed(path.to_path_buf(), e))?; - writer - .write_image_data(&pixels) - .map_err(|e| CogError::WriteToPngFailed(path.to_path_buf(), e))?; - } - Ok(result_file_buffer) -} - fn verify_requirements( decoder: &mut Decoder, model: &ModelInfo, @@ -316,76 +199,82 @@ fn verify_requirements( } #[allow(clippy::cast_possible_truncation)] -fn get_meta(path: &PathBuf) -> Result { - let tif_file = File::open(path).map_err(|e| FileError::IoError(e, path.clone()))?; - let mut decoder = Decoder::new(tif_file) - .map_err(|e| CogError::InvalidTiffFile(e, path.clone()))? - .with_limits(tiff::decoder::Limits::unlimited()); - let model = ModelInfo::decode(&mut decoder, path); +fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Result { + let nodata: Option = if let Ok(no_data) = decoder.get_tag_ascii_string(GdalNodata) { + no_data.parse().ok() + } else { + None + }; let origin = get_origin( model.tie_points.as_deref(), model.transformation.as_deref(), path, )?; - let (full_width_pixel, full_length_pixel) = decoder.dimensions().map_err(|e| { - CogError::TagsNotFound( - e, - vec![Tag::ImageWidth.to_u16(), Tag::ImageLength.to_u16()], - 0, // we are at ifd 0, the first image, haven't seek to others - path.clone(), - ) - })?; - let full_resolution = get_full_resolution( + let (full_width_pixel, full_length_pixel) = dim_in_pixel(decoder, path, 0)?; + let (full_width, full_length) = dim_in_model( + decoder, + path, + 0, model.pixel_scale.as_deref(), model.transformation.as_deref(), - path, )?; - let full_width = full_resolution[0] * f64::from(full_width_pixel); - let full_length = full_resolution[1] * f64::from(full_length_pixel); let extent = get_extent( &origin, model.transformation.as_deref(), (full_width_pixel, full_length_pixel), (full_width, full_length), ); - verify_requirements(&mut decoder, &model, path)?; - let mut zoom_and_ifd: HashMap = HashMap::new(); - let mut zoom_and_tile_across_down: HashMap = HashMap::new(); - - let nodata: Option = if let Ok(no_data) = decoder.get_tag_ascii_string(GdalNodata) { - no_data.parse().ok() - } else { - None - }; - - let images_ifd = get_images_ifd(&mut decoder, path); + let mut images = vec![]; - for (idx, image_ifd) in images_ifd.iter().enumerate() { - decoder - .seek_to_image(*image_ifd) - .map_err(|e| CogError::IfdSeekFailed(e, *image_ifd, path.clone()))?; - - let zoom = u8::try_from(images_ifd.len() - (idx + 1)) - .map_err(|_| CogError::TooManyImages(path.clone()))?; + let mut ifd_idx = 0; - let (tiles_across, tiles_down) = get_grid_dims(&mut decoder, path, *image_ifd)?; + loop { + let is_image = decoder + .get_tag_u32(Tag::NewSubfileType) + .map_or_else(|_| true, |v| v & 4 != 4); // see https://www.verypdf.com/document/tiff6/pg_0036.htm + if is_image { + //todo We should not ignore mask in the next PRs + let (tiles_across, tiles_down) = get_grid_dims(decoder, path, ifd_idx)?; + let image = Image { + ifd: ifd_idx, + across: tiles_across, + down: tiles_down, + nodata, + }; + + images.push(image); + } else { + warn!( + "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. The ifd number of this subfile is {}", + path.display(), + ifd_idx + ); + } - zoom_and_ifd.insert(zoom, *image_ifd); - zoom_and_tile_across_down.insert(zoom, (tiles_across, tiles_down)); - } + ifd_idx += 1; - if images_ifd.is_empty() { - Err(CogError::NoImagesFound(path.clone()))?; + let next_res = decoder.seek_to_image(ifd_idx); + if next_res.is_err() { + //todo add warn!() here + break; + } } - + let min_zoom = 0; + let max_zoom = (images.len() - 1) as u8; + let zoom_and_images: HashMap = images + .iter() + .map(|image| { + let zoom = max_zoom.saturating_sub((image.ifd as u8) + 1); + (zoom, image.clone()) + }) + .collect(); Ok(Meta { - min_zoom: 0, - max_zoom: images_ifd.len() as u8 - 1, + min_zoom, + max_zoom, + images: zoom_and_images, model, origin, extent, - zoom_and_ifd, - zoom_and_tile_across_down, nodata, }) } @@ -396,14 +285,14 @@ fn get_grid_dims( image_ifd: usize, ) -> Result<(u32, u32), FileError> { let (tile_width, tile_height) = (decoder.chunk_dimensions().0, decoder.chunk_dimensions().1); - let (image_width, image_length) = get_image_dims(decoder, path, image_ifd)?; + let (image_width, image_length) = dim_in_pixel(decoder, path, image_ifd)?; let tiles_across = image_width.div_ceil(tile_width); let tiles_down = image_length.div_ceil(tile_height); Ok((tiles_across, tiles_down)) } -fn get_image_dims( +fn dim_in_pixel( decoder: &mut Decoder, path: &Path, image_ifd: usize, @@ -419,33 +308,22 @@ fn get_image_dims( Ok((image_width, image_length)) } +fn dim_in_model( + decoder: &mut Decoder, + path: &Path, + image_ifd: usize, + pixel_scale: Option<&[f64]>, + transformation: Option<&[f64]>, +) -> Result<(f64, f64), FileError> { + let (image_width_pixel, image_length_pixel) = dim_in_pixel(decoder, path, image_ifd)?; -fn get_images_ifd(decoder: &mut Decoder, path: &Path) -> Vec { - let mut res = vec![]; - let mut ifd_idx = 0; - loop { - let is_image = decoder - .get_tag_u32(Tag::NewSubfileType) - .map_or_else(|_| true, |v| v & 4 != 4); // see https://www.verypdf.com/document/tiff6/pg_0036.htm - if is_image { - //todo We should not ignore mask in the next PRs - res.push(ifd_idx); - } else { - warn!( - "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. The ifd number of this subfile is {}", - path.display(), - ifd_idx - ); - } + let full_resolution = + get_full_resolution(pixel_scale, transformation, path).map_err(FileError::from)?; - ifd_idx += 1; + let width_in_model = f64::from(image_width_pixel) * full_resolution[0].abs(); + let length_in_model = f64::from(image_length_pixel) * full_resolution[1].abs(); - let next_res = decoder.seek_to_image(ifd_idx); - if next_res.is_err() { - break; - } - } - res + Ok((width_in_model, length_in_model)) } fn get_origin( @@ -567,83 +445,10 @@ mod tests { use std::path::PathBuf; use insta::assert_yaml_snapshot; - use martin_tile_utils::TileCoord; use rstest::rstest; use tiff::decoder::Decoder; use crate::cog::model::ModelInfo; - use crate::cog::source::get_tile_idx; - - #[test] - fn can_calc_tile_idx() { - assert_eq!(Some(0), get_tile_idx(TileCoord { z: 0, x: 0, y: 0 }, 3, 3)); - assert_eq!(Some(8), get_tile_idx(TileCoord { z: 0, x: 2, y: 2 }, 3, 3)); - assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 3, y: 0 }, 3, 3)); - assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 1, y: 9 }, 3, 3)); - } - - #[rstest] - // the right half should be transprent - #[case( - "../tests/fixtures/cog/expected/right_padded.png", - (0,0,0,None),None,(128,256),(256,256) - )] - // the down half should be transprent - #[case( - "../tests/fixtures/cog/expected/down_padded.png", - (0,0,0,None),None,(256,128),(256,256) - )] - // the up half should be half transprent and down half should be transprent - #[case( - "../tests/fixtures/cog/expected/down_padded_with_alpha.png", - (0,0,0,Some(128)),None,(256,128),(256,256) - )] - // the left half should be half transprent and the right half should be transprent - #[case( - "../tests/fixtures/cog/expected/right_padded_with_alpha.png", - (0,0,0,Some(128)),None,(128,256),(256,256) - )] - // should be all half transprent - #[case( - "../tests/fixtures/cog/expected/not_padded.png", - (0,0,0,Some(128)),None,(256,256),(256,256) - )] - // all padded and with a no_data whose value is 128, and all the component is 128 - // so that should be all transprent - #[case( - "../tests/fixtures/cog/expected/all_transprent.png", - (128,128,128,Some(128)),Some(128),(128,128),(256,256) - )] - fn test_padded_cases( - #[case] expected_file_path: &str, - #[case] components: (u8, u8, u8, Option), - #[case] no_value: Option, - #[case] (data_width, data_height): (u32, u32), - #[case] (tile_width, tile_height): (u32, u32), - ) { - let mut pixels = Vec::new(); - for _ in 0..(data_width * data_height) { - pixels.push(components.0); - pixels.push(components.1); - pixels.push(components.2); - if let Some(alpha) = components.3 { - pixels.push(alpha); - } - } - let componse_count = if components.3.is_some() { 4 } else { 3 }; - let png_bytes = super::rgb_to_png( - pixels, - (tile_width, tile_height), - (data_width, data_height), - componse_count, - no_value, - &PathBuf::from("not_exist.tif"), - ) - .unwrap(); - let expected = std::fs::read(expected_file_path).unwrap(); - assert_eq!(png_bytes, expected); - } - #[test] fn can_get_model_infos() { let path = PathBuf::from("../tests/fixtures/cog/rgb_u8.tif"); From 723eee76b76caa2c6b03ae2980d64e864c60d06f Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 28 May 2025 16:56:37 +0800 Subject: [PATCH 02/45] move get_index to image struct --- martin/src/cog/image.rs | 43 ++++++++++++++++++++++------------------ martin/src/cog/source.rs | 4 ++-- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index f2c6d6021..4e2ab64fa 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -29,11 +29,8 @@ impl Image { .seek_to_image(self.ifd) .map_err(|e| CogError::IfdSeekFailed(e, self.ifd, path.to_path_buf()))?; - let across = self.across; - let down = self.down; - let tile_idx; - if let Some(idx) = get_tile_idx(xyz, across, down) { + if let Some(idx) = self.get_tile_idx(xyz) { tile_idx = idx; } else { return Ok(Vec::new()); @@ -74,19 +71,20 @@ impl Image { }?; Ok(png_file_bytes) } -} -fn get_tile_idx(xyz: TileCoord, across: u32, down: u32) -> Option { - if xyz.y >= down || xyz.x >= across { - return None; - } + fn get_tile_idx(&self, xyz: TileCoord) -> Option { + let across = self.across; + let down = self.down; + if xyz.y >= down || xyz.x >= across { + return None; + } - let tile_idx = xyz.y * across + xyz.x; - if tile_idx >= across * down { - return None; + let tile_idx = xyz.y * across + xyz.x; + if tile_idx >= across * down { + return None; + } + Some(tile_idx) } - Some(tile_idx) } - fn rgb_to_png( vec: Vec, (tile_width, tile_height): (u32, u32), @@ -151,13 +149,20 @@ mod tests { use martin_tile_utils::TileCoord; use rstest::rstest; - use crate::cog::image::get_tile_idx; + use crate::cog::image::Image; + #[test] fn can_calc_tile_idx() { - assert_eq!(Some(0), get_tile_idx(TileCoord { z: 0, x: 0, y: 0 }, 3, 3)); - assert_eq!(Some(8), get_tile_idx(TileCoord { z: 0, x: 2, y: 2 }, 3, 3)); - assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 3, y: 0 }, 3, 3)); - assert_eq!(None, get_tile_idx(TileCoord { z: 0, x: 1, y: 9 }, 3, 3)); + let image = Image { + ifd: 0, + across: 3, + down: 3, + nodata: None, + }; + assert_eq!(Some(0), image.get_tile_idx(TileCoord { z: 0, x: 0, y: 0 })); + assert_eq!(Some(8), image.get_tile_idx(TileCoord { z: 0, x: 2, y: 2 })); + assert_eq!(None, image.get_tile_idx(TileCoord { z: 0, x: 3, y: 0 })); + assert_eq!(None, image.get_tile_idx(TileCoord { z: 0, x: 1, y: 9 })); } #[rstest] // the right half should be transprent diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 63d0e4e78..15d943f19 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -261,7 +261,7 @@ fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Resul } let min_zoom = 0; let max_zoom = (images.len() - 1) as u8; - let zoom_and_images: HashMap = images + let images: HashMap = images .iter() .map(|image| { let zoom = max_zoom.saturating_sub((image.ifd as u8) + 1); @@ -271,10 +271,10 @@ fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Resul Ok(Meta { min_zoom, max_zoom, - images: zoom_and_images, model, origin, extent, + images, nodata, }) } From 6b22c9b288d85a9b1de9b36668093ef5f90fba63 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 10:54:49 +0800 Subject: [PATCH 03/45] fix a bug introduced in this refactor --- martin/src/cog/source.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 15d943f19..b69b01c91 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -264,7 +264,7 @@ fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Resul let images: HashMap = images .iter() .map(|image| { - let zoom = max_zoom.saturating_sub((image.ifd as u8) + 1); + let zoom = max_zoom.saturating_sub(image.ifd as u8); (zoom, image.clone()) }) .collect(); From a6b30ff7331a57729bd2cb1bfdbebe26fccc9bec Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 10:55:30 +0800 Subject: [PATCH 04/45] revmoe #[allow(dead_code)] as aleady being used --- martin/src/cog/source.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index b69b01c91..4a8b84844 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -17,7 +17,6 @@ use super::model::ModelInfo; use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; -#[allow(dead_code)] // the unused model would be used in next PRs #[derive(Clone, Debug)] struct Meta { min_zoom: u8, From fdc881d37bdab76362d7ce62d99268a9970e950f Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 11:16:16 +0800 Subject: [PATCH 05/45] move nodata bach to meta --- martin/src/cog/image.rs | 7 +++---- martin/src/cog/source.rs | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 4e2ab64fa..d95dcc21e 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -13,7 +13,6 @@ pub struct Image { pub ifd: usize, pub across: u32, pub down: u32, - pub nodata: Option, } impl Image { #[allow(clippy::cast_sign_loss)] @@ -23,6 +22,7 @@ impl Image { &self, decoder: &mut Decoder, xyz: TileCoord, + nodata: Option, path: &Path, ) -> MartinResult { decoder @@ -52,7 +52,7 @@ impl Image { (tile_width, tile_height), (data_width, data_height), 3, - self.nodata.map(|v| v as u8), + nodata.map(|v| v as u8), path, ), (DecodingResult::U8(vec), tiff::ColorType::RGBA(_)) => rgb_to_png( @@ -60,7 +60,7 @@ impl Image { (tile_width, tile_height), (data_width, data_height), 4, - self.nodata.map(|v| v as u8), + nodata.map(|v| v as u8), path, ), (_, _) => Err(CogError::NotSupportedColorTypeAndBitDepth( @@ -157,7 +157,6 @@ mod tests { ifd: 0, across: 3, down: 3, - nodata: None, }; assert_eq!(Some(0), image.get_tile_idx(TileCoord { z: 0, x: 0, y: 0 })); assert_eq!(Some(8), image.get_tile_idx(TileCoord { z: 0, x: 2, y: 2 })); diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 4a8b84844..08feab625 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -86,7 +86,7 @@ impl CogSource { ) })?; - let bytes = image.get_tile(&mut decoder, xyz, &self.path)?; + let bytes = image.get_tile(&mut decoder, xyz, self.meta.nodata, &self.path)?; Ok(bytes) } } @@ -238,7 +238,6 @@ fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Resul ifd: ifd_idx, across: tiles_across, down: tiles_down, - nodata, }; images.push(image); From 0d4e34106e9fe0cd311d05224d8ee1286fb560fe Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 11:32:03 +0800 Subject: [PATCH 06/45] split meta fields to source --- martin/src/cog/source.rs | 184 ++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 101 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 08feab625..e0831d630 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -18,7 +18,9 @@ use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; #[derive(Clone, Debug)] -struct Meta { +pub struct CogSource { + id: String, + path: PathBuf, min_zoom: u8, max_zoom: u8, model: ModelInfo, @@ -28,18 +30,12 @@ struct Meta { extent: [f64; 4], images: HashMap, nodata: Option, -} - -#[derive(Clone, Debug)] -pub struct CogSource { - id: String, - path: PathBuf, - meta: Meta, tilejson: TileJSON, tileinfo: TileInfo, } impl CogSource { + #[allow(clippy::cast_possible_truncation)] pub fn new(id: String, path: PathBuf) -> FileResult { let tileinfo = TileInfo::new(Format::Png, martin_tile_utils::Encoding::Uncompressed); let tif_file = @@ -49,17 +45,88 @@ impl CogSource { .with_limits(tiff::decoder::Limits::unlimited()); let model = ModelInfo::decode(&mut decoder, &path); verify_requirements(&mut decoder, &model, &path.clone())?; + let nodata: Option = if let Ok(no_data) = decoder.get_tag_ascii_string(GdalNodata) { + no_data.parse().ok() + } else { + None + }; + let origin = get_origin( + model.tie_points.as_deref(), + model.transformation.as_deref(), + &path, + )?; + let (full_width_pixel, full_length_pixel) = dim_in_pixel(&mut decoder, &path, 0)?; + let (full_width, full_length) = dim_in_model( + &mut decoder, + &path, + 0, + model.pixel_scale.as_deref(), + model.transformation.as_deref(), + )?; + let extent = get_extent( + &origin, + model.transformation.as_deref(), + (full_width_pixel, full_length_pixel), + (full_width, full_length), + ); + let mut images = vec![]; + + let mut ifd_idx = 0; + + loop { + let is_image = decoder + .get_tag_u32(Tag::NewSubfileType) + .map_or_else(|_| true, |v| v & 4 != 4); // see https://www.verypdf.com/document/tiff6/pg_0036.htm + if is_image { + //todo We should not ignore mask in the next PRs + let (tiles_across, tiles_down) = get_grid_dims(&mut decoder, &path, ifd_idx)?; + let image = Image { + ifd: ifd_idx, + across: tiles_across, + down: tiles_down, + }; + + images.push(image); + } else { + warn!( + "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. The ifd number of this subfile is {}", + path.display(), + ifd_idx + ); + } + + ifd_idx += 1; - let meta = get_meta(model.clone(), &mut decoder, &path)?; + let next_res = decoder.seek_to_image(ifd_idx); + if next_res.is_err() { + //todo add warn!() here + break; + } + } + let min_zoom = 0; + let max_zoom = (images.len() - 1) as u8; + let images: HashMap = images + .iter() + .map(|image| { + let zoom = max_zoom.saturating_sub(image.ifd as u8); + (zoom, image.clone()) + }) + .collect(); let tilejson = tilejson! { tiles: vec![], - minzoom: meta.min_zoom, - maxzoom: meta.max_zoom + minzoom: min_zoom, + maxzoom: max_zoom }; Ok(CogSource { id, path, - meta, + min_zoom, + max_zoom, + model, + origin, + extent, + images, + nodata, tilejson, tileinfo, }) @@ -68,7 +135,7 @@ impl CogSource { #[allow(clippy::cast_possible_truncation)] #[allow(clippy::too_many_lines)] pub fn get_tile(&self, xyz: TileCoord) -> MartinResult { - if xyz.z < self.meta.min_zoom || xyz.z > self.meta.max_zoom { + if xyz.z < self.min_zoom || xyz.z > self.max_zoom { return Ok(Vec::new()); } let tif_file = @@ -77,16 +144,11 @@ impl CogSource { Decoder::new(tif_file).map_err(|e| CogError::InvalidTiffFile(e, self.path.clone()))?; decoder = decoder.with_limits(tiff::decoder::Limits::unlimited()); - let image = self.meta.images.get(&(xyz.z)).ok_or_else(|| { - CogError::ZoomOutOfRange( - xyz.z, - self.path.clone(), - self.meta.min_zoom, - self.meta.max_zoom, - ) + let image = self.images.get(&(xyz.z)).ok_or_else(|| { + CogError::ZoomOutOfRange(xyz.z, self.path.clone(), self.min_zoom, self.max_zoom) })?; - let bytes = image.get_tile(&mut decoder, xyz, self.meta.nodata, &self.path)?; + let bytes = image.get_tile(&mut decoder, xyz, self.nodata, &self.path)?; Ok(bytes) } } @@ -197,86 +259,6 @@ fn verify_requirements( Ok(()) } -#[allow(clippy::cast_possible_truncation)] -fn get_meta(model: ModelInfo, decoder: &mut Decoder, path: &Path) -> Result { - let nodata: Option = if let Ok(no_data) = decoder.get_tag_ascii_string(GdalNodata) { - no_data.parse().ok() - } else { - None - }; - let origin = get_origin( - model.tie_points.as_deref(), - model.transformation.as_deref(), - path, - )?; - let (full_width_pixel, full_length_pixel) = dim_in_pixel(decoder, path, 0)?; - let (full_width, full_length) = dim_in_model( - decoder, - path, - 0, - model.pixel_scale.as_deref(), - model.transformation.as_deref(), - )?; - let extent = get_extent( - &origin, - model.transformation.as_deref(), - (full_width_pixel, full_length_pixel), - (full_width, full_length), - ); - let mut images = vec![]; - - let mut ifd_idx = 0; - - loop { - let is_image = decoder - .get_tag_u32(Tag::NewSubfileType) - .map_or_else(|_| true, |v| v & 4 != 4); // see https://www.verypdf.com/document/tiff6/pg_0036.htm - if is_image { - //todo We should not ignore mask in the next PRs - let (tiles_across, tiles_down) = get_grid_dims(decoder, path, ifd_idx)?; - let image = Image { - ifd: ifd_idx, - across: tiles_across, - down: tiles_down, - }; - - images.push(image); - } else { - warn!( - "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. The ifd number of this subfile is {}", - path.display(), - ifd_idx - ); - } - - ifd_idx += 1; - - let next_res = decoder.seek_to_image(ifd_idx); - if next_res.is_err() { - //todo add warn!() here - break; - } - } - let min_zoom = 0; - let max_zoom = (images.len() - 1) as u8; - let images: HashMap = images - .iter() - .map(|image| { - let zoom = max_zoom.saturating_sub(image.ifd as u8); - (zoom, image.clone()) - }) - .collect(); - Ok(Meta { - min_zoom, - max_zoom, - model, - origin, - extent, - images, - nodata, - }) -} - fn get_grid_dims( decoder: &mut Decoder, path: &Path, From 3c2d8ab990d84351c69b796f89437cd152d799ad Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 11:43:04 +0800 Subject: [PATCH 07/45] add #[allow(dead_code)] --- martin/src/cog/source.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index e0831d630..9b3f9b099 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -18,6 +18,7 @@ use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; #[derive(Clone, Debug)] +#[allow(dead_code)] // model and origin and extent would be used in future PRs pub struct CogSource { id: String, path: PathBuf, From d3cde3e507e78f906c604bd237fd2d372ae1b820 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 May 2025 03:43:25 +0000 Subject: [PATCH 08/45] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- martin/src/cog/source.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 9b3f9b099..05b65d1ce 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -18,7 +18,7 @@ use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; #[derive(Clone, Debug)] -#[allow(dead_code)] // model and origin and extent would be used in future PRs +#[allow(dead_code)] // model and origin and extent would be used in future PRs pub struct CogSource { id: String, path: PathBuf, From 5520c18164be472957ac9e07586425fdd471db1a Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 12:33:43 +0800 Subject: [PATCH 09/45] add doc --- martin/src/cog/image.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index d95dcc21e..ef3307bbe 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -8,10 +8,17 @@ use tiff::decoder::{Decoder, DecodingResult}; use super::CogError; use crate::{MartinResult, TileData}; +/// Image represents a single image in a COG file. A tiff file may contains many iamges. +/// This type contains several useful information and method for taking tiles from the image. #[derive(Clone, Debug)] pub struct Image { + /// IFD(Image file directory) number. + /// An IFD contains information about the image, as well as pointers to the actual image data. + /// A tiff file may contains many IFDs, each IFD represents a single image. pub ifd: usize, + /// How many tiles in a row of this image pub across: u32, + // How many tiles in a column of this image pub down: u32, } impl Image { From 4fed5c0340496093dfab581a5cd6649188aa30cc Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 12:34:52 +0800 Subject: [PATCH 10/45] remove uncessary allow too many lines rule --- martin/src/cog/image.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index ef3307bbe..adbbf2489 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -24,7 +24,6 @@ pub struct Image { impl Image { #[allow(clippy::cast_sign_loss)] #[allow(clippy::cast_possible_truncation)] - #[allow(clippy::too_many_lines)] pub fn get_tile( &self, decoder: &mut Decoder, From b348116aa0f49938da43aeac3c4995043ca99ec8 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 12:35:08 +0800 Subject: [PATCH 11/45] fmt --- martin/src/cog/image.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index adbbf2489..547349111 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -21,6 +21,7 @@ pub struct Image { // How many tiles in a column of this image pub down: u32, } + impl Image { #[allow(clippy::cast_sign_loss)] #[allow(clippy::cast_possible_truncation)] @@ -91,6 +92,7 @@ impl Image { Some(tile_idx) } } + fn rgb_to_png( vec: Vec, (tile_width, tile_height): (u32, u32), From ad5867e0886f739b02ea69c06108dbae89f6afe6 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 12:44:33 +0800 Subject: [PATCH 12/45] fix typo --- martin/src/cog/image.rs | 22 +++++++++--------- ...all_transprent.png => all_transparent.png} | Bin 2 files changed, 11 insertions(+), 11 deletions(-) rename tests/fixtures/cog/expected/{all_transprent.png => all_transparent.png} (100%) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 547349111..633f5dc75 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -8,7 +8,7 @@ use tiff::decoder::{Decoder, DecodingResult}; use super::CogError; use crate::{MartinResult, TileData}; -/// Image represents a single image in a COG file. A tiff file may contains many iamges. +/// Image represents a single image in a COG file. A tiff file may contain many images. /// This type contains several useful information and method for taking tiles from the image. #[derive(Clone, Debug)] pub struct Image { @@ -74,7 +74,7 @@ impl Image { color_type, path.to_path_buf(), )), - // do others in next PRs, a lot of disscussion would be needed + // do others in next PRs, a lot of discussion would be needed }?; Ok(png_file_bytes) } @@ -172,35 +172,35 @@ mod tests { assert_eq!(None, image.get_tile_idx(TileCoord { z: 0, x: 1, y: 9 })); } #[rstest] - // the right half should be transprent + // the right half should be transparent #[case( "../tests/fixtures/cog/expected/right_padded.png", (0,0,0,None),None,(128,256),(256,256) )] - // the down half should be transprent + // the down half should be transparent #[case( "../tests/fixtures/cog/expected/down_padded.png", (0,0,0,None),None,(256,128),(256,256) )] - // the up half should be half transprent and down half should be transprent + // the up half should be half transparent and down half should be transparent #[case( "../tests/fixtures/cog/expected/down_padded_with_alpha.png", (0,0,0,Some(128)),None,(256,128),(256,256) )] - // the left half should be half transprent and the right half should be transprent + // the left half should be half transparent and the right half should be transparent #[case( "../tests/fixtures/cog/expected/right_padded_with_alpha.png", (0,0,0,Some(128)),None,(128,256),(256,256) )] - // should be all half transprent + // should be all half transparent #[case( "../tests/fixtures/cog/expected/not_padded.png", (0,0,0,Some(128)),None,(256,256),(256,256) )] // all padded and with a no_data whose value is 128, and all the component is 128 - // so that should be all transprent + // so that should be all transparent #[case( - "../tests/fixtures/cog/expected/all_transprent.png", + "../tests/fixtures/cog/expected/all_transparent.png", (128,128,128,Some(128)),Some(128),(128,128),(256,256) )] fn test_padded_cases( @@ -219,12 +219,12 @@ mod tests { pixels.push(alpha); } } - let componse_count = if components.3.is_some() { 4 } else { 3 }; + let components_count = if components.3.is_some() { 4 } else { 3 }; let png_bytes = super::rgb_to_png( pixels, (tile_width, tile_height), (data_width, data_height), - componse_count, + components_count, no_value, &PathBuf::from("not_exist.tif"), ) diff --git a/tests/fixtures/cog/expected/all_transprent.png b/tests/fixtures/cog/expected/all_transparent.png similarity index 100% rename from tests/fixtures/cog/expected/all_transprent.png rename to tests/fixtures/cog/expected/all_transparent.png From a31465fca655df89057fd3b29e821288c23cbfd3 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 12:53:38 +0800 Subject: [PATCH 13/45] improve doc --- martin/src/cog/image.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 633f5dc75..9d963383a 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -14,7 +14,6 @@ use crate::{MartinResult, TileData}; pub struct Image { /// IFD(Image file directory) number. /// An IFD contains information about the image, as well as pointers to the actual image data. - /// A tiff file may contains many IFDs, each IFD represents a single image. pub ifd: usize, /// How many tiles in a row of this image pub across: u32, From 02f69e82a3327bf0517c6f934c190625619c05e7 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 29 May 2025 15:43:39 +0800 Subject: [PATCH 14/45] add doc --- martin/src/cog/image.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 9d963383a..db970cdb3 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -22,6 +22,16 @@ pub struct Image { } impl Image { + /// Retrieves a tile from the image, decodes it, and converts it to PNG format. + /// + /// # Arguments + /// * `decoder` - A mutable reference to a TIFF decoder. + /// * `xyz` - The tile coordinates (z, x, y). + /// * [nodata](https://gdal.org/en/stable/drivers/raster/gtiff.html#nodata-value) - An optional nodata value. Pixels with this value will be made transparent. + /// * `path` - The path to the TIFF file, used for error reporting. + /// + /// # Returns + /// A `MartinResult` containing the tile data as a `Vec` (PNG bytes) or an error. #[allow(clippy::cast_sign_loss)] #[allow(clippy::cast_possible_truncation)] pub fn get_tile( From a1d38620f0b1fc17d09271f3b80d15bb925e9341 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 30 May 2025 14:13:51 +0800 Subject: [PATCH 15/45] refactor rgb_to_png --- martin/src/cog/image.rs | 73 +++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index db970cdb3..0ec1754d9 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -102,44 +102,75 @@ impl Image { } } +/// Converts a vector of RGB or RGBA bytes to PNG format, handling nodata values and padding if necessary. +/// # Arguments +/// +/// * `data` - A vector of pixel data in RGB or RGBA format. +/// * `(tile_width, tile_height)` - The dimensions of the tile. +/// * `(data_width, data_height)` - The dimensions of the chunk data decoded from this tile. +/// * `components_count` - The number of components per pixel (3 for RGB, 4 for RGBA). +/// * `nodata` - An optional nodata value. Pixels with this value will be made transparent. +/// * `path` - The path to the TIFF file, used for error reporting. +/// +/// # Returns +/// +/// A `Result` containing the PNG bytes as a `Vec` or an error. fn rgb_to_png( - vec: Vec, + data: Vec, (tile_width, tile_height): (u32, u32), (data_width, data_height): (u32, u32), - chunk_components_count: u32, + components_count: u32, nodata: Option, path: &Path, ) -> Result, CogError> { let is_padded = data_width != tile_width || data_height != tile_height; - let need_add_alpha = chunk_components_count != 4; - + let need_add_alpha = components_count != 4; + // 1. Check if the tile is padded, if so, we need to add padding part back + // The decoded might be smaller than the tile size as tiff crate always cut off the padding part + // So we need to add the padding part back if needed + // 2. Check if we need to add alpha channel, if the components count is not 4(rgba => 4 components, rgb => 3 components), we need to add an alpha channel + // 3. Check if nodata is provided, if so, we need to make the pixels with nodata value transparent + // See https://gdal.org/en/stable/drivers/raster/gtiff.html#nodata-value let pixels = if nodata.is_some() || need_add_alpha || is_padded { let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; for row in 0..data_height { - 'outer: for col in 0..data_width { - let idx_chunk = - row * data_width * chunk_components_count + col * chunk_components_count; + for col in 0..data_width { + let idx_chunk = row * data_width * components_count + col * components_count; let idx_result = row * tile_width * 4 + col * 4; - for component_idx in 0..chunk_components_count { - if nodata.eq(&Some(vec[(idx_chunk + component_idx) as usize])) { - //This pixel is nodata, just make it transparent and skip it then - let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 0; - continue 'outer; - } - result_vec[(idx_result + component_idx) as usize] = - vec[(idx_chunk + component_idx) as usize]; - } - if need_add_alpha { + let r = data[(idx_chunk) as usize]; + let g = data[(idx_chunk + 1) as usize]; + let b = data[(idx_chunk + 2) as usize]; + + if nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)) { let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 255; + result_vec[alpha_idx] = 0; + continue; } + let alpha = if need_add_alpha { + 255 + } else { + data[(idx_chunk + 3) as usize] + }; + + result_vec[idx_result as usize] = r; + result_vec[(idx_result + 1) as usize] = g; + result_vec[(idx_result + 2) as usize] = b; + result_vec[(idx_result + 3) as usize] = alpha; } } result_vec } else { - vec + data }; + Ok(encode_to_png(tile_width, tile_height, &pixels, path))? +} + +fn encode_to_png( + tile_width: u32, + tile_height: u32, + pixels: &[u8], + path: &Path, +) -> Result, CogError> { let mut result_file_buffer = Vec::new(); { let mut encoder = png::Encoder::new( @@ -153,7 +184,7 @@ fn rgb_to_png( .write_header() .map_err(|e| CogError::WritePngHeaderFailed(path.to_path_buf(), e))?; writer - .write_image_data(&pixels) + .write_image_data(pixels) .map_err(|e| CogError::WriteToPngFailed(path.to_path_buf(), e))?; } Ok(result_file_buffer) From 953cdb1c6b010bfc65438fab11cc6e98ac88e824 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Tue, 3 Jun 2025 14:52:51 +0800 Subject: [PATCH 16/45] better doc --- martin/src/cog/image.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 0ec1754d9..e6d6318bd 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -9,15 +9,15 @@ use super::CogError; use crate::{MartinResult, TileData}; /// Image represents a single image in a COG file. A tiff file may contain many images. -/// This type contains several useful information and method for taking tiles from the image. +/// This type contains several useful information and methods for taking tiles from the image. #[derive(Clone, Debug)] pub struct Image { /// IFD(Image file directory) number. /// An IFD contains information about the image, as well as pointers to the actual image data. pub ifd: usize, - /// How many tiles in a row of this image + /// Number of tiles in a row of this image pub across: u32, - // How many tiles in a column of this image + /// Number of tiles in a column of this image pub down: u32, } From 641d510804f37bd3752ebd9e2596f5f62eda7336 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Tue, 3 Jun 2025 14:54:45 +0800 Subject: [PATCH 17/45] improve image.rs --- martin/src/cog/image.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index e6d6318bd..230de5e4e 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -88,14 +88,12 @@ impl Image { Ok(png_file_bytes) } fn get_tile_idx(&self, xyz: TileCoord) -> Option { - let across = self.across; - let down = self.down; - if xyz.y >= down || xyz.x >= across { + if xyz.y >= self.down || xyz.x >= self.across { return None; } - let tile_idx = xyz.y * across + xyz.x; - if tile_idx >= across * down { + let tile_idx = xyz.y * self.across + xyz.x; + if tile_idx >= self.across * self.down { return None; } Some(tile_idx) From e47071d4522dfda940d0bf3f5a0ef9aaa066c43c Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Tue, 3 Jun 2025 15:09:14 +0800 Subject: [PATCH 18/45] better doc --- martin/src/cog/image.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 230de5e4e..013009c4b 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -12,9 +12,9 @@ use crate::{MartinResult, TileData}; /// This type contains several useful information and methods for taking tiles from the image. #[derive(Clone, Debug)] pub struct Image { - /// IFD(Image file directory) number. + /// The Number of Image file directory, generally abbreviated as IFD. /// An IFD contains information about the image, as well as pointers to the actual image data. - pub ifd: usize, + pub image_file_directory: usize, /// Number of tiles in a row of this image pub across: u32, /// Number of tiles in a column of this image @@ -83,7 +83,7 @@ impl Image { color_type, path.to_path_buf(), )), - // do others in next PRs, a lot of discussion would be needed + //todo do others in next PRs, a lot of discussion would be needed }?; Ok(png_file_bytes) } From 8f7d59e6dfdf72bd35cdf8369564a44d56419ef5 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 10:12:35 +0800 Subject: [PATCH 19/45] rename ifd --- martin/src/cog/image.rs | 14 ++++++++------ martin/src/cog/source.rs | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 013009c4b..38ed846af 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -42,8 +42,10 @@ impl Image { path: &Path, ) -> MartinResult { decoder - .seek_to_image(self.ifd) - .map_err(|e| CogError::IfdSeekFailed(e, self.ifd, path.to_path_buf()))?; + .seek_to_image(self.image_file_directory) + .map_err(|e| { + CogError::IfdSeekFailed(e, self.image_file_directory, path.to_path_buf()) + })?; let tile_idx; if let Some(idx) = self.get_tile_idx(xyz) { @@ -51,9 +53,9 @@ impl Image { } else { return Ok(Vec::new()); } - let decode_result = decoder - .read_chunk(tile_idx) - .map_err(|e| CogError::ReadChunkFailed(e, tile_idx, self.ifd, path.to_path_buf()))?; + let decode_result = decoder.read_chunk(tile_idx).map_err(|e| { + CogError::ReadChunkFailed(e, tile_idx, self.image_file_directory, path.to_path_buf()) + })?; let color_type = decoder .colortype() .map_err(|e| CogError::InvalidTiffFile(e, path.to_path_buf()))?; @@ -200,7 +202,7 @@ mod tests { #[test] fn can_calc_tile_idx() { let image = Image { - ifd: 0, + image_file_directory: 0, across: 3, down: 3, }; diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 05b65d1ce..ef6d32cb0 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -82,7 +82,7 @@ impl CogSource { //todo We should not ignore mask in the next PRs let (tiles_across, tiles_down) = get_grid_dims(&mut decoder, &path, ifd_idx)?; let image = Image { - ifd: ifd_idx, + image_file_directory: ifd_idx, across: tiles_across, down: tiles_down, }; @@ -109,7 +109,7 @@ impl CogSource { let images: HashMap = images .iter() .map(|image| { - let zoom = max_zoom.saturating_sub(image.ifd as u8); + let zoom = max_zoom.saturating_sub(image.image_file_directory as u8); (zoom, image.clone()) }) .collect(); From 08fbc71bd5c2f159291cc1ebc19d2a8a5fc30947 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 10:19:07 +0800 Subject: [PATCH 20/45] better comment --- martin/src/cog/image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 38ed846af..2258bb6de 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -63,7 +63,7 @@ impl Image { let (tile_width, tile_height) = decoder.chunk_dimensions(); let (data_width, data_height) = decoder.chunk_data_dimensions(tile_idx); - //do more research on the not u8 case, is this the right way to do it? + //FIXME: do more research on the not u8 case, is this the right way to do it? let png_file_bytes = match (decode_result, color_type) { (DecodingResult::U8(vec), tiff::ColorType::RGB(_)) => rgb_to_png( vec, From ece0b09f4f9bba29a33e97bb4a3e2798defcbe52 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 10:38:19 +0800 Subject: [PATCH 21/45] improve doc --- martin/src/cog/source.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index ef6d32cb0..26295c47f 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -289,6 +289,24 @@ fn dim_in_pixel( Ok((image_width, image_length)) } + +/// Converts pixel dimensions to model space dimensions using resolution values +/// +/// # Arguments +/// +/// * `decoder` - TIFF decoder for reading image information +/// * `path` - Image file path for error reporting +/// * `image_ifd` - Image file directory index +/// * `pixel_scale` - Optional pixel scale array [ScaleX, ScaleY, ScaleZ] +/// * `transformation` - Optional 4x4 transformation matrix (16 elements) +/// +/// # Returns +/// +/// Tuple `(width, height)` in model coordinate system units +/// +/// # Errors +/// +/// Returns `FileError` if dimensions cannot be read or resolution calculation fails fn dim_in_model( decoder: &mut Decoder, path: &Path, From b56a6a026a51140df5511ba874396aa8aca7ad08 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 14:21:34 +0800 Subject: [PATCH 22/45] improve func rgb_to_png --- martin/src/cog/image.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 2258bb6de..47e6ff789 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -137,19 +137,20 @@ fn rgb_to_png( for col in 0..data_width { let idx_chunk = row * data_width * components_count + col * components_count; let idx_result = row * tile_width * 4 + col * 4; + let r = data[(idx_chunk) as usize]; let g = data[(idx_chunk + 1) as usize]; let b = data[(idx_chunk + 2) as usize]; - if nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)) { - let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 0; - continue; - } - let alpha = if need_add_alpha { - 255 - } else { - data[(idx_chunk + 3) as usize] + let is_nodata = nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)); + + let alpha = match (need_add_alpha, is_nodata) { + // one of the components is nodata, so we make this pixel transparent + (_, true) => 0, + // The original data is rgb, not rgba. We need to add a alpha channel on it + (true, false) => 255, + // The original data is rgba, we need to copy the alpha value from it to result + (false, false) => data[idx_chunk as usize + 3], }; result_vec[idx_result as usize] = r; From 6e5b36968300cb937086278e5890c7d469e97fa8 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 14:22:29 +0800 Subject: [PATCH 23/45] better doc --- martin/src/cog/source.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 26295c47f..604a89623 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -56,7 +56,7 @@ impl CogSource { model.transformation.as_deref(), &path, )?; - let (full_width_pixel, full_length_pixel) = dim_in_pixel(&mut decoder, &path, 0)?; + let (full_width_pixel, full_length_pixel) = dimensions(&mut decoder, &path, 0)?; let (full_width, full_length) = dim_in_model( &mut decoder, &path, @@ -266,14 +266,14 @@ fn get_grid_dims( image_ifd: usize, ) -> Result<(u32, u32), FileError> { let (tile_width, tile_height) = (decoder.chunk_dimensions().0, decoder.chunk_dimensions().1); - let (image_width, image_length) = dim_in_pixel(decoder, path, image_ifd)?; + let (image_width, image_length) = dimensions(decoder, path, image_ifd)?; let tiles_across = image_width.div_ceil(tile_width); let tiles_down = image_length.div_ceil(tile_height); Ok((tiles_across, tiles_down)) } -fn dim_in_pixel( +fn dimensions( decoder: &mut Decoder, path: &Path, image_ifd: usize, @@ -297,7 +297,7 @@ fn dim_in_pixel( /// * `decoder` - TIFF decoder for reading image information /// * `path` - Image file path for error reporting /// * `image_ifd` - Image file directory index -/// * `pixel_scale` - Optional pixel scale array [ScaleX, ScaleY, ScaleZ] +/// * `pixel_scale` - Optional pixel scale array [`ScaleX`, `ScaleY`, `ScaleZ`] /// * `transformation` - Optional 4x4 transformation matrix (16 elements) /// /// # Returns @@ -314,7 +314,7 @@ fn dim_in_model( pixel_scale: Option<&[f64]>, transformation: Option<&[f64]>, ) -> Result<(f64, f64), FileError> { - let (image_width_pixel, image_length_pixel) = dim_in_pixel(decoder, path, image_ifd)?; + let (image_width_pixel, image_length_pixel) = dimensions(decoder, path, image_ifd)?; let full_resolution = get_full_resolution(pixel_scale, transformation, path).map_err(FileError::from)?; From 4c45ae864e73db3e12fb053e480f0fea6e491269 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 4 Jun 2025 16:01:57 +0800 Subject: [PATCH 24/45] improve rgb_to_png --- martin/src/cog/image.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 47e6ff789..6c53178c5 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -142,15 +142,17 @@ fn rgb_to_png( let g = data[(idx_chunk + 1) as usize]; let b = data[(idx_chunk + 2) as usize]; - let is_nodata = nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)); - - let alpha = match (need_add_alpha, is_nodata) { - // one of the components is nodata, so we make this pixel transparent - (_, true) => 0, - // The original data is rgb, not rgba. We need to add a alpha channel on it - (true, false) => 255, - // The original data is rgba, we need to copy the alpha value from it to result - (false, false) => data[idx_chunk as usize + 3], + if nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)) { + result_vec[(idx_result + 3) as usize] = 0; + // one of the components is nodata, so we set the alpha to 0 to make it transparent + continue; + } + + let alpha = if need_add_alpha { + 255 // we need to add an alpha channel, so we set it to 255(not transparent) + } else { + // if it has alpha channel already, we need to copy the alpha value + data[(idx_chunk + 3) as usize] }; result_vec[idx_result as usize] = r; From 846adc33fd4076d36ab6d4335dde178f60ddca88 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Jun 2025 08:05:04 +0000 Subject: [PATCH 25/45] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- martin/src/cog/image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 6c53178c5..2967f476d 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -147,7 +147,7 @@ fn rgb_to_png( // one of the components is nodata, so we set the alpha to 0 to make it transparent continue; } - + let alpha = if need_add_alpha { 255 // we need to add an alpha channel, so we set it to 255(not transparent) } else { From 1ece931f1c9806774d8acfd772667113a307645d Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 6 Jun 2025 11:12:36 +0800 Subject: [PATCH 26/45] refactor rgb_to_png --- martin/src/cog/image.rs | 44 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 2967f476d..5ecf3589e 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -134,38 +134,40 @@ fn rgb_to_png( let pixels = if nodata.is_some() || need_add_alpha || is_padded { let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; for row in 0..data_height { - for col in 0..data_width { + 'outer: for col in 0..data_width { let idx_chunk = row * data_width * components_count + col * components_count; let idx_result = row * tile_width * 4 + col * 4; - let r = data[(idx_chunk) as usize]; - let g = data[(idx_chunk + 1) as usize]; - let b = data[(idx_chunk + 2) as usize]; + // Copy the components one by one + for component_idx in 0..components_count { + // Before copying, check if this component == nodata. If so, do skip and it would be transparent. + // FIXME: Should we copy the RGB values anyway and just set alpha to 0? The visual result actually is the same (transparent), but the component values would differ. But it might be a little slower as we don't skip the copy + // Source pixel: [4, 1, 2, 3] nodata: Some(1) + // Do skip: + // result pixel: [4, 0, 0, 0] + // Do not skip: + // result pixel: [4, 1, 2, 0] + // So the visual result is the same, but the component values are different. - if nodata.eq(&Some(r)) || nodata.eq(&Some(g)) || nodata.eq(&Some(b)) { - result_vec[(idx_result + 3) as usize] = 0; - // one of the components is nodata, so we set the alpha to 0 to make it transparent - continue; + if nodata.eq(&Some(data[(idx_chunk + component_idx) as usize])) { + continue 'outer; + } + // Copy this component to the result vector + result_vec[(idx_result + component_idx) as usize] = + data[(idx_chunk + component_idx) as usize]; + } + // If an alpha channel needs to be added, set it to 255 (opaque) + if need_add_alpha { + let alpha_idx = (idx_result + 3) as usize; + result_vec[alpha_idx] = 255; } - - let alpha = if need_add_alpha { - 255 // we need to add an alpha channel, so we set it to 255(not transparent) - } else { - // if it has alpha channel already, we need to copy the alpha value - data[(idx_chunk + 3) as usize] - }; - - result_vec[idx_result as usize] = r; - result_vec[(idx_result + 1) as usize] = g; - result_vec[(idx_result + 2) as usize] = b; - result_vec[(idx_result + 3) as usize] = alpha; } } result_vec } else { data }; - Ok(encode_to_png(tile_width, tile_height, &pixels, path))? + encode_to_png(tile_width, tile_height, &pixels, path) } fn encode_to_png( From 0cbb55ecc29068774882e29f9f7a4e625812b81b Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 6 Jun 2025 14:36:33 +0800 Subject: [PATCH 27/45] add doc --- martin/src/cog/source.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 604a89623..f0a1257ba 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -56,8 +56,8 @@ impl CogSource { model.transformation.as_deref(), &path, )?; - let (full_width_pixel, full_length_pixel) = dimensions(&mut decoder, &path, 0)?; - let (full_width, full_length) = dim_in_model( + let (full_width_pixel, full_length_pixel) = dimensions_in_pixel(&mut decoder, &path, 0)?; + let (full_width, full_length) = dimensions_in_model( &mut decoder, &path, 0, @@ -266,14 +266,29 @@ fn get_grid_dims( image_ifd: usize, ) -> Result<(u32, u32), FileError> { let (tile_width, tile_height) = (decoder.chunk_dimensions().0, decoder.chunk_dimensions().1); - let (image_width, image_length) = dimensions(decoder, path, image_ifd)?; + let (image_width, image_length) = dimensions_in_pixel(decoder, path, image_ifd)?; let tiles_across = image_width.div_ceil(tile_width); let tiles_down = image_length.div_ceil(tile_height); Ok((tiles_across, tiles_down)) } -fn dimensions( +/// Gets image pixel dimensions from TIFF decoder +/// +/// # Arguments +/// +/// * `decoder` - TIFF decoder for reading image information +/// * `path` - Image file path for error reporting +/// * `image_ifd` - Image file directory index +/// +/// # Returns +/// +/// Returns a tuple `(width, height)` containing image dimensions in pixels +/// +/// # Errors +/// +/// Returns `FileError` if image dimension tags cannot be read +fn dimensions_in_pixel( decoder: &mut Decoder, path: &Path, image_ifd: usize, @@ -307,14 +322,14 @@ fn dimensions( /// # Errors /// /// Returns `FileError` if dimensions cannot be read or resolution calculation fails -fn dim_in_model( +fn dimensions_in_model( decoder: &mut Decoder, path: &Path, image_ifd: usize, pixel_scale: Option<&[f64]>, transformation: Option<&[f64]>, ) -> Result<(f64, f64), FileError> { - let (image_width_pixel, image_length_pixel) = dimensions(decoder, path, image_ifd)?; + let (image_width_pixel, image_length_pixel) = dimensions_in_pixel(decoder, path, image_ifd)?; let full_resolution = get_full_resolution(pixel_scale, transformation, path).map_err(FileError::from)?; From 250564889e0b1345e3d744263518b086cd28a633 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 6 Jun 2025 15:04:07 +0800 Subject: [PATCH 28/45] remove redundant check in get_tile_idx --- martin/src/cog/image.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 5ecf3589e..d146eeb1e 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -95,9 +95,6 @@ impl Image { } let tile_idx = xyz.y * self.across + xyz.x; - if tile_idx >= self.across * self.down { - return None; - } Some(tile_idx) } } From 31f3dfa2014076eb6a84f1e5c17844f953d87d7c Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 6 Jun 2025 15:26:20 +0800 Subject: [PATCH 29/45] split rgb_to_png to three funcs --- martin/src/cog/image.rs | 63 ++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index d146eeb1e..443890b85 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -99,19 +99,18 @@ impl Image { } } -/// Converts a vector of RGB or RGBA bytes to PNG format, handling nodata values and padding if necessary. -/// # Arguments +/// Converts RGB/RGBA tile data to PNG format. /// -/// * `data` - A vector of pixel data in RGB or RGBA format. -/// * `(tile_width, tile_height)` - The dimensions of the tile. -/// * `(data_width, data_height)` - The dimensions of the chunk data decoded from this tile. -/// * `components_count` - The number of components per pixel (3 for RGB, 4 for RGBA). -/// * `nodata` - An optional nodata value. Pixels with this value will be made transparent. -/// * `path` - The path to the TIFF file, used for error reporting. +/// # Arguments +/// * `data` - Raw pixel data from TIFF decoder +/// * `tile_width`, `tile_height` - Expected tile dimensions +/// * `data_width`, `data_height` - Actual data dimensions +/// * `components_count` - Number of color components (3 for RGB, 4 for RGBA) +/// * `nodata` - Optional nodata value to make transparent +/// * `path` - File path for error reporting /// /// # Returns -/// -/// A `Result` containing the PNG bytes as a `Vec` or an error. +/// PNG-encoded tile data as bytes fn rgb_to_png( data: Vec, (tile_width, tile_height): (u32, u32), @@ -120,6 +119,34 @@ fn rgb_to_png( nodata: Option, path: &Path, ) -> Result, CogError> { + let pixels = ensure_pixels_valid( + data, + (tile_width, tile_height), + (data_width, data_height), + components_count, + nodata, + ); + encode_rgba_to_png(tile_width, tile_height, &pixels, path) +} + +/// Ensures pixel data is valid for PNG encoding by handling padding, alpha channel, and nodata values. +/// +/// # Arguments +/// * `data` - Raw pixel data +/// * `tile_width`, `tile_height` - Target tile dimensions +/// * `data_width`, `data_height` - Source data dimensions +/// * `components_count` - Number of color components per pixel +/// * `nodata` - Optional value to treat as transparent +/// +/// # Returns +/// RGBA pixel data ready for PNG encoding +fn ensure_pixels_valid( + data: Vec, + (tile_width, tile_height): (u32, u32), + (data_width, data_height): (u32, u32), + components_count: u32, + nodata: Option, +) -> Vec { let is_padded = data_width != tile_width || data_height != tile_height; let need_add_alpha = components_count != 4; // 1. Check if the tile is padded, if so, we need to add padding part back @@ -128,7 +155,7 @@ fn rgb_to_png( // 2. Check if we need to add alpha channel, if the components count is not 4(rgba => 4 components, rgb => 3 components), we need to add an alpha channel // 3. Check if nodata is provided, if so, we need to make the pixels with nodata value transparent // See https://gdal.org/en/stable/drivers/raster/gtiff.html#nodata-value - let pixels = if nodata.is_some() || need_add_alpha || is_padded { + if nodata.is_some() || need_add_alpha || is_padded { let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; for row in 0..data_height { 'outer: for col in 0..data_width { @@ -163,11 +190,19 @@ fn rgb_to_png( result_vec } else { data - }; - encode_to_png(tile_width, tile_height, &pixels, path) + } } -fn encode_to_png( +/// Encodes RGBA pixel data to PNG format. +/// +/// # Arguments +/// * `tile_width`, `tile_height` - Image dimensions +/// * `pixels` - RGBA pixel data +/// * `path` - File path for error reporting +/// +/// # Returns +/// PNG-encoded image data as bytes +fn encode_rgba_to_png( tile_width: u32, tile_height: u32, pixels: &[u8], From 667587256f62caebb54f5c3b6cbead79d5795718 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 6 Jun 2025 18:53:37 -0400 Subject: [PATCH 30/45] Aditional cleanups --- martin/src/cog/config.rs | 2 +- martin/src/cog/image.rs | 63 +++++++++++++++++----------- martin/src/cog/source.rs | 91 +++++++++++++++++++--------------------- 3 files changed, 83 insertions(+), 73 deletions(-) diff --git a/martin/src/cog/config.rs b/martin/src/cog/config.rs index f636c17d0..038e16f20 100644 --- a/martin/src/cog/config.rs +++ b/martin/src/cog/config.rs @@ -27,7 +27,7 @@ impl SourceConfigExtras for CogConfig { Ok(Box::new(cog)) } - #[allow(clippy::no_effect_underscore_binding)] + #[expect(clippy::no_effect_underscore_binding)] async fn new_sources_url(&self, _id: String, _url: Url) -> FileResult> { unreachable!() } diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 443890b85..837c79a5c 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -9,19 +9,26 @@ use super::CogError; use crate::{MartinResult, TileData}; /// Image represents a single image in a COG file. A tiff file may contain many images. -/// This type contains several useful information and methods for taking tiles from the image. +/// This struct contains information and methods for taking tiles from the image. #[derive(Clone, Debug)] pub struct Image { - /// The Number of Image file directory, generally abbreviated as IFD. - /// An IFD contains information about the image, as well as pointers to the actual image data. - pub image_file_directory: usize, + /// The Image File Directory index represents IDF entry with the image pointers to the actual image data. + ifd_index: usize, /// Number of tiles in a row of this image - pub across: u32, + tiles_across: u32, /// Number of tiles in a column of this image - pub down: u32, + tiles_down: u32, } impl Image { + pub fn new(ifd_index: usize, tiles_across: u32, tiles_down: u32) -> Self { + Self { + ifd_index, + tiles_across, + tiles_down, + } + } + /// Retrieves a tile from the image, decodes it, and converts it to PNG format. /// /// # Arguments @@ -32,8 +39,7 @@ impl Image { /// /// # Returns /// A `MartinResult` containing the tile data as a `Vec` (PNG bytes) or an error. - #[allow(clippy::cast_sign_loss)] - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub fn get_tile( &self, decoder: &mut Decoder, @@ -42,19 +48,17 @@ impl Image { path: &Path, ) -> MartinResult { decoder - .seek_to_image(self.image_file_directory) - .map_err(|e| { - CogError::IfdSeekFailed(e, self.image_file_directory, path.to_path_buf()) - })?; + .seek_to_image(self.ifd_index) + .map_err(|e| CogError::IfdSeekFailed(e, self.ifd_index, path.to_path_buf()))?; let tile_idx; - if let Some(idx) = self.get_tile_idx(xyz) { + if let Some(idx) = self.get_tile_index(xyz) { tile_idx = idx; } else { return Ok(Vec::new()); } let decode_result = decoder.read_chunk(tile_idx).map_err(|e| { - CogError::ReadChunkFailed(e, tile_idx, self.image_file_directory, path.to_path_buf()) + CogError::ReadChunkFailed(e, tile_idx, self.ifd_index, path.to_path_buf()) })?; let color_type = decoder .colortype() @@ -89,12 +93,17 @@ impl Image { }?; Ok(png_file_bytes) } - fn get_tile_idx(&self, xyz: TileCoord) -> Option { - if xyz.y >= self.down || xyz.x >= self.across { + + pub fn ifd_index(&self) -> usize { + self.ifd_index + } + + fn get_tile_index(&self, xyz: TileCoord) -> Option { + if xyz.y >= self.tiles_down || xyz.x >= self.tiles_across { return None; } - let tile_idx = xyz.y * self.across + xyz.x; + let tile_idx = xyz.y * self.tiles_across + xyz.x; Some(tile_idx) } } @@ -239,14 +248,20 @@ mod tests { #[test] fn can_calc_tile_idx() { let image = Image { - image_file_directory: 0, - across: 3, - down: 3, + ifd_index: 0, + tiles_across: 3, + tiles_down: 3, }; - assert_eq!(Some(0), image.get_tile_idx(TileCoord { z: 0, x: 0, y: 0 })); - assert_eq!(Some(8), image.get_tile_idx(TileCoord { z: 0, x: 2, y: 2 })); - assert_eq!(None, image.get_tile_idx(TileCoord { z: 0, x: 3, y: 0 })); - assert_eq!(None, image.get_tile_idx(TileCoord { z: 0, x: 1, y: 9 })); + assert_eq!( + Some(0), + image.get_tile_index(TileCoord { z: 0, x: 0, y: 0 }) + ); + assert_eq!( + Some(8), + image.get_tile_index(TileCoord { z: 0, x: 2, y: 2 }) + ); + assert_eq!(None, image.get_tile_index(TileCoord { z: 0, x: 3, y: 0 })); + assert_eq!(None, image.get_tile_index(TileCoord { z: 0, x: 1, y: 9 })); } #[rstest] // the right half should be transparent diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index f0a1257ba..9a704272c 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -18,17 +18,16 @@ use crate::file_config::{FileError, FileResult}; use crate::{MartinResult, Source, TileData, UrlQuery}; #[derive(Clone, Debug)] -#[allow(dead_code)] // model and origin and extent would be used in future PRs pub struct CogSource { id: String, path: PathBuf, min_zoom: u8, max_zoom: u8, - model: ModelInfo, + _model: ModelInfo, // The geo coords of pixel(0, 0, 0) ordering in [x, y, z] - origin: [f64; 3], + _origin: [f64; 3], // [minx, miny, maxx, maxy] in its model space coordinate system - extent: [f64; 4], + _extent: [f64; 4], images: HashMap, nodata: Option, tilejson: TileJSON, @@ -36,7 +35,7 @@ pub struct CogSource { } impl CogSource { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub fn new(id: String, path: PathBuf) -> FileResult { let tileinfo = TileInfo::new(Format::Png, martin_tile_utils::Encoding::Uncompressed); let tif_file = @@ -70,37 +69,29 @@ impl CogSource { (full_width_pixel, full_length_pixel), (full_width, full_length), ); - let mut images = vec![]; - let mut ifd_idx = 0; + let mut images = vec![]; + let mut ifd_index = 0; loop { let is_image = decoder .get_tag_u32(Tag::NewSubfileType) .map_or_else(|_| true, |v| v & 4 != 4); // see https://www.verypdf.com/document/tiff6/pg_0036.htm if is_image { - //todo We should not ignore mask in the next PRs - let (tiles_across, tiles_down) = get_grid_dims(&mut decoder, &path, ifd_idx)?; - let image = Image { - image_file_directory: ifd_idx, - across: tiles_across, - down: tiles_down, - }; - - images.push(image); + // TODO: We should not ignore mask in the next PRs + images.push(get_image(&mut decoder, &path, ifd_index)?); } else { warn!( - "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. The ifd number of this subfile is {}", + "A subfile of {} is ignored in the tiff file as Martin currently does not support mask subfile in tiff. IFD={ifd_index}", path.display(), - ifd_idx ); } - ifd_idx += 1; + ifd_index += 1; - let next_res = decoder.seek_to_image(ifd_idx); + let next_res = decoder.seek_to_image(ifd_index); if next_res.is_err() { - //todo add warn!() here + // TODO: add warn!() here break; } } @@ -109,7 +100,10 @@ impl CogSource { let images: HashMap = images .iter() .map(|image| { - let zoom = max_zoom.saturating_sub(image.image_file_directory as u8); + // FIXME: explain why it is OK to use IFD index as zoom level, + // and why it would not exceed max_zoom. + // This looks like a bug for some reason. + let zoom = max_zoom.saturating_sub(image.ifd_index() as u8); (zoom, image.clone()) }) .collect(); @@ -123,18 +117,20 @@ impl CogSource { path, min_zoom, max_zoom, - model, - origin, - extent, + // FIXME: these are not yet used + _model: model, + _origin: origin, + _extent: extent, images, nodata, tilejson, tileinfo, }) } - #[allow(clippy::cast_sign_loss)] - #[allow(clippy::cast_possible_truncation)] - #[allow(clippy::too_many_lines)] + + #[expect(clippy::cast_sign_loss)] + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::too_many_lines)] pub fn get_tile(&self, xyz: TileCoord) -> MartinResult { if xyz.z < self.min_zoom || xyz.z > self.max_zoom { return Ok(Vec::new()); @@ -260,17 +256,17 @@ fn verify_requirements( Ok(()) } -fn get_grid_dims( +fn get_image( decoder: &mut Decoder, path: &Path, - image_ifd: usize, -) -> Result<(u32, u32), FileError> { + ifd_index: usize, +) -> Result { let (tile_width, tile_height) = (decoder.chunk_dimensions().0, decoder.chunk_dimensions().1); - let (image_width, image_length) = dimensions_in_pixel(decoder, path, image_ifd)?; + let (image_width, image_length) = dimensions_in_pixel(decoder, path, ifd_index)?; let tiles_across = image_width.div_ceil(tile_width); let tiles_down = image_length.div_ceil(tile_height); - Ok((tiles_across, tiles_down)) + Ok(Image::new(ifd_index, tiles_across, tiles_down)) } /// Gets image pixel dimensions from TIFF decoder @@ -279,7 +275,7 @@ fn get_grid_dims( /// /// * `decoder` - TIFF decoder for reading image information /// * `path` - Image file path for error reporting -/// * `image_ifd` - Image file directory index +/// * `ifd_index` - Image file directory index /// /// # Returns /// @@ -291,13 +287,13 @@ fn get_grid_dims( fn dimensions_in_pixel( decoder: &mut Decoder, path: &Path, - image_ifd: usize, + ifd_index: usize, ) -> Result<(u32, u32), FileError> { let (image_width, image_length) = decoder.dimensions().map_err(|e| { CogError::TagsNotFound( e, vec![Tag::ImageWidth.to_u16(), Tag::ImageLength.to_u16()], - image_ifd, + ifd_index, path.to_path_buf(), ) })?; @@ -311,7 +307,7 @@ fn dimensions_in_pixel( /// /// * `decoder` - TIFF decoder for reading image information /// * `path` - Image file path for error reporting -/// * `image_ifd` - Image file directory index +/// * `ifd_index` - Image file directory index /// * `pixel_scale` - Optional pixel scale array [`ScaleX`, `ScaleY`, `ScaleZ`] /// * `transformation` - Optional 4x4 transformation matrix (16 elements) /// @@ -325,11 +321,11 @@ fn dimensions_in_pixel( fn dimensions_in_model( decoder: &mut Decoder, path: &Path, - image_ifd: usize, + ifd_index: usize, pixel_scale: Option<&[f64]>, transformation: Option<&[f64]>, ) -> Result<(f64, f64), FileError> { - let (image_width_pixel, image_length_pixel) = dimensions_in_pixel(decoder, path, image_ifd)?; + let (image_width_pixel, image_length_pixel) = dimensions_in_pixel(decoder, path, ifd_index)?; let full_resolution = get_full_resolution(pixel_scale, transformation, path).map_err(FileError::from)?; @@ -463,29 +459,28 @@ mod tests { use tiff::decoder::Decoder; use crate::cog::model::ModelInfo; + #[test] - fn can_get_model_infos() { + fn can_get_model_info() { let path = PathBuf::from("../tests/fixtures/cog/rgb_u8.tif"); let tif_file = File::open(&path).unwrap(); let mut decoder = Decoder::new(tif_file).unwrap(); - let model = ModelInfo::decode(&mut decoder, &path); - let (pixel_scale, tie_points, transformation) = - (model.pixel_scale, model.tie_points, model.transformation); - assert_yaml_snapshot!(pixel_scale, @r###" + + assert_yaml_snapshot!(model.pixel_scale, @r" - 10 - 10 - 0 - "###); - assert_yaml_snapshot!(tie_points, @r###" + "); + assert_yaml_snapshot!(model.tie_points, @r" - 0 - 0 - 0 - 1620750.2508 - 4277012.7153 - 0 - "###); - assert_yaml_snapshot!(transformation, @"~"); + "); + assert_yaml_snapshot!(model.transformation, @"~"); } #[rstest] From 3e9b7b03bdde65a445a955e28d7c360c60446ef9 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 7 Jun 2025 22:13:39 +0800 Subject: [PATCH 31/45] remove unnecessary Clippy expectations --- martin/src/cog/config.rs | 1 - martin/src/cog/source.rs | 3 --- 2 files changed, 4 deletions(-) diff --git a/martin/src/cog/config.rs b/martin/src/cog/config.rs index 038e16f20..f2bfd3a37 100644 --- a/martin/src/cog/config.rs +++ b/martin/src/cog/config.rs @@ -27,7 +27,6 @@ impl SourceConfigExtras for CogConfig { Ok(Box::new(cog)) } - #[expect(clippy::no_effect_underscore_binding)] async fn new_sources_url(&self, _id: String, _url: Url) -> FileResult> { unreachable!() } diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 9a704272c..073d0c06b 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -128,9 +128,6 @@ impl CogSource { }) } - #[expect(clippy::cast_sign_loss)] - #[expect(clippy::cast_possible_truncation)] - #[expect(clippy::too_many_lines)] pub fn get_tile(&self, xyz: TileCoord) -> MartinResult { if xyz.z < self.min_zoom || xyz.z > self.max_zoom { return Ok(Vec::new()); From 5d675468d2a1e036bfd0b7fa3646cc5c5d3b8c96 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 7 Jun 2025 22:24:12 +0800 Subject: [PATCH 32/45] clean up --- martin/src/cog/image.rs | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 837c79a5c..aece1cc69 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -30,15 +30,6 @@ impl Image { } /// Retrieves a tile from the image, decodes it, and converts it to PNG format. - /// - /// # Arguments - /// * `decoder` - A mutable reference to a TIFF decoder. - /// * `xyz` - The tile coordinates (z, x, y). - /// * [nodata](https://gdal.org/en/stable/drivers/raster/gtiff.html#nodata-value) - An optional nodata value. Pixels with this value will be made transparent. - /// * `path` - The path to the TIFF file, used for error reporting. - /// - /// # Returns - /// A `MartinResult` containing the tile data as a `Vec` (PNG bytes) or an error. #[expect(clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub fn get_tile( &self, @@ -103,23 +94,11 @@ impl Image { return None; } - let tile_idx = xyz.y * self.tiles_across + xyz.x; - Some(tile_idx) + Some(xyz.y * self.tiles_across + xyz.x) } } /// Converts RGB/RGBA tile data to PNG format. -/// -/// # Arguments -/// * `data` - Raw pixel data from TIFF decoder -/// * `tile_width`, `tile_height` - Expected tile dimensions -/// * `data_width`, `data_height` - Actual data dimensions -/// * `components_count` - Number of color components (3 for RGB, 4 for RGBA) -/// * `nodata` - Optional nodata value to make transparent -/// * `path` - File path for error reporting -/// -/// # Returns -/// PNG-encoded tile data as bytes fn rgb_to_png( data: Vec, (tile_width, tile_height): (u32, u32), @@ -139,16 +118,6 @@ fn rgb_to_png( } /// Ensures pixel data is valid for PNG encoding by handling padding, alpha channel, and nodata values. -/// -/// # Arguments -/// * `data` - Raw pixel data -/// * `tile_width`, `tile_height` - Target tile dimensions -/// * `data_width`, `data_height` - Source data dimensions -/// * `components_count` - Number of color components per pixel -/// * `nodata` - Optional value to treat as transparent -/// -/// # Returns -/// RGBA pixel data ready for PNG encoding fn ensure_pixels_valid( data: Vec, (tile_width, tile_height): (u32, u32), @@ -203,14 +172,6 @@ fn ensure_pixels_valid( } /// Encodes RGBA pixel data to PNG format. -/// -/// # Arguments -/// * `tile_width`, `tile_height` - Image dimensions -/// * `pixels` - RGBA pixel data -/// * `path` - File path for error reporting -/// -/// # Returns -/// PNG-encoded image data as bytes fn encode_rgba_to_png( tile_width: u32, tile_height: u32, From 7fd12c70901495e5838a7e06b11d9fce1f5bda39 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 00:38:11 +0800 Subject: [PATCH 33/45] fix bug --- martin/src/cog/source.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 073d0c06b..7342a6260 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -99,11 +99,12 @@ impl CogSource { let max_zoom = (images.len() - 1) as u8; let images: HashMap = images .iter() - .map(|image| { + .enumerate() + .map(|(idx, image)| { // FIXME: explain why it is OK to use IFD index as zoom level, // and why it would not exceed max_zoom. // This looks like a bug for some reason. - let zoom = max_zoom.saturating_sub(image.ifd_index() as u8); + let zoom = max_zoom.saturating_sub(idx as u8); (zoom, image.clone()) }) .collect(); From deb23b34d76babfc0a262ac80a48a0c72af74415 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 00:39:26 +0800 Subject: [PATCH 34/45] use ifd_index() method --- martin/src/cog/image.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index aece1cc69..2d04b798a 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -39,8 +39,8 @@ impl Image { path: &Path, ) -> MartinResult { decoder - .seek_to_image(self.ifd_index) - .map_err(|e| CogError::IfdSeekFailed(e, self.ifd_index, path.to_path_buf()))?; + .seek_to_image(self.ifd_index()) + .map_err(|e| CogError::IfdSeekFailed(e, self.ifd_index(), path.to_path_buf()))?; let tile_idx; if let Some(idx) = self.get_tile_index(xyz) { @@ -49,7 +49,7 @@ impl Image { return Ok(Vec::new()); } let decode_result = decoder.read_chunk(tile_idx).map_err(|e| { - CogError::ReadChunkFailed(e, tile_idx, self.ifd_index, path.to_path_buf()) + CogError::ReadChunkFailed(e, tile_idx, self.ifd_index(), path.to_path_buf()) })?; let color_type = decoder .colortype() From 8e161fb63af67b02f7cde38b80beeb398db460db Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 00:43:02 +0800 Subject: [PATCH 35/45] remove redundant documentation comments --- martin/src/cog/source.rs | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 7342a6260..1a7a3f3de 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -268,20 +268,6 @@ fn get_image( } /// Gets image pixel dimensions from TIFF decoder -/// -/// # Arguments -/// -/// * `decoder` - TIFF decoder for reading image information -/// * `path` - Image file path for error reporting -/// * `ifd_index` - Image file directory index -/// -/// # Returns -/// -/// Returns a tuple `(width, height)` containing image dimensions in pixels -/// -/// # Errors -/// -/// Returns `FileError` if image dimension tags cannot be read fn dimensions_in_pixel( decoder: &mut Decoder, path: &Path, @@ -300,22 +286,6 @@ fn dimensions_in_pixel( } /// Converts pixel dimensions to model space dimensions using resolution values -/// -/// # Arguments -/// -/// * `decoder` - TIFF decoder for reading image information -/// * `path` - Image file path for error reporting -/// * `ifd_index` - Image file directory index -/// * `pixel_scale` - Optional pixel scale array [`ScaleX`, `ScaleY`, `ScaleZ`] -/// * `transformation` - Optional 4x4 transformation matrix (16 elements) -/// -/// # Returns -/// -/// Tuple `(width, height)` in model coordinate system units -/// -/// # Errors -/// -/// Returns `FileError` if dimensions cannot be read or resolution calculation fails fn dimensions_in_model( decoder: &mut Decoder, path: &Path, @@ -408,9 +378,6 @@ fn raster2model(i: u32, j: u32, matrix: &[f64]) -> (f64, f64) { } /// Computes the bounding box (`[min_x, min_y, max_x, max_y]`) based on the transformation matrix, origin, width and hieght. -/// -/// Applies a transformation matrix to corner pixels if provided; -/// otherwise, computes extent from origin and raster size in model units. fn get_extent( origin: &[f64; 3], transformation: Option<&[f64]>, From dc7510adaa3fd88d3d7eaab2e168d8ed36a1351a Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 14:54:48 +0800 Subject: [PATCH 36/45] remove unnecessary cloning --- martin/src/cog/source.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 1a7a3f3de..6c87f1095 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -98,14 +98,11 @@ impl CogSource { let min_zoom = 0; let max_zoom = (images.len() - 1) as u8; let images: HashMap = images - .iter() + .into_iter() .enumerate() .map(|(idx, image)| { - // FIXME: explain why it is OK to use IFD index as zoom level, - // and why it would not exceed max_zoom. - // This looks like a bug for some reason. let zoom = max_zoom.saturating_sub(idx as u8); - (zoom, image.clone()) + (zoom, image) }) .collect(); let tilejson = tilejson! { From c42be1b1dd2915a4bfce078ed22534969844673d Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 15:12:48 +0800 Subject: [PATCH 37/45] add doc about tilegrid in COG --- martin/src/cog/image.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 2d04b798a..aa9aa7663 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -1,3 +1,33 @@ +//! ## The tilegrid in COG is highly cusotomized + +//! 1. A COG may have many image(also called `subfile`, see tiff spec 6.0) , many masks, and each indexed by IFD. +//! 2. A COG must have at least one image. +//! 3. The first ifd must be an image,eg ifd ==0, must be the full resolution one, eg, the one with the most high resolution one. +//! 4. The masks(they have ifd either) is used to idenfiy which pixel is nodata. Currently we ignore them directly. + +//! 5. The tilegrid of COG is highly customized as the extent of your COG might not be just aligned with any well know tilegrid's tiles. The possibility is very very low. Unless you make it so. + +//! ### An example and formulations + +//! Say we have 5 images and 5 mask in one COG, then the customized tilegrid would be: + +//! |ifd|idx in our Vec of Image|resolution|zoom of custom tilegrid| +//! |---|---------|--------|-------------------------| +//! |0 |0|20| 4 | +//! |2 |1|40| 3 | +//! |4 |2|80| 2 | +//! |6 |3|160| 1 | +//! |8 |4|320| 0 | + +//! And the formula here: + +//! ```rust +//! let images = vec![iamge 0, iamge 1, image 2, iamge 3, image 4]; +//! let minzoom = 0; +//! let zoom_of_image = image_count -1 - idx_in_vec; +//! let maxzoom = image_count - 1; +//! ``` + use std::fs::File; use std::io::BufWriter; use std::path::Path; From e888abc19f659561ab3f67b37fd5ffb682750f02 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sun, 8 Jun 2025 15:25:56 +0800 Subject: [PATCH 38/45] ignore syntax highlighting in doc --- martin/src/cog/image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index aa9aa7663..692760ff6 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -21,7 +21,7 @@ //! And the formula here: -//! ```rust +//! ```rust, ignore //! let images = vec![iamge 0, iamge 1, image 2, iamge 3, image 4]; //! let minzoom = 0; //! let zoom_of_image = image_count -1 - idx_in_vec; From 60dd18886f532d9de552835118b74499ce194e37 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 10 Jun 2025 01:02:34 -0400 Subject: [PATCH 39/45] move docs to readme --- martin/src/cog/README.md | 29 +++++++++++++++++++++++++++++ martin/src/cog/image.rs | 30 ------------------------------ martin/src/cog/mod.rs | 2 ++ 3 files changed, 31 insertions(+), 30 deletions(-) create mode 100644 martin/src/cog/README.md diff --git a/martin/src/cog/README.md b/martin/src/cog/README.md new file mode 100644 index 000000000..da86e2180 --- /dev/null +++ b/martin/src/cog/README.md @@ -0,0 +1,29 @@ +## COG Image Representation + +* COG file is an image container representing a tile grid +* A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - `IFD`. (TODO: link to TIFF 6.0 spec) +* A COG must have at least one image. +* The first image (IFD=0) must be a full resolution image, e.g., the one with the highest resolution. +* Each image may also have a mask, which is also indexed with an IFD. The mask is used to identify which pixel is nodata. We do not support masks yet. (TODO: link) +* While uncommon, COG tile grid might be different from the common ones like Web Mercator. + +### COG structure example + +Here is an example of a tile grid for a COG file with five images and five masks. + +| ifd | image index | resolution | zoom | +|-----|-------------|------------|------| +| 0 | 0 | 20 | 4 | +| 2 | 1 | 40 | 3 | +| 4 | 2 | 80 | 2 | +| 6 | 3 | 160 | 1 | +| 8 | 4 | 320 | 0 | + +### Tile grid code representation + +```rust, ignore +let images = vec![ image_0, image_1, image_2, image_3, image_4 ]; +let minzoom = 0; +let zoom_of_image = image_count - 1 - idx_in_vec; # TODO: what is this? +let maxzoom = images.len() - 1; +``` diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 692760ff6..2d04b798a 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -1,33 +1,3 @@ -//! ## The tilegrid in COG is highly cusotomized - -//! 1. A COG may have many image(also called `subfile`, see tiff spec 6.0) , many masks, and each indexed by IFD. -//! 2. A COG must have at least one image. -//! 3. The first ifd must be an image,eg ifd ==0, must be the full resolution one, eg, the one with the most high resolution one. -//! 4. The masks(they have ifd either) is used to idenfiy which pixel is nodata. Currently we ignore them directly. - -//! 5. The tilegrid of COG is highly customized as the extent of your COG might not be just aligned with any well know tilegrid's tiles. The possibility is very very low. Unless you make it so. - -//! ### An example and formulations - -//! Say we have 5 images and 5 mask in one COG, then the customized tilegrid would be: - -//! |ifd|idx in our Vec of Image|resolution|zoom of custom tilegrid| -//! |---|---------|--------|-------------------------| -//! |0 |0|20| 4 | -//! |2 |1|40| 3 | -//! |4 |2|80| 2 | -//! |6 |3|160| 1 | -//! |8 |4|320| 0 | - -//! And the formula here: - -//! ```rust, ignore -//! let images = vec![iamge 0, iamge 1, image 2, iamge 3, image 4]; -//! let minzoom = 0; -//! let zoom_of_image = image_count -1 - idx_in_vec; -//! let maxzoom = image_count - 1; -//! ``` - use std::fs::File; use std::io::BufWriter; use std::path::Path; diff --git a/martin/src/cog/mod.rs b/martin/src/cog/mod.rs index baab4a656..dfcb179dc 100644 --- a/martin/src/cog/mod.rs +++ b/martin/src/cog/mod.rs @@ -1,3 +1,5 @@ +#![doc = include_str!("README.md")] + mod config; mod errors; mod image; From 4f5279ea8d03b0d78f8bfb0d02dafece370a915a Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 10 Jun 2025 01:04:12 -0400 Subject: [PATCH 40/45] do not change cargo files --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ea251aca..fa7f9e5da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,9 +1270,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", "clap_derive", @@ -1280,9 +1280,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", @@ -1293,9 +1293,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "anstyle", "heck", @@ -1997,7 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2118,9 +2118,9 @@ checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", "miniz_oxide", @@ -3024,7 +3024,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4429,7 +4429,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4820,7 +4820,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4833,7 +4833,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5394,9 +5394,9 @@ dependencies = [ [[package]] name = "sqlite-compressions" -version = "0.3.7" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5e2f1b577d8755cbf708cf55ef7b3cf451be9577a286f1e7c40dc1bbe0c9b3" +checksum = "74bc4b93c179134d055c0e07171e755a1ac338a0e780added5aa72f6d88f7f39" dependencies = [ "bsdiff", "flate2", @@ -5405,9 +5405,9 @@ dependencies = [ [[package]] name = "sqlite-hashes" -version = "0.10.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da3a1f764fbdce918492858d08764a4231b8f39d4307c012b56742e2ba9a365" +checksum = "e46771a6e7c04353fc993ebf4909a416d8460e2c3d77b93115e4c9cd7dc7ec91" dependencies = [ "digest", "hex", @@ -5796,7 +5796,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6729,7 +6729,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fa3705734..54134c2f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ serde_yaml = "0.9" size_format = "1.0.2" spreet = { version = "0.11", default-features = false } sqlite-compressions = { version = "0.3", default-features = false, features = ["bsdiffraw", "gzip"] } -sqlite-hashes = { version = "0.10.6", default-features = false, features = ["md5", "aggregate", "hex"] } +sqlite-hashes = { version = "0.10.1", default-features = false, features = ["md5", "aggregate", "hex"] } sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio"] } static-files = "0.2" subst = { version = "0.3", features = ["yaml"] } From b05100992aefca4de8aac360179b8b4ea822481d Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 10 Jun 2025 01:30:58 -0400 Subject: [PATCH 41/45] more cleanup --- martin/src/cog/config.rs | 8 ++--- martin/src/cog/image.rs | 71 +++++++++++++++++++++------------------- martin/src/cog/model.rs | 9 ++--- martin/src/cog/source.rs | 6 ++-- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/martin/src/cog/config.rs b/martin/src/cog/config.rs index f2bfd3a37..a04614204 100644 --- a/martin/src/cog/config.rs +++ b/martin/src/cog/config.rs @@ -22,6 +22,10 @@ impl ConfigExtras for CogConfig { } impl SourceConfigExtras for CogConfig { + fn parse_urls() -> bool { + false + } + async fn new_sources(&self, id: String, path: PathBuf) -> FileResult> { let cog = CogSource::new(id, path)?; Ok(Box::new(cog)) @@ -30,8 +34,4 @@ impl SourceConfigExtras for CogConfig { async fn new_sources_url(&self, _id: String, _url: Url) -> FileResult> { unreachable!() } - - fn parse_urls() -> bool { - false - } } diff --git a/martin/src/cog/image.rs b/martin/src/cog/image.rs index 2d04b798a..5c655e871 100644 --- a/martin/src/cog/image.rs +++ b/martin/src/cog/image.rs @@ -114,7 +114,7 @@ fn rgb_to_png( components_count, nodata, ); - encode_rgba_to_png(tile_width, tile_height, &pixels, path) + encode_rgba_as_png(tile_width, tile_height, &pixels, path) } /// Ensures pixel data is valid for PNG encoding by handling padding, alpha channel, and nodata values. @@ -126,42 +126,45 @@ fn ensure_pixels_valid( nodata: Option, ) -> Vec { let is_padded = data_width != tile_width || data_height != tile_height; - let need_add_alpha = components_count != 4; - // 1. Check if the tile is padded, if so, we need to add padding part back - // The decoded might be smaller than the tile size as tiff crate always cut off the padding part - // So we need to add the padding part back if needed - // 2. Check if we need to add alpha channel, if the components count is not 4(rgba => 4 components, rgb => 3 components), we need to add an alpha channel - // 3. Check if nodata is provided, if so, we need to make the pixels with nodata value transparent + // FIXME: why not `== 3`? + let add_alpha = components_count != 4; + // 1. Check if the tile is padded. If so, we need to add padding part back. + // The decoded value might be smaller than the tile size. + // TIFF crate always cut off the padding part, so we would need to add the padding part back. + // 2. If the components count is 3 (RGB), we need to add the alpha channel to convert it to RGBA. + // 3. Check if nodata is provided. We need to make the pixels with nodata value transparent // See https://gdal.org/en/stable/drivers/raster/gtiff.html#nodata-value - if nodata.is_some() || need_add_alpha || is_padded { + if nodata.is_some() || add_alpha || is_padded { let mut result_vec = vec![0; (tile_width * tile_height * 4) as usize]; for row in 0..data_height { 'outer: for col in 0..data_width { let idx_chunk = row * data_width * components_count + col * components_count; let idx_result = row * tile_width * 4 + col * 4; - // Copy the components one by one + // Copy component values one by one for component_idx in 0..components_count { - // Before copying, check if this component == nodata. If so, do skip and it would be transparent. - // FIXME: Should we copy the RGB values anyway and just set alpha to 0? The visual result actually is the same (transparent), but the component values would differ. But it might be a little slower as we don't skip the copy - // Source pixel: [4, 1, 2, 3] nodata: Some(1) - // Do skip: - // result pixel: [4, 0, 0, 0] - // Do not skip: - // result pixel: [4, 1, 2, 0] - // So the visual result is the same, but the component values are different. + // Before copying, check if this component == nodata. If so, skip because it's transparent. + // FIXME: Should we copy the RGB values anyway and just set alpha to 0? + // The visual result is the same (transparent), but the component values would differ. + // But it might be a little slower as we don't skip the copy. + // Source pixel: [4, 1, 2, 3] nodata: Some(1) + // Skip: + // result pixel: [4, 0, 0, 0] + // Do not skip: + // result pixel: [4, 1, 2, 0] + // So the visual result is the same, but the component values are different. - if nodata.eq(&Some(data[(idx_chunk + component_idx) as usize])) { - continue 'outer; + let value = data[(idx_chunk + component_idx) as usize]; + if let Some(v) = nodata { + if value == v { + continue 'outer; + } } // Copy this component to the result vector - result_vec[(idx_result + component_idx) as usize] = - data[(idx_chunk + component_idx) as usize]; + result_vec[(idx_result + component_idx) as usize] = value; } - // If an alpha channel needs to be added, set it to 255 (opaque) - if need_add_alpha { - let alpha_idx = (idx_result + 3) as usize; - result_vec[alpha_idx] = 255; + if add_alpha { + result_vec[(idx_result + 3) as usize] = 255; // opaque } } } @@ -172,7 +175,7 @@ fn ensure_pixels_valid( } /// Encodes RGBA pixel data to PNG format. -fn encode_rgba_to_png( +fn encode_rgba_as_png( tile_width: u32, tile_height: u32, pixels: &[u8], @@ -228,33 +231,33 @@ mod tests { // the right half should be transparent #[case( "../tests/fixtures/cog/expected/right_padded.png", - (0,0,0,None),None,(128,256),(256,256) + (0, 0, 0, None), None, (128, 256), (256, 256) )] // the down half should be transparent #[case( "../tests/fixtures/cog/expected/down_padded.png", - (0,0,0,None),None,(256,128),(256,256) + (0, 0, 0, None), None, (256, 128), (256, 256) )] - // the up half should be half transparent and down half should be transparent + // the up half should be half-transparent and down half should be transparent #[case( "../tests/fixtures/cog/expected/down_padded_with_alpha.png", - (0,0,0,Some(128)),None,(256,128),(256,256) + (0, 0, 0, Some(128)), None, (256, 128), (256, 256) )] - // the left half should be half transparent and the right half should be transparent + // the left half should be half-transparent and the right half should be transparent #[case( "../tests/fixtures/cog/expected/right_padded_with_alpha.png", - (0,0,0,Some(128)),None,(128,256),(256,256) + (0, 0, 0, Some(128)), None, (128, 256), (256, 256) )] // should be all half transparent #[case( "../tests/fixtures/cog/expected/not_padded.png", - (0,0,0,Some(128)),None,(256,256),(256,256) + (0, 0, 0, Some(128)), None, (256, 256), (256, 256) )] // all padded and with a no_data whose value is 128, and all the component is 128 // so that should be all transparent #[case( "../tests/fixtures/cog/expected/all_transparent.png", - (128,128,128,Some(128)),Some(128),(128,128),(256,256) + (128, 128, 128, Some(128)), Some(128), (128, 128), (256, 256) )] fn test_padded_cases( #[case] expected_file_path: &str, diff --git a/martin/src/cog/model.rs b/martin/src/cog/model.rs index c69202cb6..c37a78502 100644 --- a/martin/src/cog/model.rs +++ b/martin/src/cog/model.rs @@ -6,9 +6,10 @@ use tiff::tags::Tag; use super::CogError; -/// These are tags to be used for defining the relationship between raster space and model space. See [ogc doc](https://docs.ogc.org/is/19-008r4/19-008r4.html#_coordinate_transformations) for more details. +/// These tags define the relationship between raster space and model space. +/// See [ogc doc](https://docs.ogc.org/is/19-008r4/19-008r4.html#_coordinate_transformations) for details. /// -/// The the relationship may be diagrammed as: +/// The relationship may be diagrammed as: /// ```raw /// ModelPixelScaleTag /// ModelTiepointTag @@ -17,7 +18,7 @@ use super::CogError; /// ``` #[derive(Clone, Debug)] pub struct ModelInfo { - /// `ModelPixelScaleTag`, may be used to specify the size of raster pixel spacing in the model space units, when the raster space can be embedded in the model space coordinate reference system without rotation. + /// `ModelPixelScaleTag` may be used to specify the size of raster pixel spacing in the model space units, when the raster space can be embedded in the model space coordinate reference system without rotation. /// Consists of the following 3 values: `(ScaleX, ScaleY, ScaleZ)`. /// /// ```raw @@ -37,7 +38,7 @@ pub struct ModelInfo { /// ModelTiepointTag: /// Tag = 33922 (8482.H) /// Type = DOUBLE (IEEE Double precision) - /// N = 6*K, K = number of tiepoints + /// N = 6*K, K = number of tie-points /// Alias: GeoreferenceTag /// ``` /// diff --git a/martin/src/cog/source.rs b/martin/src/cog/source.rs index 6c87f1095..d80d90774 100644 --- a/martin/src/cog/source.rs +++ b/martin/src/cog/source.rs @@ -178,7 +178,7 @@ fn verify_requirements( path: &Path, ) -> Result<(), CogError> { let chunk_type = decoder.get_chunk_type(); - // see the requirement 2 in https://docs.ogc.org/is/21-026/21-026.html#_tiles + // see requirement 2 in https://docs.ogc.org/is/21-026/21-026.html#_tiles if chunk_type != ChunkType::Tile { Err(CogError::NotSupportedChunkType(path.to_path_buf()))?; } @@ -358,7 +358,7 @@ fn get_full_resolution( let mut x_res = (matrix[0] * matrix[0] + matrix[4] * matrix[4]).sqrt(); x_res = x_res.copysign(matrix[0]); let mut y_res = (matrix[1] * matrix[1] + matrix[5] * matrix[5]).sqrt(); - // A positive y_res indicates that model space Y cordinates decrease as raster space J indices increase. This is the standard vertical relationship between raster space and model space + // A positive y_res indicates that model space Y coordinates decrease as raster space J indices increase. This is the standard vertical relationship between raster space and model space y_res = y_res.copysign(-matrix[5]); Ok([x_res, y_res]) // drop the z scale directly as we don't use it } @@ -374,7 +374,7 @@ fn raster2model(i: u32, j: u32, matrix: &[f64]) -> (f64, f64) { (x, y) } -/// Computes the bounding box (`[min_x, min_y, max_x, max_y]`) based on the transformation matrix, origin, width and hieght. +/// Computes the bounding box (`[min_x, min_y, max_x, max_y]`) based on the transformation matrix, origin, width, and height. fn get_extent( origin: &[f64; 3], transformation: Option<&[f64]>, From 799a0683a3928bb6635f408030f66135c1526c71 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 10 Jun 2025 10:56:47 -0400 Subject: [PATCH 42/45] sync to main --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa7f9e5da..1ea251aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,9 +1270,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -1280,9 +1280,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -1293,9 +1293,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "anstyle", "heck", @@ -1997,7 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2118,9 +2118,9 @@ checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "miniz_oxide", @@ -3024,7 +3024,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4429,7 +4429,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4820,7 +4820,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4833,7 +4833,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5394,9 +5394,9 @@ dependencies = [ [[package]] name = "sqlite-compressions" -version = "0.3.4" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74bc4b93c179134d055c0e07171e755a1ac338a0e780added5aa72f6d88f7f39" +checksum = "5c5e2f1b577d8755cbf708cf55ef7b3cf451be9577a286f1e7c40dc1bbe0c9b3" dependencies = [ "bsdiff", "flate2", @@ -5405,9 +5405,9 @@ dependencies = [ [[package]] name = "sqlite-hashes" -version = "0.10.2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46771a6e7c04353fc993ebf4909a416d8460e2c3d77b93115e4c9cd7dc7ec91" +checksum = "2da3a1f764fbdce918492858d08764a4231b8f39d4307c012b56742e2ba9a365" dependencies = [ "digest", "hex", @@ -5796,7 +5796,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6729,7 +6729,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 54134c2f7..fa3705734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ serde_yaml = "0.9" size_format = "1.0.2" spreet = { version = "0.11", default-features = false } sqlite-compressions = { version = "0.3", default-features = false, features = ["bsdiffraw", "gzip"] } -sqlite-hashes = { version = "0.10.1", default-features = false, features = ["md5", "aggregate", "hex"] } +sqlite-hashes = { version = "0.10.6", default-features = false, features = ["md5", "aggregate", "hex"] } sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio"] } static-files = "0.2" subst = { version = "0.3", features = ["yaml"] } From 8b5ec1c2a279892d732a823817dffeb002701158 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 12 Jun 2025 12:14:45 +0800 Subject: [PATCH 43/45] update readme.md --- martin/src/cog/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/martin/src/cog/README.md b/martin/src/cog/README.md index da86e2180..84b397a27 100644 --- a/martin/src/cog/README.md +++ b/martin/src/cog/README.md @@ -1,10 +1,10 @@ ## COG Image Representation * COG file is an image container representing a tile grid -* A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - `IFD`. (TODO: link to TIFF 6.0 spec) +* A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - [`IFD`](https://download.osgeo.org/libtiff/doc/TIFF6.pdf#[{%22num%22:209,%22gen%22:0},{%22name%22:%22FitB%22}]#[{"num":76,"gen":0},{"name":"FitB"}]) * A COG must have at least one image. * The first image (IFD=0) must be a full resolution image, e.g., the one with the highest resolution. -* Each image may also have a mask, which is also indexed with an IFD. The mask is used to identify which pixel is nodata. We do not support masks yet. (TODO: link) +* [Each image may also have a mask](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles), which is also indexed with an IFD. The mask is used to defines a transparency mask[https://www.verypdf.com/document/tiff6/pg_0036.htm]. We do not support masks yet. * While uncommon, COG tile grid might be different from the common ones like Web Mercator. ### COG structure example From 1ccf0f895e3ed30d30b4bacc552a69ceeb700340 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 05:14:37 +0000 Subject: [PATCH 44/45] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- martin/src/cog/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/cog/README.md b/martin/src/cog/README.md index 84b397a27..c668d700b 100644 --- a/martin/src/cog/README.md +++ b/martin/src/cog/README.md @@ -4,7 +4,7 @@ * A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - [`IFD`](https://download.osgeo.org/libtiff/doc/TIFF6.pdf#[{%22num%22:209,%22gen%22:0},{%22name%22:%22FitB%22}]#[{"num":76,"gen":0},{"name":"FitB"}]) * A COG must have at least one image. * The first image (IFD=0) must be a full resolution image, e.g., the one with the highest resolution. -* [Each image may also have a mask](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles), which is also indexed with an IFD. The mask is used to defines a transparency mask[https://www.verypdf.com/document/tiff6/pg_0036.htm]. We do not support masks yet. +* [Each image may also have a mask](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles), which is also indexed with an IFD. The mask is used to defines a transparency mask[https://www.verypdf.com/document/tiff6/pg_0036.htm]. We do not support masks yet. * While uncommon, COG tile grid might be different from the common ones like Web Mercator. ### COG structure example From a624661c70a5bd19978c6499446cea43be510b8f Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 12 Jun 2025 14:27:09 +0800 Subject: [PATCH 45/45] format readme.md --- martin/src/cog/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/martin/src/cog/README.md b/martin/src/cog/README.md index c668d700b..01345b93e 100644 --- a/martin/src/cog/README.md +++ b/martin/src/cog/README.md @@ -1,10 +1,10 @@ ## COG Image Representation * COG file is an image container representing a tile grid -* A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - [`IFD`](https://download.osgeo.org/libtiff/doc/TIFF6.pdf#[{%22num%22:209,%22gen%22:0},{%22name%22:%22FitB%22}]#[{"num":76,"gen":0},{"name":"FitB"}]) +* A COG may have multiple images, also called subfiles, each indexed with an Image File Directory number - [`IFD`](https://download.osgeo.org/libtiff/doc/TIFF6.pdf#[{"num":209,"gen":0},{"name":"FitB"}]#[{"num":76,"gen":0},{"name":"FitB"}]#[{"num":76,"gen":0},{"name":"FitB"}]]) * A COG must have at least one image. * The first image (IFD=0) must be a full resolution image, e.g., the one with the highest resolution. -* [Each image may also have a mask](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles), which is also indexed with an IFD. The mask is used to defines a transparency mask[https://www.verypdf.com/document/tiff6/pg_0036.htm]. We do not support masks yet. +* [Each image may also have a mask](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles), which is also indexed with an IFD. The mask is used to [defines a transparency mask](https://www.verypdf.com/document/tiff6/pg_0036.htm). We do not support masks yet. * While uncommon, COG tile grid might be different from the common ones like Web Mercator. ### COG structure example @@ -12,7 +12,7 @@ Here is an example of a tile grid for a COG file with five images and five masks. | ifd | image index | resolution | zoom | -|-----|-------------|------------|------| +| --- | ----------- | ---------- | ---- | | 0 | 0 | 20 | 4 | | 2 | 1 | 40 | 3 | | 4 | 2 | 80 | 2 |