Add lz4 compression to planetiler (#1104)

onthegomap · Nov 23, 2024 · 8a1b54f · 8a1b54f
1 parent 396fa7e
commit 8a1b54f
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 3 deletions.
diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml
@@ -181,6 +181,11 @@
       <artifactId>parquet-floor</artifactId>
       <version>1.48</version>
     </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <version>1.8.0</version>
+    </dependency>
 
   </dependencies>
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java
@@ -22,6 +22,9 @@
 import java.util.function.IntUnaryOperator;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
 import org.locationtech.jts.geom.Envelope;
 
 /**
@@ -74,8 +77,24 @@ private static byte[] readBlobContent(Fileformat.Blob blob) {
         throw new FileFormatException("PBF blob contains incomplete compressed data.");
       }
       inflater.end();
+    } else if (blob.hasLz4Data()) {
+      final int decompressedLength = blob.getRawSize();
+      LZ4Factory factory = LZ4Factory.fastestInstance();
+      LZ4FastDecompressor decompressor = factory.fastDecompressor();
+      blobData = new byte[decompressedLength];
+      try {
+        int compressedBytesRead =
+          decompressor.decompress(blob.getLz4Data().toByteArray(), 0, blobData, 0, decompressedLength);
+        int compressedBytesExpected = blob.getLz4Data().size();
+        if (compressedBytesRead != compressedBytesExpected) {
+          throw new FileFormatException("Unable to decompress PBF blob. read %d compressed bytes but expected %d"
+            .formatted(decompressedLength, compressedBytesExpected));
+        }
+      } catch (LZ4Exception e) {
+        throw new FileFormatException("Unable to decompress PBF blob.", e);
+      }
     } else {
-      throw new FileFormatException("PBF blob uses unsupported compression, only raw or zlib may be used.");
+      throw new FileFormatException("PBF blob uses unsupported compression, only lz4, zlib, or raw may be used.");
     }
 
     return blobData;

diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -2088,10 +2088,13 @@ private static TileCompression extractTileCompression(String args) {
     "--tile-compression=none",
     "--tile-compression=gzip",
     "--output-layerstats",
-    "--max-point-buffer=1"
+    "--max-point-buffer=1",
+    "--osm-test-path=monaco-latest.lz4.osm.pbf",
   })
   void testPlanetilerRunner(String args) throws Exception {
-    Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf");
+    var argParsed = Arguments.fromArgs(args.split(" "));
+    Path originalOsm =
+      TestUtils.pathToResource(argParsed.getString("osm-test-path", "osm-test-path", "monaco-latest.osm.pbf"));
     Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf");
     final TileCompression tileCompression = extractTileCompression(args);
 

diff --git a/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf b/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf