Complex packing

Cargo.toml

@@ -11,11 +11,12 @@ chrono = "0.4"
 openjpeg-sys = { version = "1.0.3", optional = true }
 png = { version = "0.17.2", optional = true }
 num = { version = "0.2", default-features = false }
+itertools = "0.10.5"
 
 [profile.release]
 debug = true
 
 [features]
 default = ["png", "jpeg"]
 png = ["dep:png"]
-jpeg = ["dep:openjpeg-sys"]
+jpeg = ["dep:openjpeg-sys"]

examples/message-dump/main.rs

@@ -1,7 +1,6 @@
 extern crate chrono;
 extern crate gribberish;
 
-use chrono::prelude::*;
 use gribberish::message::{Message, read_messages};
 use std::env;
 use std::fs::File;
@@ -29,29 +28,33 @@ fn main() {
     println!("GRIB2 file read: {}", grib_path);
     println!("Message count: {}", messages.len());
     println!(
-        "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+        "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
         "Message #",
         "Variable",
+        "Name",
         "Units",
+        "Product Template Id",
         "Date",
         "Region",
         "Grid",
         "Data Template Id",
-        "Data Point Count"
+        "Data Point Count",
     );
     println!("------------------------------------------------------------------------------------------------------------");
 
     messages.iter().enumerate().for_each(|m| {
         println!(
-            "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+            "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
             m.0,
             match m.1.variable_abbrev() { Ok(p) => p, Err(_) => m.1.parameter_index().unwrap_or("--".into())},
+            match m.1.variable_name() {Ok(p) => p, Err(_) => m.1.parameter_index().unwrap_or("--".into())},
             match m.1.unit() {Ok(p) => p, Err(_) => "--".into()},
+            match m.1.product_template_id() {Ok(p) => format!("{p}"), Err(_) => "--".into()},
             match m.1.forecast_date() { Ok(d) => format!("{}", d), Err(_) => "--".into()},
             match m.1.location_region() {Ok(r) => format!("{:?}", r), Err(_) => "--".into()},
             match m.1.location_grid_dimensions() {Ok(r) => format!("{:?}", r), Err(_) => "--".into()},
-            match m.1.data_template_number() {Ok(t) => format!("{}", t), Err(_) => "--".into()},
-            match m.1.data_point_count() {Ok(c) => format!("{}", c), Err(_) => "--".into()},
+            match m.1.data_template_number() {Ok(t) => format!("{t}"), Err(_) => "--".into()},
+            match m.1.data_point_count() {Ok(c) => format!("{c}"), Err(_) => "--".into()},
         );
     });
 }

node/examples/gfsnap/index.mjs

@@ -1,16 +1,24 @@
 import fs from 'fs';
-import { parseMessagesFromBuffer } from 'gribberishjs';
-import { createCanvas, Image } from 'canvas';
+import { GribMessageFactory } from 'gribberishjs';
 import { Resvg } from '@resvg/resvg-js';
 import * as d3 from 'd3';
 
-const gribData = fs.readFileSync('/Users/matthewiannucci/Downloads/MRMS_MergedReflectivityQCComposite_00.50_20230106-000439.grib2');
-const gribMessages = parseMessagesFromBuffer(gribData);
-const message = gribMessages.find(g => g.varAbbrev === 'MergedReflectivityQCComposite');
+// WAVE
+// const gribPath = './data/gfswave.20221222.t18z.atlocn.0p16.f064.grib2'
+// const gribVariable = 'HTSGW@groundorwater_1'
+// RADAR
+// const gribPath = '/Users/matthewiannucci/Downloads/MRMS_MergedReflectivityQCComposite_00.50_20230106-000439.grib2'
+// const gribVariable = 'MergedReflectivityQCComposite'
+const gribPath = '/Users/matthewiannucci/Downloads/gfs.t18z.pgrb2.0p25.f186.grib2';
+const gribVariable = 'GUST@groundorwater_0';
 
-// const gribData = fs.readFileSync('./data/gfswave.20221222.t18z.atlocn.0p16.f064.grib2');
-// const gribMessages = parseMessagesFromBuffer(gribData);
-// const message = gribMessages.find(g => g.varAbbrev === 'HTSGW');
+const gribData = fs.readFileSync(gribPath);
+const messageFactory = GribMessageFactory.fromBuffer(gribData);
+
+// console.log(messageFactory.availableMessages);
+// process.exit(0);
+
+const message = messageFactory.getMessage(gribVariable);
 
 if (message !== undefined) {
   console.log('Found matching grib message, contouring...');
@@ -27,19 +35,22 @@ const height = message.latitudes.length;
 const width = message.longitudes.length;
 
 const values = message.data;
+const max = d3.max(values);
+const min = d3.min(values);
+const range = max - min;
+const steps = 20;
+
 for (let i = 0; i < values.length; ++i) {
   if (isNaN(values[i])) {
-    values[i] = -99999;
+    values[i] = -9999999;
   }
 }
 
-const max = d3.max(values);
-
 // const blurredValues = d3.blur2({ data: swhMessage.data, width }, 0.5).data;
 const contours = d3
   .contours()
   .size([width, height])
-  .thresholds(Array.from({ length: 10 }, (_, i) => i / 10 * max));
+  .thresholds(Array.from({ length: steps }, (_, i) => min + (i / steps * range)));
 
 const color = d3.scaleSequential([max, 0], d3.interpolateRdBu);
 
@@ -70,14 +81,14 @@ const svgout = `
 `;
 
 console.log('Writing to SVG file...');
-fs.writeFileSync('vector.svg', svgout);
+fs.writeFileSync(`${gribVariable}.svg`, svgout);
 
 console.log('Rendering svg to image...');
 const resvg = new Resvg(svgout)
 const pngData = resvg.render()
 const pngBuffer = pngData.asPng()
 
 console.log('Writing to PNG file...');
-fs.writeFileSync("./rendered.png", pngBuffer);
+fs.writeFileSync(`./${gribVariable}.png`, pngBuffer);
 
 console.log('Operation Successful!');

node/index.d.ts

@@ -13,7 +13,6 @@ export class GribMessage {
   get varName(): string
   get varAbbrev(): string
   get units(): string
-  get arrayIndex(): number
   get forecastDate(): Date
   get referenceDate(): Date
   get proj(): string
@@ -25,3 +24,8 @@ export class GribMessage {
   get longitudes(): Float64Array
   get data(): Float64Array
 }
+export class GribMessageFactory {
+  static fromBuffer(buffer: Buffer): GribMessageFactory
+  get availableMessages(): Array<string>
+  getMessage(key: string): GribMessage
+}

node/index.js

@@ -237,7 +237,8 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { GribMessage, parseMessagesFromBuffer } = nativeBinding
+const { GribMessage, parseMessagesFromBuffer, GribMessageFactory } = nativeBinding
 
 module.exports.GribMessage = GribMessage
 module.exports.parseMessagesFromBuffer = parseMessagesFromBuffer
+module.exports.GribMessageFactory = GribMessageFactory

node/src/lib.rs

@@ -1,8 +1,10 @@
 #![deny(clippy::all)]
 
+use std::collections::HashMap;
+
 use gribberish::{
   data_message::DataMessage,
-  message::{self, read_message, read_messages, Message},
+  message::{scan_messages, read_message, read_messages},
 };
 use napi::{
   bindgen_prelude::{Array, Buffer, Float64Array},
@@ -29,83 +31,90 @@ impl GribMessage {
   pub fn parse_from_buffer(buffer: Buffer, offset: u32) -> Self {
     let buf: Vec<u8> = buffer.into();
     let message = read_message(&buf, offset as usize).unwrap();
-    let message = DataMessage::try_from(message).unwrap();
+    let message = DataMessage::try_from(&message).unwrap();
+
+    GribMessage { inner: message }
+  }
+
+  pub fn parse_from_bytes(data: &[u8], offset: usize) -> Self {
+    let message = read_message(&data, offset).unwrap();
+    let message = DataMessage::try_from(&message).unwrap();
 
     GribMessage { inner: message }
   }
 
   #[napi(getter)]
   pub fn var_name(&self) -> &str {
-    self.inner.name.as_str()
+    self.inner.metadata.name.as_str()
   }
 
   #[napi(getter)]
   pub fn var_abbrev(&self) -> &str {
-    self.inner.var.as_str()
+    self.inner.metadata.var.as_str()
   }
 
   #[napi(getter)]
   pub fn units(&self) -> &str {
-    self.inner.units.as_str()
+    self.inner.metadata.units.as_str()
   }
 
-  #[napi(getter)]
-  pub fn array_index(&self) -> u32 {
-    self.inner.array_index.unwrap_or(0) as u32
-  }
+  // #[napi(getter)]
+  // pub fn array_index(&self) -> u32 {
+  //   self.inner.metadata.array_index.unwrap_or(0) as u32
+  // }
 
   #[napi(getter)]
   pub fn forecast_date(&self) -> chrono::DateTime<chrono::Utc> {
-    self.inner.forecast_date
+    self.inner.metadata.forecast_date
   }
 
   #[napi(getter)]
   pub fn reference_date(&self) -> chrono::DateTime<chrono::Utc> {
-    self.inner.reference_date
+    self.inner.metadata.reference_date
   }
 
   #[napi(getter)]
   pub fn proj(&self) -> &str {
-    self.inner.proj.as_str()
+    self.inner.metadata.proj.as_str()
   }
 
   #[napi(getter)]
   pub fn crs(&self) -> &str {
-    self.inner.crs.as_str()
+    self.inner.metadata.crs.as_str()
   }
 
   #[napi(getter)]
   pub fn bbox(&self) -> Vec<f64> {
-    let bbox = &self.inner.bbox;
+    let bbox = &self.inner.metadata.bbox;
     vec![bbox.0, bbox.1, bbox.2, bbox.3]
   }
 
   #[napi(getter)]
   pub fn grid_shape(&self) -> GridShape {
-    let (rows, cols) = self.inner.grid_shape();
+    let (rows, cols) = self.inner.metadata.grid_shape();
     GridShape {
-      rows: rows as u32, 
-      cols: cols as u32
+      rows: rows as u32,
+      cols: cols as u32,
     }
   }
 
   #[napi(getter)]
   pub fn grid_resolution(&self) -> GridShape {
-    let (rows, cols) = self.inner.grid_resolution;
+    let (rows, cols) = self.inner.metadata.grid_resolution;
     GridShape {
-      rows: rows as u32, 
-      cols: cols as u32
+      rows: rows as u32,
+      cols: cols as u32,
     }
   }
 
   #[napi(getter)]
   pub fn latitudes(&self) -> Float64Array {
-    Float64Array::new(self.inner.latitude.clone())
+    Float64Array::new(self.inner.metadata.latitude.clone())
   }
 
   #[napi(getter)]
   pub fn longitudes(&self) -> Float64Array {
-    Float64Array::new(self.inner.longitude.clone())
+    Float64Array::new(self.inner.metadata.longitude.clone())
   }
 
   #[napi(getter)]
@@ -122,11 +131,38 @@ pub fn parse_messages_from_buffer(buffer: Buffer, env: Env) -> Array {
   let mut arr = env.create_array(0).unwrap();
   messages.into_iter().for_each(|gm| {
     let grib_message = GribMessage {
-      inner: DataMessage::try_from(gm).unwrap(),
+      inner: DataMessage::try_from(&gm).unwrap(),
     };
 
     arr.insert(grib_message).unwrap();
   });
 
   arr
 }
+
+#[napi]
+pub struct GribMessageFactory {
+  data: Vec<u8>,
+  mapping: HashMap<String, (usize, usize)>,
+}
+
+#[napi]
+impl GribMessageFactory {
+  #[napi(factory)]
+  pub fn from_buffer(buffer: Buffer) -> Self {
+    let data: Vec<u8> = buffer.into();
+    let mapping = scan_messages(&data);
+
+    GribMessageFactory { data, mapping }
+  }
+
+  #[napi(getter)]
+  pub fn available_messages(&self) -> Vec<String> {
+    self.mapping.keys().map(|k| k.into()).collect()
+  }
+
+  #[napi]
+  pub fn get_message(&self, key: String) -> GribMessage {
+    GribMessage::parse_from_bytes(&self.data, self.mapping[&key].1)
+  }
+}