New v2 serializer and improved v2 streaming parser

Co-authored-by: Divy Srivastava <[email protected]>

Cargo.toml

@@ -2,25 +2,44 @@
 name = "eszip"
 version = "0.14.1"
 authors = ["Ryan Dahl <[email protected]>"]
-edition = "2018"
+edition = "2021"
 description = "A utility that can download JavaScript and TypeScript module graphs and store them locally in a special zip file"
 license = "MIT"
 
 [lib]
 name = "eszip"
 path = "src/lib.rs"
 
+[[example]]
+name = "eszip_builder"
+path = "src/examples/builder.rs"
+
+[[example]]
+name = "eszip_viewer"
+path = "src/examples/viewer.rs"
+
+[[example]]
+name = "eszip_run"
+path = "src/examples/run.rs"
+
+
 [dependencies]
-base64 = "0.13"
-deno_ast = { version = "0.5", features = ["codegen", "dep_graph", "proposal", "react", "sourcemap", "transforms", "typescript", "visit"] }
-futures = "0.3"
-reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
+anyhow = "1"
+base64 = "0.13.0"
+deno_ast = { version = "0.9.0", features = ["utils", "transpiling", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] }
+deno_graph = "0.18.0"
+futures = "0.3.19"
 serde = "1"
 serde_json = "1"
-thiserror = "1"
-tokio = { version = "1", features = ["full"] }
-url = { version = "2", features = ["serde"] }
-data-url = "0.1.0"
+sha2 = "0.10.1"
+tokio = { version = "1", features = ["io-std", "io-util"] }
+thiserror = "1.0.30"
+url = "2.2.2"
 
 [dev-dependencies]
+clap = "3"
+deno_console = "0.32.0"
+deno_core = "0.114.0"
 indicatif = "0.16"
+tokio = { version = "1", features = ["full"] }
+reqwest = "0.11.9"

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright 2021 Deno Land Inc.
+Copyright 2021-2022 Deno Land Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -1,4 +1,30 @@
 # eszip
 
-A library that can download JavaScript and TypeScript module graphs and store
-them locally in a special zip file.
+The eszip format lets you losslessly serialize an ECMAScript module graph
+(represented by [`deno_graph::ModuleGraph`][module_graph]) into a single compact
+file.
+
+The eszip file format is designed to be compact and streaming capable. This
+allows for efficient loading of large ECMAScript module graphs.
+
+[module_graph]: https://docs.rs/deno_graph/latest/deno_graph/struct.ModuleGraph.html
+
+## Examples
+
+### Creating an eszip
+
+```shell
+cargo run --example eszip_builder https://deno.land/std/http/file_server.ts file_server.eszip2
+```
+
+### Viewing the contents of an eszip
+
+```shell
+cargo run --example eszip_viewer file_server.eszip2
+```
+
+### Loading the eszip into V8
+
+```shell
+cargo run --example eszip_run file_server.eszip2 https://deno.land/std/http/file_server.ts
+```

src/error.rs

@@ -1,44 +1,28 @@
 use thiserror::Error;
 
-use crate::parser::ParseError;
-use crate::resolve_import::ModuleResolutionError;
+#[derive(Debug, Error)]
+pub enum ParseError {
+  #[error("invalid eszip v1: {0}")]
+  InvalidV1Json(serde_json::Error),
+  #[error("invalid eszip v1 version: got {0}, expected 1")]
+  InvalidV1Version(u32),
+  #[error("invalid eszip v2")]
+  InvalidV2,
+  #[error("invalid eszip v2 header hash")]
+  InvalidV2HeaderHash,
+  #[error("invalid specifier in eszip v2 header at offset {0}")]
+  InvalidV2Specifier(usize),
+  #[error("invalid entry kind {0} in eszip v2 header at offset {0}")]
+  InvalidV2EntryKind(u8, usize),
+  #[error("invalid module kind {0} in eszip v2 header at offset {0}")]
+  InvalidV2ModuleKind(u8, usize),
+  #[error("invalid eszip v2 header: {0}")]
+  InvalidV2Header(&'static str),
+  #[error("invalid eszip v2 source offset ({0})")]
+  InvalidV2SourceOffset(usize),
+  #[error("invalid eszip v2 source hash (specifier {0})")]
+  InvalidV2SourceHash(String),
 
-#[derive(Error, Debug)]
-pub enum Error {
-  #[error("module with specifier '{specifier}' not found")]
-  NotFound { specifier: String },
   #[error(transparent)]
-  Parse(#[from] ParseError),
-  #[error(transparent)]
-  ModuleResolution(#[from] ModuleResolutionError),
-  #[error(
-    "invalid redirect for '{specifier}': missing or invalid Location header"
-  )]
-  InvalidRedirect { specifier: String },
-  #[error("failed to fetch '{specifier}': {inner}")]
-  Download {
-    specifier: String,
-    inner: reqwest::Error,
-  },
-  #[error(transparent)]
-  Other(Box<dyn std::error::Error + Sync + Send + 'static>),
-  #[error("invalid data url '{specifier}': '{error}'")]
-  InvalidDataUrl { specifier: String, error: String },
-  #[error("scheme '{scheme}' is not supported: '{specifier}'")]
-  InvalidScheme { scheme: String, specifier: String },
-}
-
-pub fn reqwest_error(specifier: String, error: reqwest::Error) -> Error {
-  if error.is_connect()
-    || error.is_decode()
-    || error.is_status()
-    || error.is_timeout()
-  {
-    Error::Download {
-      specifier,
-      inner: error,
-    }
-  } else {
-    Error::Other(Box::new(error))
-  }
+  Io(#[from] std::io::Error),
 }

src/examples/builder.rs

@@ -0,0 +1,96 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use reqwest::StatusCode;
+use url::Url;
+
+#[tokio::main]
+async fn main() {
+  let args = std::env::args().collect::<Vec<_>>();
+  let url = args.get(1).unwrap();
+  let url = Url::parse(&url).unwrap();
+  let out = args.get(2).unwrap();
+
+  let mut loader = Loader;
+  let graph = deno_graph::create_code_graph(
+    vec![url],
+    false,
+    None,
+    &mut loader,
+    None,
+    None,
+    None,
+    None,
+  )
+  .await;
+
+  graph.valid().unwrap();
+
+  let eszip = eszip::EsZipV2::from_graph(graph).unwrap();
+  let bytes = eszip.into_bytes();
+
+  std::fs::write(out, bytes).unwrap();
+}
+
+struct Loader;
+
+impl deno_graph::source::Loader for Loader {
+  fn load(
+    &mut self,
+    specifier: &deno_graph::ModuleSpecifier,
+    is_dynamic: bool,
+  ) -> deno_graph::source::LoadFuture {
+    let specifier = specifier.clone();
+
+    Box::pin(async move {
+      if is_dynamic {
+        return Ok(None);
+      }
+
+      match specifier.scheme() {
+        "data" => deno_graph::source::load_data_url(&specifier),
+        "file" => {
+          let path =
+            tokio::fs::canonicalize(specifier.to_file_path().unwrap()).await?;
+          let content = tokio::fs::read(&path).await?;
+          let content = String::from_utf8(content)?;
+          Ok(Some(deno_graph::source::LoadResponse {
+            specifier: Url::from_file_path(&path).unwrap(),
+            maybe_headers: None,
+            content: Arc::new(content),
+          }))
+        }
+        "http" | "https" => {
+          let resp = reqwest::get(specifier.as_str()).await?;
+          if resp.status() == StatusCode::NOT_FOUND {
+            Ok(None)
+          } else {
+            let resp = resp.error_for_status()?;
+            let mut headers = HashMap::new();
+            for key in resp.headers().keys() {
+              let key_str = key.to_string();
+              let values = resp.headers().get_all(key);
+              let values_str = values
+                .iter()
+                .filter_map(|e| e.to_str().ok())
+                .collect::<Vec<&str>>()
+                .join(",");
+              headers.insert(key_str, values_str);
+            }
+            let url = resp.url().clone();
+            let content = resp.text().await?;
+            Ok(Some(deno_graph::source::LoadResponse {
+              specifier: url,
+              maybe_headers: Some(headers),
+              content: Arc::new(content),
+            }))
+          }
+        }
+        _ => Err(anyhow::anyhow!(
+          "unsupported scheme: {}",
+          specifier.scheme()
+        )),
+      }
+    })
+  }
+}

src/examples/run.rs

@@ -0,0 +1,89 @@
+use std::rc::Rc;
+
+use deno_core::error::type_error;
+use eszip::EsZipV2;
+use futures::FutureExt;
+use url::Url;
+
+#[tokio::main]
+async fn main() {
+  let args = std::env::args().collect::<Vec<_>>();
+  let path = args.get(1).unwrap();
+  let url = args.get(2).unwrap();
+  let url = Url::parse(&url).unwrap();
+
+  let file = tokio::fs::File::open(path).await.unwrap();
+  let bufreader = tokio::io::BufReader::new(file);
+  let (eszip, loader) = eszip::EsZipV2::parse(bufreader).await.unwrap();
+
+  let loader_fut = loader.map(|r| r.map_err(anyhow::Error::new));
+
+  let fut = async move {
+    let mut runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
+      module_loader: Some(Rc::new(Loader(eszip))),
+      extensions: vec![deno_console::init()],
+      ..Default::default()
+    });
+
+    let mod_id = runtime.load_main_module(&url, None).await?;
+
+    let fut = runtime
+      .mod_evaluate(mod_id)
+      .map(|r| r.map_err(anyhow::Error::new));
+
+    let (_, r) = tokio::try_join!(runtime.run_event_loop(false), fut)?;
+
+    r
+  };
+
+  tokio::try_join!(loader_fut, fut).unwrap();
+}
+
+struct Loader(EsZipV2);
+
+impl deno_core::ModuleLoader for Loader {
+  fn resolve(
+    &self,
+    specifier: &str,
+    referrer: &str,
+    _is_main: bool,
+  ) -> Result<deno_core::ModuleSpecifier, anyhow::Error> {
+    let specifier = deno_core::resolve_import(specifier, referrer)?;
+    Ok(specifier)
+  }
+
+  fn load(
+    &self,
+    module_specifier: &deno_core::ModuleSpecifier,
+    _maybe_referrer: Option<deno_core::ModuleSpecifier>,
+    is_dyn_import: bool,
+  ) -> std::pin::Pin<Box<deno_core::ModuleSourceFuture>> {
+    let module_specifier = module_specifier.clone();
+
+    let res = self
+      .0
+      .get_module(module_specifier.as_str())
+      .ok_or_else(|| type_error("module not found"));
+
+    Box::pin(async move {
+      if is_dyn_import {
+        return Err(type_error("dynamic import not supported"));
+      }
+
+      let module = res?;
+
+      let source = module.source().await;
+      let source = std::str::from_utf8(&source).unwrap();
+
+      Ok(deno_core::ModuleSource {
+        code: source.to_string(),
+        module_type: match module.kind {
+          eszip::ModuleKind::JS => deno_core::ModuleType::JavaScript,
+          eszip::ModuleKind::JSON => deno_core::ModuleType::Json,
+        },
+        module_url_found: module.specifier,
+        module_url_specified: module_specifier.to_string(),
+      })
+    })
+  }
+}