feat: eszip v2

Cargo.toml

@@ -2,25 +2,45 @@
 name = "eszip"
 version = "0.14.1"
 authors = ["Ryan Dahl <[email protected]>"]
-edition = "2018"
+edition = "2021"
 description = "A utility that can download JavaScript and TypeScript module graphs and store them locally in a special zip file"
 license = "MIT"
 
 [lib]
 name = "eszip"
 path = "src/lib.rs"
 
+[[example]]
+name = "eszip_builder"
+path = "src/examples/builder.rs"
+
+[[example]]
+name = "eszip_viewer"
+path = "src/examples/viewer.rs"
+
+[[example]]
+name = "eszip_load"
+path = "src/examples/load.rs"
+
+
 [dependencies]
-base64 = "0.13"
-deno_ast = { version = "0.5", features = ["codegen", "dep_graph", "proposal", "react", "sourcemap", "transforms", "typescript", "visit"] }
-futures = "0.3"
-reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
+anyhow = "1"
+base64 = "0.13.0"
+deno_ast = { version = "0.10.0", features = ["utils", "transpiling", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] }
+deno_graph = "0.21.0"
+futures = "0.3.19"
 serde = "1"
 serde_json = "1"
-thiserror = "1"
-tokio = { version = "1", features = ["full"] }
-url = { version = "2", features = ["serde"] }
-data-url = "0.1.0"
+sha2 = "0.10.1"
+tokio = { version = "1", features = ["io-std", "io-util"] }
+thiserror = "1.0.30"
+url = "2.2.2"
 
 [dev-dependencies]
+clap = "3"
+deno_console = "0.32.0"
+deno_core = "0.114.0"
+import_map = "0.6.0"
 indicatif = "0.16"
+tokio = { version = "1", features = ["full"] }
+reqwest = { version = "0.11.9", features = ["rustls-tls"] }

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright 2021 Deno Land Inc.
+Copyright 2021-2022 Deno Land Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -1,4 +1,58 @@
 # eszip
 
-A library that can download JavaScript and TypeScript module graphs and store
-them locally in a special zip file.
+The eszip format lets you losslessly serialize an ECMAScript module graph
+(represented by [`deno_graph::ModuleGraph`][module_graph]) into a single compact
+file.
+
+The eszip file format is designed to be compact and streaming capable. This
+allows for efficient loading of large ECMAScript module graphs.
+
+[module_graph]: https://docs.rs/deno_graph/latest/deno_graph/struct.ModuleGraph.html
+
+## Examples
+
+### Creating an eszip
+
+```shell
+cargo run --example eszip_builder https://deno.land/std/http/file_server.ts file_server.eszip2
+```
+
+### Viewing the contents of an eszip
+
+```shell
+cargo run --example eszip_viewer file_server.eszip2
+```
+
+### Loading the eszip into V8
+
+```shell
+cargo run --example eszip_load file_server.eszip2 https://deno.land/std/http/file_server.ts
+```
+
+## File format
+
+The file format looks as follows:
+
+```
+Eszip:
+| Magic (8) | Header size (4) | Header (n) | Header hash (32) | Sources size (4) | Sources (n) | SourceMaps size (4) | SourceMaps (n) |
+
+Header:
+( | Specifier size (4) | Specifier (n) | Entry type (1) | Entry (n) | )*
+
+Entry (redirect):
+| Specifier size (4) | Specifier (n) |
+
+Entry (module):
+| Source offset (4) | Source size (4) | SourceMap offset (4) | SourceMap size (4) | Module type (1) |
+
+Sources:
+( | Source (n) | Hash (32) | )*
+
+SourceMaps:
+( | SourceMap (n) | Hash (32) | )*
+```
+
+There is one optimization for empty source / source map entries. If both the
+offset and size are set to 0, no entry and no hash is present in the data
+sections for that module.

src/error.rs

@@ -1,44 +1,28 @@
 use thiserror::Error;
 
-use crate::parser::ParseError;
-use crate::resolve_import::ModuleResolutionError;
+#[derive(Debug, Error)]
+pub enum ParseError {
+  #[error("invalid eszip v1: {0}")]
+  InvalidV1Json(serde_json::Error),
+  #[error("invalid eszip v1 version: got {0}, expected 1")]
+  InvalidV1Version(u32),
+  #[error("invalid eszip v2")]
+  InvalidV2,
+  #[error("invalid eszip v2 header hash")]
+  InvalidV2HeaderHash,
+  #[error("invalid specifier in eszip v2 header at offset {0}")]
+  InvalidV2Specifier(usize),
+  #[error("invalid entry kind {0} in eszip v2 header at offset {0}")]
+  InvalidV2EntryKind(u8, usize),
+  #[error("invalid module kind {0} in eszip v2 header at offset {0}")]
+  InvalidV2ModuleKind(u8, usize),
+  #[error("invalid eszip v2 header: {0}")]
+  InvalidV2Header(&'static str),
+  #[error("invalid eszip v2 source offset ({0})")]
+  InvalidV2SourceOffset(usize),
+  #[error("invalid eszip v2 source hash (specifier {0})")]
+  InvalidV2SourceHash(String),
 
-#[derive(Error, Debug)]
-pub enum Error {
-  #[error("module with specifier '{specifier}' not found")]
-  NotFound { specifier: String },
   #[error(transparent)]
-  Parse(#[from] ParseError),
-  #[error(transparent)]
-  ModuleResolution(#[from] ModuleResolutionError),
-  #[error(
-    "invalid redirect for '{specifier}': missing or invalid Location header"
-  )]
-  InvalidRedirect { specifier: String },
-  #[error("failed to fetch '{specifier}': {inner}")]
-  Download {
-    specifier: String,
-    inner: reqwest::Error,
-  },
-  #[error(transparent)]
-  Other(Box<dyn std::error::Error + Sync + Send + 'static>),
-  #[error("invalid data url '{specifier}': '{error}'")]
-  InvalidDataUrl { specifier: String, error: String },
-  #[error("scheme '{scheme}' is not supported: '{specifier}'")]
-  InvalidScheme { scheme: String, specifier: String },
-}
-
-pub fn reqwest_error(specifier: String, error: reqwest::Error) -> Error {
-  if error.is_connect()
-    || error.is_decode()
-    || error.is_status()
-    || error.is_timeout()
-  {
-    Error::Download {
-      specifier,
-      inner: error,
-    }
-  } else {
-    Error::Other(Box::new(error))
-  }
+  Io(#[from] std::io::Error),
 }

src/examples/builder.rs

@@ -0,0 +1,154 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use deno_ast::EmitOptions;
+use deno_graph::source::ResolveResponse;
+use import_map::ImportMap;
+use reqwest::StatusCode;
+use url::Url;
+
+#[tokio::main]
+async fn main() {
+  let args = std::env::args().collect::<Vec<_>>();
+  let url = args.get(1).unwrap();
+  let url = Url::parse(url).unwrap();
+  let out = args.get(2).unwrap();
+  let maybe_import_map = args.get(3).map(|url| Url::parse(url).unwrap());
+
+  let mut loader = Loader;
+  let (maybe_import_map, maybe_import_map_data) =
+    if let Some(import_map_url) = maybe_import_map {
+      let resp =
+        deno_graph::source::Loader::load(&mut loader, &import_map_url, false)
+          .await
+          .unwrap()
+          .unwrap();
+      let import_map =
+        ImportMap::from_json_with_diagnostics(&resp.specifier, &resp.content)
+          .unwrap();
+      (
+        Some(import_map.import_map),
+        Some((resp.specifier, resp.content)),
+      )
+    } else {
+      (None, None)
+    };
+
+  let graph = deno_graph::create_code_graph(
+    vec![(url, deno_graph::ModuleKind::Esm)],
+    false,
+    None,
+    &mut loader,
+    Some(&Resolver(maybe_import_map)),
+    None,
+    None,
+    None,
+  )
+  .await;
+
+  graph.valid().unwrap();
+
+  let mut eszip =
+    eszip::EszipV2::from_graph(graph, EmitOptions::default()).unwrap();
+  if let Some((import_map_specifier, import_map_content)) =
+    maybe_import_map_data
+  {
+    eszip.add_import_map(
+      import_map_specifier.to_string(),
+      Arc::new(import_map_content.as_bytes().to_vec()),
+    )
+  }
+  for specifier in &eszip.ordered_modules {
+    println!("source: {specifier}")
+  }
+
+  let bytes = eszip.into_bytes();
+
+  std::fs::write(out, bytes).unwrap();
+}
+
+#[derive(Debug)]
+struct Resolver(Option<ImportMap>);
+
+impl deno_graph::source::Resolver for Resolver {
+  fn resolve(
+    &self,
+    specifier: &str,
+    referrer: &deno_graph::ModuleSpecifier,
+  ) -> ResolveResponse {
+    if let Some(import_map) = &self.0 {
+      match import_map.resolve(specifier, referrer) {
+        Ok(specifier) => ResolveResponse::Specifier(specifier),
+        Err(err) => ResolveResponse::Err(err.into()),
+      }
+    } else {
+      match deno_graph::resolve_import(specifier, referrer) {
+        Ok(specifier) => ResolveResponse::Specifier(specifier),
+        Err(err) => ResolveResponse::Err(err.into()),
+      }
+    }
+  }
+}
+
+struct Loader;
+
+impl deno_graph::source::Loader for Loader {
+  fn load(
+    &mut self,
+    specifier: &deno_graph::ModuleSpecifier,
+    is_dynamic: bool,
+  ) -> deno_graph::source::LoadFuture {
+    let specifier = specifier.clone();
+
+    Box::pin(async move {
+      if is_dynamic {
+        return Ok(None);
+      }
+
+      match specifier.scheme() {
+        "data" => deno_graph::source::load_data_url(&specifier),
+        "file" => {
+          let path =
+            tokio::fs::canonicalize(specifier.to_file_path().unwrap()).await?;
+          let content = tokio::fs::read(&path).await?;
+          let content = String::from_utf8(content)?;
+          Ok(Some(deno_graph::source::LoadResponse {
+            specifier: Url::from_file_path(&path).unwrap(),
+            maybe_headers: None,
+            content: Arc::new(content),
+          }))
+        }
+        "http" | "https" => {
+          let resp = reqwest::get(specifier.as_str()).await?;
+          if resp.status() == StatusCode::NOT_FOUND {
+            Ok(None)
+          } else {
+            let resp = resp.error_for_status()?;
+            let mut headers = HashMap::new();
+            for key in resp.headers().keys() {
+              let key_str = key.to_string();
+              let values = resp.headers().get_all(key);
+              let values_str = values
+                .iter()
+                .filter_map(|e| e.to_str().ok())
+                .collect::<Vec<&str>>()
+                .join(",");
+              headers.insert(key_str, values_str);
+            }
+            let url = resp.url().clone();
+            let content = resp.text().await?;
+            Ok(Some(deno_graph::source::LoadResponse {
+              specifier: url,
+              maybe_headers: Some(headers),
+              content: Arc::new(content),
+            }))
+          }
+        }
+        _ => Err(anyhow::anyhow!(
+          "unsupported scheme: {}",
+          specifier.scheme()
+        )),
+      }
+    })
+  }
+}