s3: mirror content type from HTTP (#82)

sjtug · Apr 23, 2021 · 7d023c1 · 7d023c1
1 parent 228ad26
commit 7d023c1
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 11 deletions.
diff --git a/src/index_pipe.rs b/src/index_pipe.rs
@@ -281,6 +281,7 @@ where
                 },
                 length: content.len() as u64,
                 modified_at: unix_time(),
+                content_type: None, // use `text/html` by default
             })
         } else {
             self.source.get_object(snapshot, mission).await

diff --git a/src/s3.rs b/src/s3.rs
@@ -291,6 +291,7 @@ impl S3Metadata for SnapshotMeta {
 
 fn get_mime(key: &str) -> Option<String> {
     // TODO: add more types from https://github.com/nginx/nginx/blob/master/conf/mime.types
+    // TODO: the correct way is to mirror content-type from remote as-is, or to read MIME type
     if key.ends_with(".htm") || key.ends_with(".html") || key.ends_with(".shtml") {
         Some("text/html; charset=utf-8".to_string())
     } else {
@@ -316,6 +317,7 @@ where
             mut object,
             length,
             modified_at,
+            content_type,
         } = byte_stream;
 
         let body = object.as_stream();
@@ -330,7 +332,7 @@ where
             body: Some(rusoto_s3::StreamingBody::new(body)),
             metadata: Some(metadata),
             content_length: Some(length as i64),
-            content_type: get_mime(snapshot.key()),
+            content_type: content_type.or_else(|| get_mime(snapshot.key())),
             ..Default::default()
         };
 

diff --git a/src/stream_pipe.rs b/src/stream_pipe.rs
@@ -70,6 +70,7 @@ pub struct ByteStream {
     pub object: ByteObject,
     pub length: u64,
     pub modified_at: u64,
+    pub content_type: Option<String>,
 }
 
 pub struct ByteStreamPipe<Source> {
@@ -146,16 +147,13 @@ where
         let mut total_bytes: u64 = 0;
         let content_length = response.content_length();
         let snapshot_modified_at = snapshot.last_modified();
-        let http_modified_at = std::str::from_utf8(
-            response
-                .headers()
-                .get(reqwest::header::LAST_MODIFIED)
-                .unwrap()
-                .as_bytes(),
-        )
-        .ok()
-        .and_then(|header| DateTime::parse_from_rfc2822(&header).ok())
-        .map(|x| x.timestamp() as u64);
+        let http_modified_at = response
+            .headers()
+            .get(reqwest::header::LAST_MODIFIED)
+            .map(|x| x.as_bytes())
+            .and_then(|x| std::str::from_utf8(x).ok())
+            .and_then(|header| DateTime::parse_from_rfc2822(&header).ok())
+            .map(|x| x.timestamp() as u64);
 
         let modified_at = if self.use_snapshot_last_modified {
             snapshot_modified_at
@@ -179,6 +177,13 @@ where
             }
         }
 
+        let content_type = response
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .map(|x| x.as_bytes())
+            .and_then(|x| std::str::from_utf8(x).ok())
+            .map(|x| x.to_string());
+
         debug!(logger, "download: {} {:?}", transfer_url.0, content_length);
 
         let mut stream = response.bytes_stream();
@@ -210,6 +215,7 @@ where
             },
             length: total_bytes,
             modified_at,
+            content_type,
         })
     }
 }