feat(gw): support UnixFS 1.5 on deserialized responses as Last-Modifi…

…ed (#659)

bare minimum support for Last-Modified and If-Modified-Since
if mtime present in dag-pb

CHANGELOG.md

@@ -21,6 +21,7 @@ The following emojis are used to highlight certain changes:
 ### Changed
 
 - `chunker` refactored to reduce overall memory use by reducing heap fragmentation [#649](https://github.com/ipfs/boxo/pull/649)
+- `gateway` deserialized responses will have `Last-Modified` set to value from optional UnixFS 1.5 modification time field (if present in DAG) and a matching `If-Modified-Since` will return `304 Not Modified` (UnixFS 1.5 files only) [#659](https://github.com/ipfs/boxo/pull/659)
 
 ### Removed
 

gateway/backend_blocks.go

@@ -156,6 +156,12 @@ func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, range
 		return md, nil, err
 	}
 
+	// Set modification time in ContentPathMetadata if found in dag-pb's optional mtime field (UnixFS 1.5)
+	mtime := f.ModTime()
+	if !mtime.IsZero() {
+		md.ModTime = mtime
+	}
+
 	if d, ok := f.(files.Directory); ok {
 		dir, err := uio.NewDirectoryFromNode(bb.dagService, nd)
 		if err != nil {
@@ -231,6 +237,12 @@ func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (Con
 		return ContentPathMetadata{}, nil, err
 	}
 
+	// Set modification time in ContentPathMetadata if found in dag-pb's optional mtime field (UnixFS 1.5)
+	mtime := fileNode.ModTime()
+	if !mtime.IsZero() {
+		md.ModTime = mtime
+	}
+
 	sz, err := fileNode.Size()
 	if err != nil {
 		return ContentPathMetadata{}, nil, err

gateway/gateway.go

@@ -209,7 +209,8 @@ type ContentPathMetadata struct {
 	PathSegmentRoots     []cid.Cid
 	LastSegment          path.ImmutablePath
 	LastSegmentRemainder []string
-	ContentType          string // Only used for UnixFS requests
+	ContentType          string    // Only used for UnixFS requests
+	ModTime              time.Time // Optional, non-zero values may be present in UnixFS 1.5 DAGs
 }
 
 // ByteRange describes a range request within a UnixFS file. "From" and "To" mostly

gateway/gateway_test.go

@@ -256,6 +256,60 @@ func TestHeaders(t *testing.T) {
 		test(dagCborResponseFormat, dagCborPath)
 	})
 
+	// We have UnixFS1.5 tests in TestHeadersUnixFSModeModTime, here we test default behavior (DAG without modtime)
+	t.Run("If-Modified-Since is noop against DAG without optional UnixFS 1.5 mtime", func(t *testing.T) {
+		test := func(responseFormat string, path string) {
+			t.Run(responseFormat, func(t *testing.T) {
+				// Make regular request and read Last-Modified
+				url := ts.URL + path
+				req := mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				res := mustDoWithoutRedirect(t, req)
+				_, err := io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				require.Equal(t, http.StatusOK, res.StatusCode)
+				lastModified := res.Header.Get("Last-Modified")
+				require.Empty(t, lastModified)
+
+				// Make second request with If-Modified-Since far in past and expect normal response
+				req = mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				req.Header.Add("If-Modified-Since", "Mon, 13 Jun 2000 22:18:32 GMT")
+				res = mustDoWithoutRedirect(t, req)
+				_, err = io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				require.Equal(t, http.StatusOK, res.StatusCode)
+			})
+		}
+
+		test("", dirPath)
+		test("text/html", dirPath)
+		test(carResponseFormat, dirPath)
+		test(rawResponseFormat, dirPath)
+		test(tarResponseFormat, dirPath)
+
+		test("", hamtFilePath)
+		test("text/html", hamtFilePath)
+		test(carResponseFormat, hamtFilePath)
+		test(rawResponseFormat, hamtFilePath)
+		test(tarResponseFormat, hamtFilePath)
+
+		test("", filePath)
+		test("text/html", filePath)
+		test(carResponseFormat, filePath)
+		test(rawResponseFormat, filePath)
+		test(tarResponseFormat, filePath)
+
+		test("", dagCborPath)
+		test("text/html", dagCborPath+"/")
+		test(carResponseFormat, dagCborPath)
+		test(rawResponseFormat, dagCborPath)
+		test(dagJsonResponseFormat, dagCborPath)
+		test(dagCborResponseFormat, dagCborPath)
+	})
+
 	t.Run("X-Ipfs-Roots contains expected values", func(t *testing.T) {
 		test := func(responseFormat string, path string, roots string) {
 			t.Run(responseFormat, func(t *testing.T) {
@@ -495,6 +549,112 @@ func TestHeaders(t *testing.T) {
 	})
 }
 
+// Testing a DAG with (optional) UnixFS1.5 modification time
+func TestHeadersUnixFSModeModTime(t *testing.T) {
+	t.Parallel()
+
+	ts, _, root := newTestServerAndNode(t, "unixfs-dir-with-mode-mtime.car")
+	var (
+		rootCID  = root.String() // "bafybeidbcy4u6y55gsemlubd64zk53xoxs73ifd6rieejxcr7xy46mjvky"
+		filePath = "/ipfs/" + rootCID + "/file1"
+		dirPath  = "/ipfs/" + rootCID + "/dir1/"
+	)
+
+	t.Run("If-Modified-Since matching UnixFS 1.5 modtime returns Not Modified", func(t *testing.T) {
+		test := func(responseFormat string, path string, entityType string, supported bool) {
+			t.Run(fmt.Sprintf("%s/%s support=%t", responseFormat, entityType, supported), func(t *testing.T) {
+				// Make regular request and read Last-Modified
+				url := ts.URL + path
+				req := mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				res := mustDoWithoutRedirect(t, req)
+				_, err := io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				require.Equal(t, http.StatusOK, res.StatusCode)
+				lastModified := res.Header.Get("Last-Modified")
+				if supported {
+					assert.NotEmpty(t, lastModified)
+				} else {
+					assert.Empty(t, lastModified)
+					lastModified = "Mon, 13 Jun 2022 22:18:32 GMT" // manually set value for use in next steps
+				}
+
+				ifModifiedSinceTime, err := time.Parse(time.RFC1123, lastModified)
+				require.NoError(t, err)
+				oneHourBefore := ifModifiedSinceTime.Add(-1 * time.Hour).Truncate(time.Second)
+				oneHourAfter := ifModifiedSinceTime.Add(1 * time.Hour).Truncate(time.Second)
+				oneHourBeforeStr := oneHourBefore.Format(time.RFC1123)
+				oneHourAfterStr := oneHourAfter.Format(time.RFC1123)
+				lastModifiedStr := ifModifiedSinceTime.Format(time.RFC1123)
+
+				// Make second request with If-Modified-Since and value read from response to first request
+				req = mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				req.Header.Add("If-Modified-Since", lastModifiedStr)
+				res = mustDoWithoutRedirect(t, req)
+				_, err = io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				if supported {
+					// 304 on exact match, can skip body
+					assert.Equal(t, http.StatusNotModified, res.StatusCode)
+				} else {
+					assert.Equal(t, http.StatusOK, res.StatusCode)
+				}
+
+				// Make third request with If-Modified-Since 1h before value read from response to first request
+				// and expect HTTP 200
+				req = mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				req.Header.Add("If-Modified-Since", oneHourBeforeStr)
+				res = mustDoWithoutRedirect(t, req)
+				_, err = io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				// always return 200 with body because mtime from unixfs is after value from If-Modified-Since
+				// so it counts as an update
+				assert.Equal(t, http.StatusOK, res.StatusCode)
+
+				// Make third request with If-Modified-Since 1h after value read from response to first request
+				// and expect HTTP 200
+				req = mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				req.Header.Add("If-Modified-Since", oneHourAfterStr)
+				res = mustDoWithoutRedirect(t, req)
+				_, err = io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				if supported {
+					// 304 because mtime from unixfs is before value from If-Modified-Since
+					// so no update, can skip body
+					assert.Equal(t, http.StatusNotModified, res.StatusCode)
+				} else {
+					assert.Equal(t, http.StatusOK, res.StatusCode)
+				}
+			})
+		}
+
+		file, dir := "file", "directory"
+		// supported on file-based web responses
+		test("", filePath, file, true)
+		test("text/html", filePath, file, true)
+
+		// not supported on other formats
+		// we may implement support for If-Modified-Since for below request types
+		// if users raise the need, but If-None-Match is way better
+		test(carResponseFormat, filePath, file, false)
+		test(rawResponseFormat, filePath, file, false)
+		test(tarResponseFormat, filePath, file, false)
+
+		test("", dirPath, dir, false)
+		test("text/html", dirPath, dir, false)
+		test(carResponseFormat, dirPath, dir, false)
+		test(rawResponseFormat, dirPath, dir, false)
+		test(tarResponseFormat, dirPath, dir, false)
+	})
+}
+
 func TestGoGetSupport(t *testing.T) {
 	ts, _, root := newTestServerAndNode(t, "fixtures.car")
 

gateway/handler.go

@@ -300,6 +300,11 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Detect when If-Modified-Since HTTP header + UnixFS 1.5 allow returning HTTP 304 Not Modified.
+	if i.handleIfModifiedSince(w, r, rq) {
+		return
+	}
+
 	// Support custom response formats passed via ?format or Accept HTTP header
 	switch responseFormat {
 	case "", jsonResponseFormat, cborResponseFormat:
@@ -410,18 +415,25 @@ func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath
 		}
 
 		if lastMod.IsZero() {
-			// Otherwise, we set Last-Modified to the current time to leverage caching heuristics
+			// If no lastMod, set Last-Modified to the current time to leverage caching heuristics
 			// built into modern browsers: https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768
 			modtime = time.Now()
 		} else {
+			// set Last-Modified to a meaningful value e.g. one read from dag-pb (UnixFS 1.5, mtime field)
+			// or the last time DNSLink / IPNS Record was modified / resoved or cache
 			modtime = lastMod
 		}
+
 	} else {
 		w.Header().Set("Cache-Control", immutableCacheControl)
-		modtime = noModtime // disable Last-Modified
 
-		// TODO: consider setting Last-Modified if UnixFS V1.5 ever gets released
-		// with metadata: https://github.com/ipfs/kubo/issues/6920
+		if lastMod.IsZero() {
+			// (noop) skip Last-Modified on immutable response
+			modtime = noModtime
+		} else {
+			// set Last-Modified to value read from dag-pb (UnixFS 1.5, mtime field)
+			modtime = lastMod
+		}
 	}
 
 	return modtime
@@ -507,6 +519,21 @@ func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathM
 	w.Header().Set("X-Ipfs-Roots", rootCidList)
 }
 
+// lastModifiedMatch returns true if we can respond with HTTP 304 Not Modified
+// It compares If-Modified-Since with logical modification time read from DAG
+// (e.g. UnixFS 1.5 modtime, if present)
+func lastModifiedMatch(ifModifiedSinceHeader string, lastModified time.Time) bool {
+	if ifModifiedSinceHeader == "" || lastModified.IsZero() {
+		return false
+	}
+	ifModifiedSinceTime, err := time.Parse(time.RFC1123, ifModifiedSinceHeader)
+	if err != nil {
+		return false
+	}
+	// ignoring fractional seconds (as HTTP dates don't include fractional seconds)
+	return !lastModified.Truncate(time.Second).After(ifModifiedSinceTime)
+}
+
 // etagMatch evaluates if we can respond with HTTP 304 Not Modified
 // It supports multiple weak and strong etags passed in If-None-Match string
 // including the wildcard one.
@@ -745,6 +772,50 @@ func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, rq *
 	return false
 }
 
+func (i *handler) handleIfModifiedSince(w http.ResponseWriter, r *http.Request, rq *requestData) bool {
+	// Detect when If-Modified-Since HTTP header allows returning HTTP 304 Not Modified
+	ifModifiedSince := r.Header.Get("If-Modified-Since")
+	if ifModifiedSince == "" {
+		return false
+	}
+
+	// Resolve path to be able to read pathMetadata.ModTime
+	pathMetadata, err := i.backend.ResolvePath(r.Context(), rq.immutablePath)
+	if err != nil {
+		var forwardedPath path.ImmutablePath
+		var continueProcessing bool
+		if isWebRequest(rq.responseFormat) {
+			forwardedPath, continueProcessing = i.handleWebRequestErrors(w, r, rq.mostlyResolvedPath(), rq.immutablePath, rq.contentPath, err, rq.logger)
+			if continueProcessing {
+				pathMetadata, err = i.backend.ResolvePath(r.Context(), forwardedPath)
+			}
+		}
+		if !continueProcessing || err != nil {
+			err = fmt.Errorf("failed to resolve %s: %w", debugStr(rq.contentPath.String()), err)
+			i.webError(w, r, err, http.StatusInternalServerError)
+			return true
+		}
+	}
+
+	// Currently we only care about optional mtime from UnixFS 1.5 (dag-pb)
+	// but other sources of this metadata could be added in the future
+	lastModified := pathMetadata.ModTime
+	if lastModifiedMatch(ifModifiedSince, lastModified) {
+		w.WriteHeader(http.StatusNotModified)
+		return true
+	}
+
+	// Check if the resolvedPath is an immutable path.
+	_, err = path.NewImmutablePath(pathMetadata.LastSegment)
+	if err != nil {
+		i.webError(w, r, err, http.StatusInternalServerError)
+		return true
+	}
+
+	rq.pathMetadata = &pathMetadata
+	return false
+}
+
 // check if request was for one of known explicit formats,
 // or should use the default, implicit Web+UnixFS behaviors.
 func isWebRequest(responseFormat string) bool {

gateway/handler_defaults.go

@@ -94,6 +94,14 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h
 
 	setIpfsRootsHeader(w, rq, &pathMetadata)
 
+	// On deserialized responses, we prefer Last-Modified from pathMetadata
+	// (mtime in UnixFS 1.5 DAG). This also applies to /ipns/, because value
+	// from dag-pb, if present, is more meaningful than lastMod inferred from
+	// namesys.
+	if !pathMetadata.ModTime.IsZero() {
+		rq.lastMod = pathMetadata.ModTime
+	}
+
 	resolvedPath := pathMetadata.LastSegment
 	switch mc.Code(resolvedPath.RootCid().Prefix().Codec) {
 	case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: