diff --git a/kurtosis-devnet/op-program-svc/README.md b/kurtosis-devnet/op-program-svc/README.md index 4e59ff1a1918a..5ffca686b0454 100644 --- a/kurtosis-devnet/op-program-svc/README.md +++ b/kurtosis-devnet/op-program-svc/README.md @@ -1,4 +1,24 @@ -# Trigger new build: +# op-program-svc + +This small service is a temporary measure until we come up with a better way +of generating/serving prestate files based on chain information. + +# API + +The API is intentionally extremely simple: +- `POST /`: generate new prestates from provided inputs +- `GET /HASH.(bin.gz|json)`: get prestate data +- `GET /info.json`: get prestates mapping + +The idea is for this service to be basically a function +(chains_specs, deptsets) -> prestates. + +In the future, we definitely want to replace the implementation of that +function (see implementation notes below) + +## Trigger new build: + +Example using curl ``` $ curl -X POST -H "Content-Type: multipart/form-data" \ @@ -9,3 +29,30 @@ $ curl -X POST -H "Content-Type: multipart/form-data" \ -F "files[]=@depsets.json" \ http://localhost:8080 ``` + +## Retrieve prestates mapping + +``` +$ curl -q http://localhost:8080/info.json +{ + "prestate": "0x03f4b7435fec731578c72635d8e8180f7b48703073d038fc7f8c494eeed1ce19", + "prestate_interop": "0x034731331d519c93fc0562643e0728c43f8e45a0af1160ad4c57c4e5141d2bbb", + "prestate_mt64": "0x0325bb0ca8521b468bb8234d8ba54b1b74db60e2b5bc75d0077a0fe2098b6b45" +} +``` + +## Implementation notes + +Unfortunately, op-program-client relies on embedded (using `//go:embed`) +configuration files to store unannounced chain configs. + +This means that in the context of devnets, we need to store the configs +(which are available only mid-deployment) into the **source tree** and +trigger a late build step. + +So effectively, we need to package the relevant part of the sources into +a container, deploy that one alongside the devnet, and run that build step +on demand. + +This is ugly, unsafe, easy to run a DOS against,... we need to do better. +But for now this is what we have. \ No newline at end of file diff --git a/kurtosis-devnet/op-program-svc/build.go b/kurtosis-devnet/op-program-svc/build.go index f4e3582ab3459..7da7a4a23d87b 100644 --- a/kurtosis-devnet/op-program-svc/build.go +++ b/kurtosis-devnet/op-program-svc/build.go @@ -147,6 +147,7 @@ func (b *Builder) ExecuteBuild() ([]byte, error) { return output.Bytes(), nil } +// This is a convenience hack to natively support the file format of op-deployer func (b *Builder) normalizeFilename(filename string) string { // Get just the filename without directories filename = filepath.Base(filename) @@ -156,8 +157,15 @@ func (b *Builder) normalizeFilename(filename string) string { if numStr := strings.TrimSuffix(parts[1], ".json"); numStr != parts[1] { // Check if the number part is actually numeric if _, err := strconv.Atoi(numStr); err == nil { - // It matches the pattern and has a valid number, reorder to NUMBER-PREFIX.json - return numStr + "-" + parts[0] + ".json" + // Handle specific cases + switch parts[0] { + case "genesis": + return fmt.Sprintf("%s-genesis-l2.json", numStr) + case "rollup": + return fmt.Sprintf("%s-rollup.json", numStr) + + } + // For all other cases, leave the filename unchanged } } } diff --git a/kurtosis-devnet/op-program-svc/build_test.go b/kurtosis-devnet/op-program-svc/build_test.go index df3b6f879f7c6..70a813f1f3b39 100644 --- a/kurtosis-devnet/op-program-svc/build_test.go +++ b/kurtosis-devnet/op-program-svc/build_test.go @@ -167,9 +167,19 @@ func TestNormalizeFilename(t *testing.T) { expected string }{ { - name: "standard format", + name: "standard format - unchanged", input: "prefix-123.json", - expected: "123-prefix.json", + expected: "prefix-123.json", + }, + { + name: "genesis format", + input: "genesis-123.json", + expected: "123-genesis-l2.json", + }, + { + name: "rollup format", + input: "rollup-456.json", + expected: "456-rollup.json", }, { name: "no number", diff --git a/kurtosis-devnet/op-program-svc/defaults.go b/kurtosis-devnet/op-program-svc/defaults.go index aba44060ae0e1..baf756a0d423d 100644 --- a/kurtosis-devnet/op-program-svc/defaults.go +++ b/kurtosis-devnet/op-program-svc/defaults.go @@ -48,6 +48,10 @@ func (fs *DefaultFileSystem) Join(elem ...string) string { return filepath.Join(elem...) } +func (fs *DefaultFileSystem) Stat(name string) (fs.FileInfo, error) { + return os.Stat(name) +} + // commandWrapper wraps exec.Cmd to implement CommandRunner type commandWrapper struct { *exec.Cmd diff --git a/kurtosis-devnet/op-program-svc/fs.go b/kurtosis-devnet/op-program-svc/fs.go index 7408760cf67b8..c79ef00773d69 100644 --- a/kurtosis-devnet/op-program-svc/fs.go +++ b/kurtosis-devnet/op-program-svc/fs.go @@ -17,6 +17,11 @@ import ( "time" ) +const ( + depsetsFilename = "depsets.json" + infoFilename = "info.json" +) + // proofFileSystem implements http.FileSystem, mapping hash-based virtual paths to actual files type proofFileSystem struct { root string @@ -69,7 +74,7 @@ func (f *infoFile) Readdir(count int) ([]fs.FileInfo, error) { func (f *infoFile) Stat() (fs.FileInfo, error) { return virtualFileInfo{ - name: "info.json", + name: infoFilename, size: int64(len(f.content)), mode: 0644, modTime: time.Now(), @@ -120,8 +125,11 @@ func (d *proofDir) Readdir(count int) ([]fs.FileInfo, error) { d.proofMutex.RLock() defer d.proofMutex.RUnlock() + // Calculate total number of entries + totalEntries := len(d.proofFiles)*2 + 1 // hash.json, hash.bin.gz files + info.json + // If we've already read all entries - if d.pos >= len(d.proofFiles)*2+1 { + if d.pos >= totalEntries { if count <= 0 { return nil, nil } @@ -137,15 +145,15 @@ func (d *proofDir) Readdir(count int) ([]fs.FileInfo, error) { start := d.pos end := start + count - if count <= 0 || end > len(d.proofFiles)*2+1 { - end = len(d.proofFiles)*2 + 1 + if count <= 0 || end > totalEntries { + end = totalEntries } for i := start; i < end; i++ { - // Special case for info.json (last entry) + // Special case for info.json (second to last entry) if i == len(d.proofFiles)*2 { entries = append(entries, virtualFileInfo{ - name: "info.json", + name: infoFilename, size: 0, // Size will be determined when actually opening the file mode: 0644, modTime: time.Now(), @@ -228,7 +236,7 @@ func (fs *proofFileSystem) Open(name string) (http.File, error) { name = strings.TrimPrefix(filepath.Clean(name), "/") // Special case for info.json - if name == "info.json" { + if name == infoFilename { fs.proofMutex.RLock() defer fs.proofMutex.RUnlock() return newInfoFile(fs.proofFiles), nil @@ -279,12 +287,15 @@ func (fs *proofFileSystem) scanProofFiles() error { proofRegexp := regexp.MustCompile(`^prestate-proof(.*)\.json$`) for _, entry := range entries { + log.Printf("entry: %s", entry.Name()) if entry.IsDir() { + log.Printf("entry is a directory: %s", entry.Name()) continue } matches := proofRegexp.FindStringSubmatch(entry.Name()) if matches == nil { + log.Printf("Warning: ignoring non-proof file %s", entry.Name()) continue } diff --git a/kurtosis-devnet/op-program-svc/fs_test.go b/kurtosis-devnet/op-program-svc/fs_test.go index 951c292c1a47a..67a1f204dfb3d 100644 --- a/kurtosis-devnet/op-program-svc/fs_test.go +++ b/kurtosis-devnet/op-program-svc/fs_test.go @@ -130,15 +130,11 @@ func TestProofFileSystem(t *testing.T) { } // Verify info.json is included in the directory listing - foundInfoJson := false for _, file := range files { if file.Name() == "info.json" { - foundInfoJson = true - break + return } } - if !foundInfoJson { - t.Error("info.json not found in directory listing") - } + t.Error("info.json not found in directory listing") }) } diff --git a/kurtosis-devnet/op-program-svc/interfaces.go b/kurtosis-devnet/op-program-svc/interfaces.go index 701890459c883..0d93ecb8349ed 100644 --- a/kurtosis-devnet/op-program-svc/interfaces.go +++ b/kurtosis-devnet/op-program-svc/interfaces.go @@ -20,6 +20,7 @@ type FS interface { ReadDir(name string) ([]fs.DirEntry, error) ReadFile(name string) ([]byte, error) Join(elem ...string) string + Stat(name string) (fs.FileInfo, error) // Additional FileSystem operations MkdirAll(path string, perm os.FileMode) error diff --git a/kurtosis-devnet/op-program-svc/mocks.go b/kurtosis-devnet/op-program-svc/mocks.go index 6d52f31009aa1..8fa108cb0c873 100644 --- a/kurtosis-devnet/op-program-svc/mocks.go +++ b/kurtosis-devnet/op-program-svc/mocks.go @@ -67,21 +67,23 @@ func (m *MockFile) Readdir(count int) ([]fs.FileInfo, error) { return nil, fmt.Errorf("not implemented") } -// MockFS implements both FS and FileSystem interfaces for testing +// MockFS implements FS interface for testing type MockFS struct { - Files map[string]*MockFile - MkdirCalls []string - CreateCalls []string - JoinCalls [][]string - ShouldFail bool + Files map[string]*MockFile + ShouldFail bool + StatFailPaths map[string]bool // Paths that should fail for Stat + JoinCalls [][]string + MkdirCalls []string + CreateCalls []string } func NewMockFS() *MockFS { return &MockFS{ - Files: make(map[string]*MockFile), - MkdirCalls: make([]string, 0), - CreateCalls: make([]string, 0), - JoinCalls: make([][]string, 0), + Files: make(map[string]*MockFile), + StatFailPaths: make(map[string]bool), + JoinCalls: make([][]string, 0), + MkdirCalls: make([]string, 0), + CreateCalls: make([]string, 0), } } @@ -142,6 +144,16 @@ func (m *MockFS) Join(elem ...string) string { return filepath.Join(elem...) } +func (m *MockFS) Stat(name string) (fs.FileInfo, error) { + if m.ShouldFail { + return nil, fmt.Errorf("mock stat error") + } + if m.StatFailPaths[name] { + return nil, fmt.Errorf("file not found: %s", name) + } + return m.Files[name], nil +} + // MockWriteCloser implements io.WriteCloser for testing type MockWriteCloser struct { bytes.Buffer diff --git a/kurtosis-devnet/op-program-svc/server.go b/kurtosis-devnet/op-program-svc/server.go index 401e92ff9e4dc..ad8e16a15b52e 100644 --- a/kurtosis-devnet/op-program-svc/server.go +++ b/kurtosis-devnet/op-program-svc/server.go @@ -77,7 +77,7 @@ func (s *server) handleUpload(w http.ResponseWriter, r *http.Request) { // Check if we need to rebuild if currentHash == s.lastBuildHash { log.Printf("Hash matches last build, skipping") - w.WriteHeader(http.StatusOK) + w.WriteHeader(http.StatusNotModified) fmt.Fprintf(w, "Files unchanged, skipping build") return } @@ -111,8 +111,8 @@ func (s *server) handleUpload(w http.ResponseWriter, r *http.Request) { // Update the last successful build hash s.lastBuildHash = currentHash - // Redirect to the proofs endpoint - http.Redirect(w, r, "/proofs", http.StatusSeeOther) + log.Printf("Build successful, last build hash: %s", currentHash) + w.WriteHeader(http.StatusOK) } func (s *server) processMultipartForm(r *http.Request) ([]*multipart.FileHeader, string, error) { diff --git a/kurtosis-devnet/op-program-svc/server_test.go b/kurtosis-devnet/op-program-svc/server_test.go index f66cea90bd0a7..5831e37cfa096 100644 --- a/kurtosis-devnet/op-program-svc/server_test.go +++ b/kurtosis-devnet/op-program-svc/server_test.go @@ -108,12 +108,9 @@ func TestHandleUpload_Success(t *testing.T) { w := httptest.NewRecorder() srv.handleUpload(w, req) - if w.Code != http.StatusSeeOther { - t.Errorf("Expected status code %d, got %d", http.StatusSeeOther, w.Code) - } - - if location := w.Header().Get("Location"); location != "/proofs" { - t.Errorf("Expected redirect to /proofs, got %s", location) + // We now expect 200 OK instead of a redirect + if w.Code != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, w.Code) } } @@ -206,9 +203,9 @@ func TestHandleUpload_ScanError(t *testing.T) { w := httptest.NewRecorder() srv.handleUpload(w, req) - // Even with scan error, we should still redirect - if w.Code != http.StatusSeeOther { - t.Errorf("Expected status code %d, got %d", http.StatusSeeOther, w.Code) + // Even with scan error, we should still return 200 OK + if w.Code != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, w.Code) } } @@ -229,8 +226,9 @@ func TestHandleUpload_UnchangedFiles(t *testing.T) { w1 := httptest.NewRecorder() srv.handleUpload(w1, req1) - if w1.Code != http.StatusSeeOther { - t.Errorf("Expected status code %d, got %d", http.StatusSeeOther, w1.Code) + // First request should return 200 OK + if w1.Code != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, w1.Code) } // Second request with same files @@ -242,8 +240,9 @@ func TestHandleUpload_UnchangedFiles(t *testing.T) { w2 := httptest.NewRecorder() srv.handleUpload(w2, req2) - if w2.Code != http.StatusOK { - t.Errorf("Expected status code %d, got %d", http.StatusOK, w2.Code) + // Second request with unchanged files should return 304 Not Modified + if w2.Code != http.StatusNotModified { + t.Errorf("Expected status code %d, got %d", http.StatusNotModified, w2.Code) } if !strings.Contains(w2.Body.String(), "Files unchanged") {