feat(tdf): implement tdf3 encrypt and decrypt

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/bufbuild/protovalidate-go v0.4.3
 	github.com/creasty/defaults v1.7.0
 	github.com/go-chi/cors v1.2.1
+	github.com/golang-jwt/jwt/v4 v4.5.0
+	github.com/google/uuid v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1
 	github.com/jackc/pgx/v5 v5.5.0
@@ -51,7 +53,6 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/cel-go v0.18.2 // indirect
-	github.com/google/uuid v1.4.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect

internal/archive/tdf3_reader.go

@@ -9,8 +9,8 @@ type TDFReader struct {
 }
 
 const (
-	tdfManifestFileName = "0.manifest.json"
-	tdfPayloadFileName  = "0.payload"
+	TDFManifestFileName = "0.manifest.json"
+	TDFPayloadFileName  = "0.payload"
 )
 
 // NewTDFReader Create tdf reader instance.
@@ -28,7 +28,7 @@ func NewTDFReader(readSeeker io.ReadSeeker) (TDFReader, error) {
 
 // Manifest Return the manifest of the tdf.
 func (tdfReader TDFReader) Manifest() (string, error) {
-	fileContent, err := tdfReader.archiveReader.ReadAllFileData(tdfManifestFileName)
+	fileContent, err := tdfReader.archiveReader.ReadAllFileData(TDFManifestFileName)
 	if err != nil {
 		return "", err
 	}
@@ -37,7 +37,7 @@ func (tdfReader TDFReader) Manifest() (string, error) {
 
 // ReadPayload Return the payload of given length from index.
 func (tdfReader TDFReader) ReadPayload(index, length int64) ([]byte, error) {
-	buf, err := tdfReader.archiveReader.ReadFileData(tdfPayloadFileName, index, length)
+	buf, err := tdfReader.archiveReader.ReadFileData(TDFPayloadFileName, index, length)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (tdfReader TDFReader) ReadPayload(index, length int64) ([]byte, error) {
 
 // PayloadSize Return the size of the payload.
 func (tdfReader TDFReader) PayloadSize() (int64, error) {
-	size, err := tdfReader.archiveReader.ReadFileSize(tdfPayloadFileName)
+	size, err := tdfReader.archiveReader.ReadFileSize(TDFPayloadFileName)
 	if err != nil {
 		return -1, err
 	}

internal/archive/tdf3_writer.go

@@ -4,6 +4,7 @@ import "io"
 
 type TDFWriter struct {
 	archiveWriter *Writer
+	totalBytes    int64
 }
 
 // NewTDFWriter Create tdf writer instance.
@@ -20,25 +21,25 @@ func (tdfWriter *TDFWriter) SetPayloadSize(payloadSize int64) error {
 		tdfWriter.archiveWriter.EnableZip64()
 	}
 
-	return tdfWriter.archiveWriter.AddHeader(tdfPayloadFileName, payloadSize)
+	return tdfWriter.archiveWriter.AddHeader(TDFPayloadFileName, payloadSize)
 }
 
-// AppendManifest Add the manifest to tdf3 archive.
+// AppendManifest Add the manifest to tdf archive.
 func (tdfWriter *TDFWriter) AppendManifest(manifest string) error {
-	err := tdfWriter.archiveWriter.AddHeader(tdfManifestFileName, int64(len(manifest)))
+	err := tdfWriter.archiveWriter.AddHeader(TDFManifestFileName, int64(len(manifest)))
 	if err != nil {
 		return err
 	}
 
 	return tdfWriter.archiveWriter.AddData([]byte(manifest))
 }
 
-// AppendPayload Add payload to tdf3 archive.
+// AppendPayload Add payload to sdk archive.
 func (tdfWriter *TDFWriter) AppendPayload(data []byte) error {
 	return tdfWriter.archiveWriter.AddData(data)
 }
 
-// Close Completed adding all the files in zip archive.
-func (tdfWriter *TDFWriter) Close() error {
-	return tdfWriter.archiveWriter.Close()
+// Finish Finished adding all the files in zip archive.
+func (tdfWriter *TDFWriter) Finish() (int64, error) {
+	return tdfWriter.archiveWriter.Finish()
 }

internal/archive/tdf3_writer_reader_test.go

@@ -10,12 +10,14 @@ import (
 type TDF3Entry struct {
 	manifest    string
 	payloadSize int64
+	tdfSize     int64
 }
 
 var TDF3Tests = []TDF3Entry{ //nolint:gochecknoglobals // This global is used as test harness for other tests
 	{
 		manifest:    "some manifest",
 		payloadSize: oneKB,
+		tdfSize:     1291,
 	},
 	{
 		manifest: `{
@@ -63,6 +65,7 @@ var TDF3Tests = []TDF3Entry{ //nolint:gochecknoglobals // This global is used as
 	}
 }`,
 		payloadSize: 10 * oneMB,
+		tdfSize:     10487693,
 	},
 	{
 		manifest: `{
@@ -110,6 +113,7 @@ var TDF3Tests = []TDF3Entry{ //nolint:gochecknoglobals // This global is used as
 	}
 }`,
 		payloadSize: 3 * oneGB,
+		tdfSize:     3145729933,
 	},
 	{
 		manifest: `{
@@ -157,6 +161,7 @@ var TDF3Tests = []TDF3Entry{ //nolint:gochecknoglobals // This global is used as
 	}
 }`,
 		payloadSize: 10 * oneGB,
+		tdfSize:     10485762121,
 	},
 }
 
@@ -190,12 +195,6 @@ func writeTDFs(t *testing.T) {
 		}(writer)
 
 		tdf3Writer := NewTDFWriter(writer)
-		defer func(tdf3Writer *TDFWriter) {
-			err := tdf3Writer.Close()
-			if err != nil {
-				t.Fatalf("Fail to close tdf3 writer: %v", err)
-			}
-		}(tdf3Writer)
 
 		// write payload
 		totalBytes := tdf3Entry.payloadSize
@@ -225,6 +224,15 @@ func writeTDFs(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Fail to add payload to tdf3 writer: %v", err)
 		}
+
+		tdfSize, err := tdf3Writer.Finish()
+		if err != nil {
+			t.Fatalf("Fail to close tdf3 writer: %v", err)
+		}
+
+		if tdfSize != tdf3Entry.tdfSize {
+			t.Errorf("tdf size test failed expected %v, got %v", tdfSize, tdf3Entry.tdfSize)
+		}
 	}
 }
 

internal/archive/writer.go

@@ -62,6 +62,7 @@ type Writer struct {
 	fileInfoEntries []FileInfo
 	writeState      WriteState
 	isZip64         bool
+	totalBytes      int64
 }
 
 // NewWriter Create tdf3 writer instance.
@@ -137,13 +138,13 @@ func (writer *Writer) AddData(data []byte) error {
 			return fmt.Errorf("binary.Write failed: %w", err)
 		}
 
-		_, err = writer.writer.Write(buf.Bytes())
+		err = writer.writeData(buf.Bytes())
 		if err != nil {
 			return fmt.Errorf("io.Writer.Write failed: %w", err)
 		}
 
 		// write the file name
-		_, err = writer.writer.Write([]byte(writer.FileInfo.filename))
+		err = writer.writeData([]byte(writer.FileInfo.filename))
 		if err != nil {
 			return fmt.Errorf("io.Writer.Write failed: %w", err)
 		}
@@ -161,7 +162,7 @@ func (writer *Writer) AddData(data []byte) error {
 				return fmt.Errorf("binary.Write failed: %w", err)
 			}
 
-			_, err = writer.writer.Write(buf.Bytes())
+			err = writer.writeData(buf.Bytes())
 			if err != nil {
 				return fmt.Errorf("io.Writer.Write failed: %w", err)
 			}
@@ -176,7 +177,7 @@ func (writer *Writer) AddData(data []byte) error {
 	}
 
 	// now write the contents
-	_, err := writer.writer.Write(data)
+	err := writer.writeData(data)
 	if err != nil {
 		return fmt.Errorf("io.Writer.Write failed: %w", err)
 	}
@@ -214,7 +215,7 @@ func (writer *Writer) AddData(data []byte) error {
 				return fmt.Errorf("binary.Write failed: %w", err)
 			}
 
-			_, err = writer.writer.Write(buf.Bytes())
+			err = writer.writeData(buf.Bytes())
 			if err != nil {
 				return fmt.Errorf("io.Writer.Write failed: %w", err)
 			}
@@ -238,7 +239,7 @@ func (writer *Writer) AddData(data []byte) error {
 				return fmt.Errorf("binary.Write failed: %w", err)
 			}
 
-			_, err = writer.writer.Write(buf.Bytes())
+			err = writer.writeData(buf.Bytes())
 			if err != nil {
 				return fmt.Errorf("io.Writer.Write failed: %w", err)
 			}
@@ -256,19 +257,19 @@ func (writer *Writer) AddData(data []byte) error {
 	return nil
 }
 
-// Close Completed adding all the files in zip archive.
-func (writer *Writer) Close() error {
+// Finish Finished adding all the files in zip archive.
+func (writer *Writer) Finish() (int64, error) {
 	err := writer.writeCentralDirectory()
 	if err != nil {
-		return err
+		return writer.totalBytes, err
 	}
 
 	err = writer.writeEndOfCentralDirectory()
 	if err != nil {
-		return fmt.Errorf("io.Writer.Write failed: %w", err)
+		return writer.totalBytes, fmt.Errorf("io.Writer.Write failed: %w", err)
 	}
 
-	return nil
+	return writer.totalBytes, nil
 }
 
 // WriteCentralDirectory write central directory struct into archive.
@@ -313,13 +314,13 @@ func (writer *Writer) writeCentralDirectory() error {
 			return fmt.Errorf("binary.Write failed: %w", err)
 		}
 
-		_, err = writer.writer.Write(buf.Bytes())
+		err = writer.writeData(buf.Bytes())
 		if err != nil {
 			return fmt.Errorf("io.Writer.Write failed: %w", err)
 		}
 
 		// write the filename
-		_, err = writer.writer.Write([]byte(writer.fileInfoEntries[i].filename))
+		err = writer.writeData([]byte(writer.fileInfoEntries[i].filename))
 		if err != nil {
 			return fmt.Errorf("io.Writer.Write failed: %w", err)
 		}
@@ -339,7 +340,7 @@ func (writer *Writer) writeCentralDirectory() error {
 				return fmt.Errorf("binary.Write failed: %w", err)
 			}
 
-			_, err = writer.writer.Write(buf.Bytes())
+			err = writer.writeData(buf.Bytes())
 			if err != nil {
 				return fmt.Errorf("io.Writer.Write failed: %w", err)
 			}
@@ -391,7 +392,7 @@ func (writer *Writer) writeEndOfCentralDirectory() error {
 		return fmt.Errorf("binary.Write failed: %w", err)
 	}
 
-	_, err = writer.writer.Write(buf.Bytes())
+	err = writer.writeData(buf.Bytes())
 	if err != nil {
 		return fmt.Errorf("io.Writer.Write failed: %w", err)
 	}
@@ -420,7 +421,7 @@ func (writer *Writer) WriteZip64EndOfCentralDirectory() error {
 		return fmt.Errorf("binary.Write failed: %w", err)
 	}
 
-	_, err = writer.writer.Write(buf.Bytes())
+	err = writer.writeData(buf.Bytes())
 	if err != nil {
 		return fmt.Errorf("io.Writer.Write failed: %w", err)
 	}
@@ -444,10 +445,11 @@ func (writer *Writer) WriteZip64EndOfCentralDirectoryLocator() error {
 		return fmt.Errorf("binary.Write failed: %w", err)
 	}
 
-	_, err = writer.writer.Write(buf.Bytes())
+	err = writer.writeData(buf.Bytes())
 	if err != nil {
 		return fmt.Errorf("io.Writer.Write failed: %w", err)
 	}
+
 	return nil
 }
 
@@ -458,3 +460,13 @@ func (writer *Writer) getTimeDateUnMSDosFormat() (uint16, uint16) {
 	dateInDos := (t.Year()-80)<<9 | int((t.Month()+1)<<5) | t.Day()
 	return uint16(timeInDos), uint16(dateInDos)
 }
+
+func (writer *Writer) writeData(data []byte) error {
+	n, err := writer.writer.Write(data)
+	if err != nil {
+		return err
+	}
+
+	writer.totalBytes += int64(n)
+	return nil
+}

internal/archive/writer_test.go

@@ -24,7 +24,8 @@ type ZipEntryInfo struct {
 }
 
 var ArchiveTests = []struct { //nolint:gochecknoglobals // This global is used as test harness for other tests
-	files []ZipEntryInfo
+	files       []ZipEntryInfo
+	archiveSize int64
 }{
 	{
 		[]ZipEntryInfo{
@@ -41,6 +42,7 @@ var ArchiveTests = []struct { //nolint:gochecknoglobals // This global is used a
 				10,
 			},
 		},
+		358,
 	},
 	{
 		[]ZipEntryInfo{
@@ -69,6 +71,7 @@ var ArchiveTests = []struct { //nolint:gochecknoglobals // This global is used a
 				oneKB,
 			},
 		},
+		6778,
 	},
 	{
 		[]ZipEntryInfo{
@@ -97,6 +100,7 @@ var ArchiveTests = []struct { //nolint:gochecknoglobals // This global is used a
 				oneMB + oneKB,
 			},
 		},
+		526397048,
 	},
 
 	{
@@ -114,6 +118,7 @@ var ArchiveTests = []struct { //nolint:gochecknoglobals // This global is used a
 				tenGB,
 			},
 		},
+		12582912572,
 	},
 }
 
@@ -191,10 +196,14 @@ func customZip(t *testing.T) {
 			}
 		}
 
-		err = archiveWriter.Close()
+		archiveSize, err := archiveWriter.Finish()
 		if err != nil {
 			t.Fatalf("Fail to close to archive: %v", err)
 		}
+
+		if archiveSize != test.archiveSize {
+			t.Errorf("archive size test failed expected %v, got %v", archiveSize, test.archiveSize)
+		}
 	}
 }