From d89a56cb7185dc3d2b60b8fecd83b7ef1aa28abc Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 18:12:08 -0600 Subject: [PATCH 01/14] feat: add examples for CLI usage Refactor encrypt and decrypt --- cmd/tdf-decrypt.go | 19 +--- cmd/tdf-encrypt.go | 25 ++---- docs/man/auth/_index.md | 6 +- docs/man/auth/clear-client-credentials.md | 3 +- docs/man/auth/client-credentials.md | 23 +++++ docs/man/auth/login.md | 9 ++ docs/man/auth/logout.md | 10 +++ docs/man/auth/print-access-token.md | 11 ++- docs/man/decrypt/_index.md | 33 ++++--- docs/man/encrypt/_index.md | 56 ++++++++++-- go.mod | 2 +- go.sum | 2 + pkg/handlers/nano-tdf.go | 44 --------- pkg/handlers/tdf.go | 105 +++++++++++++++------- 14 files changed, 213 insertions(+), 135 deletions(-) delete mode 100644 pkg/handlers/nano-tdf.go diff --git a/cmd/tdf-decrypt.go b/cmd/tdf-decrypt.go index 54a972e4..2de0d18c 100644 --- a/cmd/tdf-decrypt.go +++ b/cmd/tdf-decrypt.go @@ -1,7 +1,6 @@ package cmd import ( - "bytes" "errors" "fmt" "os" @@ -14,15 +13,11 @@ import ( var TDF = "tdf" func dev_tdfDecryptCmd(cmd *cobra.Command, args []string) { - c := cli.New(cmd, args) + c := cli.New(cmd, args, cli.WithPrintJson()) h := NewHandler(c) defer h.Close() output := c.Flags.GetOptionalString("out") - tdfType := c.Flags.GetOptionalString("tdf-type") - if tdfType == "" { - tdfType = TDF3 - } // check for piped input piped := readPipedStdin() @@ -39,16 +34,7 @@ func dev_tdfDecryptCmd(cmd *cobra.Command, args []string) { cli.ExitWithError("Must provide ONE of the following to decrypt: [file argument, stdin input]", errors.New("no input provided")) } - var decrypted *bytes.Buffer - var err error - switch tdfType { - case TDF3: - decrypted, err = h.DecryptTDF(bytesToDecrypt) - case NANO: - decrypted, err = h.DecryptNanoTDF(bytesToDecrypt) - default: - cli.ExitWithError("Failed to decrypt", fmt.Errorf("unrecognized tdf-type: %s", tdfType)) - } + decrypted, err := h.DecryptBytes(bytesToDecrypt) if err != nil { cli.ExitWithError("Failed to decrypt file", err) } @@ -80,6 +66,7 @@ func init() { decryptCmd.GetDocFlag("out").Default, decryptCmd.GetDocFlag("out").Description, ) + // deprecated flag decryptCmd.Flags().StringP( decryptCmd.GetDocFlag("tdf-type").Name, decryptCmd.GetDocFlag("tdf-type").Shorthand, diff --git a/cmd/tdf-encrypt.go b/cmd/tdf-encrypt.go index c677c80d..b63a648c 100644 --- a/cmd/tdf-encrypt.go +++ b/cmd/tdf-encrypt.go @@ -1,7 +1,6 @@ package cmd import ( - "bytes" "fmt" "io" "log/slog" @@ -16,15 +15,16 @@ import ( ) const ( - TDF3 = "tdf3" - NANO = "nano" - Size_1MB = 1024 * 1024 + TDFTYPE_ZTDF = "ztdf" + TDF3 = "tdf3" + NANO = "nano" + Size_1MB = 1024 * 1024 ) var attrValues []string func dev_tdfEncryptCmd(cmd *cobra.Command, args []string) { - c := cli.New(cmd, args) + c := cli.New(cmd, args, cli.WithPrintJson()) h := NewHandler(c) defer h.Close() @@ -39,9 +39,6 @@ func dev_tdfEncryptCmd(cmd *cobra.Command, args []string) { fileMimeType := c.Flags.GetOptionalString("mime-type") attrValues = c.Flags.GetStringSlice("attr", attrValues, cli.FlagsStringSliceOptions{Min: 0}) tdfType := c.Flags.GetOptionalString("tdf-type") - if tdfType == "" { - tdfType = TDF3 - } kasURLPath := c.Flags.GetOptionalString("kas-url-path") piped := readPipedStdin() @@ -90,17 +87,7 @@ func dev_tdfEncryptCmd(cmd *cobra.Command, args []string) { ) // Do the encryption - var encrypted *bytes.Buffer - var err error - switch tdfType { - case TDF3: - encrypted, err = h.EncryptBytes(bytesSlice, attrValues, fileMimeType, kasURLPath) - case NANO: - ecdsaBinding := c.Flags.GetOptionalBool("ecdsa-binding") - encrypted, err = h.EncryptNanoBytes(bytesSlice, attrValues, kasURLPath, ecdsaBinding) - default: - cli.ExitWithError("Failed to encrypt", fmt.Errorf("unrecognized tdf-type: %s", tdfType)) - } + encrypted, err := h.EncryptBytes(tdfType, bytesSlice, attrValues, fileMimeType, kasURLPath, c.Flags.GetOptionalBool("ecdsa-binding")) if err != nil { cli.ExitWithError("Failed to encrypt", err) } diff --git a/docs/man/auth/_index.md b/docs/man/auth/_index.md index 858ee028..909ba22f 100644 --- a/docs/man/auth/_index.md +++ b/docs/man/auth/_index.md @@ -5,4 +5,8 @@ command: name: auth --- -This command will allow you to manage your local authentication session with the OpenTDF platform. \ No newline at end of file +> [!NOTE] +> Requires experimental profiles feature. (Linux not yet supported. Windows is brittle.) + +The auth commands facilitate the process of authenticating the user with the system using profiles to store the +credentials. diff --git a/docs/man/auth/clear-client-credentials.md b/docs/man/auth/clear-client-credentials.md index 43d32b1d..d24b0ae4 100644 --- a/docs/man/auth/clear-client-credentials.md +++ b/docs/man/auth/clear-client-credentials.md @@ -9,4 +9,5 @@ command: default: false --- -This command has been deprecated. Use the `profile` subcommand to manage profiles and credentials. +> [!WARNING] +> Deprecated. Use the `profile` subcommand to manage profiles and credentials. diff --git a/docs/man/auth/client-credentials.md b/docs/man/auth/client-credentials.md index 47ccedd7..c957e3d3 100644 --- a/docs/man/auth/client-credentials.md +++ b/docs/man/auth/client-credentials.md @@ -9,5 +9,28 @@ command: - client-secret --- +> [!NOTE] +> Requires experimental profiles feature. +> +> | OS | Keychain | State | +> | --- | --- | --- | +> | macOS | Keychain | Stable | +> | Windows | Credential Manager | Alpha | +> | Linux | Secret Service | Not yet supported | + Allows the user to login in via Client Credentials flow. The client credentials will be stored safely in the OS keyring for future use. + +## Examples + +Authenticate with client credentials (secret provided interactively) + +```shell +opentdf auth client-credentials --client-id +``` + +Authenticate with client credentials (secret provided as argument) + +```shell +opentdf auth client-credentials --client-id --client-secret +``` diff --git a/docs/man/auth/login.md b/docs/man/auth/login.md index 7e5ba257..4a8be1ca 100644 --- a/docs/man/auth/login.md +++ b/docs/man/auth/login.md @@ -10,6 +10,15 @@ command: required: false --- +> [!NOTE] +> Requires experimental profiles feature. +> +> | OS | Keychain | State | +> | --- | --- | --- | +> | macOS | Keychain | Stable | +> | Windows | Credential Manager | Alpha | +> | Linux | Secret Service | Not yet supported | + Authenticate for use of the OpenTDF Platform through a browser (required). Provide a specific public 'client-id' known to support the Auth Code PKCE flow and recognized diff --git a/docs/man/auth/logout.md b/docs/man/auth/logout.md index b295f3ab..6a920f54 100644 --- a/docs/man/auth/logout.md +++ b/docs/man/auth/logout.md @@ -5,5 +5,15 @@ command: name: logout --- + +> [!NOTE] +> Requires experimental profiles feature. +> +> | OS | Keychain | State | +> | --- | --- | --- | +> | macOS | Keychain | Stable | +> | Windows | Credential Manager | Alpha | +> | Linux | Secret Service | Not yet supported | + Removes any auth credentials (Client Credentials or an Access Token from a login) from the current profile. diff --git a/docs/man/auth/print-access-token.md b/docs/man/auth/print-access-token.md index 376f97c9..36628402 100644 --- a/docs/man/auth/print-access-token.md +++ b/docs/man/auth/print-access-token.md @@ -9,4 +9,13 @@ command: default: false --- -Retrieves a new OIDC Access Token using the client credentials from the OS-specific keychain and prints to stdout if found. +> [!NOTE] +> Requires experimental profiles feature. +> +> | OS | Keychain | State | +> | --- | --- | --- | +> | macOS | Keychain | Stable | +> | Windows | Credential Manager | Alpha | +> | Linux | Secret Service | Not yet supported | + +Retrieves a new OIDC Access Token using the client credentials and prints to stdout if found. diff --git a/docs/man/decrypt/_index.md b/docs/man/decrypt/_index.md index d378846a..dcb39a39 100644 --- a/docs/man/decrypt/_index.md +++ b/docs/man/decrypt/_index.md @@ -9,25 +9,32 @@ command: default: '' - name: tdf-type shorthand: t - description: The type of tdf to decrypt as - enum: - - tdf3 - - nano - default: tdf3 + description: Deprecated. TDF type is now auto-detected. + default: '' --- Decrypt a Trusted Data Format (TDF) file and output the contents to stdout or a file in the current working directory. The first argument is the TDF file with path from the current working directory being decrypted. -## Examples: +## Examples + +Various ways to decrypt a TDF file + +```shell +# decrypt file and write to standard output +otdfctl decrypt hello.txt.tdf -```bash -# specify the TDF to decrypt then output decrypted contents -otdfctl decrypt hello.txt.tdf # write to stdout -otdfctl decrypt hello.txt.tdf > hello.txt # consume stdout to write to hello.txt file -otdfctl decrypt hello.txt.tdf -o hello.txt # write to hello.txt file instead of stdout +# decrypt file and write to hello.txt file +otdfctl decrypt hello.txt.tdf -o hello.txt -# pipe the TDF to decrypt -cat hello.txt.tdf | otdfctl decrypt > hello.txt +# decrypt piped TDF content and write to hello.txt file +cat hello.txt.tdf | otdfctl decrypt -o hello.txt ``` + +Advanced piping is supported + +```shell +$ echo "hello world" | otdfctl encrypt | otdfctl decrypt | cat +hello world +``` \ No newline at end of file diff --git a/docs/man/encrypt/_index.md b/docs/man/encrypt/_index.md index f6d74d8c..2f982f2b 100644 --- a/docs/man/encrypt/_index.md +++ b/docs/man/encrypt/_index.md @@ -17,6 +17,7 @@ command: description: The type of tdf to encrypt as. TDF3 supports structured manifests and larger payloads. Nano has a smaller footprint and more performant, but does not support structured manifests or large payloads. enum: - tdf3 + - ztdf - nano default: tdf3 - name: ecdsa-binding @@ -28,17 +29,54 @@ command: Build a Trusted Data Format (TDF) with encrypted content from a specified file or input from stdin utilizing OpenTDF platform. -## Examples: +## Examples -```bash +Various ways to encrypt a file + +```shell # output to stdout -echo "some text" | otdfctl encrypt otdfctl encrypt hello.txt -# pipe stdout to a bucket -echo "my secret" | otdfctl encrypt | aws s3 cp - s3://my-bucket/secret.txt.tdf -# output hello.txt.tdf in root directory -echo "hello world" | otdfctl encrypt -o hello.txt -cat hello.txt | otdfctl encrypt -o hello.txt -cat hello.txt | otdfctl encrypt -o hello.txt.tdf #.tdf extension is only added once +# output to hello.txt.tdf +otdfctl encrypt hello.txt --out hello.txt.tdf + +# encrypt piped content and write to hello.txt.tdf +cat hello.txt | otdfctl encrypt --out hello.txt.tdf +``` + +Automatically append .tdf to the output file name + +```shell +$ cat hello.txt | otdfctl encrypt --out hello.txt; ls +hello.txt hello.txt.tdf + +$ cat hello.txt | otdfctl encrypt --out hello.txt.tdf; ls +hello.txt hello.txt.tdf +``` + +Advanced piping is supported + +```shell +$ echo "hello world" | otdfctl encrypt | otdfctl decrypt | cat +hello world +``` + +## Attributes + +Attributes can be added to the encrypted data. The attribute value is a Fully Qualified Name (FQN) that is used to +restrict access to the data based on entity entitlements. + +```shell +# output to hello.txt.tdf with attribute +otdfctl encrypt hello.txt --out hello.txt.tdf --attr https://example.com/attr/attr1/value/value1 +``` + +## NanoTDF + +NanoTDF is a lightweight TDF format that is more performant and has a smaller footprint than TDF3. NanoTDF does not +support structured manifests or large payloads. + +```shell +# output to nano.tdf +otdfctl encrypt hello.txt --tdf-type nano --out hello.txt.tdf ``` diff --git a/go.mod b/go.mod index 00a1647a..fe99281e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/opentdf/platform/lib/flattening v0.1.1 github.com/opentdf/platform/protocol/go v0.2.18 - github.com/opentdf/platform/sdk v0.3.15 + github.com/opentdf/platform/sdk v0.3.17 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 1ffd87ba..c4916477 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,8 @@ github.com/opentdf/platform/protocol/go v0.2.18 h1:s+TVZkOPGCzy7WyObtJWJNaFeOGDU github.com/opentdf/platform/protocol/go v0.2.18/go.mod h1:WqDcnFQJb0v8ivRQPidbehcL8ils5ZSZYXkuv0nyvsI= github.com/opentdf/platform/sdk v0.3.15 h1:RFSZ93SlBiZfFY6JAFLWbv4JC/H/KwC1acxyxCjPNnM= github.com/opentdf/platform/sdk v0.3.15/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs= +github.com/opentdf/platform/sdk v0.3.17 h1:Uo/kTMneB18i0gZNfTRtvw34bGLFUc8BEnA/BMK0VVs= +github.com/opentdf/platform/sdk v0.3.17/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/pkg/handlers/nano-tdf.go b/pkg/handlers/nano-tdf.go deleted file mode 100644 index 8f6b2203..00000000 --- a/pkg/handlers/nano-tdf.go +++ /dev/null @@ -1,44 +0,0 @@ -package handlers - -import ( - "bytes" - "io" -) - -func (h Handler) EncryptNanoBytes(b []byte, values []string, kasUrlPath string, ecdsaBinding bool) (*bytes.Buffer, error) { - var encrypted []byte - enc := bytes.NewBuffer(encrypted) - - nanoTDFConfig, err := h.sdk.NewNanoTDFConfig() - if err != nil { - return nil, err - } - - err = nanoTDFConfig.SetKasURL(h.platformEndpoint + kasUrlPath) - if err != nil { - return nil, err - } - err = nanoTDFConfig.SetAttributes(values) - if err != nil { - return nil, err - } - if ecdsaBinding { - nanoTDFConfig.EnableECDSAPolicyBinding() - } - - // TODO: validate values are FQNs or return an error [https://github.com/opentdf/platform/issues/515] - _, err = h.sdk.CreateNanoTDF(enc, bytes.NewReader(b), *nanoTDFConfig) - if err != nil { - return nil, err - } - return enc, nil -} - -func (h Handler) DecryptNanoTDF(toDecrypt []byte) (*bytes.Buffer, error) { - outBuf := bytes.Buffer{} - _, err := h.sdk.ReadNanoTDF(io.Writer(&outBuf), bytes.NewReader(toDecrypt)) - if err != nil { - return nil, err - } - return &outBuf, nil -} diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index 6e1b4ae2..62198c0b 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -18,44 +18,89 @@ var ( minBytesLength = 3 ) -func (h Handler) EncryptBytes(b []byte, values []string, mimeType string, kasUrlPath string) (*bytes.Buffer, error) { +const ( + TDF_TYPE_ZTDF = "ztdf" + TDF_TYPE_TDF3 = "tdf3" // alias for TDF + TDF_TYPE_NANO = "nano" +) + +type TDFInspect struct { + NanoHeader *sdk.NanoTDFHeader + ZTDFManifest *sdk.Manifest + Attributes []string + UnencryptedMetadata []byte +} + +func (h Handler) EncryptBytes(tdfType string, b []byte, values []string, mimeType string, kasUrlPath string, ecdsaBinding bool) (*bytes.Buffer, error) { var encrypted []byte enc := bytes.NewBuffer(encrypted) - // TODO: validate values are FQNs or return an error [https://github.com/opentdf/platform/issues/515] - _, err := h.sdk.CreateTDF(enc, bytes.NewReader(b), - sdk.WithDataAttributes(values...), - sdk.WithKasInformation(sdk.KASInfo{ - URL: h.platformEndpoint + kasUrlPath, - }), - sdk.WithMimeType(mimeType), - ) - if err != nil { - return nil, err - } - return enc, nil -} + switch tdfType { -func (h Handler) DecryptTDF(toDecrypt []byte) (*bytes.Buffer, error) { - tdfreader, err := h.sdk.LoadTDF(bytes.NewReader(toDecrypt)) - if err != nil { - return nil, err - } + // Encrypt the data as a ZTDF + case "", TDF_TYPE_TDF3, TDF_TYPE_ZTDF: + if ecdsaBinding { + return nil, errors.New("ECDSA policy binding is not supported for ZTDF") + } - buf := new(bytes.Buffer) - _, err = io.Copy(buf, tdfreader) - //nolint:errorlint // callers intended to test error equality directly - if err != nil && err != io.EOF { - return nil, err + _, err := h.sdk.CreateTDF(enc, bytes.NewReader(b), + sdk.WithDataAttributes(values...), + sdk.WithKasInformation(sdk.KASInfo{ + URL: h.platformEndpoint + kasUrlPath, + }), + sdk.WithMimeType(mimeType), + ) + return enc, err + + // Encrypt the data as a Nano TDF + case TDF_TYPE_NANO: + nanoTDFConfig, err := h.sdk.NewNanoTDFConfig() + if err != nil { + return nil, err + } + // set the KAS URL + if err = nanoTDFConfig.SetKasURL(h.platformEndpoint + kasUrlPath); err != nil { + return nil, err + } + // set the attributes + if err = nanoTDFConfig.SetAttributes(values); err != nil { + return nil, err + } + // enable ECDSA policy binding + if ecdsaBinding { + nanoTDFConfig.EnableECDSAPolicyBinding() + } + // create the nano TDF + if _, err = h.sdk.CreateNanoTDF(enc, bytes.NewReader(b), *nanoTDFConfig); err != nil { + return nil, err + } + return enc, nil + default: + return nil, errors.New("unknown TDF type") } - return buf, nil } -type TDFInspect struct { - NanoHeader *sdk.NanoTDFHeader - ZTDFManifest *sdk.Manifest - Attributes []string - UnencryptedMetadata []byte +func (h Handler) DecryptBytes(toDecrypt []byte) (*bytes.Buffer, error) { + out := &bytes.Buffer{} + pt := io.Writer(out) + ec := bytes.NewReader(toDecrypt) + switch sdk.GetTdfType(ec) { + case sdk.Nano: + if _, err := h.sdk.ReadNanoTDF(pt, ec); err != nil { + return nil, err + } + case sdk.Standard: + r, err := h.sdk.LoadTDF(ec) + if err != nil { + return nil, err + } + if _, err = io.Copy(pt, r); err != nil && err != io.EOF { + return nil, err + } + default: + return nil, errors.New("unknown TDF type") + } + return out, nil } func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { From 012b170713e18332e6b89f8bec4bb0a0e78a31cd Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 21:24:38 -0600 Subject: [PATCH 02/14] bump jose --- go.mod | 2 +- go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fe99281e..936c0b12 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/creasty/defaults v1.8.0 github.com/evertras/bubble-table v0.16.1 github.com/gabriel-vasile/mimetype v1.4.5 + github.com/go-jose/go-jose/v3 v3.0.3 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.6.0 github.com/opentdf/platform/lib/flattening v0.1.1 @@ -49,7 +50,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index c4916477..3676750a 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,6 @@ github.com/opentdf/platform/lib/ocrypto v0.1.6 h1:rd4ctCZOE/c3qDJORtkSK9tw6dEXb+ github.com/opentdf/platform/lib/ocrypto v0.1.6/go.mod h1:ne+l8Q922OdzA0xesK3XJmfECBnn5vLSGYU3/3OhiHM= github.com/opentdf/platform/protocol/go v0.2.18 h1:s+TVZkOPGCzy7WyObtJWJNaFeOGDUTuSmAsq3omvugY= github.com/opentdf/platform/protocol/go v0.2.18/go.mod h1:WqDcnFQJb0v8ivRQPidbehcL8ils5ZSZYXkuv0nyvsI= -github.com/opentdf/platform/sdk v0.3.15 h1:RFSZ93SlBiZfFY6JAFLWbv4JC/H/KwC1acxyxCjPNnM= -github.com/opentdf/platform/sdk v0.3.15/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs= github.com/opentdf/platform/sdk v0.3.17 h1:Uo/kTMneB18i0gZNfTRtvw34bGLFUc8BEnA/BMK0VVs= github.com/opentdf/platform/sdk v0.3.17/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= From d36d4afe1f7f788f4b7daa115894ca8933e166d4 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 21:28:42 -0600 Subject: [PATCH 03/14] fix spell check --- docs/man/auth/client-credentials.md | 2 +- docs/man/auth/login.md | 2 +- docs/man/auth/logout.md | 2 +- docs/man/auth/print-access-token.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/man/auth/client-credentials.md b/docs/man/auth/client-credentials.md index c957e3d3..97f97e20 100644 --- a/docs/man/auth/client-credentials.md +++ b/docs/man/auth/client-credentials.md @@ -14,7 +14,7 @@ command: > > | OS | Keychain | State | > | --- | --- | --- | -> | macOS | Keychain | Stable | +> | MacOS | Keychain | Stable | > | Windows | Credential Manager | Alpha | > | Linux | Secret Service | Not yet supported | diff --git a/docs/man/auth/login.md b/docs/man/auth/login.md index 4a8be1ca..db134900 100644 --- a/docs/man/auth/login.md +++ b/docs/man/auth/login.md @@ -15,7 +15,7 @@ command: > > | OS | Keychain | State | > | --- | --- | --- | -> | macOS | Keychain | Stable | +> | MacOS | Keychain | Stable | > | Windows | Credential Manager | Alpha | > | Linux | Secret Service | Not yet supported | diff --git a/docs/man/auth/logout.md b/docs/man/auth/logout.md index 6a920f54..f92e2a68 100644 --- a/docs/man/auth/logout.md +++ b/docs/man/auth/logout.md @@ -11,7 +11,7 @@ command: > > | OS | Keychain | State | > | --- | --- | --- | -> | macOS | Keychain | Stable | +> | MacOS | Keychain | Stable | > | Windows | Credential Manager | Alpha | > | Linux | Secret Service | Not yet supported | diff --git a/docs/man/auth/print-access-token.md b/docs/man/auth/print-access-token.md index 36628402..50774446 100644 --- a/docs/man/auth/print-access-token.md +++ b/docs/man/auth/print-access-token.md @@ -14,7 +14,7 @@ command: > > | OS | Keychain | State | > | --- | --- | --- | -> | macOS | Keychain | Stable | +> | MacOS | Keychain | Stable | > | Windows | Credential Manager | Alpha | > | Linux | Secret Service | Not yet supported | From b701f88dd37dc5a32fcf01438bc59cd58d40bf17 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 21:28:53 -0600 Subject: [PATCH 04/14] fix spellcheck --- .github/spellcheck.ignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/spellcheck.ignore b/.github/spellcheck.ignore index 1b819ef1..ef70ddd5 100644 --- a/.github/spellcheck.ignore +++ b/.github/spellcheck.ignore @@ -22,6 +22,7 @@ NPM Namespace namespace's Nano +NanoTDF OIDC OpenTDF OpenID @@ -106,4 +107,6 @@ upsert uri with-client-creds with-client-creds-file -yaml \ No newline at end of file +yaml +ZTDF +ztdf \ No newline at end of file From 3a288d87cff2b54d1af4d9674b2b5101fafaef7d Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 21:37:05 -0600 Subject: [PATCH 05/14] fix lint issues --- pkg/handlers/tdf.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index 62198c0b..6e4c1148 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -36,7 +36,6 @@ func (h Handler) EncryptBytes(tdfType string, b []byte, values []string, mimeTyp enc := bytes.NewBuffer(encrypted) switch tdfType { - // Encrypt the data as a ZTDF case "", TDF_TYPE_TDF3, TDF_TYPE_ZTDF: if ecdsaBinding { @@ -94,9 +93,12 @@ func (h Handler) DecryptBytes(toDecrypt []byte) (*bytes.Buffer, error) { if err != nil { return nil, err } + //nolint:errorlint // callers intended to test error equality directly if _, err = io.Copy(pt, r); err != nil && err != io.EOF { return nil, err } + case sdk.Invalid: + return nil, errors.New("invalid TDF") default: return nil, errors.New("unknown TDF type") } From 704aed5472204cdde5a5d657c35e4601efc9dd98 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Mon, 4 Nov 2024 21:53:57 -0600 Subject: [PATCH 06/14] fix reader --- pkg/handlers/tdf.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index 6e4c1148..460d98d1 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -83,7 +83,10 @@ func (h Handler) DecryptBytes(toDecrypt []byte) (*bytes.Buffer, error) { out := &bytes.Buffer{} pt := io.Writer(out) ec := bytes.NewReader(toDecrypt) - switch sdk.GetTdfType(ec) { + tdfType := sdk.GetTdfType(ec) + // reset the reader to the beginning + ec.Reset(toDecrypt) + switch tdfType { case sdk.Nano: if _, err := h.sdk.ReadNanoTDF(pt, ec); err != nil { return nil, err From db9bc148c42a75a28e8b20e8302f50cfdf221891 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Wed, 6 Nov 2024 13:19:45 -0600 Subject: [PATCH 07/14] new line --- docs/man/decrypt/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/man/decrypt/_index.md b/docs/man/decrypt/_index.md index dcb39a39..48d0dd74 100644 --- a/docs/man/decrypt/_index.md +++ b/docs/man/decrypt/_index.md @@ -37,4 +37,4 @@ Advanced piping is supported ```shell $ echo "hello world" | otdfctl encrypt | otdfctl decrypt | cat hello world -``` \ No newline at end of file +``` From 79a12838684caaed59ed12c7f3b5a87668bc6637 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 12:29:34 -0600 Subject: [PATCH 08/14] Add assertions --- pkg/handlers/tdf.go | 114 ++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index 460d98d1..6314e4d0 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -2,6 +2,7 @@ package handlers import ( "bytes" + "encoding/json" "errors" "fmt" "io" @@ -15,7 +16,7 @@ var ( ErrTDFInspectFailNotInspectable = errors.New("file or input is not inspectable") ErrTDFUnableToReadAttributes = errors.New("unable to read attributes from TDF") ErrTDFUnableToReadUnencryptedMetadata = errors.New("unable to read unencrypted metadata from TDF") - minBytesLength = 3 + ErrTDFUnableToReadAssertions = errors.New("unable to read assertions") ) const ( @@ -31,7 +32,7 @@ type TDFInspect struct { UnencryptedMetadata []byte } -func (h Handler) EncryptBytes(tdfType string, b []byte, values []string, mimeType string, kasUrlPath string, ecdsaBinding bool) (*bytes.Buffer, error) { +func (h Handler) EncryptBytes(tdfType string, b []byte, values []string, mimeType string, kasUrlPath string, ecdsaBinding bool, assertions string) (*bytes.Buffer, error) { var encrypted []byte enc := bytes.NewBuffer(encrypted) @@ -42,13 +43,24 @@ func (h Handler) EncryptBytes(tdfType string, b []byte, values []string, mimeTyp return nil, errors.New("ECDSA policy binding is not supported for ZTDF") } - _, err := h.sdk.CreateTDF(enc, bytes.NewReader(b), + opts := []sdk.TDFOption{ sdk.WithDataAttributes(values...), sdk.WithKasInformation(sdk.KASInfo{ URL: h.platformEndpoint + kasUrlPath, }), sdk.WithMimeType(mimeType), - ) + } + + var assertionConfigs []sdk.AssertionConfig + if assertions != "" { + err := json.Unmarshal([]byte(assertions), &assertionConfigs) + if err != nil { + return nil, errors.Join(ErrTDFUnableToReadAssertions, err) + } + opts = append(opts, sdk.WithAssertions(assertionConfigs...)) + } + + _, err := h.sdk.CreateTDF(enc, bytes.NewReader(b), opts...) return enc, err // Encrypt the data as a Nano TDF @@ -83,10 +95,7 @@ func (h Handler) DecryptBytes(toDecrypt []byte) (*bytes.Buffer, error) { out := &bytes.Buffer{} pt := io.Writer(out) ec := bytes.NewReader(toDecrypt) - tdfType := sdk.GetTdfType(ec) - // reset the reader to the beginning - ec.Reset(toDecrypt) - switch tdfType { + switch sdk.GetTdfType(ec) { case sdk.Nano: if _, err := h.sdk.ReadNanoTDF(pt, ec); err != nil { return nil, err @@ -109,60 +118,49 @@ func (h Handler) DecryptBytes(toDecrypt []byte) (*bytes.Buffer, error) { } func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { - if len(toInspect) < minBytesLength { - return TDFInspect{}, []error{fmt.Errorf("tdf too small [%d] bytes", len(toInspect))} - } - switch { - case bytes.Equal([]byte("PK"), toInspect[0:2]): - return h.InspectZTDF(toInspect) - case bytes.Equal([]byte("L1L"), toInspect[0:3]): - return h.InspectNanoTDF(toInspect) - } - return TDFInspect{}, []error{fmt.Errorf("tdf format unrecognized")} -} - -func (h Handler) InspectZTDF(toInspect []byte) (TDFInspect, []error) { - // grouping errors so we don't impact the piping of the data - errs := []error{} + b := bytes.NewReader(toInspect) + switch sdk.GetTdfType(b) { + case sdk.Standard: + // grouping errors so we don't impact the piping of the data + errs := []error{} - tdfreader, err := h.sdk.LoadTDF(bytes.NewReader(toInspect)) - if err != nil { - if strings.Contains(err.Error(), "zip: not a valid zip file") { - return TDFInspect{}, []error{ErrTDFInspectFailNotInspectable} + tdfreader, err := h.sdk.LoadTDF(bytes.NewReader(toInspect)) + if err != nil { + if strings.Contains(err.Error(), "zip: not a valid zip file") { + return TDFInspect{}, []error{ErrTDFInspectFailNotInspectable} + } + return TDFInspect{}, []error{errors.Join(ErrTDFInspectFailNotValidTDF, err)} } - return TDFInspect{}, []error{errors.Join(ErrTDFInspectFailNotValidTDF, err)} - } - - attributes, err := tdfreader.DataAttributes() - if err != nil { - errs = append(errs, errors.Join(ErrTDFUnableToReadAttributes, err)) - } - unencryptedMetadata, err := tdfreader.UnencryptedMetadata() - if err != nil { - errs = append(errs, errors.Join(ErrTDFUnableToReadUnencryptedMetadata, err)) - } + attributes, err := tdfreader.DataAttributes() + if err != nil { + errs = append(errs, errors.Join(ErrTDFUnableToReadAttributes, err)) + } - m := tdfreader.Manifest() - return TDFInspect{ - ZTDFManifest: &m, - Attributes: attributes, - UnencryptedMetadata: unencryptedMetadata, - }, errs -} + unencryptedMetadata, err := tdfreader.UnencryptedMetadata() + if err != nil { + errs = append(errs, errors.Join(ErrTDFUnableToReadUnencryptedMetadata, err)) + } -//nolint:gosec,mnd // SDK should secure lengths of inputs/outputs -func (h Handler) InspectNanoTDF(toInspect []byte) (TDFInspect, []error) { - header, size, err := sdk.NewNanoTDFHeaderFromReader(bytes.NewReader(toInspect)) - if err != nil { - return TDFInspect{}, []error{errors.Join(ErrTDFInspectFailNotValidTDF, err)} - } - r := TDFInspect{ - NanoHeader: &header, - } - remainder := uint32(len(toInspect)) - size - if remainder < 18 { - return r, []error{ErrTDFInspectFailNotValidTDF} + m := tdfreader.Manifest() + return TDFInspect{ + ZTDFManifest: &m, + Attributes: attributes, + UnencryptedMetadata: unencryptedMetadata, + }, errs + case sdk.Nano: + header, size, err := sdk.NewNanoTDFHeaderFromReader(b) + if err != nil { + return TDFInspect{}, []error{errors.Join(ErrTDFInspectFailNotValidTDF, err)} + } + r := TDFInspect{ + NanoHeader: &header, + } + remainder := uint32(len(toInspect)) - size + if remainder < 18 { + return r, []error{ErrTDFInspectFailNotValidTDF} + } + return r, nil } - return r, nil + return TDFInspect{}, []error{fmt.Errorf("tdf format unrecognized")} } From 246bb2ee02ef4c2a8ea4d03cbe94160cdc33efb7 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 14:50:10 -0600 Subject: [PATCH 09/14] move example --- docs/man/encrypt/_index.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/man/encrypt/_index.md b/docs/man/encrypt/_index.md index 6ef3e5ae..a851a05c 100644 --- a/docs/man/encrypt/_index.md +++ b/docs/man/encrypt/_index.md @@ -14,12 +14,12 @@ command: description: The MIME type of the input data. If not provided, the MIME type is inferred from the input data. - name: tdf-type shorthand: t - description: The type of tdf to encrypt as. TDF3 supports structured manifests and larger payloads. Nano has a smaller footprint and more performant, but does not support structured manifests or large payloads. + description: The type of tdf to encrypt as. ZTDF supports structured manifests and larger payloads. NanoTDF has a smaller footprint and more performant, but does not support structured manifests or large payloads. (tdf3 is an alias for ztdf) enum: - - tdf3 - ztdf + - tdf3 - nano - default: tdf3 + default: ztdf - name: ecdsa-binding description: For nano type containers only, enables ECDSA policy binding - name: kas-url-path @@ -27,7 +27,7 @@ command: default: /kas - name: with-assertions description: > - EXPERIMENTAL: JSON string containing list of assertions to be applied during encryption. example - '[{"id":"assertion1","type":"handling","scope":"tdo","appliesToState":"encrypted","statement":{"format":"json+stanag5636","schema":"urn:nato:stanag:5636:A:1:elements:json","value":"{\"ocl\":\"2024-10-21T20:47:36Z\"}"}}]' + EXPERIMENTAL: JSON string of assertions to bind metadata to the TDF. See examples for more information. --- Build a Trusted Data Format (TDF) with encrypted content from a specified file or input from stdin utilizing OpenTDF platform. @@ -83,3 +83,15 @@ support structured manifests or large payloads. # output to nano.tdf otdfctl encrypt hello.txt --tdf-type nano --out hello.txt.tdf ``` + +## ZTDF Assertions (experimental) + +Assertions are a way to bind metadata to the TDF data object in a cryptographically secure way. + +### STANAG 5636 + +The following example demonstrates how to bind a STANAG 5636 metadata assertion to the TDF data object. + +```shell +otdfctl encrypt hello.txt --out hello.txt.tdf --with-assertions '[{"id":"assertion1","type":"handling","scope":"tdo","appliesToState":"encrypted","statement":{"format":"json+stanag5636","schema":"urn:nato:stanag:5636:A:1:elements:json","value":"{\"ocl\":\"2024-10-21T20:47:36Z\"}"}}]' +``` From 0f4bf5b08c6a6c2ebaf998731b2c8f98bbc1454f Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 14:56:28 -0600 Subject: [PATCH 10/14] update sdk --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 936c0b12..7a8fee32 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.6.0 github.com/opentdf/platform/lib/flattening v0.1.1 - github.com/opentdf/platform/protocol/go v0.2.18 - github.com/opentdf/platform/sdk v0.3.17 + github.com/opentdf/platform/protocol/go v0.2.20 + github.com/opentdf/platform/sdk v0.3.18 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 3676750a..e24f1b18 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,12 @@ github.com/opentdf/platform/lib/ocrypto v0.1.6 h1:rd4ctCZOE/c3qDJORtkSK9tw6dEXb+ github.com/opentdf/platform/lib/ocrypto v0.1.6/go.mod h1:ne+l8Q922OdzA0xesK3XJmfECBnn5vLSGYU3/3OhiHM= github.com/opentdf/platform/protocol/go v0.2.18 h1:s+TVZkOPGCzy7WyObtJWJNaFeOGDUTuSmAsq3omvugY= github.com/opentdf/platform/protocol/go v0.2.18/go.mod h1:WqDcnFQJb0v8ivRQPidbehcL8ils5ZSZYXkuv0nyvsI= +github.com/opentdf/platform/protocol/go v0.2.20 h1:FPU1ZcXvPm/QeE2nqgbD/HMTOCICQSD0DoncQbAZ1ws= +github.com/opentdf/platform/protocol/go v0.2.20/go.mod h1:TWIuf387VeR3q0TL4nAMKQTWEqqID+8Yjao76EX9Dto= github.com/opentdf/platform/sdk v0.3.17 h1:Uo/kTMneB18i0gZNfTRtvw34bGLFUc8BEnA/BMK0VVs= github.com/opentdf/platform/sdk v0.3.17/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs= +github.com/opentdf/platform/sdk v0.3.18 h1:IY6fNrOfQD9lF/hZp9ewZsH0PMuLe17HlSE1A5kyIWc= +github.com/opentdf/platform/sdk v0.3.18/go.mod h1:u+XZhVRsMq5blukCFCHcjk6HLCp4Y5mmIQu7GhtKQ3E= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= From 4fc28c316d33e448292e2620ca7ee3232286f010 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 15:36:45 -0600 Subject: [PATCH 11/14] Lint and spelling --- .github/spellcheck.ignore | 1 + pkg/handlers/tdf.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/spellcheck.ignore b/.github/spellcheck.ignore index 4590fbdc..bd302959 100644 --- a/.github/spellcheck.ignore +++ b/.github/spellcheck.ignore @@ -116,3 +116,4 @@ appliesToState stanag nato ocl +cryptographically \ No newline at end of file diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index a316122a..a825d4ab 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -161,6 +161,9 @@ func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { return r, []error{ErrTDFInspectFailNotValidTDF} } return r, nil + case sdk.Invalid: + return TDFInspect{}, []error{ErrTDFInspectFailNotValidTDF} + default: + return TDFInspect{}, []error{fmt.Errorf("tdf format unrecognized")} } - return TDFInspect{}, []error{fmt.Errorf("tdf format unrecognized")} } From 62d96567d4f277538024370034dbaa2b51652604 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 15:39:16 -0600 Subject: [PATCH 12/14] Lint --- pkg/handlers/tdf.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index a825d4ab..5634afbd 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -156,8 +156,9 @@ func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { r := TDFInspect{ NanoHeader: &header, } - remainder := uint32(len(toInspect)) - size - if remainder < 18 { + remainder := uint32(len(toInspect) - int(size)) + eighteen := uint32(18) // TODO: Rename. Not sure what this value is at present + if remainder < eighteen { return r, []error{ErrTDFInspectFailNotValidTDF} } return r, nil From d9dd1548c29cc8528e9858a72a3441891bf13c0c Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 16:09:19 -0600 Subject: [PATCH 13/14] fix lint --- pkg/handlers/tdf.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/handlers/tdf.go b/pkg/handlers/tdf.go index 5634afbd..f6250f72 100644 --- a/pkg/handlers/tdf.go +++ b/pkg/handlers/tdf.go @@ -117,6 +117,9 @@ func (h Handler) DecryptBytes(toDecrypt []byte, disableAssertionCheck bool) (*by return out, nil } +// TODO: Rename. Not sure what this value is at present +const inspectTDFEighteen = 18 + func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { b := bytes.NewReader(toInspect) switch sdk.GetTdfType(b) { @@ -156,9 +159,8 @@ func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { r := TDFInspect{ NanoHeader: &header, } - remainder := uint32(len(toInspect) - int(size)) - eighteen := uint32(18) // TODO: Rename. Not sure what this value is at present - if remainder < eighteen { + remainder := len(toInspect) - int(size) + if remainder < inspectTDFEighteen { return r, []error{ErrTDFInspectFailNotValidTDF} } return r, nil From 3d287c25e484cbbaa6945edbf811503b926b68c2 Mon Sep 17 00:00:00 2001 From: Ryan Schumacher Date: Thu, 7 Nov 2024 16:11:30 -0600 Subject: [PATCH 14/14] Update docs/man/encrypt/_index.md Co-authored-by: Jake Van Vorhis <83739412+jakedoublev@users.noreply.github.com> --- docs/man/encrypt/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/man/encrypt/_index.md b/docs/man/encrypt/_index.md index a851a05c..2b6c76b8 100644 --- a/docs/man/encrypt/_index.md +++ b/docs/man/encrypt/_index.md @@ -76,7 +76,7 @@ otdfctl encrypt hello.txt --out hello.txt.tdf --attr https://example.com/attr/at ## NanoTDF -NanoTDF is a lightweight TDF format that is more performant and has a smaller footprint than TDF3. NanoTDF does not +NanoTDF is a lightweight TDF format that is more performant and has a smaller footprint than ZTDF. NanoTDF does not support structured manifests or large payloads. ```shell