Added base64 support

Author: mikefarah

pkg/yqlib/decoder.go

@@ -13,6 +13,7 @@ const (
 	YamlInputFormat = 1 << iota
 	XMLInputFormat
 	PropertiesInputFormat
+	Base64InputFormat
 )
 
 type Decoder interface {

pkg/yqlib/decoder_base64.go

@@ -0,0 +1,44 @@
+package yqlib
+
+import (
+	"bytes"
+	"encoding/base64"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type base64Decoder struct {
+	reader   io.Reader
+	finished bool
+	encoding base64.Encoding
+}
+
+func NewBase64Decoder() Decoder {
+	return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
+}
+
+func (dec *base64Decoder) Init(reader io.Reader) {
+	dec.reader = reader
+	dec.finished = false
+}
+
+func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
+	if dec.finished {
+		return io.EOF
+	}
+	base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(base64Reader); err != nil {
+		return err
+	}
+	if buf.Len() == 0 {
+		dec.finished = true
+		return io.EOF
+	}
+	rootYamlNode.Kind = yaml.ScalarNode
+	rootYamlNode.Tag = "!!str"
+	rootYamlNode.Value = buf.String()
+	return nil
+}

pkg/yqlib/doc/operators/encode-decode.md

@@ -15,13 +15,16 @@ These operators are useful to process yaml documents that have stringified embed
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
+| Base64 | @base64d | @base64 |
 
 
 CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 
 
+Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 
@@ -354,3 +357,66 @@ b:
   foo: bar
 ```
 
+## Encode a string to base64
+Given a sample.yml file of:
+```yaml
+coolData: a special string
+```
+then
+```bash
+yq '.coolData | @base64' sample.yml
+```
+will output
+```yaml
+YSBzcGVjaWFsIHN0cmluZw==
+```
+
+## Encode a yaml document to base64
+Pipe through @yaml first to convert to a string, then use @base64 to encode it.
+
+Given a sample.yml file of:
+```yaml
+a: apple
+```
+then
+```bash
+yq '@yaml | @base64' sample.yml
+```
+will output
+```yaml
+YTogYXBwbGUK
+```
+
+## Decode a base64 encoded string
+Decoded data is assumed to be a string.
+
+Given a sample.yml file of:
+```yaml
+coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==
+```
+then
+```bash
+yq '.coolData | @base64d' sample.yml
+```
+will output
+```yaml
+Works with UTF-16 😊
+```
+
+## Decode a base64 encoded yaml document
+Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.
+
+Given a sample.yml file of:
+```yaml
+coolData: YTogYXBwbGUK
+```
+then
+```bash
+yq '.coolData |= (@base64d | from_yaml)' sample.yml
+```
+will output
+```yaml
+coolData:
+  a: apple
+```
+

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -15,9 +15,12 @@ These operators are useful to process yaml documents that have stringified embed
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
+| Base64 | @base64d | @base64 |
 
 
 CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 
+
+Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.

pkg/yqlib/doc/usage/base64.md

@@ -0,0 +1,23 @@
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Decode Base64
+Decoded data is assumed to be a string.
+
+Given a sample.yml file of:
+```yml
+V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==
+```
+then
+```bash
+yq -p=props sample.properties
+```
+will output
+```yaml
+V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig: =
+```
+

pkg/yqlib/encoder_base64.go

@@ -0,0 +1,38 @@
+package yqlib
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type base64Encoder struct {
+	encoding base64.Encoding
+}
+
+func NewBase64Encoder() Encoder {
+	return &base64Encoder{encoding: *base64.StdEncoding}
+}
+
+func (e *base64Encoder) CanHandleAliases() bool {
+	return false
+}
+
+func (e *base64Encoder) PrintDocumentSeparator(writer io.Writer) error {
+	return nil
+}
+
+func (e *base64Encoder) PrintLeadingContent(writer io.Writer, content string) error {
+	return nil
+}
+
+func (e *base64Encoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
+	node := unwrapDoc(originalNode)
+	if guessTagFromCustomType(node) != "!!str" {
+		return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	}
+	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(originalNode.Value))))
+	return err
+}

pkg/yqlib/expression_tokeniser.go

@@ -363,6 +363,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 2}))
 	lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 0}))
 
+	lexer.Add([]byte(`@base64`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: Base64OutputFormat}))
+	lexer.Add([]byte(`@base64d`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: Base64InputFormat}))
+
 	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))

pkg/yqlib/operator_encoder_decoder.go

@@ -24,6 +24,8 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 		return NewYamlEncoder(indent, false, true, true)
 	case XMLOutputFormat:
 		return NewXMLEncoder(indent, XMLPreferences.AttributePrefix, XMLPreferences.ContentName)
+	case Base64OutputFormat:
+		return NewBase64Encoder()
 	}
 	panic("invalid encoder")
 }
@@ -103,6 +105,8 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		decoder = NewYamlDecoder()
 	case XMLInputFormat:
 		decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)
+	case Base64InputFormat:
+		decoder = NewBase64Decoder()
 	}
 
 	var results = list.New()

pkg/yqlib/operator_encoder_decoder_test.go

@@ -200,6 +200,41 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n    foo: bar\n",
 		},
 	},
+	{
+		description: "Encode a string to base64",
+		document:    "coolData: a special string",
+		expression:  ".coolData | @base64",
+		expected: []string{
+			"D0, P[coolData], (!!str)::YSBzcGVjaWFsIHN0cmluZw==\n",
+		},
+	},
+	{
+		description:    "Encode a yaml document to base64",
+		subdescription: "Pipe through @yaml first to convert to a string, then use @base64 to encode it.",
+		document:       "a: apple",
+		expression:     "@yaml | @base64",
+		expected: []string{
+			"D0, P[], (!!str)::YTogYXBwbGUK\n",
+		},
+	},
+	{
+		description:    "Decode a base64 encoded string",
+		subdescription: "Decoded data is assumed to be a string.",
+		document:       "coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==",
+		expression:     ".coolData | @base64d",
+		expected: []string{
+			"D0, P[coolData], (!!str)::Works with UTF-16 😊\n",
+		},
+	},
+	{
+		description:    "Decode a base64 encoded yaml document",
+		subdescription: "Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.",
+		document:       "coolData: YTogYXBwbGUK",
+		expression:     ".coolData |= (@base64d | from_yaml)",
+		expected: []string{
+			"D0, P[], (doc)::coolData:\n    a: apple\n",
+		},
+	},
 }
 
 func TestEncoderDecoderOperatorScenarios(t *testing.T) {

pkg/yqlib/printer.go

@@ -26,6 +26,7 @@ const (
 	CSVOutputFormat
 	TSVOutputFormat
 	XMLOutputFormat
+	Base64OutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {