From c51271399c688abd0f4719c46000a2ed74eec993 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Sun, 2 Apr 2023 12:20:01 +0200 Subject: [PATCH] interp: Add skip_gaps option for tovalue/-V Skips gap fields in struct and arrays. Gaps fields are bit ranges that a decoder did not add any fields for. Note that skipping gaps in arrays will affect indexes. --- doc/usage.md | 26 ++++++++++++++-- pkg/interp/decode.go | 48 +++++++++++++++++++++++------- pkg/interp/interp.go | 7 +++-- pkg/interp/options.jq | 4 ++- pkg/interp/testdata/args.fqtest | 1 + pkg/interp/testdata/options.fqtest | 1 + pkg/interp/testdata/tovalue.fqtest | 22 ++++++++++---- pkg/scalar/scalar_gen.go | 16 +++++----- pkg/scalar/scalar_gen.go.tmpl | 2 +- 9 files changed, 96 insertions(+), 31 deletions(-) diff --git a/doc/usage.md b/doc/usage.md index a9bca220d..ac6a906e6 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -796,16 +796,36 @@ fq has some general options in addition to decode and decoders specific options. `` is fuzzily parsed based on the type of the option. Ex: a string can be specified as `-o name=string` or `-o name="string"`. -### `bits_format` +### `-o bits_format=` -How to represent raw bits as JSON. +How to represent raw binary as JSON. -- `-o bits_foramt=string` String with raw bytes (zero bit padded). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON. +- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON. - `-o bits_format=md5` MD5 hex string (zero bit padded). - `-o bits_format=base64` Base64 string. - `-p bits_foramt=truncate` Truncated string. - `-o bits_format=snippet` Truncated Base64 string prefixed with bit length. +```sh +$ fq -V -o bits_format=base64 . file` +``` +In query +```jq +tovalue({bits_format: "md5"}) +``` + +### `-o skip_gaps=` + +Skip gaps fields (`gap0` etc) when using `tovalue` or `-V`. Note that this might affect array indexes if one more more gaps fields are skipped in an array. + +```sh +$ fq -V -o skip_gaps=true . file` +``` +In query +```jq +tovalue({skip_gaps: true}) +``` + ## Color and unicode output fq by default tries to use colors if possible, this can be disabled with `-M`. You can also diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go index ab05f7596..97136d209 100644 --- a/pkg/interp/decode.go +++ b/pkg/interp/decode.go @@ -163,9 +163,12 @@ func (i *Interp) _registry(c any) any { } } -func (i *Interp) _toValue(c any, opts map[string]any) any { +func (i *Interp) _toValue(c any, om map[string]any) any { return toValue( - func() Options { return OptionsFromValue(opts) }, + func() *Options { + opts := OptionsFromValue(om) + return &opts + }, c, ) } @@ -307,7 +310,7 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any { // TODO: make more efficient somehow? shallow values but might be hard // when things like tovalue.key should behave like a jq value and not a decode value etc -func toValue(optsFn func() Options, v any) any { +func toValue(optsFn func() *Options, v any) any { nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) { switch v := v.(type) { case JQValueEx: @@ -540,7 +543,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any { case "_gap": switch vv := dv.V.(type) { case Scalarable: - return vv.ScalarGap() + return vv.ScalarIsGap() default: return false } @@ -606,7 +609,7 @@ func (v decodeValue) JQValueKey(name string) any { func (v decodeValue) JQValueHas(key any) any { return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas) } -func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any { +func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any { if !v.bitsFormat { return v.JQValueToGoJQ() } @@ -695,13 +698,26 @@ func (v ArrayDecodeValue) JQValueHas(key any) any { return intKey >= 0 && intKey < len(v.Compound.Children) }) } -func (v ArrayDecodeValue) JQValueToGoJQ() any { - vs := make([]any, len(v.Compound.Children)) - for i, f := range v.Compound.Children { - vs[i] = makeDecodeValue(f, decodeValueValue) +func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { + opts := optsFn() + + vs := make([]any, 0, len(v.Compound.Children)) + for _, f := range v.Compound.Children { + switch s := f.V.(type) { + case Scalarable: + if s.ScalarIsGap() && opts.SkipGaps { + // skip, note for arrays this will affect indexes + continue + } + } + + vs = append(vs, makeDecodeValue(f, decodeValueValue)) } return vs } +func (v ArrayDecodeValue) JQValueToGoJQ() any { + return v.JQValueToGoJQEx(func() *Options { return &Options{} }) +} // decode value struct @@ -767,10 +783,22 @@ func (v StructDecodeValue) JQValueHas(key any) any { }, ) } -func (v StructDecodeValue) JQValueToGoJQ() any { +func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { + opts := optsFn() + vm := make(map[string]any, len(v.Compound.Children)) for _, f := range v.Compound.Children { + switch s := f.V.(type) { + case Scalarable: + if s.ScalarIsGap() && opts.SkipGaps { + continue + } + } + vm[f.Name] = makeDecodeValue(f, decodeValueValue) } return vm } +func (v StructDecodeValue) JQValueToGoJQ() any { + return v.JQValueToGoJQEx(func() *Options { return &Options{} }) +} diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index e9ab6f5c1..1f12e5999 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -82,7 +82,7 @@ type Scalarable interface { ScalarValue() any ScalarSym() any ScalarDescription() string - ScalarGap() bool + ScalarIsGap() bool ScalarDisplayFormat() scalar.DisplayFormat } @@ -210,7 +210,7 @@ type Display interface { type JQValueEx interface { gojq.JQValue - JQValueToGoJQEx(optsFn func() Options) any + JQValueToGoJQEx(optsFn func() *Options) any } func valuePath(v *decode.Value) []any { @@ -682,7 +682,7 @@ func (i *Interp) _printColorJSON(c any, v any) gojq.Iter { Color: opts.Color, Tab: false, Indent: indent, - ValueFn: func(v any) any { return toValue(func() Options { return opts }, v) }, + ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) }, Colors: colorjson.Colors{ Reset: []byte(ansi.Reset.SetString), Null: []byte(opts.Decorator.Null.SetString), @@ -1034,6 +1034,7 @@ type Options struct { DisplayBytes int Addrbase int Sizebase int + SkipGaps bool Decorator Decorator BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error) diff --git a/pkg/interp/options.jq b/pkg/interp/options.jq index 2f4ce8c7b..c650f78e5 100644 --- a/pkg/interp/options.jq +++ b/pkg/interp/options.jq @@ -59,6 +59,7 @@ def _opt_build_default_fixed: raw_output: ($stdout.is_terminal | not), raw_string: false, repl: false, + skip_gaps: false, sizebase: 10, show_formats: false, show_help: false, @@ -101,9 +102,10 @@ def _opt_options: raw_output: "boolean", raw_string: "boolean", repl: "boolean", - sizebase: "number", show_formats: "boolean", show_help: "boolean", + sizebase: "number", + skip_gaps: "boolean", slurp: "boolean", string_input: "boolean", unicode: "boolean", diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest index f2003699e..61795a5a6 100644 --- a/pkg/interp/testdata/args.fqtest +++ b/pkg/interp/testdata/args.fqtest @@ -99,6 +99,7 @@ repl false show_formats false show_help options sizebase 10 +skip_gaps false slurp false string_input false unicode false diff --git a/pkg/interp/testdata/options.fqtest b/pkg/interp/testdata/options.fqtest index 8409062a3..2b07b35da 100644 --- a/pkg/interp/testdata/options.fqtest +++ b/pkg/interp/testdata/options.fqtest @@ -83,6 +83,7 @@ $ fq -n options "show_formats": false, "show_help": false, "sizebase": 10, + "skip_gaps": false, "slurp": false, "string_input": false, "unicode": false, diff --git a/pkg/interp/testdata/tovalue.fqtest b/pkg/interp/testdata/tovalue.fqtest index c50b69685..3ab21bc28 100644 --- a/pkg/interp/testdata/tovalue.fqtest +++ b/pkg/interp/testdata/tovalue.fqtest @@ -2,9 +2,21 @@ $ fq -i . test.mp3 mp3> .headers[0] | tovalue.header.magic "ID3" +mp3> [tobytes, "abc"] | mp3 | tovalue, tovalue({skip_gaps: true}) | keys +[ + "footers", + "frames", + "gap0", + "headers" +] +[ + "footers", + "frames", + "headers" +] +mp3> "abc" | mpeg_ts | tovalue, tovalue({skip_gaps: true}) | keys +[ + "gap0" +] +[] mp3> ^D -$ fq -i -null> "aaa" | mp3_frame | .gap0 | tovalue, tovalue({sizebase: 2}) -"aaa" -"aaa" -null> ^D diff --git a/pkg/scalar/scalar_gen.go b/pkg/scalar/scalar_gen.go index 4e1e8b0e2..af0956bfc 100644 --- a/pkg/scalar/scalar_gen.go +++ b/pkg/scalar/scalar_gen.go @@ -27,7 +27,7 @@ func (s Any) ScalarValue() any { } func (s Any) ScalarSym() any { return s.Sym } func (s Any) ScalarDescription() string { return s.Description } -func (s Any) ScalarGap() bool { return s.Gap } +func (s Any) ScalarIsGap() bool { return s.Gap } func (s Any) ScalarDisplayFormat() DisplayFormat { return 0 } func AnyActual(v any) AnyMapper { @@ -224,7 +224,7 @@ func (s BigInt) ScalarValue() any { } func (s BigInt) ScalarSym() any { return s.Sym } func (s BigInt) ScalarDescription() string { return s.Description } -func (s BigInt) ScalarGap() bool { return s.Gap } +func (s BigInt) ScalarIsGap() bool { return s.Gap } func (s BigInt) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat } func BigIntActual(v *big.Int) BigIntMapper { @@ -420,7 +420,7 @@ func (s BitBuf) ScalarValue() any { } func (s BitBuf) ScalarSym() any { return s.Sym } func (s BitBuf) ScalarDescription() string { return s.Description } -func (s BitBuf) ScalarGap() bool { return s.Gap } +func (s BitBuf) ScalarIsGap() bool { return s.Gap } func (s BitBuf) ScalarDisplayFormat() DisplayFormat { return 0 } func BitBufActual(v bitio.ReaderAtSeeker) BitBufMapper { @@ -616,7 +616,7 @@ func (s Bool) ScalarValue() any { } func (s Bool) ScalarSym() any { return s.Sym } func (s Bool) ScalarDescription() string { return s.Description } -func (s Bool) ScalarGap() bool { return s.Gap } +func (s Bool) ScalarIsGap() bool { return s.Gap } func (s Bool) ScalarDisplayFormat() DisplayFormat { return 0 } func BoolActual(v bool) BoolMapper { @@ -812,7 +812,7 @@ func (s Flt) ScalarValue() any { } func (s Flt) ScalarSym() any { return s.Sym } func (s Flt) ScalarDescription() string { return s.Description } -func (s Flt) ScalarGap() bool { return s.Gap } +func (s Flt) ScalarIsGap() bool { return s.Gap } func (s Flt) ScalarDisplayFormat() DisplayFormat { return 0 } func FltActual(v float64) FltMapper { @@ -1009,7 +1009,7 @@ func (s Sint) ScalarValue() any { } func (s Sint) ScalarSym() any { return s.Sym } func (s Sint) ScalarDescription() string { return s.Description } -func (s Sint) ScalarGap() bool { return s.Gap } +func (s Sint) ScalarIsGap() bool { return s.Gap } func (s Sint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat } func SintActual(v int64) SintMapper { @@ -1205,7 +1205,7 @@ func (s Str) ScalarValue() any { } func (s Str) ScalarSym() any { return s.Sym } func (s Str) ScalarDescription() string { return s.Description } -func (s Str) ScalarGap() bool { return s.Gap } +func (s Str) ScalarIsGap() bool { return s.Gap } func (s Str) ScalarDisplayFormat() DisplayFormat { return 0 } func StrActual(v string) StrMapper { @@ -1402,7 +1402,7 @@ func (s Uint) ScalarValue() any { } func (s Uint) ScalarSym() any { return s.Sym } func (s Uint) ScalarDescription() string { return s.Description } -func (s Uint) ScalarGap() bool { return s.Gap } +func (s Uint) ScalarIsGap() bool { return s.Gap } func (s Uint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat } func UintActual(v uint64) UintMapper { diff --git a/pkg/scalar/scalar_gen.go.tmpl b/pkg/scalar/scalar_gen.go.tmpl index 0a7c5992c..cda46a66d 100644 --- a/pkg/scalar/scalar_gen.go.tmpl +++ b/pkg/scalar/scalar_gen.go.tmpl @@ -31,7 +31,7 @@ import ( } func (s {{$name}}) ScalarSym() any { return s.Sym } func (s {{$name}}) ScalarDescription() string { return s.Description } - func (s {{$name}}) ScalarGap() bool { return s.Gap } + func (s {{$name}}) ScalarIsGap() bool { return s.Gap } {{- if $t.display_format}} func (s {{$name}}) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat } {{- else }}