Skip to content

Commit

Permalink
ISSUE-144: allow schema object property name contains any character (#…
Browse files Browse the repository at this point in the history 
…145)

Perviously we have a validation forcing all output object property name to be only
alpha-numeric (plus underscore). Turns out we have use cases where the property name
might contains special characters such as `'$'` or `'%'`, or `'.'`.

Loosen up the restriction and dealing with all special character in property name
(thus in FQDN) properly with escaping/unescaping.
  • Loading branch information
@jf-tech
jf-tech authored Apr 28, 2021
1 parent 2c284f5 commit 3cd5f73
Show file tree
Hide file tree
Showing 10 changed files with 18 additions and 25 deletions.
2 changes: 1 addition & 1 deletion extensions/omniv21/samples/json/.snapshots/Test1_Single_Object
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"RawRecord": "{\"items\":[{\"item_price\":12.34,\"item_sku\":\"ab123\",\"number_purchased\":5},{\"item_price\":3.12,\"item_sku\":\"ck763-23\",\"number_purchased\":2}],\"order_id\":\"1234567\",\"tracking_number\":\"1z9999999999999999\"}",
"RawRecordHash": "6ef8f474-8b41-3366-9356-ef44a84c1439",
"TransformedRecord": {
"$order.id": "1234567",
"items": [
{
"sku": "AB123",
Expand All @@ -13,7 +14,6 @@
"total_price": 6.24
}
],
"order_id": "1234567",
"tracking_number": "1Z9999999999999999"
}
}
Expand Down
2 changes: 1 addition & 1 deletion extensions/omniv21/samples/json/1_single_object.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
},
"transform_declarations": {
"FINAL_OUTPUT": { "object": {
"order_id": { "xpath": "order_id" },
"$order.id": { "xpath": "order_id" },
"tracking_number": { "custom_func": {
"name": "upper",
"args": [ { "xpath": "tracking_number" } ]
Expand Down
14 changes: 7 additions & 7 deletions extensions/omniv21/transform/.snapshots/TestValidateTransformDeclarations-success
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
{
"object": {
"$field_13 with space. and other non-alphanumeric chars": {
"custom_parse": "test_custom_parse",
"fqdn": "FINAL_OUTPUT.$field_13 with space%. and other non-alphanumeric chars",
"kind": "custom_parse",
"parent": "FINAL_OUTPUT"
},
"field1": {
"const": "value1",
"fqdn": "FINAL_OUTPUT.field1",
Expand Down Expand Up @@ -200,12 +206,6 @@
],
"parent": "FINAL_OUTPUT"
},
"field_13": {
"custom_parse": "test_custom_parse",
"fqdn": "FINAL_OUTPUT.field_13",
"kind": "custom_parse",
"parent": "FINAL_OUTPUT"
},
"field_9": {
"xpath": "1/2/3",
"object": {
Expand All @@ -227,14 +227,14 @@
"fqdn": "FINAL_OUTPUT",
"kind": "object",
"children": [
"FINAL_OUTPUT.$field_13 with space%. and other non-alphanumeric chars",
"FINAL_OUTPUT.field1",
"FINAL_OUTPUT.field2",
"FINAL_OUTPUT.field3",
"FINAL_OUTPUT.field6",
"FINAL_OUTPUT.field_10",
"FINAL_OUTPUT.field_11",
"FINAL_OUTPUT.field_12",
"FINAL_OUTPUT.field_13",
"FINAL_OUTPUT.field_9"
],
"parent": "(nil)"
Expand Down
2 changes: 1 addition & 1 deletion extensions/omniv21/transform/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ func (p *parseCtx) parseObject(n *idr.Node, decl *Decl) (interface{}, error) {
// value returned by p.ParseNode is already normalized, thus this
// normalizeAndSaveValue won't fail.
_ = normalizeAndSaveValue(childDecl, childValue, func(normalizedValue interface{}) {
obj[strs.LastNameletOfFQDN(childDecl.fqdn)] = normalizedValue
obj[strs.LastNameletOfFQDNWithEsc(childDecl.fqdn)] = normalizedValue
})
}
return normalizeAndReturnValue(decl, obj)
Expand Down
3 changes: 2 additions & 1 deletion extensions/omniv21/transform/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ func (ctx *validateCtx) validateXPath(fqdn string, decl *Decl, templateRefStack
func (ctx *validateCtx) validateObject(fqdn string, decl *Decl, templateRefStack []string) error {
for childName, childDecl := range decl.Object {
childDecl, err := ctx.validateDecl(
strs.BuildFQDN(fqdn, childName), childDecl, templateRefStack)
// childName can contain '.' or '%', it needs to be escaped.
strs.BuildFQDN(fqdn, strs.BuildFQDNWithEsc(childName)), childDecl, templateRefStack)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion extensions/omniv21/transform/validate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func TestValidateTransformDeclarations(t *testing.T) {
"field_10": { "xpath_dynamic": { "const": "X/Y/Z" }, "template": "template10" },
"field_11": { "template": "template11" },
"field_12": { "template": "template12" },
"field_13": { "custom_parse": "test_custom_parse" }
"$field_13 with space. and other non-alphanumeric chars": { "custom_parse": "test_custom_parse" }
}},
"template9": { "xpath": "1/2/3", "object": {
"field9": { "xpath": "4/5/6" }
Expand Down
2 changes: 1 addition & 1 deletion extensions/omniv21/validation/transformDeclarations.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion extensions/omniv21/validation/transformDeclarations.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
"value_object": {
"type": "object",
"patternProperties": {
"^[_a-zA-Z0-9]+$": {
"^.+$": {
"oneOf": [
{ "$ref": "#/definitions/const" },
{ "$ref": "#/definitions/external" },
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ require (
github.com/go-chi/chi v4.1.2+incompatible
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/google/uuid v1.1.2
github.com/jf-tech/go-corelib v0.0.13
github.com/jf-tech/go-corelib v0.0.14
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.6.1
Expand Down
12 changes: 2 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antchfx/xmlquery v1.3.1 h1:nIKWdtnhrXtj0/IRUAAw2I7TfpHUa3zMnHvNmPXFg+w=
github.com/antchfx/xmlquery v1.3.1/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg=
github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antchfx/xpath v1.1.11 h1:WOFtK8TVAjLm3lbgqeP0arlHpvCEeTANeWZ/csPpJkQ=
github.com/antchfx/xpath v1.1.11/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
Expand All @@ -22,7 +21,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -64,8 +62,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jf-tech/go-corelib v0.0.13 h1:4aDwS09bdfRb/loU6Va+kvqwNJq5irR5WjKVpzJxdag=
github.com/jf-tech/go-corelib v0.0.13/go.mod h1:0+Fejzd53JtexKE5VI8I06WiBNATLIURRJgPrv4Yysg=
github.com/jf-tech/go-corelib v0.0.14 h1:PXS6ApXGhZk+9TTxVFiQe9YYJV5liAzjmvotY7l4dCA=
github.com/jf-tech/go-corelib v0.0.14/go.mod h1:0+Fejzd53JtexKE5VI8I06WiBNATLIURRJgPrv4Yysg=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
Expand Down Expand Up @@ -107,7 +105,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
Expand Down Expand Up @@ -145,7 +142,6 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
Expand All @@ -159,9 +155,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand All @@ -175,14 +169,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down

0 comments on commit 3cd5f73

Please sign in to comment.