From c5f96f2fe820933481984d8feb15d725f1158d2b Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 14:03:44 +0200 Subject: [PATCH 1/9] feat: draft dnslink support --- .github/workflows/test.yml | 2 + Makefile | 1 - cmd/gateway-conformance/main.go | 27 ++++++- dnslinkgen.go | 39 ++++++++++ fixtures/t0109-dnslink.yml | 6 ++ go.mod | 2 +- go.sum | 36 +-------- tests/t0109_gateway_web_redirects_test.go | 95 ++++++++++++++++++++++- tooling/dnslink/dnslink.go | 52 +++++++++++++ tooling/dnslink/merge.go | 63 +++++++++++++++ tooling/fixtures/list.go | 21 ++++- tooling/specs/specs.go | 2 + 12 files changed, 302 insertions(+), 44 deletions(-) create mode 100644 dnslinkgen.go create mode 100644 fixtures/t0109-dnslink.yml create mode 100644 tooling/dnslink/dnslink.go create mode 100644 tooling/dnslink/merge.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51777a329..a031b5e95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,8 @@ jobs: ipfs init; ./gateway-conformance/kubo-config.example.sh; - uses: ipfs/start-ipfs-daemon-action@v1 + env: + IPFS_NS_MAP: 12 # TODO - name: Provision Kubo Gateway run: | find ./fixtures -name '*.car' -exec ipfs dag import {} \; diff --git a/Makefile b/Makefile index fdec29df2..da79019b9 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ provision-cargateway: ./fixtures.car provision-kubo: find ./fixtures -name '*.car' -exec ipfs dag import {} \; -# tools fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 93cc3f1b3..720c4c6be 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -12,6 +12,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" + "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/fixtures" "github.com/urfave/cli/v2" ) @@ -173,19 +174,39 @@ func main() { return err } - files, err := fixtures.List() + fxs, err := fixtures.List() if err != nil { return err } merged := cCtx.Bool("merged") if merged { - err = car.Merge(files, filepath.Join(directory, "fixtures.car")) + err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car")) + if err != nil { + return err + } + + err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) + if err != nil { + return err + } + + err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env")) if err != nil { return err } } else { - err = copyFiles(files, directory) + err = copyFiles(fxs.CarFiles, directory) + if err != nil { + return err + } + + err = copyFiles(fxs.ConfigFiles, directory) + if err != nil { + return err + } + + err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env")) if err != nil { return err } diff --git a/dnslinkgen.go b/dnslinkgen.go new file mode 100644 index 000000000..837da5fa5 --- /dev/null +++ b/dnslinkgen.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/ipfs/gateway-conformance/tooling/dnslink" + "github.com/ipfs/gateway-conformance/tooling/fixtures" +) + +func main() { + if len(os.Args) < 2 { + fmt.Println("Usage: dnslinkgen ") + os.Exit(1) + } + + domain := os.Args[1] + + fxs, err := fixtures.List() + if err != nil { + log.Fatal(err) + } + + configs := fxs.ConfigFiles + aggMap, err := dnslink.Aggregate(configs) + if err != nil { + log.Fatal(err) + } + + // print k=v on stdout + var kvs []string + for k, v := range aggMap { + kvs = append(kvs, fmt.Sprintf("%s%s:%s", k, domain, v)) + } + + fmt.Println("export IPFS_NS_MAP=\"" + strings.Join(kvs, ",") + "\"") +} diff --git a/fixtures/t0109-dnslink.yml b/fixtures/t0109-dnslink.yml new file mode 100644 index 000000000..ce9947ca2 --- /dev/null +++ b/fixtures/t0109-dnslink.yml @@ -0,0 +1,6 @@ +dnslinks: + custom-dnslink: + subdomain: dnslink-enabled-on-fqdn + # this is the cid of the folder + # t0109-redirects.car:/examples/ + path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj \ No newline at end of file diff --git a/go.mod b/go.mod index 8088f2391..9fd9b449d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/ipld/go-ipld-prime v0.20.0 github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.25.3 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -21,7 +22,6 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/goleak v1.1.12 // indirect golang.org/x/sync v0.1.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index dad4a8d01..f18a3caf0 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -21,8 +19,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -49,8 +45,6 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc= -github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU= github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= @@ -65,8 +59,6 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= -github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= @@ -115,8 +107,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= @@ -137,8 +127,6 @@ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0 github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -164,12 +152,8 @@ github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2 github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -226,8 +210,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= -github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= @@ -238,8 +220,6 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 h1:XYEgH2nJgsrcrj32p+SAbx6T3s/6QknOXezXtz7kzbg= github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -248,12 +228,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.15.0 h1:NIl24d4eiLJPM0vKn4HjLYM+UZf6gSfi9Z+NmCxkWbk= go.opentelemetry.io/otel v1.15.0/go.mod h1:qfwLEbWhLPk5gyWrne4XnF0lC8wtywbuJbgfAE3zbek= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.15.0 h1:5Fwje4O2ooOxkfyqI/kJwxWotggDLix4BSAvpE1wlpo= go.opentelemetry.io/otel/trace v1.15.0/go.mod h1:CUsmE2Ht1CRkvE8OsMESvraoZrrcgD1J2W8GV1ev0Y4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -265,8 +241,6 @@ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= @@ -281,12 +255,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -301,7 +271,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -319,8 +289,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -346,8 +314,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index e5471acf0..b6c1e8e25 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -7,13 +7,13 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestRedirectsFileSupport(t *testing.T) { fixture := car.MustOpenUnixfsCar("t0109-redirects.car") - redirectDir := fixture.MustGetNode("examples") redirectDirCID := redirectDir.Base32Cid() @@ -52,6 +52,7 @@ func TestRedirectsFileSupport(t *testing.T) { Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", Request: Request(). DoNotFollowRedirects(). + Header("Host", u.Host). URL("%s/redirect-one", redirectDirBaseURL), Response: Expect(). Status(301). @@ -311,7 +312,97 @@ func TestRedirectsFileSupport(t *testing.T) { } } -// TODO: dnslink tests +func TestRedirectsFileSupportWithDNSLink(t *testing.T) { + dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml") + dnsLink := dnsLinks.Get("custom-dnslink") + + gatewayURL := SubdomainGatewayURL + u, err := url.Parse(gatewayURL) + if err != nil { + t.Fatal(err) + } + + dnsLinkBaseUrl := fmt.Sprintf("%s://%s.%s", u.Scheme, dnsLink, u.Host) + + tests := SugarTests{ + // # make sure test setup is valid (fail if CoreAPI is unable to resolve) + // test_expect_success "spoofed DNSLink record resolves in cli" " + // ipfs resolve /ipns/$DNSLINK_FQDN > result && + // test_should_contain \"$REDIRECTS_DIR_CID\" result && + // ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result && + // test_should_contain \"index.html\" result + // " + // SKIPPED + + // test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // ' + { + Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + URL("%s/redirect-one", dnsLinkBaseUrl), + Response: Expect(). + Status(301). + Headers( + Header("Location", "/one.html"), + ), + }, + // # ensure custom 404 works and has the same cache headers as regular /ipns/ paths + // test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response && + // test_should_contain "404 Not Found" response && + // test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response && + // test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_not_contain "immutable" response && + // test_should_contain "Date: " response && + // test_should_contain "my 404" response + // ' + { + Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file", + Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`, + Request: Request(). + URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl), + Response: Expect(). + Status(404). + Headers( + Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""), + Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"), + Header("Cache-Control").Not().Contains("immutable"), + Header("Date").Exists(), + ). + Body( + // TODO: I like the readable part here, maybe rewrite to load the file. + Contains("my 404"), + ), + }, + // test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" ' + // curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_not_contain "one.html" response && + // test_should_not_contain "301 Moved Permanently" response && + // test_should_not_contain "Location:" response + // ' + // TODO(lidel): this test seems to validate some kubo behavior not really gateway. + // { + // Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled", + // Request: Request(). + // URL("%s/redirect-one", noDnsLinkBaseUrl), + // Response: Expect(). + // // TODO: add "status not equal to 301" check. + // // TODO: what `test_should_not_contain "one.html" response` actually means? No location correct? + // Headers( + // Header("Location").Not().Exists(), + // ), + // }, + } + + if specs.DNSLinkResolver.IsEnabled() { + Run(t, unwrapTests(t, tests)) + } else { + t.Skip("subdomain gateway disabled") + } +} func unwrapTests(t *testing.T, tests SugarTests) SugarTests { t.Helper() diff --git a/tooling/dnslink/dnslink.go b/tooling/dnslink/dnslink.go new file mode 100644 index 000000000..55b9b6e77 --- /dev/null +++ b/tooling/dnslink/dnslink.go @@ -0,0 +1,52 @@ +package dnslink + +import ( + "fmt" + "os" + "path" + + "github.com/ipfs/gateway-conformance/tooling/fixtures" + "gopkg.in/yaml.v3" +) + +type DNSLinks struct { + DNSLinks map[string]DNSLink `yaml:"dnslinks"` +} + +type DNSLink struct { + Subdomain string `yaml:"subdomain"` + Path string `yaml:"path"` +} + +func OpenDNSLink(absPath string) (*DNSLinks, error) { + data, err := os.ReadFile(absPath) + if err != nil { + return nil, err + } + + var dnsLinks DNSLinks + err = yaml.Unmarshal(data, &dnsLinks) + if err != nil { + return nil, err + } + + return &dnsLinks, nil +} + +func MustOpenDNSLink(file string) *DNSLinks { + fixturePath := path.Join(fixtures.Dir(), file) + dnsLinks, err := OpenDNSLink(fixturePath) + if err != nil { + panic(err) + } + + return dnsLinks +} + +func (d *DNSLinks) Get(id string) string { + dnsLink, ok := d.DNSLinks[id] + if !ok { + panic(fmt.Errorf("dnslink %s not found", id)) + } + return dnsLink.Subdomain +} diff --git a/tooling/dnslink/merge.go b/tooling/dnslink/merge.go new file mode 100644 index 000000000..9bc127f02 --- /dev/null +++ b/tooling/dnslink/merge.go @@ -0,0 +1,63 @@ +package dnslink + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +func Aggregate(inputPaths []string) (map[string]string, error) { + aggMap := make(map[string]string) + + for _, file := range inputPaths { + dnsLinks, err := OpenDNSLink(file) + if err != nil { + return nil, fmt.Errorf("error loading file %s: %v", file, err) + } + + for _, link := range dnsLinks.DNSLinks { + if _, ok := aggMap[link.Subdomain]; ok { + return nil, fmt.Errorf("collision detected for subdomain %s", link.Subdomain) + } + + aggMap[link.Subdomain] = link.Path + } + } + + return aggMap, nil +} + +func Merge(inputPaths []string, outputPath string) error { + kvs, err := Aggregate(inputPaths) + if err != nil { + return err + } + + j, err := json.MarshalIndent(kvs, "", " ") + if err != nil { + return err + } + + err = os.WriteFile(outputPath, j, 0644) + return err +} + +func AsEnv(inputPaths []string, outputPath string) error { + kvs, err := Aggregate(inputPaths) + if err != nil { + return err + } + + var kvsStr []string + for k, v := range kvs { + kvsStr = append(kvsStr, fmt.Sprintf("%s:%s", k, v)) + } + + env := strings.Join(kvsStr, ",") + env = "export IPFS_NS_MAP=\"" + env + "\"" + + err = os.WriteFile(outputPath, []byte(env), 0644) + + return err +} diff --git a/tooling/fixtures/list.go b/tooling/fixtures/list.go index d58cbec83..4151ff2d8 100644 --- a/tooling/fixtures/list.go +++ b/tooling/fixtures/list.go @@ -13,8 +13,14 @@ func Dir() string { return path.Join(home, "fixtures") } -func List() ([]string, error) { +type Fixtures struct { + CarFiles []string + ConfigFiles []string +} + +func List() (*Fixtures, error) { var carFiles []string + var yamlFiles []string err := filepath.WalkDir(Dir(), func(path string, d os.DirEntry, err error) error { if err != nil { @@ -31,6 +37,14 @@ func List() ([]string, error) { carFiles = append(carFiles, path) } + // if we have a yaml file, append: + if filepath.Ext(path) == ".yml" { + path, err := filepath.Abs(path) + if err != nil { + return err + } + yamlFiles = append(yamlFiles, path) + } return nil }) @@ -39,5 +53,8 @@ func List() ([]string, error) { return nil, err } - return carFiles, nil + return &Fixtures{ + CarFiles: carFiles, + ConfigFiles: yamlFiles, + }, nil } diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go index a44ff42b5..6258ce7e5 100644 --- a/tooling/specs/specs.go +++ b/tooling/specs/specs.go @@ -26,11 +26,13 @@ type Spec string const ( SubdomainGateway Spec = "subdomain-gateway" + DNSLinkResolver Spec = "dnslink-resolver" ) // All specs should be listed here. var specMaturity = map[Spec]maturity{ SubdomainGateway: stable, + DNSLinkResolver: stable, } func (s Spec) IsMature() bool { From c6928ae0a95a6bc00eda0d932cf49d8139920d57 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 7 Apr 2023 09:58:41 +0200 Subject: [PATCH 2/9] feat: add helpers lib --- tests/t0109_gateway_web_redirects_test.go | 82 +-------------------- tooling/helpers/subdomain.go | 90 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 79 deletions(-) create mode 100644 tooling/helpers/subdomain.go diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index b6c1e8e25..94b47df00 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/dnslink" + "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) @@ -306,7 +307,7 @@ func TestRedirectsFileSupport(t *testing.T) { } if specs.SubdomainGateway.IsEnabled() { - Run(t, unwrapTests(t, tests)) + Run(t, helpers.UnwrapSubdomainTests(t, tests)) } else { t.Skip("subdomain gateway disabled") } @@ -398,85 +399,8 @@ func TestRedirectsFileSupportWithDNSLink(t *testing.T) { } if specs.DNSLinkResolver.IsEnabled() { - Run(t, unwrapTests(t, tests)) + Run(t, helpers.UnwrapSubdomainTests(t, tests)) } else { t.Skip("subdomain gateway disabled") } } - -func unwrapTests(t *testing.T, tests SugarTests) SugarTests { - t.Helper() - - var out SugarTests - for _, test := range tests { - out = append(out, unwrapTestForGateway(t, test)...) - } - return out -} - -func unwrapTestForGateway(t *testing.T, test SugarTest) SugarTests { - t.Helper() - - baseURL := test.Request.GetURL() - req := test.Request - expected := test.Response - - u, err := url.Parse(baseURL) - if err != nil { - t.Fatal(err) - } - // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed - // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is - // actually living on http://127.0.0.1:8080 or somewhere else. - // - // The test knows two addresses: - // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. - // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. - - // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` - host := u.Host - - // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` - u.Host = GatewayHost - rawURL := u.String() - - return SugarTests{ - { - Name: fmt.Sprintf("%s (direct HTTP)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), - Request: req. - URL(rawURL). - DoNotFollowRedirects(). - Headers( - Header("Host", host), - ), - Response: expected, - }, - { - Name: fmt.Sprintf("%s (HTTP proxy)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, "HTTP proxy (hostname is passed via URL)"), - Request: req. - URL(baseURL). - Proxy(GatewayURL). - DoNotFollowRedirects(), - Response: expected, - }, - { - Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, `HTTP proxy - In HTTP/1.x, the pseudo-method CONNECT, - can be used to convert an HTTP connection into a tunnel to a remote host - https://tools.ietf.org/html/rfc7231#section-4.3.6 - `), - Request: req. - URL(baseURL). - Proxy(GatewayURL). - WithProxyTunnel(). - DoNotFollowRedirects(). - Headers( - Header("Host", host), - ), - Response: expected, - }, - } -} diff --git a/tooling/helpers/subdomain.go b/tooling/helpers/subdomain.go new file mode 100644 index 000000000..7049d3e3e --- /dev/null +++ b/tooling/helpers/subdomain.go @@ -0,0 +1,90 @@ +package helpers + +import ( + "fmt" + "net/url" + "testing" + + "github.com/ipfs/gateway-conformance/tooling/test" +) + +/** + * UnwrapSubdomainTests takes a list of tests and returns a (larger) list of tests + * that will run on the subdomain gateway. + */ +func UnwrapSubdomainTests(t *testing.T, tests test.SugarTests) test.SugarTests { + t.Helper() + + var out test.SugarTests + for _, test := range tests { + out = append(out, unwrapSubdomainTest(t, test)...) + } + return out +} + +func unwrapSubdomainTest(t *testing.T, unwraped test.SugarTest) test.SugarTests { + t.Helper() + + baseURL := unwraped.Request.GetURL() + req := unwraped.Request + expected := unwraped.Response + + u, err := url.Parse(baseURL) + if err != nil { + t.Fatal(err) + } + // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed + // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is + // actually living on http://127.0.0.1:8080 or somewhere else. + // + // The test knows two addresses: + // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. + // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. + + // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` + host := u.Host + + // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` + u.Host = test.GatewayHost + rawURL := u.String() + + return test.SugarTests{ + { + Name: fmt.Sprintf("%s (direct HTTP)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), + Request: req. + URL(rawURL). + DoNotFollowRedirects(). + Headers( + test.Header("Host", host), + ), + Response: expected, + }, + { + Name: fmt.Sprintf("%s (HTTP proxy)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "HTTP proxy (hostname is passed via URL)"), + Request: req. + URL(baseURL). + Proxy(test.GatewayURL). + DoNotFollowRedirects(), + Response: expected, + }, + { + Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, `HTTP proxy + In HTTP/1.x, the pseudo-method CONNECT, + can be used to convert an HTTP connection into a tunnel to a remote host + https://tools.ietf.org/html/rfc7231#section-4.3.6 + `), + Request: req. + URL(baseURL). + Proxy(test.GatewayURL). + WithProxyTunnel(). + DoNotFollowRedirects(). + Headers( + test.Header("Host", host), + ), + Response: expected, + }, + } +} From cf14faf7082a78084c6c268ec8bd19edfc51ea31 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 7 Apr 2023 11:19:43 +0200 Subject: [PATCH 3/9] feat: setup ci ipns map --- .github/workflows/test.yml | 5 +++-- Makefile | 2 +- cmd/gateway-conformance/main.go | 7 +------ kubo-config.example.sh | 7 ++++++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a031b5e95..b6ca3272a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,9 +27,10 @@ jobs: run: | ipfs init; ./gateway-conformance/kubo-config.example.sh; + IPFS_NS_MAP=$(cat ./fixtures/dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') + echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + # note: the IPFS_NS_MAP set above will be passed the daemon: - uses: ipfs/start-ipfs-daemon-action@v1 - env: - IPFS_NS_MAP: 12 # TODO - name: Provision Kubo Gateway run: | find ./fixtures -name '*.car' -exec ipfs dag import {} \; diff --git a/Makefile b/Makefile index da79019b9..02cb059f4 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ provision-cargateway: ./fixtures.car # cd go-libipfs/examples/car && go install car -c ./fixtures.car & -provision-kubo: +provision-kubo: fixtures.car find ./fixtures -name '*.car' -exec ipfs dag import {} \; fixtures.car: gateway-conformance diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 720c4c6be..a56ac45bc 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -190,11 +190,6 @@ func main() { if err != nil { return err } - - err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env")) - if err != nil { - return err - } } else { err = copyFiles(fxs.CarFiles, directory) if err != nil { @@ -206,7 +201,7 @@ func main() { return err } - err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env")) + err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) if err != nil { return err } diff --git a/kubo-config.example.sh b/kubo-config.example.sh index d1873534a..8a266ad37 100755 --- a/kubo-config.example.sh +++ b/kubo-config.example.sh @@ -9,4 +9,9 @@ ipfs config --json Gateway.PublicGateways '{ "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns", "/api"] } -}' \ No newline at end of file +}' + +IPFS_NS_MAP=$(cat ./dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') + +echo "Set the following IPFS_NS_MAP before starting the kubo daemon:" +echo "IPFS_NS_MAP=${IPFS_NS_MAP}" From 9f908f53350d2bd715c6354c56cd371104b7e414 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 7 Apr 2023 12:02:43 +0200 Subject: [PATCH 4/9] feat: add RunIfSpecsAreEnabled --- tests/t0109_gateway_web_redirects_test.go | 21 +++++++++++---------- tooling/specs/specs.go | 4 +++- tooling/test/test.go | 23 +++++++++++++++++++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 94b47df00..847a137d5 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -306,11 +306,11 @@ func TestRedirectsFileSupport(t *testing.T) { }...) } - if specs.SubdomainGateway.IsEnabled() { - Run(t, helpers.UnwrapSubdomainTests(t, tests)) - } else { - t.Skip("subdomain gateway disabled") - } + RunIfSpecsAreEnabled( + t, + helpers.UnwrapSubdomainTests(t, tests), + specs.SubdomainGateway, + ) } func TestRedirectsFileSupportWithDNSLink(t *testing.T) { @@ -398,9 +398,10 @@ func TestRedirectsFileSupportWithDNSLink(t *testing.T) { // }, } - if specs.DNSLinkResolver.IsEnabled() { - Run(t, helpers.UnwrapSubdomainTests(t, tests)) - } else { - t.Skip("subdomain gateway disabled") - } + RunIfSpecsAreEnabled( + t, + helpers.UnwrapSubdomainTests(t, tests), + specs.DNSLinkResolver, + specs.SubdomainGateway, + ) } diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go index 6258ce7e5..cc009d9d9 100644 --- a/tooling/specs/specs.go +++ b/tooling/specs/specs.go @@ -1,6 +1,8 @@ package specs -import "fmt" +import ( + "fmt" +) type maturity string diff --git a/tooling/test/test.go b/tooling/test/test.go index 388194bc3..5b62e2994 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/specs" ) type SugarTest struct { @@ -19,9 +20,27 @@ type SugarTest struct { type SugarTests []SugarTest -func Run(t *testing.T, tests SugarTests) { - // NewDialer() +func RunIfSpecsAreEnabled( + t *testing.T, + tests SugarTests, + required ...specs.Spec, +) { + missing := []specs.Spec{} + for _, spec := range required { + if !spec.IsEnabled() { + missing = append(missing, spec) + } + } + + if len(missing) > 0 { + t.Skipf("skipping tests, missing specs: %v", missing) + return + } + Run(t, tests) +} + +func Run(t *testing.T, tests SugarTests) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { method := test.Request.Method_ From 566f4816d6693f6e075a343c2de7e23a40ca4743 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 7 Apr 2023 12:13:21 +0200 Subject: [PATCH 5/9] fix: remove dead dnslink code --- dnslinkgen.go | 39 --------------------------------------- tooling/dnslink/merge.go | 22 +--------------------- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 dnslinkgen.go diff --git a/dnslinkgen.go b/dnslinkgen.go deleted file mode 100644 index 837da5fa5..000000000 --- a/dnslinkgen.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "strings" - - "github.com/ipfs/gateway-conformance/tooling/dnslink" - "github.com/ipfs/gateway-conformance/tooling/fixtures" -) - -func main() { - if len(os.Args) < 2 { - fmt.Println("Usage: dnslinkgen ") - os.Exit(1) - } - - domain := os.Args[1] - - fxs, err := fixtures.List() - if err != nil { - log.Fatal(err) - } - - configs := fxs.ConfigFiles - aggMap, err := dnslink.Aggregate(configs) - if err != nil { - log.Fatal(err) - } - - // print k=v on stdout - var kvs []string - for k, v := range aggMap { - kvs = append(kvs, fmt.Sprintf("%s%s:%s", k, domain, v)) - } - - fmt.Println("export IPFS_NS_MAP=\"" + strings.Join(kvs, ",") + "\"") -} diff --git a/tooling/dnslink/merge.go b/tooling/dnslink/merge.go index 9bc127f02..2d7a91356 100644 --- a/tooling/dnslink/merge.go +++ b/tooling/dnslink/merge.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "os" - "strings" ) func Aggregate(inputPaths []string) (map[string]string, error) { @@ -41,23 +40,4 @@ func Merge(inputPaths []string, outputPath string) error { err = os.WriteFile(outputPath, j, 0644) return err -} - -func AsEnv(inputPaths []string, outputPath string) error { - kvs, err := Aggregate(inputPaths) - if err != nil { - return err - } - - var kvsStr []string - for k, v := range kvs { - kvsStr = append(kvsStr, fmt.Sprintf("%s:%s", k, v)) - } - - env := strings.Join(kvsStr, ",") - env = "export IPFS_NS_MAP=\"" + env + "\"" - - err = os.WriteFile(outputPath, []byte(env), 0644) - - return err -} +} \ No newline at end of file From 8c31b065c99d766b12bad9172697f5ce2d999d3f Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Tue, 9 May 2023 11:00:02 +0200 Subject: [PATCH 6/9] fix: disable kubo test --- tests/t0109_gateway_web_redirects_test.go | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 847a137d5..7652adbc2 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -236,18 +236,18 @@ func TestRedirectsFileSupport(t *testing.T) { // test_should_not_contain "my 404" response // // ' - { - // TODO: how to test this correctly? - Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", - Hint: ` - We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. - `, - Request: Request(). - URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), - Response: Expect(). - Status(404). - Body(Not(Contains(custom404.ReadFile()))), - }, + // { + // // TODO: confirm with lidel: this should be skipped + // Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", + // Hint: ` + // We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. + // `, + // Request: Request(). + // URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), + // Response: Expect(). + // Status(404). + // Body(Not(Contains(custom404.ReadFile()))), + // }, }...) // # Invalid file, containing forced redirect From b51d4e395e9c54a98b14917bff93748aef96b402 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Tue, 9 May 2023 16:39:21 +0200 Subject: [PATCH 7/9] feat: introduce fixture schema.json --- fixtures/fixture.schema.json | 26 ++++++++++++++++++++++++++ fixtures/t0109-dnslink.yml | 1 + 2 files changed, 27 insertions(+) create mode 100644 fixtures/fixture.schema.json diff --git a/fixtures/fixture.schema.json b/fixtures/fixture.schema.json new file mode 100644 index 000000000..ae26a75bf --- /dev/null +++ b/fixtures/fixture.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "dnslinks": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "subdomain": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": [ + "subdomain", + "path" + ], + "additionalProperties": false + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/fixtures/t0109-dnslink.yml b/fixtures/t0109-dnslink.yml index ce9947ca2..d65b5f638 100644 --- a/fixtures/t0109-dnslink.yml +++ b/fixtures/t0109-dnslink.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=fixture.schema.json dnslinks: custom-dnslink: subdomain: dnslink-enabled-on-fqdn From c3bc2ef809f7b350ab0483fdaf4f25b59854bd9a Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Tue, 9 May 2023 16:43:36 +0200 Subject: [PATCH 8/9] fix: use MustGet for consistency --- tests/t0109_gateway_web_redirects_test.go | 2 +- tooling/dnslink/dnslink.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 7652adbc2..ad7aed5da 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -315,7 +315,7 @@ func TestRedirectsFileSupport(t *testing.T) { func TestRedirectsFileSupportWithDNSLink(t *testing.T) { dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml") - dnsLink := dnsLinks.Get("custom-dnslink") + dnsLink := dnsLinks.MustGet("custom-dnslink") gatewayURL := SubdomainGatewayURL u, err := url.Parse(gatewayURL) diff --git a/tooling/dnslink/dnslink.go b/tooling/dnslink/dnslink.go index 55b9b6e77..2a8778ab9 100644 --- a/tooling/dnslink/dnslink.go +++ b/tooling/dnslink/dnslink.go @@ -43,7 +43,7 @@ func MustOpenDNSLink(file string) *DNSLinks { return dnsLinks } -func (d *DNSLinks) Get(id string) string { +func (d *DNSLinks) MustGet(id string) string { dnsLink, ok := d.DNSLinks[id] if !ok { panic(fmt.Errorf("dnslink %s not found", id)) From 83c417ccc5965a73556246375bd4b2b4c6529a17 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Tue, 9 May 2023 16:46:32 +0200 Subject: [PATCH 9/9] feat: export the DNS link mapping to be reused --- .github/workflows/test.yml | 2 +- kubo-config.example.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6ca3272a..6d626f03c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: - name: Configure Kubo Gateway run: | ipfs init; - ./gateway-conformance/kubo-config.example.sh; + source ./gateway-conformance/kubo-config.example.sh; IPFS_NS_MAP=$(cat ./fixtures/dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV # note: the IPFS_NS_MAP set above will be passed the daemon: diff --git a/kubo-config.example.sh b/kubo-config.example.sh index 8a266ad37..14cf41969 100755 --- a/kubo-config.example.sh +++ b/kubo-config.example.sh @@ -11,7 +11,7 @@ ipfs config --json Gateway.PublicGateways '{ } }' -IPFS_NS_MAP=$(cat ./dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') +export IPFS_NS_MAP=$(cat ./dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') echo "Set the following IPFS_NS_MAP before starting the kubo daemon:" echo "IPFS_NS_MAP=${IPFS_NS_MAP}"