Skip to content

Commit

Permalink
feat: draft dnslink support
Browse files Browse the repository at this point in the history
  • Loading branch information
laurentsenta committed Apr 7, 2023
1 parent 8b224e1 commit 279e24a
Show file tree
Hide file tree
Showing 12 changed files with 304 additions and 8 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ jobs:
ipfs init;
./gateway-conformance/kubo-config.example.sh;
- uses: ipfs/start-ipfs-daemon-action@v1
env:
IPFS_NS_MAP: 12 # TODO
- name: Provision Kubo Gateway
run: |
find ./fixtures -name '*.car' -exec ipfs dag import {} \;
Expand Down
1 change: 0 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ provision-cargateway: ./fixtures.car
provision-kubo:
find ./fixtures -name '*.car' -exec ipfs dag import {} \;

# tools
fixtures.car: gateway-conformance
./gateway-conformance extract-fixtures --merged=true --dir=.

Expand Down
27 changes: 24 additions & 3 deletions cmd/gateway-conformance/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/car"
"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/fixtures"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -173,19 +174,39 @@ func main() {
return err
}

files, err := fixtures.List()
fxs, err := fixtures.List()
if err != nil {
return err
}

merged := cCtx.Bool("merged")
if merged {
err = car.Merge(files, filepath.Join(directory, "fixtures.car"))
err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car"))
if err != nil {
return err
}

err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json"))
if err != nil {
return err
}

err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env"))
if err != nil {
return err
}
} else {
err = copyFiles(files, directory)
err = copyFiles(fxs.CarFiles, directory)
if err != nil {
return err
}

err = copyFiles(fxs.ConfigFiles, directory)
if err != nil {
return err
}

err = dnslink.AsEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.env"))
if err != nil {
return err
}
Expand Down
39 changes: 39 additions & 0 deletions dnslinkgen.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package main

import (
"fmt"
"log"
"os"
"strings"

"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/fixtures"
)

func main() {
if len(os.Args) < 2 {
fmt.Println("Usage: dnslinkgen <domain>")
os.Exit(1)
}

domain := os.Args[1]

fxs, err := fixtures.List()
if err != nil {
log.Fatal(err)
}

configs := fxs.ConfigFiles
aggMap, err := dnslink.Aggregate(configs)
if err != nil {
log.Fatal(err)
}

// print k=v on stdout
var kvs []string
for k, v := range aggMap {
kvs = append(kvs, fmt.Sprintf("%s%s:%s", k, domain, v))
}

fmt.Println("export IPFS_NS_MAP=\"" + strings.Join(kvs, ",") + "\"")
}
6 changes: 6 additions & 0 deletions fixtures/t0109-dnslink.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
dnslinks:
custom-dnslink:
subdomain: dnslink-enabled-on-fqdn
# this is the cid of the folder
# t0109-redirects.car:/examples/
path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/goleak v1.1.12 // indirect
golang.org/x/sync v0.1.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
Expand Down Expand Up @@ -64,5 +65,6 @@ require (
golang.org/x/sys v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v2 v2.4.0
lukechampine.com/blake3 v1.1.7 // indirect
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
95 changes: 93 additions & 2 deletions tests/t0109_gateway_web_redirects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ import (

"github.com/ipfs/gateway-conformance/tooling/car"
. "github.com/ipfs/gateway-conformance/tooling/check"
"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/specs"
. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestRedirectsFileSupport(t *testing.T) {
fixture := car.MustOpenUnixfsCar("t0109-redirects.car")

redirectDir := fixture.MustGetNode("examples")
redirectDirCID := redirectDir.Base32Cid()

Expand Down Expand Up @@ -52,6 +52,7 @@ func TestRedirectsFileSupport(t *testing.T) {
Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file",
Request: Request().
DoNotFollowRedirects().
Header("Host", u.Host).
URL("%s/redirect-one", redirectDirBaseURL),
Response: Expect().
Status(301).
Expand Down Expand Up @@ -311,7 +312,97 @@ func TestRedirectsFileSupport(t *testing.T) {
}
}

// TODO: dnslink tests
func TestRedirectsFileSupportWithDNSLink(t *testing.T) {
dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml")
dnsLink := dnsLinks.Get("custom-dnslink")

gatewayURL := SubdomainGatewayURL
u, err := url.Parse(gatewayURL)
if err != nil {
t.Fatal(err)
}

dnsLinkBaseUrl := fmt.Sprintf("%s://%s.%s", u.Scheme, dnsLink, u.Host)

tests := SugarTests{
// # make sure test setup is valid (fail if CoreAPI is unable to resolve)
// test_expect_success "spoofed DNSLink record resolves in cli" "
// ipfs resolve /ipns/$DNSLINK_FQDN > result &&
// test_should_contain \"$REDIRECTS_DIR_CID\" result &&
// ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result &&
// test_should_contain \"index.html\" result
// "
// SKIPPED

// test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" '
// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
// test_should_contain "301 Moved Permanently" response &&
// test_should_contain "Location: /one.html" response
// '
{
Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file",
Request: Request().
URL("%s/redirect-one", dnsLinkBaseUrl),
Response: Expect().
Status(301).
Headers(
Header("Location", "/one.html"),
),
},
// # ensure custom 404 works and has the same cache headers as regular /ipns/ paths
// test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" '
// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response &&
// test_should_contain "404 Not Found" response &&
// test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response &&
// test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response &&
// test_should_not_contain "immutable" response &&
// test_should_contain "Date: " response &&
// test_should_contain "my 404" response
// '
{
Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file",
Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`,
Request: Request().
URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl),
Response: Expect().
Status(404).
Headers(
Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""),
Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"),
Header("Cache-Control").Not().Contains("immutable"),
Header("Date").Exists(),
).
Body(
// TODO: I like the readable part here, maybe rewrite to load the file.
Contains("my 404"),
),
},
// test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" '
// curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
// test_should_not_contain "one.html" response &&
// test_should_not_contain "301 Moved Permanently" response &&
// test_should_not_contain "Location:" response
// '
// TODO(lidel): this test seems to validate some kubo behavior not really gateway.
// {
// Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled",
// Request: Request().
// URL("%s/redirect-one", noDnsLinkBaseUrl),
// Response: Expect().
// // TODO: add "status not equal to 301" check.
// // TODO: what `test_should_not_contain "one.html" response` actually means? No location correct?
// Headers(
// Header("Location").Not().Exists(),
// ),
// },
}

if specs.DNSLinkResolver.IsEnabled() {
Run(t, unwrapTests(t, tests).Build())
} else {
t.Skip("subdomain gateway disabled")
}
}

func unwrapTests(t *testing.T, tests SugarTests) SugarTests {
t.Helper()
Expand Down
52 changes: 52 additions & 0 deletions tooling/dnslink/dnslink.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package dnslink

import (
"fmt"
"os"
"path"

"github.com/ipfs/gateway-conformance/tooling/fixtures"
"gopkg.in/yaml.v3"
)

type DNSLinks struct {
DNSLinks map[string]DNSLink `yaml:"dnslinks"`
}

type DNSLink struct {
Subdomain string `yaml:"subdomain"`
Path string `yaml:"path"`
}

func OpenDNSLink(absPath string) (*DNSLinks, error) {
data, err := os.ReadFile(absPath)
if err != nil {
return nil, err
}

var dnsLinks DNSLinks
err = yaml.Unmarshal(data, &dnsLinks)
if err != nil {
return nil, err
}

return &dnsLinks, nil
}

func MustOpenDNSLink(file string) *DNSLinks {
fixturePath := path.Join(fixtures.Dir(), file)
dnsLinks, err := OpenDNSLink(fixturePath)
if err != nil {
panic(err)
}

return dnsLinks
}

func (d *DNSLinks) Get(id string) string {
dnsLink, ok := d.DNSLinks[id]
if !ok {
panic(fmt.Errorf("dnslink %s not found", id))
}
return dnsLink.Subdomain
}
63 changes: 63 additions & 0 deletions tooling/dnslink/merge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package dnslink

import (
"encoding/json"
"fmt"
"os"
"strings"
)

func Aggregate(inputPaths []string) (map[string]string, error) {
aggMap := make(map[string]string)

for _, file := range inputPaths {
dnsLinks, err := OpenDNSLink(file)
if err != nil {
return nil, fmt.Errorf("error loading file %s: %v", file, err)
}

for _, link := range dnsLinks.DNSLinks {
if _, ok := aggMap[link.Subdomain]; ok {
return nil, fmt.Errorf("collision detected for subdomain %s", link.Subdomain)
}

aggMap[link.Subdomain] = link.Path
}
}

return aggMap, nil
}

func Merge(inputPaths []string, outputPath string) error {
kvs, err := Aggregate(inputPaths)
if err != nil {
return err
}

j, err := json.MarshalIndent(kvs, "", " ")
if err != nil {
return err
}

err = os.WriteFile(outputPath, j, 0644)
return err
}

func AsEnv(inputPaths []string, outputPath string) error {
kvs, err := Aggregate(inputPaths)
if err != nil {
return err
}

var kvsStr []string
for k, v := range kvs {
kvsStr = append(kvsStr, fmt.Sprintf("%s:%s", k, v))
}

env := strings.Join(kvsStr, ",")
env = "export IPFS_NS_MAP=\"" + env + "\""

err = os.WriteFile(outputPath, []byte(env), 0644)

return err
}
Loading

0 comments on commit 279e24a

Please sign in to comment.