Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 81 additions & 34 deletions src/compute-plane-services/nvca/cmd/export-cluster-pubkeys/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,11 @@ import (
)

type options struct {
out io.Writer
outputFormat string
egressCIDRs []string
jwksURI string
out io.Writer
outputFormat string
egressCIDRs []string
jwksURI string
forceStaticKeys bool
}

func main() {
Expand Down Expand Up @@ -92,6 +93,12 @@ func main() {
Usage: `Override the JWKS URI from the OIDC configuration`,
Destination: &opts.jwksURI,
},
&cli.BoolFlag{
Name: "static-pubkeys",
Value: false,
Usage: `Always emit jwt_validation_pubkeys instead of jwks_url (overrides auto-detection for public EKS OIDC)`,
Destination: &opts.forceStaticKeys,
},
},
Action: func(c *cli.Context) error {
ctx := c.Context
Expand Down Expand Up @@ -141,7 +148,34 @@ type vaultJWTConfig struct {
BoundCIDRs []string `json:"bound_cidrs" yaml:"-"`
Issuer string `json:"bound_issuer" yaml:"bound_issuer"`
SupportedSigAlgs []string `json:"jwt_supported_algs" yaml:"jwt_supported_algs"`
PubkeysPEM []string `json:"jwt_validation_pubkeys" yaml:"jwt_validation_pubkeys"`
JWKSURL string `json:"jwks_url,omitempty" yaml:"jwks_url,omitempty"`
PubkeysPEM []string `json:"jwt_validation_pubkeys,omitempty" yaml:"jwt_validation_pubkeys,omitempty"`
}

// publicEKSOIDCAuthority returns the normalized HTTPS authority for a public EKS OIDC URL.
func publicEKSOIDCAuthority(rawURL string) (string, bool) {
u, err := url.Parse(rawURL)
if err != nil || !u.IsAbs() || u.Scheme != "https" {
return "", false
}

host := strings.ToLower(u.Hostname())
if !strings.HasPrefix(host, "oidc.eks.") || !strings.HasSuffix(host, ".amazonaws.com") {
return "", false
}

if port := u.Port(); port != "" && port != "443" {
return host + ":" + port, true
}
return host, true
}

// useVaultJWKSURL reports whether Vault should fetch signing keys dynamically from a
// publicly reachable JWKS endpoint instead of using static jwt_validation_pubkeys.
func useVaultJWKSURL(issuer, jwksURIStr string) bool {
issuerAuthority, issuerOK := publicEKSOIDCAuthority(issuer)
jwksAuthority, jwksOK := publicEKSOIDCAuthority(jwksURIStr)
return issuerOK && jwksOK && issuerAuthority == jwksAuthority
}

type streamer interface {
Expand Down Expand Up @@ -224,45 +258,54 @@ func run(ctx context.Context, s streamer, opts options, k8sServerHost string) (e
return fmt.Errorf("parse JWKS URI: %w", err)
}

jwksRC, err := s.streamGetURI(ctx, jwksURL)
if err != nil {
if os.IsTimeout(err) || errors.Is(err, syscall.ECONNREFUSED) {
// If the JWKS URL times out, try the /openid/v1/jwks endpoint from the KUBECONFIG
retryJWKSURLStr := fmt.Sprintf("%s/openid/v1/jwks", k8sServerHost)
log.Printf("get JWKS from cluster failed due to timeout: %v, "+
"retrying with the cluster's server endpoint (from KUBECONFIG) --jwks-uri=\"%s\" flag\n",
err,
retryJWKSURLStr)
jwksURL, err = url.Parse(retryJWKSURLStr)
if err != nil {
return fmt.Errorf("parse JWKS URI: %w", err)
}
jwksRC, err = s.streamGetURI(ctx, jwksURL)
if err != nil {
useJWKSURL := !opts.forceStaticKeys && useVaultJWKSURL(oidcCfg.Issuer, jwksURIStr)
var pubkeyPEMStrs []string
if !useJWKSURL {
jwksRC, err := s.streamGetURI(ctx, jwksURL)
if err != nil {
if os.IsTimeout(err) || errors.Is(err, syscall.ECONNREFUSED) {
// If the JWKS URL times out, try the /openid/v1/jwks endpoint from the KUBECONFIG
retryJWKSURLStr := fmt.Sprintf("%s/openid/v1/jwks", k8sServerHost)
log.Printf("get JWKS from cluster failed due to timeout: %v, "+
"retrying with the cluster's server endpoint (from KUBECONFIG) --jwks-uri=\"%s\" flag\n",
err,
retryJWKSURLStr)
jwksURL, err = url.Parse(retryJWKSURLStr)
if err != nil {
return fmt.Errorf("parse JWKS URI: %w", err)
}
jwksRC, err = s.streamGetURI(ctx, jwksURL)
if err != nil {
return fmt.Errorf("get JWKS from cluster: %w", err)
}
} else {
return fmt.Errorf("get JWKS from cluster: %w", err)
}
} else {
return fmt.Errorf("get JWKS from cluster: %w", err)
}
}
defer jwksRC.Close()
defer jwksRC.Close()

pubkeyPEMStrs, err := jwksToPEMStrs(jwksRC)
if err != nil {
return fmt.Errorf("convert JWKS to PEM: %w", err)
pubkeyPEMStrs, err = jwksToPEMStrs(jwksRC)
if err != nil {
return fmt.Errorf("convert JWKS to PEM: %w", err)
}
} else {
log.Printf("using Vault jwks_url %s (public EKS OIDC; Vault fetches keys dynamically)", jwksURIStr)
}

switch opts.outputFormat {
case "yaml":
pubkeyPEMStrsBase64 := make([]string, len(pubkeyPEMStrs))
for i, pkpem := range pubkeyPEMStrs {
pubkeyPEMStrsBase64[i] = base64.StdEncoding.EncodeToString([]byte(pkpem))
}

vaultJWTCfg := vaultJWTConfig{
Issuer: oidcCfg.Issuer,
SupportedSigAlgs: oidcCfg.SupportedSigAlgs,
PubkeysPEM: pubkeyPEMStrsBase64,
}
if useJWKSURL {
vaultJWTCfg.JWKSURL = jwksURIStr
} else {
pubkeyPEMStrsBase64 := make([]string, len(pubkeyPEMStrs))
for i, pkpem := range pubkeyPEMStrs {
pubkeyPEMStrsBase64[i] = base64.StdEncoding.EncodeToString([]byte(pkpem))
}
vaultJWTCfg.PubkeysPEM = pubkeyPEMStrsBase64
}

vaultJWTMount := newVaultJWTMount(
Expand All @@ -277,7 +320,11 @@ func run(ctx context.Context, s streamer, opts options, k8sServerHost string) (e
BoundCIDRs: opts.egressCIDRs,
Issuer: oidcCfg.Issuer,
SupportedSigAlgs: oidcCfg.SupportedSigAlgs,
PubkeysPEM: pubkeyPEMStrs,
}
if useJWKSURL {
vaultJWTCfg.JWKSURL = jwksURIStr
} else {
vaultJWTCfg.PubkeysPEM = pubkeyPEMStrs
}
if err := encodeJSON(opts.out, vaultJWTCfg); err != nil {
return fmt.Errorf("write Vault JWT config: %w", err)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
jose "github.com/go-jose/go-jose/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
Expand Down Expand Up @@ -264,6 +265,59 @@ func TestRunWithMockStreamer(t *testing.T) {
},
expectedError: false,
},
{
name: "EKS uses jwks_url without fetching JWKS",
oidcConfig: `{
"issuer": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
"id_token_signing_alg_values_supported": ["RS256"],
"jwks_uri": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys"
}`,
opts: options{
outputFormat: "json",
egressCIDRs: []string{"10.0.0.0/24"},
out: &bytes.Buffer{},
},
expectedError: false,
},
{
name: "EKS uses jwks_url in YAML output",
oidcConfig: `{
"issuer": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
"id_token_signing_alg_values_supported": ["RS256"],
"jwks_uri": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys"
}`,
opts: options{
outputFormat: "yaml",
egressCIDRs: []string{"10.0.0.0/24"},
out: &bytes.Buffer{},
},
expectedError: false,
},
{
name: "EKS static pubkeys with force flag",
oidcConfig: `{
"issuer": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
"id_token_signing_alg_values_supported": ["RS256"],
"jwks_uri": "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys"
}`,
jwksResponse: `{
"keys": [{
"kty": "RSA",
"kid": "test-key",
"use": "sig",
"alg": "RS256",
"n": "test-n",
"e": "AQAB"
}]
}`,
opts: options{
outputFormat: "json",
egressCIDRs: []string{"10.0.0.0/24"},
forceStaticKeys: true,
out: &bytes.Buffer{},
},
expectedError: false,
},
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

for _, tt := range tests {
Expand All @@ -273,6 +327,7 @@ func TestRunWithMockStreamer(t *testing.T) {
"/.well-known/openid-configuration": tt.oidcConfig,
"/keys": tt.jwksResponse,
"/custom-keys": tt.jwksResponse,
"/id/ABC123/keys": tt.jwksResponse,
},
errors: tt.errors,
delays: tt.delays,
Expand All @@ -288,12 +343,87 @@ func TestRunWithMockStreamer(t *testing.T) {
assert.NoError(t, err)
if buf, ok := tt.opts.out.(*bytes.Buffer); ok {
assert.NotEmpty(t, buf.String())
switch tt.name {
case "EKS uses jwks_url without fetching JWKS":
var cfg vaultJWTConfig
require.NoError(t, json.Unmarshal(buf.Bytes(), &cfg))
assert.Equal(t, "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys", cfg.JWKSURL)
assert.Empty(t, cfg.PubkeysPEM)
case "EKS uses jwks_url in YAML output":
var mount vaultJWTMountYAML
require.NoError(t, yaml.Unmarshal(buf.Bytes(), &mount))
assert.Equal(t, "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys", mount.Config.JWKSURL)
assert.Empty(t, mount.Config.PubkeysPEM)
case "EKS static pubkeys with force flag":
var cfg vaultJWTConfig
require.NoError(t, json.Unmarshal(buf.Bytes(), &cfg))
assert.Empty(t, cfg.JWKSURL)
assert.NotEmpty(t, cfg.PubkeysPEM)
}
}
}
})
}
}

func TestUseVaultJWKSURL(t *testing.T) {
tests := []struct {
name string
issuer string
jwksURI string
expected bool
}{
{
name: "EKS absolute jwks uri",
issuer: "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
jwksURI: "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys",
expected: true,
},
{
name: "k3d cluster internal issuer",
issuer: "https://kubernetes.default.svc.cluster.local",
jwksURI: "https://kubernetes.default.svc.cluster.local/openid/v1/jwks",
expected: false,
},
{
name: "relative jwks uri",
issuer: "https://test-issuer",
jwksURI: "/keys",
expected: false,
},
{
name: "azure aks issuer",
issuer: "https://westus.oic.prod-aks.azure.com/tenant",
jwksURI: "https://westus.oic.prod-aks.azure.com/tenant/discovery/v2.0/keys",
expected: false,
},
{
name: "EKS jwks host with non-EKS issuer",
issuer: "https://kubernetes.default.svc.cluster.local",
jwksURI: "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123/keys",
expected: false,
},
{
name: "EKS issuer with non-EKS jwks override",
issuer: "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
jwksURI: "https://test-server:6443/openid/v1/jwks",
expected: false,
},
{
name: "mismatched EKS region authorities",
issuer: "https://oidc.eks.us-east-2.amazonaws.com/id/ABC123",
jwksURI: "https://oidc.eks.us-west-2.amazonaws.com/id/ABC123/keys",
expected: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, useVaultJWKSURL(tt.issuer, tt.jwksURI))
})
}
}

func TestK8sStreamer(t *testing.T) {
// Create a test server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down
Loading