Skip to content

Commit a118316

Browse files
elizabethhealyelizabethhealy
authored
feat(core): Introduce ERS mode, ability to connect to remote ERS (#1735)
1 parent 5dc1893 commit a118316

File tree

14 files changed

+385
-32
lines changed

14 files changed

+385
-32
lines changed

‎Contributing.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ For end-users/consumers, see [here](./Consuming.md).
1616
1. Note: You will have to add the ``localhost.crt`` as a trusted certificate to do TLS authentication at ``localhost:8443``.
1717
3. Create an OpenTDF config file: `opentdf.yaml`
1818
1. The `opentdf-dev.yaml` file is the more secure starting point, but you will likely need to modify it to match your environment. This configuration is recommended as it is more secure but it does require valid development keypairs.
19-
2. The `opentdf-example-no-kas.yaml` file is simpler to run but less secure. This file configures the platform to startup without a KAS instances and without endpoint authentication.
19+
2. The `opentdf-core-mode.yaml` file is simpler to run but less secure. This file configures the platform to startup without a KAS instances, without a built-in ERS instance, and without endpoint authentication.
2020
4. Provision keycloak: `go run github.com/opentdf/platform/service provision keycloak`. Updates the local Keycloak configuration for local testing and development by creating a realm, roles, a client, and users.
2121
5. Run the server: `go run github.com/opentdf/platform/service start`. Runs the OpenTDF platform capabilities as a monolithic service.
2222
1. _Alt_ use the hot-reload development environment `air`

‎docs/configuration.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,12 @@ Root level key `sdk_config`
3434

3535
| Field | Description | Default | Environment Variable |
3636
| -------- | -------------| -------- | -------------------- |
37-
| `endpoint` | The core platform endpoint to connect to | | OPENTDF_SDK_CONFIG_ENDPOINT |
38-
| `plaintext` | Use a plaintext grpc connection | `false` | OPENTDF_SDK_CONFIG_PLAINTEXT |
37+
| `core.endpoint` | The core platform endpoint to connect to | | OPENTDF_SDK_CONFIG_ENDPOINT |
38+
| `core.plaintext` | Use a plaintext grpc connection | `false` | OPENTDF_SDK_CONFIG_PLAINTEXT |
39+
| `core.insecure` | Use an insecure tls connection | `false` | |
40+
| `entityresolution.endpoint` | The entityresolution endpoint to connect to | | |
41+
| `entityresolution.plaintext` | Use a plaintext ERS grpc connection | `false` | |
42+
| `entityresolution.insecure` | Use an insecure tls connection | `false` | |
3943
| `client_id` | OAuth client id | | OPENTDF_SDK_CONFIG_CLIENT_ID |
4044
| `client_secret` | The clients credentials | | OPENTDF_SDK_CONFIG_CLIENT_SECRET |
4145

‎opentdf-example-no-kas.yaml‎ renamed to ‎opentdf-core-mode.yaml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
1+
# configures the platform to startup without a KAS instances, without a built-in ERS instance, and without endpoint authentication
2+
# build off of this config file if you are running your ERS and KAS instances seperately or if you only need the policy features
13
mode: core
4+
sdk_config:
5+
entityresolution:
6+
endpoint: http://localhost:8181
7+
plaintext: true
8+
client_id: opentdf
9+
client_secret: secret
210
logger:
311
level: debug
412
type: text
@@ -44,4 +52,4 @@ server:
4452
maxage: 3600
4553
grpc:
4654
reflectionEnabled: true # Default is false
47-
port: 8080
55+
port: 8383

‎opentdf-ers-mode.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# configures the platform to run just the entity resolution service and the well-known service
2+
# primarily to be used for testing and development of external ERS connections
3+
mode: entityresolution
4+
logger:
5+
level: debug
6+
type: text
7+
output: stdout
8+
services:
9+
entityresolution:
10+
log_level: info
11+
url: http://localhost:8888/auth
12+
clientid: 'tdf-entity-resolution'
13+
clientsecret: 'secret'
14+
realm: 'opentdf'
15+
legacykeycloak: true
16+
inferid:
17+
from:
18+
email: true
19+
username: true
20+
server:
21+
auth:
22+
enabled: true
23+
enforceDPoP: false
24+
public_client_id: 'opentdf-public'
25+
audience: 'http://localhost:8080'
26+
issuer: http://localhost:8888/auth/realms/opentdf
27+
policy:
28+
## Default policy for all requests
29+
default: #"role:standard"
30+
## Dot notation is used to access nested claims (i.e. realm_access.roles)
31+
claim: # realm_access.roles
32+
## Maps the external role to the opentdf role
33+
## Note: left side is used in the policy, right side is the external role
34+
map:
35+
# standard: opentdf-standard
36+
# admin: opentdf-admin
37+
# org-admin: opentdf-org-admin
38+
39+
## Custom policy (see examples https://github.com/casbin/casbin/tree/master/examples)
40+
csv: #|
41+
# p, role:org-admin, policy:attributes, *, *, allow
42+
# p, role:org-admin, policy:subject-mappings, *, *, allow
43+
# p, role:org-admin, policy:resource-mappings, *, *, allow
44+
# p, role:org-admin, policy:kas-registry, *, *, allow
45+
# p, role:org-admin, policy:unsafe, *, *, allow
46+
47+
## Custom model (see https://casbin.org/docs/syntax-for-models/)
48+
model: #|
49+
# [request_definition]
50+
# r = sub, res, act, obj
51+
#
52+
# [policy_definition]
53+
# p = sub, res, act, obj, eft
54+
#
55+
# [role_definition]
56+
# g = _, _
57+
#
58+
# [policy_effect]
59+
# e = some(where (p.eft == allow)) && !some(where (p.eft == deny))
60+
#
61+
# [matchers]
62+
# m = g(r.sub, p.sub) && globOrRegexMatch(r.res, p.res) && globOrRegexMatch(r.act, p.act) && globOrRegexMatch(r.obj, p.obj)
63+
cors:
64+
enabled: false
65+
# "*" to allow any origin or a specific domain like "https://yourdomain.com"
66+
allowedorigins:
67+
- '*'
68+
# List of methods. Examples: "GET,POST,PUT"
69+
allowedmethods:
70+
- GET
71+
- POST
72+
- PATCH
73+
- PUT
74+
- DELETE
75+
- OPTIONS
76+
# List of headers that are allowed in a request
77+
allowedheaders:
78+
- ACCEPT
79+
- Authorization
80+
- Content-Type
81+
- X-CSRF-Token
82+
- X-Request-ID
83+
# List of response headers that browsers are allowed to access
84+
exposedheaders:
85+
- Link
86+
# Sets whether credentials are included in the CORS request
87+
allowcredentials: true
88+
# Sets the maximum age (in seconds) of a specific CORS preflight request
89+
maxage: 3600
90+
grpc:
91+
reflectionEnabled: true # Default is false
92+
port: 8282

‎opentdf-kas-mode.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# configures the platform to run only the KAS and well known service
2+
# build off this config file if you intend on running a (or multiple) seperate kas instance(s)
3+
mode: kas
4+
sdk_config:
5+
core:
6+
endpoint: http://localhost:8080
7+
plaintext: true
8+
client_id: opentdf
9+
client_secret: secret
10+
logger:
11+
level: debug
12+
type: text
13+
output: stdout
14+
services:
15+
kas:
16+
keyring:
17+
- kid: e1
18+
alg: ec:secp256r1
19+
- kid: e1
20+
alg: ec:secp256r1
21+
legacy: true
22+
- kid: r1
23+
alg: rsa:2048
24+
- kid: r1
25+
alg: rsa:2048
26+
legacy: true
27+
server:
28+
tls:
29+
enabled: false
30+
cert: ./keys/platform.crt
31+
key: ./keys/platform-key.pem
32+
auth:
33+
enabled: true
34+
enforceDPoP: false
35+
public_client_id: 'opentdf-public'
36+
audience: 'http://localhost:8080'
37+
issuer: http://localhost:8888/auth/realms/opentdf
38+
policy:
39+
## Default policy for all requests
40+
default: #"role:standard"
41+
## Dot notation is used to access nested claims (i.e. realm_access.roles)
42+
claim: # realm_access.roles
43+
## Maps the external role to the opentdf role
44+
## Note: left side is used in the policy, right side is the external role
45+
map:
46+
# standard: opentdf-standard
47+
# admin: opentdf-admin
48+
49+
## Custom policy (see examples https://github.com/casbin/casbin/tree/master/examples)
50+
csv: #|
51+
# p, role:admin, *, *, allow
52+
53+
## Custom model (see https://casbin.org/docs/syntax-for-models/)
54+
model: #|
55+
# [request_definition]
56+
# r = sub, res, act, obj
57+
#
58+
# [policy_definition]
59+
# p = sub, res, act, obj, eft
60+
#
61+
# [role_definition]
62+
# g = _, _
63+
#
64+
# [policy_effect]
65+
# e = some(where (p.eft == allow)) && !some(where (p.eft == deny))
66+
#
67+
# [matchers]
68+
# m = g(r.sub, p.sub) && globOrRegexMatch(r.res, p.res) && globOrRegexMatch(r.act, p.act) && globOrRegexMatch(r.obj, p.obj)
69+
cors:
70+
enabled: false
71+
# "*" to allow any origin or a specific domain like "https://yourdomain.com"
72+
allowedorigins:
73+
- '*'
74+
# List of methods. Examples: "GET,POST,PUT"
75+
allowedmethods:
76+
- GET
77+
- POST
78+
- PATCH
79+
- PUT
80+
- DELETE
81+
- OPTIONS
82+
# List of headers that are allowed in a request
83+
allowedheaders:
84+
- ACCEPT
85+
- Authorization
86+
- Content-Type
87+
- X-CSRF-Token
88+
- X-Request-ID
89+
# List of response headers that browsers are allowed to access
90+
exposedheaders:
91+
- Link
92+
# Sets whether credentials are included in the CORS request
93+
allowcredentials: true
94+
# Sets the maximum age (in seconds) of a specific CORS preflight request
95+
maxage: 3600
96+
grpc:
97+
reflectionEnabled: true # Default is false
98+
cryptoProvider:
99+
type: standard
100+
standard:
101+
keys:
102+
- kid: r1
103+
alg: rsa:2048
104+
private: kas-private.pem
105+
cert: kas-cert.pem
106+
- kid: e1
107+
alg: ec:secp256r1
108+
private: kas-ec-private.pem
109+
cert: kas-ec-cert.pem
110+
port: 8181

‎sdk/options.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type config struct {
3535
customAccessTokenSource auth.AccessTokenSource
3636
oauthAccessTokenSource oauth2.TokenSource
3737
coreConn *grpc.ClientConn
38+
entityResolutionConn *grpc.ClientConn
3839
collectionStore *collectionStore
3940
}
4041

@@ -135,10 +136,9 @@ func WithCustomAuthorizationConnection(conn *grpc.ClientConn) Option {
135136
}
136137
}
137138

138-
// Deprecated: Use WithCustomCoreConnection instead
139139
func WithCustomEntityResolutionConnection(conn *grpc.ClientConn) Option {
140140
return func(c *config) {
141-
c.coreConn = conn
141+
c.entityResolutionConn = conn
142142
}
143143
}
144144

‎sdk/sdk.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ type SDK struct {
7474
func New(platformEndpoint string, opts ...Option) (*SDK, error) {
7575
var (
7676
platformConn *grpc.ClientConn // Connection to the platform
77+
ersConn *grpc.ClientConn // Connection to ERS (possibly remote)
7778
err error
7879
)
7980

@@ -169,6 +170,12 @@ func New(platformEndpoint string, opts ...Option) (*SDK, error) {
169170
}
170171
}
171172

173+
if cfg.entityResolutionConn != nil {
174+
ersConn = cfg.entityResolutionConn
175+
} else {
176+
ersConn = platformConn
177+
}
178+
172179
return &SDK{
173180
config: *cfg,
174181
collectionStore: cfg.collectionStore,
@@ -183,7 +190,7 @@ func New(platformEndpoint string, opts ...Option) (*SDK, error) {
183190
Unsafe: unsafe.NewUnsafeServiceClient(platformConn),
184191
KeyAccessServerRegistry: kasregistry.NewKeyAccessServerRegistryServiceClient(platformConn),
185192
Authorization: authorization.NewAuthorizationServiceClient(platformConn),
186-
EntityResoution: entityresolution.NewEntityResolutionServiceClient(platformConn),
193+
EntityResoution: entityresolution.NewEntityResolutionServiceClient(ersConn),
187194
wellknownConfiguration: wellknownconfiguration.NewWellKnownServiceClient(platformConn),
188195
}, nil
189196
}

‎service/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ require (
2424
github.com/opentdf/platform/lib/flattening v0.1.1
2525
github.com/opentdf/platform/lib/ocrypto v0.1.7
2626
github.com/opentdf/platform/protocol/go v0.2.20
27-
github.com/opentdf/platform/sdk v0.3.21
27+
github.com/opentdf/platform/sdk v0.3.22
2828
github.com/pressly/goose/v3 v3.19.1
2929
github.com/spf13/cobra v1.8.1
3030
github.com/spf13/viper v1.18.2

‎service/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,8 @@ github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdl
287287
github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg=
288288
github.com/opentdf/platform/protocol/go v0.2.20 h1:FPU1ZcXvPm/QeE2nqgbD/HMTOCICQSD0DoncQbAZ1ws=
289289
github.com/opentdf/platform/protocol/go v0.2.20/go.mod h1:TWIuf387VeR3q0TL4nAMKQTWEqqID+8Yjao76EX9Dto=
290-
github.com/opentdf/platform/sdk v0.3.21 h1:18oZk8t32luXBL2lhRa3qvjTY17Y3PmA0Wp1F8tdkqc=
291-
github.com/opentdf/platform/sdk v0.3.21/go.mod h1:KpT/m5zXQ19WqhGePKfIC39Ly8LOipKdKGbJ1B/59a8=
290+
github.com/opentdf/platform/sdk v0.3.22 h1:nxmu7i+dmKuRQKVi5EIjOVdEFzzu/zkaA5LmGPPtPzw=
291+
github.com/opentdf/platform/sdk v0.3.22/go.mod h1:KpT/m5zXQ19WqhGePKfIC39Ly8LOipKdKGbJ1B/59a8=
292292
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
293293
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
294294
github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4=

‎service/internal/config/config.go‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ type Config struct {
4343

4444
// SDKConfig represents the configuration for the SDK.
4545
type SDKConfig struct {
46-
// Endpoint is the URL of the Core Platform endpoint.
47-
Endpoint string `mapstructure:"endpoint" json:"endpoint"`
46+
// Connection to the Core Platform
47+
CorePlatformConnection Connection `mapstructure:"core" json:"core"`
4848

49-
// Plaintext specifies whether the SDK should use plaintext communication.
50-
Plaintext bool `mapstructure:"plaintext" json:"plaintext" default:"false" validate:"boolean"`
49+
// Connection to an ERS if not in the core platform
50+
EntityResolutionConnection Connection `mapstructure:"entityresolution" json:"entityresolution"`
5151

5252
// ClientID is the client ID used for client credentials grant.
5353
// It is required together with ClientSecret.
@@ -58,6 +58,17 @@ type SDKConfig struct {
5858
ClientSecret string `mapstructure:"client_secret" json:"client_secret" validate:"required_with=ClientID"`
5959
}
6060

61+
type Connection struct {
62+
// Endpoint is the URL of the platform or service.
63+
Endpoint string `mapstructure:"endpoint" json:"endpoint"`
64+
65+
// Plaintext specifies whether the SDK should use plaintext communication.
66+
Plaintext bool `mapstructure:"plaintext" json:"plaintext" default:"false" validate:"boolean"`
67+
68+
// Insecure specifies whether the SDK should use insecure TLS communication.
69+
Insecure bool `mapstructure:"insecure" json:"insecure" default:"false" validate:"boolean"`
70+
}
71+
6172
type Error string
6273

6374
func (e Error) Error() string {
@@ -137,8 +148,14 @@ func (c *Config) LogValue() slog.Value {
137148

138149
func (c SDKConfig) LogValue() slog.Value {
139150
return slog.GroupValue(
140-
slog.String("endpoint", c.Endpoint),
141-
slog.Bool("plaintext", c.Plaintext),
151+
slog.Group("core",
152+
"endpoint", c.CorePlatformConnection.Endpoint,
153+
"plaintext", c.CorePlatformConnection.Plaintext,
154+
"insecure", c.CorePlatformConnection.Insecure),
155+
slog.Group("entityresolution",
156+
"endpoint", c.EntityResolutionConnection.Endpoint,
157+
"plaintext", c.EntityResolutionConnection.Plaintext,
158+
"insecure", c.EntityResolutionConnection.Insecure),
142159
slog.String("client_id", c.ClientID),
143160
slog.String("client_secret", "[REDACTED]"),
144161
)

0 commit comments

Comments
 (0)