Skip to content

NIFI-16191: Add OAuth support for ConfluentSchemaRegistry controller service #11534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
takraj wants to merge 1 commit into
base: main
Choose a base branch
from
Open

NIFI-16191: Add OAuth support for ConfluentSchemaRegistry controller service #11534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...ion-bundles/nifi-confluent-platform-bundle/nifi-confluent-schema-registry-service/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@
<artifactId>nifi-web-client</artifactId>
<version>2.12.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-oauth2-provider-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-confluent-protobuf-message-name-resolver</artifactId>
Expand Down
21 changes: 19 additions & 2 deletions ...rvice/src/main/java/org/apache/nifi/confluent/schemaregistry/ConfluentSchemaRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.migration.PropertyConfiguration;
import org.apache.nifi.oauth2.OAuth2AccessTokenProvider;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.schema.access.SchemaField;
import org.apache.nifi.schema.access.SchemaNotFoundException;
Expand Down Expand Up @@ -140,6 +141,14 @@ public class ConfluentSchemaRegistry extends AbstractControllerService implement
.sensitive(true)
.build();

static final PropertyDescriptor OAUTH2_ACCESS_TOKEN_PROVIDER = new PropertyDescriptor.Builder()
.name("OAuth2 Access Token Provider")
.description("OAuth2 Access Token Provider used for Bearer authentication to Confluent Schema Registry")
.identifiesControllerService(OAuth2AccessTokenProvider.class)
.required(true)
.dependsOn(AUTHENTICATION_TYPE, AuthenticationType.OAUTH2.toString())
.build();

private static final List<PropertyDescriptor> PROPERTY_DESCRIPTORS = List.of(
SCHEMA_REGISTRY_URLS,
SSL_CONTEXT,
Expand All @@ -148,7 +157,8 @@ public class ConfluentSchemaRegistry extends AbstractControllerService implement
CACHE_EXPIRATION,
AUTHENTICATION_TYPE,
USERNAME,
PASSWORD
PASSWORD,
OAUTH2_ACCESS_TOKEN_PROVIDER
);

private volatile SchemaRegistryClient client;
Expand Down Expand Up @@ -182,9 +192,16 @@ public void onEnabled(final ConfigurationContext context) {

final SSLContextProvider sslContextProvider = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextProvider.class);

final AuthenticationType authenticationType = AuthenticationType.valueOf(context.getProperty(AUTHENTICATION_TYPE).getValue());
final String username = context.getProperty(USERNAME).getValue();
final String password = context.getProperty(PASSWORD).getValue();

OAuth2AccessTokenProvider oauth2AccessTokenProvider = null;
if (AuthenticationType.OAUTH2.equals(authenticationType)) {
oauth2AccessTokenProvider = context.getProperty(OAUTH2_ACCESS_TOKEN_PROVIDER).asControllerService(OAuth2AccessTokenProvider.class);
oauth2AccessTokenProvider.getAccessDetails();
}

// generate a map of http headers where the key is the remainder of the property name after
// the request header prefix
final Map<String, String> httpHeaders =
Expand All @@ -197,7 +214,7 @@ public void onEnabled(final ConfigurationContext context) {
);

final SchemaRegistryClient restClient = new RestSchemaRegistryClient(baseUrls, timeoutMillis,
sslContextProvider, username, password, getLogger(), httpHeaders);
sslContextProvider, username, password, oauth2AccessTokenProvider, getLogger(), httpHeaders);

final int cacheSize = context.getProperty(CACHE_SIZE).asInteger();
final long cacheExpiration = context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.NANOSECONDS);
Expand Down
4 changes: 3 additions & 1 deletion ...ice/src/main/java/org/apache/nifi/confluent/schemaregistry/client/AuthenticationType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,7 @@
public enum AuthenticationType {
BASIC,

NONE
NONE,

OAUTH2
}
202 changes: 135 additions & 67 deletions ...c/main/java/org/apache/nifi/confluent/schemaregistry/client/RestSchemaRegistryClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.avro.AvroTypeUtil;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.oauth2.OAuth2AccessTokenProvider;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.schemaregistry.services.SchemaDefinition;
import org.apache.nifi.schemaregistry.services.StandardSchemaDefinition;
Expand Down Expand Up @@ -54,8 +55,10 @@
import javax.net.ssl.X509KeyManager;
import javax.net.ssl.X509TrustManager;

import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_OK;
import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
import static org.apache.nifi.schemaregistry.services.SchemaDefinition.SchemaType;

/**
Expand All @@ -73,6 +76,9 @@ public class RestSchemaRegistryClient implements SchemaRegistryClient {
private final List<String> baseUrls;
private final ComponentLog logger;
private final Map<String, String> httpHeaders;
private final String username;
private final String password;
private final OAuth2AccessTokenProvider oauth2AccessTokenProvider;
private final WebClientService webClientService;

private static final ObjectMapper objectMapper = new ObjectMapper();
Expand All @@ -90,24 +96,21 @@ public class RestSchemaRegistryClient implements SchemaRegistryClient {
private static final String APPLICATION_JSON_CONTENT_TYPE = "application/json";
private static final String BASIC_CREDENTIALS_FORMAT = "%s:%s";
private static final String BASIC_AUTHORIZATION_FORMAT = "Basic %s";
private static final String BEARER_AUTHORIZATION_FORMAT = "Bearer %s";

public RestSchemaRegistryClient(final List<String> baseUrls,
final int timeoutMillis,
final SSLContextProvider sslContextProvider,
final String username,
final String password,
final OAuth2AccessTokenProvider oauth2AccessTokenProvider,
final ComponentLog logger,
final Map<String, String> httpHeaders) {
this.baseUrls = new ArrayList<>(baseUrls);
this.httpHeaders = new HashMap<>(httpHeaders);

if (StringUtils.isNoneBlank(username, password)) {
final String credentials = BASIC_CREDENTIALS_FORMAT.formatted(username, password);
final byte[] credentialsEncoded = credentials.getBytes(StandardCharsets.UTF_8);
final String authorization = Base64.getEncoder().encodeToString(credentialsEncoded);
final String basicAuthorization = BASIC_AUTHORIZATION_FORMAT.formatted(authorization);
this.httpHeaders.put(HttpHeaderName.AUTHORIZATION.getHeaderName(), basicAuthorization);
}
this.username = username;
this.password = password;
this.oauth2AccessTokenProvider = oauth2AccessTokenProvider;

final StandardWebClientService standardWebClientService = new StandardWebClientService();
final Duration timeout = Duration.ofMillis(timeoutMillis);
Expand Down Expand Up @@ -184,7 +187,7 @@ public RecordSchema getSchema(final int schemaId) throws SchemaNotFoundException

if (subjectsJson != null) {
final ArrayNode subjectsList = (ArrayNode) subjectsJson;
for (JsonNode subject: subjectsList) {
for (JsonNode subject : subjectsList) {
final String searchName = subject.asText();
try {
// get complete schema (name + id + version) using the subject name API
Expand Down Expand Up @@ -234,7 +237,7 @@ public RecordSchema getSchema(final int schemaId) throws SchemaNotFoundException
try {
final JsonNode subjectsAllJson = fetchJsonResponse("/subjects", "subjects array");
final ArrayNode subjectsAllList = (ArrayNode) subjectsAllJson;
for (JsonNode subject: subjectsAllList) {
for (JsonNode subject : subjectsAllList) {
try {
final String searchName = subject.asText();
completeSchema = postJsonResponse("/subjects/" + searchName, schemaJson, "schema id: " + schemaId);
Expand Down Expand Up @@ -380,49 +383,64 @@ private String getSchemaPath(final long schemaId) {

private JsonNode postJsonResponse(final String pathSuffix, final JsonNode schema, final String schemaDescription) throws SchemaNotFoundException {
String errorMessage = null;
for (final String baseUrl: baseUrls) {
for (final String baseUrl : baseUrls) {
final String path = getPath(pathSuffix);
final String trimmedBase = getTrimmedBase(baseUrl);
final String url = trimmedBase + path;
final URI uri = URI.create(url);

logger.debug("POST JSON response URL {}", url);

HttpRequestBodySpec requestBodySpec = webClientService.post()
.uri(uri)
.header(HttpHeaderName.ACCEPT.getHeaderName(), APPLICATION_JSON_CONTENT_TYPE)
.header(HttpHeaderName.CONTENT_TYPE.getHeaderName(), SCHEMA_REGISTRY_CONTENT_TYPE);
boolean oauthTokenRefreshed = false;
while (true) {
HttpRequestBodySpec requestBodySpec = webClientService.post()
.uri(uri)
.header(HttpHeaderName.ACCEPT.getHeaderName(), APPLICATION_JSON_CONTENT_TYPE)
.header(HttpHeaderName.CONTENT_TYPE.getHeaderName(), SCHEMA_REGISTRY_CONTENT_TYPE);

for (final Map.Entry<String, String> header : httpHeaders.entrySet()) {
requestBodySpec = requestBodySpec.header(header.getKey(), header.getValue());
}
requestBodySpec = applyRequestHeaders(requestBodySpec);

final String requestBody = schema.toString();
try (HttpResponseEntity responseEntity = requestBodySpec.body(requestBody).retrieve()) {
final int responseCode = responseEntity.statusCode();
final String requestBody = schema.toString();
try (HttpResponseEntity responseEntity = requestBodySpec.body(requestBody).retrieve()) {
final int responseCode = responseEntity.statusCode();

switch (responseCode) {
case HTTP_OK:
try (InputStream responseBody = responseEntity.body()) {
final JsonNode jsonResponse = objectMapper.readTree(responseBody);

if (logger.isDebugEnabled()) {
logger.debug("JSON Response: {}", jsonResponse);
}
switch (responseCode) {
case HTTP_OK:
try (InputStream responseBody = responseEntity.body()) {
final JsonNode jsonResponse = objectMapper.readTree(responseBody);

return jsonResponse;
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response Body from URL [%s]".formatted(url), e);
}
case HTTP_NOT_FOUND:
logger.debug("Could not find Schema {} from Registry {}", schemaDescription, baseUrl);
continue;
if (logger.isDebugEnabled()) {
logger.debug("JSON Response: {}", jsonResponse);
}

default:
errorMessage = readErrorResponseBody(responseEntity);
return jsonResponse;
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response Body from URL [%s]".formatted(url), e);
}
case HTTP_UNAUTHORIZED:
case HTTP_FORBIDDEN:
if (!oauthTokenRefreshed && refreshOAuthAccessToken()) {
oauthTokenRefreshed = true;
continue;
}
errorMessage = readErrorResponseBody(responseEntity);
break;
case HTTP_NOT_FOUND:
logger.debug("Could not find Schema {} from Registry {}", schemaDescription, baseUrl);
errorMessage = null;
break;

default:
errorMessage = readErrorResponseBody(responseEntity);
}
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response from URL [%s]".formatted(url), e);
}
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response from URL [%s]".formatted(url), e);
break;
}

if (errorMessage == null) {
continue;
}
}

Expand All @@ -441,44 +459,94 @@ private JsonNode fetchJsonResponse(final String pathSuffix, final String schemaD

logger.debug("GET JSON response URL {}", url);

HttpRequestBodySpec requestBodySpec = webClientService.get()
.uri(uri)
.header(HttpHeaderName.ACCEPT.getHeaderName(), APPLICATION_JSON_CONTENT_TYPE);
boolean oauthTokenRefreshed = false;
while (true) {
HttpRequestBodySpec requestBodySpec = webClientService.get()
.uri(uri)
.header(HttpHeaderName.ACCEPT.getHeaderName(), APPLICATION_JSON_CONTENT_TYPE);

for (final Map.Entry<String, String> header : httpHeaders.entrySet()) {
requestBodySpec = requestBodySpec.header(header.getKey(), header.getValue());
}
try (HttpResponseEntity responseEntity = requestBodySpec.retrieve()) {
final int responseCode = responseEntity.statusCode();
requestBodySpec = applyRequestHeaders(requestBodySpec);
try (HttpResponseEntity responseEntity = requestBodySpec.retrieve()) {
final int responseCode = responseEntity.statusCode();

switch (responseCode) {
case HTTP_OK:
try (InputStream responseBody = responseEntity.body()) {
final JsonNode jsonResponse = objectMapper.readTree(responseBody);
switch (responseCode) {
case HTTP_OK:
try (InputStream responseBody = responseEntity.body()) {
final JsonNode jsonResponse = objectMapper.readTree(responseBody);

if (logger.isDebugEnabled()) {
logger.debug("JSON Response {}", jsonResponse);
}

return jsonResponse;
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Schema Response Body from URL [%s]".formatted(url), e);
}
case HTTP_NOT_FOUND:
logger.debug("Could not find Schema {} from Registry {}", schemaDescription, baseUrl);
continue;
if (logger.isDebugEnabled()) {
logger.debug("JSON Response {}", jsonResponse);
}

default:
errorMessage = readErrorResponseBody(responseEntity);
return jsonResponse;
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Schema Response Body from URL [%s]".formatted(url), e);
}
case HTTP_UNAUTHORIZED:
case HTTP_FORBIDDEN:
if (!oauthTokenRefreshed && refreshOAuthAccessToken()) {
oauthTokenRefreshed = true;
continue;
}
errorMessage = readErrorResponseBody(responseEntity);
break;
case HTTP_NOT_FOUND:
logger.debug("Could not find Schema {} from Registry {}", schemaDescription, baseUrl);
errorMessage = null;
break;

default:
errorMessage = readErrorResponseBody(responseEntity);
}
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response from URL [%s]".formatted(url), e);
}
} catch (final IOException e) {
throw new SchemaNotFoundException("Failed to read Response from URL [%s]".formatted(url), e);
break;
}

if (errorMessage == null) {
continue;
}
}
throw new SchemaNotFoundException("Failed to retrieve Schema with " + schemaDescription
+ " from any of the Confluent Schema Registry URL's provided; failure response message: " + errorMessage);
}

private HttpRequestBodySpec applyRequestHeaders(final HttpRequestBodySpec requestBodySpec) {
HttpRequestBodySpec updatedRequest = requestBodySpec;
for (final Map.Entry<String, String> header : httpHeaders.entrySet()) {
updatedRequest = updatedRequest.header(header.getKey(), header.getValue());
}
if (!httpHeaders.containsKey(HttpHeaderName.AUTHORIZATION.getHeaderName())) {
updatedRequest = applyAuthorizationHeader(updatedRequest);
}
return updatedRequest;
}

private HttpRequestBodySpec applyAuthorizationHeader(final HttpRequestBodySpec requestBodySpec) {
if (oauth2AccessTokenProvider != null) {
final String accessToken = oauth2AccessTokenProvider.getAccessDetails().getAccessToken();
return requestBodySpec.header(HttpHeaderName.AUTHORIZATION.getHeaderName(),
BEARER_AUTHORIZATION_FORMAT.formatted(accessToken));
}
if (StringUtils.isNoneBlank(username, password)) {
final String credentials = BASIC_CREDENTIALS_FORMAT.formatted(username, password);
final byte[] credentialsEncoded = credentials.getBytes(StandardCharsets.UTF_8);
final String authorization = Base64.getEncoder().encodeToString(credentialsEncoded);
return requestBodySpec.header(HttpHeaderName.AUTHORIZATION.getHeaderName(),
BASIC_AUTHORIZATION_FORMAT.formatted(authorization));
}
return requestBodySpec;
}

private boolean refreshOAuthAccessToken() {
if (oauth2AccessTokenProvider == null) {
return false;
}
oauth2AccessTokenProvider.refreshAccessDetails();
return true;
}

private String getTrimmedBase(String baseUrl) {
return baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl;
}
Expand Down
Loading
Loading