diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMultiTenantManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMultiTenantManager.java index b2dbba1d323e..07b3458d782c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMultiTenantManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMultiTenantManager.java @@ -62,6 +62,16 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public interface OMMultiTenantManager { + + String OZONE_TENANT_RANGER_POLICY_DESCRIPTION = + "Created by Ozone. WARNING: " + + "Changes will be lost when this tenant is deleted."; + + String OZONE_TENANT_RANGER_ROLE_DESCRIPTION = + "Managed by Ozone. WARNING: " + + "Changes will be overridden. " + + "Use Ozone tenant CLI to manage users in this tenant role instead."; + /* TODO: Outdated * Init multi-tenant manager. Performs initialization e.g. * - Initialize Multi-Tenant-Gatekeeper-Plugin @@ -341,15 +351,6 @@ static boolean checkAndEnableMultiTenancy( return true; } - String OZONE_TENANT_RANGER_POLICY_DESCRIPTION = - "Created by Ozone. WARNING: " - + "Changes will be lost when this tenant is deleted."; - - String OZONE_TENANT_RANGER_ROLE_DESCRIPTION = - "Managed by Ozone. WARNING: " - + "Changes will be overridden. " - + "Use Ozone tenant CLI to manage users in this tenant role instead."; - /** * Returns default VolumeAccess policy given tenant and role names. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 19a77fea7a44..d7b361330e2f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -31,18 +31,6 @@ public class OMPerformanceMetrics { private static final String SOURCE_NAME = OMPerformanceMetrics.class.getSimpleName(); - public static OMPerformanceMetrics register() { - MetricsSystem ms = DefaultMetricsSystem.instance(); - return ms.register(SOURCE_NAME, - "OzoneManager Request Performance", - new OMPerformanceMetrics()); - } - - public static void unregister() { - MetricsSystem ms = DefaultMetricsSystem.instance(); - ms.unregisterSource(SOURCE_NAME); - } - @Metric(about = "Overall lookupKey in nanoseconds") private MutableRate lookupLatencyNs; @@ -160,6 +148,18 @@ public static void unregister() { @Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms") private MutableGaugeLong openKeyCleanupServiceLatencyMs; + public static OMPerformanceMetrics register() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + return ms.register(SOURCE_NAME, + "OzoneManager Request Performance", + new OMPerformanceMetrics()); + } + + public static void unregister() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(SOURCE_NAME); + } + public void addLookupLatency(long latencyInNs) { lookupLatencyNs.add(latencyInNs); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java index d8b7a12946aa..cdea1ce6e680 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java @@ -46,15 +46,6 @@ public final class OMPolicyProvider extends PolicyProvider { private static final Supplier SUPPLIER = MemoizedSupplier.valueOf(OMPolicyProvider::new); - private OMPolicyProvider() { - } - - @Private - @Unstable - public static OMPolicyProvider getInstance() { - return SUPPLIER.get(); - } - private static final List OM_SERVICES = Arrays.asList( new Service(OZONE_OM_SECURITY_CLIENT_PROTOCOL_ACL, @@ -67,6 +58,15 @@ public static OMPolicyProvider getInstance() { ReconfigureProtocol.class) ); + private OMPolicyProvider() { + } + + @Private + @Unstable + public static OMPolicyProvider getInstance() { + return SUPPLIER.get(); + } + @Override public Service[] getServices() { return OM_SERVICES.toArray(new Service[0]); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 4953d3b35610..c551c5a8540f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -50,15 +50,6 @@ * Helper class for fetching List Status for a path. */ public class OzoneListStatusHelper { - /** - * Interface to get the File Status for a path. - */ - @FunctionalInterface - public interface GetFileStatusHelper { - OzoneFileStatus apply(OmKeyArgs args, String clientAddress, - boolean skipFileNotFoundError) throws IOException; - } - private static final Logger LOG = LoggerFactory.getLogger(OzoneListStatusHelper.class); @@ -79,6 +70,15 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, this.omDefaultReplication = omDefaultReplication; } + /** + * Interface to get the File Status for a path. + */ + @FunctionalInterface + public interface GetFileStatusHelper { + OzoneFileStatus apply(OmKeyArgs args, String clientAddress, + boolean skipFileNotFoundError) throws IOException; + } + public Collection listStatusFSO(OmKeyArgs args, String startKey, long numEntries, String clientAddress, boolean allowPartialPrefixes) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index df3cef96a54b..d265e6dfd8eb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -481,22 +481,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private String omHostName; - /** - * OM Startup mode. - */ - public enum StartupOption { - REGUALR, - BOOTSTRAP, - FORCE_BOOTSTRAP - } - - private enum State { - INITIALIZED, - BOOTSTRAPPING, - RUNNING, - STOPPED - } - // Used in MiniOzoneCluster testing private State omState; private Thread emptier; @@ -5157,4 +5141,20 @@ public void compactOMDB(String columnFamily) throws IOException { public OMExecutionFlow getOmExecutionFlow() { return omExecutionFlow; } + + /** + * OM Startup mode. + */ + public enum StartupOption { + REGUALR, + BOOTSTRAP, + FORCE_BOOTSTRAP + } + + private enum State { + INITIALIZED, + BOOTSTRAPPING, + RUNNING, + STOPPED + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 99d749a9fb56..f761f31db6a2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -83,6 +83,8 @@ public class SstFilteringService extends BackgroundService private AtomicBoolean running; + private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock(); + public static boolean isSstFiltered(OzoneConfiguration ozoneConfiguration, SnapshotInfo snapshotInfo) { Path sstFilteredFile = Paths.get(OmSnapshotManager.getSnapshotPath(ozoneConfiguration, snapshotInfo), SST_FILTERED_FILE); @@ -101,9 +103,6 @@ public SstFilteringService(long interval, TimeUnit unit, long serviceTimeout, running = new AtomicBoolean(false); } - private final BootstrapStateHandler.Lock lock = - new BootstrapStateHandler.Lock(); - @Override public void start() { running.set(true); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java index cf7e3e0f2588..b13a8e0b607a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java @@ -34,6 +34,13 @@ @Metrics(about = "OzoneManager HA Metrics", context = OzoneConsts.OZONE) public final class OMHAMetrics implements MetricsSource { + public static final String SOURCE_NAME = OMHAMetrics.class.getSimpleName(); + private final OMHAMetricsInfo omhaMetricsInfo = new OMHAMetricsInfo(); + private MetricsRegistry metricsRegistry; + + private String currNodeId; + private String leaderId; + /** * Private nested class to hold the values * of MetricsInfo for OMHAMetrics. @@ -72,14 +79,6 @@ public void setNodeId(String nodeId) { } } - public static final String SOURCE_NAME = - OMHAMetrics.class.getSimpleName(); - private final OMHAMetricsInfo omhaMetricsInfo = new OMHAMetricsInfo(); - private MetricsRegistry metricsRegistry; - - private String currNodeId; - private String leaderId; - private OMHAMetrics(String currNodeId, String leaderId) { this.currNodeId = currNodeId; this.leaderId = leaderId; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java index 80c20f7af6dc..e6185f3d65a0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java @@ -37,16 +37,18 @@ * This class is used for OM Audit logs. */ public final class OMAuditLogger { - private OMAuditLogger() { - } - private static final Map CMD_AUDIT_ACTION_MAP = new HashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(OMAuditLogger.class); + private static final Map CMD_AUDIT_ACTION_MAP = new HashMap<>(); + static { init(); } + private OMAuditLogger() { + } + private static void init() { CMD_AUDIT_ACTION_MAP.put(Type.CreateVolume, OMAction.CREATE_VOLUME); CMD_AUDIT_ACTION_MAP.put(Type.DeleteVolume, OMAction.DELETE_VOLUME); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index c7f17e16bb33..bf7dd2df7b0f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -73,6 +73,35 @@ public final class OzoneManagerDoubleBuffer { private static final Logger LOG = LoggerFactory.getLogger(OzoneManagerDoubleBuffer.class); + private Queue currentBuffer; + private Queue readyBuffer; + /** + * Limit the number of un-flushed transactions for {@link OzoneManagerStateMachine}. + */ + private final Semaphore unFlushedTransactions; + + /** To flush the buffers. */ + private final Daemon daemon; + /** Is the {@link #daemon} running? */ + private final AtomicBoolean isRunning = new AtomicBoolean(false); + /** Notify flush operations are completed by the {@link #daemon}. */ + private final FlushNotifier flushNotifier; + + private final OMMetadataManager omMetadataManager; + + private final Consumer updateLastAppliedIndex; + + private final S3SecretManager s3SecretManager; + + private final boolean isTracingEnabled; + + private final OzoneManagerDoubleBufferMetrics metrics = OzoneManagerDoubleBufferMetrics.create(); + + /** Accumulative count (for testing and debug only). */ + private final AtomicLong flushedTransactionCount = new AtomicLong(); + /** The number of flush iterations (for testing and debug only). */ + private final AtomicLong flushIterations = new AtomicLong(); + /** Entry for {@link #currentBuffer} and {@link #readyBuffer}. */ private static class Entry { private final TermIndex termIndex; @@ -160,35 +189,6 @@ static Semaphore newSemaphore(int permits) { return permits > 0 ? new Semaphore(permits) : null; } - private Queue currentBuffer; - private Queue readyBuffer; - /** - * Limit the number of un-flushed transactions for {@link OzoneManagerStateMachine}. - */ - private final Semaphore unFlushedTransactions; - - /** To flush the buffers. */ - private final Daemon daemon; - /** Is the {@link #daemon} running? */ - private final AtomicBoolean isRunning = new AtomicBoolean(false); - /** Notify flush operations are completed by the {@link #daemon}. */ - private final FlushNotifier flushNotifier; - - private final OMMetadataManager omMetadataManager; - - private final Consumer updateLastAppliedIndex; - - private final S3SecretManager s3SecretManager; - - private final boolean isTracingEnabled; - - private final OzoneManagerDoubleBufferMetrics metrics = OzoneManagerDoubleBufferMetrics.create(); - - /** Accumulative count (for testing and debug only). */ - private final AtomicLong flushedTransactionCount = new AtomicLong(); - /** The number of flush iterations (for testing and debug only). */ - private final AtomicLong flushIterations = new AtomicLong(); - private OzoneManagerDoubleBuffer(Builder b) { this.currentBuffer = new ConcurrentLinkedQueue<>(); this.readyBuffer = new ConcurrentLinkedQueue<>(); @@ -613,21 +613,6 @@ public void awaitFlush() throws InterruptedException { } static class FlushNotifier { - static class Entry { - private final CompletableFuture future = new CompletableFuture<>(); - private int count; - - private CompletableFuture await() { - count++; - return future; - } - - private int complete() { - Preconditions.assertTrue(future.complete(count)); - return future.join(); - } - } - /** The size of the map is at most two since it uses {@link #flushCount} + 2 in {@link #await()} .*/ private final Map flushFutures = new TreeMap<>(); private int awaitCount; @@ -652,5 +637,20 @@ synchronized int notifyFlush() { LOG.debug("notifyFlush {}, awaitCount: {} -> {}", flush, await, awaitCount); return await; } + + static class Entry { + private final CompletableFuture future = new CompletableFuture<>(); + private int count; + + private CompletableFuture await() { + count++; + return future; + } + + private int complete() { + Preconditions.assertTrue(future.complete(count)); + return future.join(); + } + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServerConfig.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServerConfig.java index d862366b6f82..a3703dfe63bc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServerConfig.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServerConfig.java @@ -44,14 +44,6 @@ public class OzoneManagerRatisServerConfig { ) private long logAppenderWaitTimeMin; - public long getLogAppenderWaitTimeMin() { - return logAppenderWaitTimeMin; - } - - public void setLogAppenderWaitTimeMin(long logAppenderWaitTimeMin) { - this.logAppenderWaitTimeMin = logAppenderWaitTimeMin; - } - @Config(key = "retrycache.expirytime", defaultValue = "300s", type = ConfigType.TIME, @@ -60,6 +52,14 @@ public void setLogAppenderWaitTimeMin(long logAppenderWaitTimeMin) { ) private long retryCacheTimeout = Duration.ofSeconds(300).toMillis(); + public long getLogAppenderWaitTimeMin() { + return logAppenderWaitTimeMin; + } + + public void setLogAppenderWaitTimeMin(long logAppenderWaitTimeMin) { + this.logAppenderWaitTimeMin = logAppenderWaitTimeMin; + } + public long getRetryCacheTimeout() { return retryCacheTimeout; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java index 31f0eabada84..4322e543fae3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java @@ -55,6 +55,11 @@ public class OMKeySetTimesRequest extends OMKeyRequest { private static final Logger LOG = LoggerFactory.getLogger(OMKeySetTimesRequest.class); + private final String volumeName; + private final String bucketName; + private final String keyName; + private final long modificationTime; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { OMRequest request = super.preExecute(ozoneManager); @@ -81,11 +86,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private final String volumeName; - private final String bucketName; - private final String keyName; - private final long modificationTime; - public OMKeySetTimesRequest(OMRequest omRequest, BucketLayout bucketLayout) { super(omRequest, bucketLayout); OzoneManagerProtocolProtos.SetTimesRequest setTimesRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 2b4985f0b785..fa09e6bb3ec1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -57,6 +57,10 @@ public class OMKeyAddAclRequest extends OMKeyAclRequest { private static final Logger LOG = LoggerFactory.getLogger(OMKeyAddAclRequest.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -70,10 +74,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeyAddAclRequest(OMRequest omRequest, OzoneManager ozoneManager) { super(omRequest); OzoneManagerProtocolProtos.AddAclRequest addAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequestWithFSO.java index 28a66eb55294..bc33b0a9de1f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequestWithFSO.java @@ -48,6 +48,10 @@ public class OMKeyAddAclRequestWithFSO extends OMKeyAclRequestWithFSO { private static final Logger LOG = LoggerFactory.getLogger(OMKeyAddAclRequestWithFSO.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OzoneManagerProtocolProtos.OMRequest preExecute( OzoneManager ozoneManager) throws IOException { @@ -60,10 +64,6 @@ public OzoneManagerProtocolProtos.OMRequest preExecute( .setUserInfo(getUserInfo()).build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeyAddAclRequestWithFSO( OzoneManagerProtocolProtos.OMRequest omReq, BucketLayout bucketLayout) { super(omReq, bucketLayout); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index be3c5facbbcf..b34086410464 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -57,6 +57,10 @@ public class OMKeyRemoveAclRequest extends OMKeyAclRequest { private static final Logger LOG = LoggerFactory.getLogger(OMKeyRemoveAclRequest.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -70,10 +74,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeyRemoveAclRequest(OMRequest omRequest, OzoneManager ozoneManager) { super(omRequest); OzoneManagerProtocolProtos.RemoveAclRequest removeAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequestWithFSO.java index 3744764e99ac..f216cf7d86f0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequestWithFSO.java @@ -48,6 +48,10 @@ public class OMKeyRemoveAclRequestWithFSO extends OMKeyAclRequestWithFSO { private static final Logger LOG = LoggerFactory.getLogger(OMKeyRemoveAclRequestWithFSO.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OzoneManagerProtocolProtos.OMRequest preExecute( OzoneManager ozoneManager) throws IOException { @@ -62,10 +66,6 @@ public OzoneManagerProtocolProtos.OMRequest preExecute( .build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeyRemoveAclRequestWithFSO( OzoneManagerProtocolProtos.OMRequest omRequest, BucketLayout bucketLayout) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index c9a304fd4de9..d713dd1dc714 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -58,6 +58,10 @@ public class OMKeySetAclRequest extends OMKeyAclRequest { private static final Logger LOG = LoggerFactory.getLogger(OMKeySetAclRequest.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -71,10 +75,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeySetAclRequest(OMRequest omRequest, OzoneManager ozoneManager) { super(omRequest); OzoneManagerProtocolProtos.SetAclRequest setAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequestWithFSO.java index 68ba967ca292..6cdefe8e4fe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequestWithFSO.java @@ -49,6 +49,10 @@ public class OMKeySetAclRequestWithFSO extends OMKeyAclRequestWithFSO { private static final Logger LOG = LoggerFactory.getLogger(OMKeySetAclRequestWithFSO.class); + private String path; + private List ozoneAcls; + private OzoneObj obj; + @Override public OzoneManagerProtocolProtos.OMRequest preExecute( OzoneManager ozoneManager) throws IOException { @@ -61,10 +65,6 @@ public OzoneManagerProtocolProtos.OMRequest preExecute( .setUserInfo(getUserInfo()).build(); } - private String path; - private List ozoneAcls; - private OzoneObj obj; - public OMKeySetAclRequestWithFSO( OzoneManagerProtocolProtos.OMRequest omReq, BucketLayout bucketLayout) { super(omReq, bucketLayout); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java index 91a6060b22f4..32a362e32d3f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java @@ -47,14 +47,6 @@ */ public abstract class OMVolumeAclRequest extends OMVolumeRequest { - /** - * Volume ACL operation. - */ - public interface VolumeAclOp extends - CheckedBiConsumer, OmVolumeArgs, IOException> { - // just a shortcut to avoid having to repeat long list of generic parameters - } - private final VolumeAclOp omVolumeAclOp; OMVolumeAclRequest(OzoneManagerProtocolProtos.OMRequest omRequest, @@ -200,4 +192,12 @@ abstract OMClientResponse onFailure(OMResponse.Builder omResponse, */ abstract void onComplete(Result result, Exception ex, long trxnLogIndex, AuditLogger auditLogger, Map auditMap); + + /** + * Volume ACL operation. + */ + public interface VolumeAclOp extends + CheckedBiConsumer, OmVolumeArgs, IOException> { + // just a shortcut to avoid having to repeat long list of generic parameters + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java index 88b07ff053bb..c3baca532097 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java @@ -50,6 +50,10 @@ public class OMVolumeAddAclRequest extends OMVolumeAclRequest { private static final VolumeAclOp VOLUME_ADD_ACL_OP = (acls, volArgs) -> volArgs.addAcl(acls.get(0)); + private final List ozoneAcls; + private final String volumeName; + private final OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -63,10 +67,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private final List ozoneAcls; - private final String volumeName; - private final OzoneObj obj; - public OMVolumeAddAclRequest(OMRequest omRequest) { super(omRequest, VOLUME_ADD_ACL_OP); OzoneManagerProtocolProtos.AddAclRequest addAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java index 05f470d56c50..6482770954b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java @@ -50,6 +50,10 @@ public class OMVolumeRemoveAclRequest extends OMVolumeAclRequest { private static final VolumeAclOp VOLUME_REMOVE_ACL_OP = (acls, volArgs) -> volArgs.removeAcl(acls.get(0)); + private final List ozoneAcls; + private final String volumeName; + private final OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -63,10 +67,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private final List ozoneAcls; - private final String volumeName; - private final OzoneObj obj; - public OMVolumeRemoveAclRequest(OMRequest omRequest) { super(omRequest, VOLUME_REMOVE_ACL_OP); OzoneManagerProtocolProtos.RemoveAclRequest removeAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java index 242c3e3ffa30..49031363cf84 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java @@ -50,6 +50,10 @@ public class OMVolumeSetAclRequest extends OMVolumeAclRequest { private static final VolumeAclOp VOLUME_SET_ACL_OP = (acls, volArgs) -> volArgs.setAcls(acls); + private final List ozoneAcls; + private final String volumeName; + private final OzoneObj obj; + @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { long modificationTime = Time.now(); @@ -63,10 +67,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .build(); } - private final List ozoneAcls; - private final String volumeName; - private final OzoneObj obj; - public OMVolumeSetAclRequest(OMRequest omRequest) { super(omRequest, VOLUME_SET_ACL_OP); OzoneManagerProtocolProtos.SetAclRequest setAclRequest = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java index ea25e37dcce2..693c0d6fc9d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java @@ -25,6 +25,10 @@ * Provider of {@link S3SecretCache}. */ public interface S3SecretCacheProvider { + /** + * In-memory cache implementation. + */ + S3SecretCacheProvider IN_MEMORY = conf -> new S3InMemoryCache(); /** * Returns S3 secret cache instance constructed by provided configuration. @@ -33,11 +37,4 @@ public interface S3SecretCacheProvider { * @return S3 secret cache instance. */ S3SecretCache get(Configuration conf); - - /** - * In-memory cache implementation. - */ - S3SecretCacheProvider IN_MEMORY = conf -> { - return new S3InMemoryCache(); - }; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OMRangerBGSyncService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OMRangerBGSyncService.java index 4bb672f39a8a..7c4609bb05bd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OMRangerBGSyncService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OMRangerBGSyncService.java @@ -100,85 +100,6 @@ public class OMRangerBGSyncService extends BackgroundService { private volatile boolean isServiceStarted = false; - static class BGRole { - private final String name; - private String id; - private final HashSet userSet; - - BGRole(String n) { - this.name = n; - userSet = new HashSet<>(); - } - - public void setId(String id) { - this.id = id; - } - - public String getId() { - return id; - } - - public void addUserPrincipal(String userPrincipal) { - userSet.add(userPrincipal); - } - - public HashSet getUserSet() { - return userSet; - } - - @Override - public int hashCode() { - return Objects.hash(name, id, userSet); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - BGRole bgRole = (BGRole) o; - return name.equals(bgRole.name) - && id.equals(bgRole.id) - && userSet.equals(bgRole.userSet); - } - } - - enum PolicyType { - BUCKET_NAMESPACE_POLICY, - BUCKET_POLICY - } - - /** - * Helper class that stores the tenant name and policy type. - */ - static class PolicyInfo { - - private final String tenantId; - private final PolicyType policyType; - - PolicyInfo(String tenantId, PolicyType policyType) { - this.tenantId = tenantId; - this.policyType = policyType; - } - - public String getTenantId() { - return tenantId; - } - - public PolicyType getPolicyType() { - return policyType; - } - - @Override - public String toString() { - return "PolicyInfo{" + - "tenantId='" + tenantId + '\'' + ", policyType=" + policyType + '}'; - } - } - // This map keeps all the policies found in OM DB. These policies should be // in Ranger. If not, the default policy will be (re)created. // @@ -868,4 +789,82 @@ private void pushOMDBRoleToRanger(String roleName) throws IOException { public long getRangerSyncRunCount() { return runCount.get(); } + + static class BGRole { + private final String name; + private String id; + private final HashSet userSet; + + BGRole(String n) { + this.name = n; + userSet = new HashSet<>(); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void addUserPrincipal(String userPrincipal) { + userSet.add(userPrincipal); + } + + public HashSet getUserSet() { + return userSet; + } + + @Override + public int hashCode() { + return Objects.hash(name, id, userSet); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BGRole bgRole = (BGRole) o; + return name.equals(bgRole.name) + && id.equals(bgRole.id) + && userSet.equals(bgRole.userSet); + } + } + + enum PolicyType { + BUCKET_NAMESPACE_POLICY, + BUCKET_POLICY + } + + /** + * Helper class that stores the tenant name and policy type. + */ + static class PolicyInfo { + + private final String tenantId; + private final PolicyType policyType; + + PolicyInfo(String tenantId, PolicyType policyType) { + this.tenantId = tenantId; + this.policyType = policyType; + } + + public String getTenantId() { + return tenantId; + } + + public PolicyType getPolicyType() { + return policyType; + } + + @Override + public String toString() { + return "PolicyInfo{tenantId='" + tenantId + '\'' + ", policyType=" + policyType + '}'; + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java index 1df5c8390224..4118894e7223 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java @@ -39,9 +39,6 @@ final class AWSV4AuthValidator { LoggerFactory.getLogger(AWSV4AuthValidator.class); private static final String HMAC_SHA256_ALGORITHM = "HmacSHA256"; - private AWSV4AuthValidator() { - } - /** * ThreadLocal cache of Mac instances. */ @@ -56,6 +53,9 @@ private AWSV4AuthValidator() { } }); + private AWSV4AuthValidator() { + } + public static String hash(String payload) throws NoSuchAlgorithmException { MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(payload.getBytes(StandardCharsets.UTF_8));