Skip to content

HDDS-8253. Set ozone.metadata.dirs to temporary dir if not defined in S3 Gateway #4455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 10 commits into from
Apr 18, 2023
Merged

HDDS-8253. Set ozone.metadata.dirs to temporary dir if not defined in S3 Gateway #4455

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e2a7d30
HDDS-8253: Set ozone.metadata.dirs if not defined to tmp dir for s3Ga…
swamirishi Mar 23, 2023
cea29f3
HDDS-8253: Default Metadir in Current Working Directory
swamirishi Mar 24, 2023
99273ee
HDDS-8253: Set Basedir instead of Metadata dir
swamirishi Mar 29, 2023
669e89b
HDDS-8253: Set Basedir instead of Metadata dir
swamirishi Mar 29, 2023
d9da3b1
HDDS-8253. Use shutdown hook manager
swamirishi Apr 17, 2023
eda47dc
HDDS-8253. Fix checkstyle issue
swamirishi Apr 17, 2023
7429fcd
HDDS-8253. Fix checkstyle issue
swamirishi Apr 17, 2023
df38baf
HDDS-8253. Address review comments to move tmpMetaDatadir
swamirishi Apr 17, 2023
d992797
HDDS-8253. Fix comments
swamirishi Apr 17, 2023
71694af
HDDS-8253. Fix checkstyle issue
swamirishi Apr 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,19 @@
*/
package org.apache.hadoop.ozone.s3;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;

import org.apache.hadoop.hdds.StringUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
Expand All @@ -38,6 +43,7 @@
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;

import static org.apache.hadoop.hdds.StringUtils.startupShutdownMessage;
import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_KERBEROS_PRINCIPAL_KEY;
Expand All @@ -63,13 +69,35 @@ public static void main(String[] args) throws Exception {
new Gateway().run(args);
}

private void setHttpBaseDir()
throws IOException {
if (StringUtils.isEmpty(ozoneConfiguration.get(
OzoneConfigKeys.OZONE_HTTP_BASEDIR))) {
//Setting ozone.http.basedir to cwd if not set so that server setup
// doesn't fail.
File tmpMetaDir = Files.createTempDirectory(Paths.get(""),
"ozone_s3g_tmp_base_dir").toFile();
ShutdownHookManager.get().addShutdownHook(() -> {
try {
FileUtils.deleteDirectory(tmpMetaDir);
} catch (IOException e) {
LOG.error("Failed to cleanup temporary S3 Gateway Metadir {}",
tmpMetaDir.getAbsolutePath(), e);
}
}, 0);
ozoneConfiguration.set(OzoneConfigKeys.OZONE_HTTP_BASEDIR,
tmpMetaDir.getAbsolutePath());
}
}

@Override
public Void call() throws Exception {
ozoneConfiguration = createOzoneConfiguration();
TracingUtil.initTracing("S3gateway", ozoneConfiguration);
OzoneConfigurationHolder.setConfiguration(ozoneConfiguration);
UserGroupInformation.setConfiguration(ozoneConfiguration);
loginS3GUser(ozoneConfiguration);
setHttpBaseDir();
httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
metrics = S3GatewayMetrics.create();
start();
Expand All @@ -87,7 +115,7 @@ public Void call() throws Exception {
public void start() throws IOException {
String[] originalArgs = getCmd().getParseResult().originalArgs()
.toArray(new String[0]);
StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO,
startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO,
Gateway.class, originalArgs, LOG, ozoneConfiguration);

LOG.info("Starting Ozone S3 gateway");
Expand Down