Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@
import io.trino.spi.PageSorter;
import io.trino.spi.Plugin;
import io.trino.spi.connector.CatalogHandle;
import io.trino.spi.connector.Connector;
import io.trino.spi.connector.ConnectorFactory;
import io.trino.spi.exchange.ExchangeManager;
import io.trino.spi.session.PropertyMetadata;
Expand Down Expand Up @@ -786,6 +787,13 @@ public CatalogManager getCatalogManager()
return catalogManager;
}

public Connector getConnector(String catalogName)
{
return catalogManager
.getConnectorServices(getCatalogHandle(catalogName))
.getConnector();
}

public LocalQueryRunner printPlan()
{
printPlan = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static io.trino.filesystem.azure.AzureUtils.handleAzureException;
import static io.trino.filesystem.azure.AzureUtils.isFileNotFoundException;
import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
import static java.util.UUID.randomUUID;
Expand Down Expand Up @@ -123,6 +124,9 @@ public void deleteFile(Location location)
client.delete();
}
catch (RuntimeException e) {
if (isFileNotFoundException(e)) {
return;
}
throw handleAzureException(e, "deleting file", azureLocation);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,26 @@ private AzureUtils() {}
public static IOException handleAzureException(RuntimeException exception, String action, AzureLocation location)
throws IOException
{
if (exception instanceof BlobStorageException blobStorageException) {
if (BlobErrorCode.BLOB_NOT_FOUND.equals(blobStorageException.getErrorCode())) {
throw withCause(new FileNotFoundException(location.toString()), exception);
}
}
if (exception instanceof DataLakeStorageException dataLakeStorageException) {
if ("PathNotFound".equals(dataLakeStorageException.getErrorCode())) {
throw withCause(new FileNotFoundException(location.toString()), exception);
}
if (isFileNotFoundException(exception)) {
throw withCause(new FileNotFoundException(location.toString()), exception);
}
if (exception instanceof AzureException) {
throw new IOException("Azure service error %s file: %s".formatted(action, location), exception);
}
throw new IOException("Error %s file: %s".formatted(action, location), exception);
}

public static boolean isFileNotFoundException(RuntimeException exception)
{
if (exception instanceof BlobStorageException blobStorageException) {
return BlobErrorCode.BLOB_NOT_FOUND.equals(blobStorageException.getErrorCode());
}
if (exception instanceof DataLakeStorageException dataLakeStorageException) {
return "PathNotFound" .equals(dataLakeStorageException.getErrorCode());
}
return false;
}

private static <T extends Throwable> T withCause(T throwable, Throwable cause)
{
throwable.initCause(cause);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import static com.google.cloud.storage.Storage.BlobListOption.pageSize;
import static com.google.common.collect.Iterables.partition;
import static io.airlift.concurrent.MoreFutures.getFutureValue;
import static io.trino.filesystem.gcs.GcsUtils.getBlobOrThrow;
import static io.trino.filesystem.gcs.GcsUtils.getBlob;
import static io.trino.filesystem.gcs.GcsUtils.handleGcsException;
import static java.util.Objects.requireNonNull;

Expand Down Expand Up @@ -99,8 +99,7 @@ public void deleteFile(Location location)
{
GcsLocation gcsLocation = new GcsLocation(location);
checkIsValidFile(gcsLocation);
Blob blob = getBlobOrThrow(storage, gcsLocation);
blob.delete();
getBlob(storage, gcsLocation).ifPresent(Blob::delete);
}

@Override
Expand All @@ -112,7 +111,8 @@ public void deleteFiles(Collection<Location> locations)
for (List<Location> locationBatch : partition(locations, batchSize)) {
StorageBatch batch = storage.batch();
for (Location location : locationBatch) {
batch.delete(getBlobOrThrow(storage, new GcsLocation(location)).getBlobId());
getBlob(storage, new GcsLocation(location))
.ifPresent(blob -> batch.delete(blob.getBlobId()));
}
batchFutures.add(executorService.submit(batch::submit));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,6 @@ protected final boolean supportsRenameFile()
return false;
}

@Override
protected final boolean deleteFileFailsIfNotExists()
{
return false;
}

@Override
protected final void verifyFileSystemIsEmpty()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,10 @@ public interface TrinoFileSystem

/**
* Deletes the specified file. The file location path cannot be empty, and must not end with
* a slash or whitespace. If the file is a director, an exception is raised.
* a slash or whitespace. If the file is a director, an exception is raised. If the file does
* not exist, this method is a noop.
*
* @throws IllegalArgumentException if location is not valid for this file system
* @throws IOException if the file does not exist (optional) or was not deleted
*/
void deleteFile(Location location)
throws IOException;
Expand All @@ -90,9 +90,9 @@ void deleteFile(Location location)
* Delete specified files. This operation is <b>not</b> required to be atomic, so if an error
* occurs, all, some, or, none of the files may be deleted. This operation may be faster than simply
* looping over the locations as some file systems support batch delete operations natively.
* If a file does not exist, it is ignored.
*
* @throws IllegalArgumentException if location is not valid for this file system
* @throws IOException if a file does not exist (optional) or was not deleted
*/
default void deleteFiles(Collection<Location> locations)
throws IOException
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
Expand All @@ -34,6 +35,7 @@
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static io.trino.filesystem.local.LocalUtils.handleException;
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import static java.util.UUID.randomUUID;

/**
* A hierarchical file system for testing.
Expand Down Expand Up @@ -75,6 +77,8 @@ public void deleteFile(Location location)
try {
Files.delete(filePath);
}
catch (NoSuchFileException ignored) {
}
catch (IOException e) {
throw handleException(location, e);
}
Expand Down Expand Up @@ -223,7 +227,23 @@ public Set<Location> listDirectories(Location location)
public Optional<Location> createTemporaryDirectory(Location targetPath, String temporaryPrefix, String relativePrefix)
throws IOException
{
throw new IOException("Local file system does not support creating temporary directories");
// allow for absolute or relative temporary prefix
Location temporary;
if (temporaryPrefix.startsWith("/")) {
String prefix = temporaryPrefix;
while (prefix.startsWith("/")) {
prefix = prefix.substring(1);
}
temporary = Location.of("local:///").appendPath(prefix);
}
else {
temporary = targetPath.appendPath(temporaryPrefix);
}

temporary = temporary.appendPath(randomUUID().toString());

createDirectory(temporary);
return Optional.of(temporary);
}

private Path toFilePath(Location location)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import io.trino.filesystem.TrinoOutputFile;
import io.trino.filesystem.memory.MemoryOutputFile.OutputBlob;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.util.Iterator;
Expand Down Expand Up @@ -96,9 +95,7 @@ public void overwriteBlob(Slice data)
public void deleteFile(Location location)
throws IOException
{
if (blobs.remove(toBlobKey(location)) == null) {
throw new FileNotFoundException(location.toString());
}
blobs.remove(toBlobKey(location));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,6 @@ protected boolean supportsRenameFile()
return true;
}

protected boolean deleteFileFailsIfNotExists()
{
return true;
}

protected boolean normalizesListFilesResult()
{
return false;
Expand Down Expand Up @@ -671,16 +666,8 @@ void testDeleteFile()
.hasMessageContaining(createLocation("foo/").toString());

try (TempBlob tempBlob = randomBlobLocation("delete")) {
if (deleteFileFailsIfNotExists()) {
// deleting a non-existent file is an error
assertThatThrownBy(() -> getFileSystem().deleteFile(tempBlob.location()))
.isInstanceOf(FileNotFoundException.class)
.hasMessageContaining(tempBlob.location().toString());
}
else {
// deleting a non-existent file is a no-op
getFileSystem().deleteFile(tempBlob.location());
}
// deleting a non-existent file is a no-op
getFileSystem().deleteFile(tempBlob.location());

tempBlob.createOrOverwrite("delete me");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,8 @@ public void deleteFile(Location location)
}
return null;
}
catch (FileNotFoundException e) {
stats.getDeleteFileCalls().recordException(e);
throw new FileNotFoundException(location.toString());
catch (FileNotFoundException ignored) {
return null;
}
catch (IOException e) {
stats.getDeleteFileCalls().recordException(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,6 @@ protected final boolean supportsCreateExclusive()
return false;
}

@Override
protected final boolean deleteFileFailsIfNotExists()
{
return false;
}

@Override
protected boolean normalizesListFilesResult()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,12 +188,13 @@ protected QueryRunner createQueryRunner()
registerTableFromResources(table.tableName(), table.resourcePath(), queryRunner);
});

queryRunner.installPlugin(new TestingHivePlugin());
queryRunner.installPlugin(new TestingHivePlugin(queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data")));

queryRunner.createCatalog(
"hive",
"hive",
ImmutableMap.<String, String>builder()
.put("hive.metastore", "thrift")
.put("hive.metastore.uri", "thrift://" + hiveHadoop.getHiveMetastoreEndpoint())
.put("hive.allow-drop-table", "true")
.putAll(hiveStorageConfiguration())
Expand Down
Loading