From e6613248479b9c2f3f309695693ebeb9337e3e41 Mon Sep 17 00:00:00 2001 From: xiaoxmeng Date: Sat, 20 Apr 2024 22:44:32 -0700 Subject: [PATCH] [native] Remove old legecy code of temp directory --- .../main/operators/tests/BroadcastTest.cpp | 16 +++++----- .../operators/tests/UnsafeRowShuffleTest.cpp | 4 +-- .../presto_cpp/main/tests/TaskManagerTest.cpp | 30 ++++++++++--------- .../main/types/tests/PlanConverterTest.cpp | 2 +- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp index ede4762a81099..dcd22da8d6456 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp @@ -164,13 +164,13 @@ class BroadcastTest : public exec::test::OperatorTestBase { auto tempDirectoryPath = exec::test::TempDirectoryPath::create(); auto [serdeRowType, broadcastFilePaths] = - executeBroadcastWrite(data, tempDirectoryPath->path, serdeLayout); + executeBroadcastWrite(data, tempDirectoryPath->getPath(), serdeLayout); // Expect one file for each request. ASSERT_EQ(broadcastFilePaths.size(), 1); // Validate file path prefix is consistent. - ASSERT_EQ(broadcastFilePaths.back().find(tempDirectoryPath->path), 0); + ASSERT_EQ(broadcastFilePaths.back().find(tempDirectoryPath->getPath()), 0); // Read back broadcast data from broadcast file. auto result = readFromFile(broadcastFilePaths.back(), serdeRowType); @@ -184,7 +184,7 @@ class BroadcastTest : public exec::test::OperatorTestBase { // Read back result. auto [broadcastReadCursor, broadcastReadResults] = executeBroadcastRead( - serdeRowType, tempDirectoryPath->path, broadcastFilePaths); + serdeRowType, tempDirectoryPath->getPath(), broadcastFilePaths); // Assert its same as data. velox::exec::test::assertEqualResults(expected, broadcastReadResults); @@ -268,14 +268,14 @@ TEST_F(BroadcastTest, endToEndWithNoRows) { std::vector broadcastFilePaths; // Execute write. - auto results = executeBroadcastWrite({data}, tempDirectoryPath->path); + auto results = executeBroadcastWrite({data}, tempDirectoryPath->getPath()); // Assert no file path returned. ASSERT_EQ(broadcastFilePaths.size(), 0); auto fileSystem = - velox::filesystems::getFileSystem(tempDirectoryPath->path, nullptr); - auto files = fileSystem->list(tempDirectoryPath->path); + velox::filesystems::getFileSystem(tempDirectoryPath->getPath(), nullptr); + auto files = fileSystem->list(tempDirectoryPath->getPath()); // Assert no file was generated in broadcast directory path. ASSERT_EQ(files.size(), 0); @@ -297,14 +297,14 @@ TEST_F(BroadcastTest, endToEndWithMultipleWriteNodes) { // Execute write. for (auto data : dataVector) { auto [serdeRowType, results] = - executeBroadcastWrite({data}, tempDirectoryPath->path); + executeBroadcastWrite({data}, tempDirectoryPath->getPath()); broadcastFilePaths.emplace_back(results[0]); } // Read back result. auto [taskCursorReadNode, broadcastReadResults] = executeBroadcastRead( asRowType(dataVector[0]->type()), - tempDirectoryPath->path, + tempDirectoryPath->getPath(), broadcastFilePaths); // Validate BroadcastExchange reads back output of both writes. diff --git a/presto-native-execution/presto_cpp/main/operators/tests/UnsafeRowShuffleTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/UnsafeRowShuffleTest.cpp index 5a78e9909e1f1..b6b1e7c7da595 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/UnsafeRowShuffleTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/UnsafeRowShuffleTest.cpp @@ -636,7 +636,7 @@ class UnsafeRowShuffleTest : public exec::test::OperatorTestBase { // Create a local file system storage based shuffle. velox::filesystems::registerLocalFileSystem(); auto rootDirectory = velox::exec::test::TempDirectoryPath::create(); - auto rootPath = rootDirectory->path; + auto rootPath = rootDirectory->getPath(); const std::string shuffleWriteInfo = localShuffleWriteInfo(rootPath, numPartitions); @@ -979,7 +979,7 @@ TEST_F(UnsafeRowShuffleTest, persistentShuffle) { // Create a local file system storage based shuffle. velox::filesystems::registerLocalFileSystem(); auto rootDirectory = velox::exec::test::TempDirectoryPath::create(); - auto rootPath = rootDirectory->path; + auto rootPath = rootDirectory->getPath(); auto data = makeRowVector({ makeFlatVector({1, 2, 3, 4, 5, 6}), diff --git a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp index 92cd67de070e4..b2141b09d2696 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp @@ -398,7 +398,8 @@ class TaskManagerTest : public testing::Test { protocol::TaskSource source; source.planNodeId = sourceId; for (auto& filePath : filePaths) { - source.splits.emplace_back(makeSplit(filePath->path, splitSequenceId++)); + source.splits.emplace_back( + makeSplit(filePath->getPath(), splitSequenceId++)); } source.noMoreSplits = noMoreSplits; return source; @@ -571,15 +572,16 @@ class TaskManagerTest : public testing::Test { static std::shared_ptr setupSpillPath() { auto spillDirectory = exec::test::TempDirectoryPath::create(); auto sysConfigFilePath = - fmt::format("{}/config.properties", spillDirectory->path); + fmt::format("{}/config.properties", spillDirectory->getPath()); auto fileSystem = filesystems::getFileSystem(sysConfigFilePath, nullptr); auto sysConfigFile = fileSystem->openFileForWrite(sysConfigFilePath); SystemConfig::instance()->setValue( - std::string(SystemConfig::kSpillerSpillPath), spillDirectory->path); + std::string(SystemConfig::kSpillerSpillPath), + spillDirectory->getPath()); sysConfigFile->close(); auto nodeConfigFilePath = - fmt::format("{}/node.properties", spillDirectory->path); + fmt::format("{}/node.properties", spillDirectory->getPath()); auto nodeConfigFile = fileSystem->openFileForWrite(nodeConfigFilePath); nodeConfigFile->append(fmt::format( "{}={}\n{}={}", @@ -647,7 +649,7 @@ TEST_F(TaskManagerTest, tableScanAllSplitsAtOnce) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -673,7 +675,7 @@ TEST_F(TaskManagerTest, fecthFromFinishedTask) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -790,7 +792,7 @@ TEST_F(TaskManagerTest, taskCleanupWithPendingResultData) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } auto planFragment = exec::test::PlanBuilder() @@ -845,7 +847,7 @@ TEST_F(TaskManagerTest, tableScanOneSplitAtATime) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -879,7 +881,7 @@ TEST_F(TaskManagerTest, tableScanMultipleTasks) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -954,7 +956,7 @@ TEST_F(TaskManagerTest, countAggregation) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -967,7 +969,7 @@ TEST_F(TaskManagerTest, distributedSort) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (int i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -1039,7 +1041,7 @@ TEST_F(TaskManagerTest, outOfQueryUserMemory) { auto filePaths = makeFilePaths(5); auto vectors = makeVectors(filePaths.size(), 1'000); for (auto i = 0; i < filePaths.size(); i++) { - writeToFile(filePaths[i]->path, vectors[i]); + writeToFile(filePaths[i]->getPath(), vectors[i]); } duckDbQueryRunner_.createTable("tmp", vectors); @@ -1161,7 +1163,7 @@ TEST_F(TaskManagerTest, aggregationSpill) { } std::vector vectors; for (int i = 0; i < filePaths.size(); ++i) { - writeToFile(filePaths[i]->path, batches[i]); + writeToFile(filePaths[i]->getPath(), batches[i]); std::move( batches[i].begin(), batches[i].end(), std::back_inserter(vectors)); } @@ -1177,7 +1179,7 @@ TEST_F(TaskManagerTest, aggregationSpill) { int32_t spillPct{0}; if (doSpill) { spillDirectory = TaskManagerTest::setupSpillPath(); - taskManager_->setBaseSpillDirectory(spillDirectory->path); + taskManager_->setBaseSpillDirectory(spillDirectory->getPath()); spillPct = 100; queryConfigs.emplace(core::QueryConfig::kSpillEnabled, "true"); queryConfigs.emplace(core::QueryConfig::kAggregationSpillEnabled, "true"); diff --git a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp index 5329a799ba75d..ccf5cacc852e5 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp @@ -185,7 +185,7 @@ TEST_F(PlanConverterTest, batchPlanConversion) { " \"rootPath\": \"{}\",\n" " \"numPartitions\": {}\n" "}}", - exec::test::TempDirectoryPath::create()->path, + exec::test::TempDirectoryPath::create()->getPath(), 10)), std::make_shared("/tmp"));