diff --git a/velox-connector/src/connector/tests/ClpConfigTest.cpp b/velox-connector/src/connector/tests/ClpConfigTest.cpp deleted file mode 100644 index d9b6c61..0000000 --- a/velox-connector/src/connector/tests/ClpConfigTest.cpp +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "gtest/gtest.h" -#include "velox/common/base/tests/GTestUtils.h" -#include "velox/common/config/Config.h" -#include "connector/ClpConfig.h" -#include "connector/search_lib/ClpPackageS3AuthProvider.h" - -namespace facebook::velox::connector::clp { - -namespace { - -class ClpEnvironmentVariableGuard { - public: - void saveOriginalEnvironmentVariables() { - originalEnv_.clear(); - originalKeys_.clear(); - -#if defined(_WIN32) - LPCH envStrings = GetEnvironmentStringsA(); - if (!envStrings) - return; - - LPCH var = envStrings; - while (*var) { - std::string entry(var); - auto pos = entry.find('='); - if (pos != std::string::npos) { - std::string key = entry.substr(0, pos); - std::string value = entry.substr(pos + 1); - originalEnv_[key] = value; - originalKeys_.insert(key); - } - var += entry.size() + 1; - } - FreeEnvironmentStringsA(envStrings); -#elif defined(__unix__) || defined(__APPLE__) - for (char** current = environ; *current; ++current) { - std::string entry(*current); - auto pos = entry.find('='); - if (pos != std::string::npos) { - std::string key = entry.substr(0, pos); - std::string value = entry.substr(pos + 1); - originalEnv_[key] = value; - originalKeys_.insert(key); - } - } -#else - VELOX_UNSUPPORTED("Unsupported OS"); -#endif - } - - void restoreEnvironmentVariables() { -#if defined(_WIN32) - // Remove added vars and restore originals - for (auto const& [key, value] : originalEnv_) { - _putenv_s(key.c_str(), value.c_str()); // restore original - } - - // Collect variables to unset - std::vector keysToUnset; - for (char** env = _environ; *env; ++env) { - std::string entry(*env); - auto pos = entry.find('='); - if (pos != std::string::npos) { - std::string key = entry.substr(0, pos); - if (originalKeys_.find(key) == originalKeys_.end()) { - keysToUnset.push_back(key); - } - } - } - - // Unset them - for (const auto& key : keysToUnset) { - _putenv((key + "=").c_str()); - } -#elif defined(__unix__) || defined(__APPLE__) - // Restore original variables - for (auto const& [key, value] : originalEnv_) { - setenv(key.c_str(), value.c_str(), 1); - } - - // Collect keys to unset - std::vector keysToUnset; - for (char** env = environ; *env; ++env) { - std::string entry(*env); - auto pos = entry.find('='); - if (pos != std::string::npos) { - std::string key = entry.substr(0, pos); - if (originalKeys_.find(key) == originalKeys_.end()) { - keysToUnset.push_back(key); - } - } - } - - // Unset them - for (const auto& key : keysToUnset) { - unsetenv(key.c_str()); - } -#else - VELOX_UNSUPPORTED("Unsupported OS"); -#endif - } - - private: - std::map originalEnv_; - std::set originalKeys_; -}; - -class ClpConfigTest : public testing::Test { - public: - explicit ClpConfigTest() - : envVarGuard_(std::make_unique()) {} - - void SetUp() override { - envVarGuard_->saveOriginalEnvironmentVariables(); - } - - void TearDown() override { - envVarGuard_->restoreEnvironmentVariables(); - } - - std::unique_ptr buildClpConfig( - std::unordered_map configMap) { - auto config = - std::make_shared(std::move(configMap)); - return std::make_unique(config); - } - - private: - std::unique_ptr envVarGuard_; -}; - -class ClpS3AuthProviderBaseTest : public ClpConfigTest { - public: - /// Checks whether an environment variable matches a given value. - /// - /// @param key The name of the environment variable to check. - /// @param expectedValue Optional expected value to compare against. If - /// std::nullopt, the function returns true if the variable is not - /// defined. - /// @return True if: - /// - expectedValue is std::nullopt and the variable is not defined, - /// or - /// - expectedValue is set and matches the variable's current value. - bool checkEnvironmentVariableEquals( - std::string_view key, - std::optional expectedValue) { - const char* actualValue = std::getenv(std::string(key).c_str()); - if (nullptr == actualValue) { - return !expectedValue.has_value(); - } - return expectedValue.has_value() && - std::string_view(actualValue) == expectedValue.value(); - } -}; - -class ClpPackageS3AuthProviderTest : public ClpS3AuthProviderBaseTest { - public: - std::unique_ptr buildClpPackageS3AuthProvider( - std::unordered_map configMap) { - auto config = - std::make_shared(std::move(configMap)); - return std::make_unique(config); - } -}; - -} // namespace - -TEST_F(ClpConfigTest, invalidAuthProvider) { - const std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "dummy-provider"}}); - VELOX_ASSERT_UNSUPPORTED_THROW( - buildClpConfig(configMap), - "Unsupported s3 auth provider type: dummy-provider."); -} - -TEST_F(ClpS3AuthProviderBaseTest, caseInsensitiveAuthProvider) { - const std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "ClP_PaCkAgE"}, - {ClpPackageS3AuthProvider::kAccessKeyId, "aaaaaa"}, - {ClpPackageS3AuthProvider::kEndPoint, "http://aaaaaa"}, - {ClpPackageS3AuthProvider::kSecretAccessKey, "cccccc"}}); - VELOX_CHECK_NOT_NULL(buildClpConfig(configMap)); -} - -TEST_F(ClpPackageS3AuthProviderTest, readAndExportAwsAuthEnvironmentVariables) { - const std::string cTestAccessKeyId{"aaaaaa"}; - const std::string cTestBucket{"test-bucket"}; - const std::string cTestEndPoint{"http://localhost:9000"}; - const std::string cTestSecretAccessKey{"bbbbbb"}; - const std::string cTestSessionToken{"cccccc"}; - - // Test all properties - std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "clp_package"}, - {ClpPackageS3AuthProvider::kAccessKeyId, cTestAccessKeyId}, - {ClpPackageS3AuthProvider::kBucket, cTestBucket}, - {ClpPackageS3AuthProvider::kEndPoint, cTestEndPoint}, - {ClpPackageS3AuthProvider::kSecretAccessKey, cTestSecretAccessKey}, - {ClpPackageS3AuthProvider::kSessionToken, cTestSessionToken}}); - auto clpPackageS3AuthProvider = buildClpPackageS3AuthProvider(configMap); - VELOX_CHECK(clpPackageS3AuthProvider->exportAuthEnvironmentVariables()); - VELOX_CHECK(checkEnvironmentVariableEquals( - ClpPackageS3AuthProvider::kEnvAwsAccessKeyId, cTestAccessKeyId)); - VELOX_CHECK(checkEnvironmentVariableEquals( - ClpPackageS3AuthProvider::kEnvAwsSecretAccessKey, cTestSecretAccessKey)); - VELOX_CHECK(checkEnvironmentVariableEquals( - ClpPackageS3AuthProvider::kEnvAwsSessionToken, cTestSessionToken)); - - // Test auth without the session token - configMap = { - {"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "clp_package"}, - {ClpPackageS3AuthProvider::kAccessKeyId, cTestAccessKeyId}, - {ClpPackageS3AuthProvider::kEndPoint, cTestEndPoint}, - {ClpPackageS3AuthProvider::kSecretAccessKey, cTestSecretAccessKey}}; - clpPackageS3AuthProvider = buildClpPackageS3AuthProvider(configMap); - VELOX_CHECK(clpPackageS3AuthProvider->exportAuthEnvironmentVariables()); - VELOX_CHECK(checkEnvironmentVariableEquals( - ClpPackageS3AuthProvider::kEnvAwsSessionToken, std::nullopt)); -} - -// Tests URL construction for S3-compatible storage (e.g., MinIO) using -// path-style URLs with bucket. -TEST_F(ClpPackageS3AuthProviderTest, constructS3UrlForPathStyleWithBucket) { - const std::string cTestAccessKeyId{"aaaaaa"}; - const std::string cTestBucket{"logs"}; - const std::string cTestEndPoint{"http://172.26.105.44:9000"}; - const std::string cTestSecretAccessKey{"bbbbbb"}; - const std::string cTestSplitPath{"archives/default/abc123"}; - - std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "clp_package"}, - {ClpPackageS3AuthProvider::kAccessKeyId, cTestAccessKeyId}, - {ClpPackageS3AuthProvider::kBucket, cTestBucket}, - {ClpPackageS3AuthProvider::kEndPoint, cTestEndPoint}, - {ClpPackageS3AuthProvider::kSecretAccessKey, cTestSecretAccessKey}}); - auto clpPackageS3AuthProvider = buildClpPackageS3AuthProvider(configMap); - VELOX_CHECK(clpPackageS3AuthProvider->exportAuthEnvironmentVariables()); - - auto url = clpPackageS3AuthProvider->constructS3Url(cTestSplitPath); - VELOX_CHECK_EQ(url, "http://172.26.105.44:9000/logs/archives/default/abc123"); -} - -// Tests URL construction for AWS S3 using virtual-hosted style URLs without -// bucket config. -TEST_F(ClpPackageS3AuthProviderTest, constructS3UrlForAwsVirtualHostedStyle) { - const std::string cTestAccessKeyId{"aaaaaa"}; - const std::string cTestEndPoint{"https://logs.s3.us-east-1.amazonaws.com"}; - const std::string cTestSecretAccessKey{"bbbbbb"}; - const std::string cTestSplitPath{"archives/default/abc123"}; - - std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "clp_package"}, - {ClpPackageS3AuthProvider::kAccessKeyId, cTestAccessKeyId}, - {ClpPackageS3AuthProvider::kEndPoint, cTestEndPoint}, - {ClpPackageS3AuthProvider::kSecretAccessKey, cTestSecretAccessKey}}); - auto clpPackageS3AuthProvider = buildClpPackageS3AuthProvider(configMap); - VELOX_CHECK(clpPackageS3AuthProvider->exportAuthEnvironmentVariables()); - - auto url = clpPackageS3AuthProvider->constructS3Url(cTestSplitPath); - VELOX_CHECK_EQ( - url, "https://logs.s3.us-east-1.amazonaws.com/archives/default/abc123"); -} - -// Tests URL construction for AWS S3 using path-style URLs with bucket config. -TEST_F(ClpPackageS3AuthProviderTest, constructS3UrlForAwsPathStyleWithBucket) { - const std::string cTestAccessKeyId{"aaaaaa"}; - const std::string cTestBucket{"logs"}; - const std::string cTestEndPoint{"https://s3.us-east-1.amazonaws.com"}; - const std::string cTestSecretAccessKey{"bbbbbb"}; - const std::string cTestSplitPath{"archives/default/abc123"}; - - std::unordered_map configMap( - {{"clp.storage-type", "s3"}, - {ClpConfig::kAuthProvider, "clp_package"}, - {ClpPackageS3AuthProvider::kAccessKeyId, cTestAccessKeyId}, - {ClpPackageS3AuthProvider::kBucket, cTestBucket}, - {ClpPackageS3AuthProvider::kEndPoint, cTestEndPoint}, - {ClpPackageS3AuthProvider::kSecretAccessKey, cTestSecretAccessKey}}); - auto clpPackageS3AuthProvider = buildClpPackageS3AuthProvider(configMap); - VELOX_CHECK(clpPackageS3AuthProvider->exportAuthEnvironmentVariables()); - - auto url = clpPackageS3AuthProvider->constructS3Url(cTestSplitPath); - VELOX_CHECK_EQ( - url, "https://s3.us-east-1.amazonaws.com/logs/archives/default/abc123"); -} - -} // namespace facebook::velox::connector::clp diff --git a/velox-connector/src/connector/tests/ClpConnectorTest.cpp b/velox-connector/src/connector/tests/ClpConnectorTest.cpp deleted file mode 100644 index 26299d8..0000000 --- a/velox-connector/src/connector/tests/ClpConnectorTest.cpp +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "velox/common/base/Fs.h" -#include "connector/ClpColumnHandle.h" -#include "connector/ClpConnector.h" -#include "connector/ClpConnectorSplit.h" -#include "connector/ClpTableHandle.h" -#include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/OperatorTestBase.h" -#include "velox/exec/tests/utils/PlanBuilder.h" -#include "velox/type/Timestamp.h" -#include "velox/type/Type.h" - -namespace { - -using namespace facebook::velox; -using namespace facebook::velox::connector::clp; - -using facebook::velox::exec::test::PlanBuilder; - -// Epoch seconds and nanoseconds for the timestamp "2025-04-30T08:50:05Z" -constexpr int64_t kTestTimestampSeconds{1746003005}; -constexpr uint64_t kTestTimestampNanoseconds{0ULL}; - -class ClpConnectorTest : public exec::test::OperatorTestBase { - public: - const std::string kClpConnectorId = "test-clp"; - - void SetUp() override { - OperatorTestBase::SetUp(); - connector::clp::ClpConnectorFactory factory; - auto clpConnector = factory.newConnector( - kClpConnectorId, - std::make_shared( - std::unordered_map{}), - nullptr, - nullptr); - connector::registerConnector(clpConnector); - } - - void TearDown() override { - connector::unregisterConnector(kClpConnectorId); - OperatorTestBase::TearDown(); - } - - exec::Split makeClpSplit( - const std::string& splitPath, - ClpConnectorSplit::SplitType type, - std::shared_ptr kqlQuery) { - return exec::Split( - std::make_shared( - kClpConnectorId, splitPath, static_cast(type), kqlQuery)); - } - - RowVectorPtr getResults( - const core::PlanNodePtr& planNode, - std::vector&& splits) { - return exec::test::AssertQueryBuilder(planNode) - .splits(std::move(splits)) - .copyResults(pool()); - } - - static std::string getExampleFilePath(const std::string& filePath) { - std::string current_path = fs::current_path().string(); - return current_path + "/examples/" + filePath; - } -}; - -TEST_F(ClpConnectorTest, test1NoPushdown) { - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder() - .startTableScan() - .outputType( - ROW({"requestId", "userId", "method"}, - {VARCHAR(), VARCHAR(), VARCHAR()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_1")) - .assignments({ - {"requestId", - std::make_shared( - "requestId", "requestId", VARCHAR())}, - {"userId", - std::make_shared( - "userId", "userId", VARCHAR())}, - {"method", - std::make_shared( - "method", "method", VARCHAR())}, - }) - .endTableScan() - .filter("method = 'GET'") - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// requestId - makeFlatVector( - {"req-100", "req-105", "req-107", "req-109", "req-102"}), - // userId - makeNullableFlatVector( - {"user201", "user204", "user202", "user203", std::nullopt}), - // method - makeFlatVector({ - "GET", - "GET", - "GET", - "GET", - "GET", - })}); - test::assertEqualVectors(expected, output); - - // The IR stream will be deserialized in order, so the expected vector is - // different - auto irExpected = makeRowVector( - {// requestId - makeFlatVector( - {"req-100", "req-102", "req-105", "req-107", "req-109"}), - // userId - makeNullableFlatVector( - {"user201", std::nullopt, "user204", "user202", "user203"}), - // method - makeFlatVector({ - "GET", - "GET", - "GET", - "GET", - "GET", - })}); - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - test::assertEqualVectors(irExpected, irOutput); -} - -TEST_F(ClpConnectorTest, test1Pushdown) { - auto kqlQuery = - std::make_shared("method: \"POST\" AND status: 200"); - auto plan = - PlanBuilder() - .startTableScan() - .outputType( - ROW({"requestId", "userId", "path"}, - {VARCHAR(), VARCHAR(), VARCHAR()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_1")) - .assignments({ - {"requestId", - std::make_shared( - "requestId", "requestId", VARCHAR())}, - {"userId", - std::make_shared( - "userId", "userId", VARCHAR())}, - {"path", - std::make_shared("path", "path", VARCHAR())}, - }) - .endTableScan() - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// requestId - makeFlatVector({"req-106"}), - // userId - makeNullableFlatVector({std::nullopt}), - // path - makeFlatVector({"/auth/login"})}); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - test::assertEqualVectors(expected, irOutput); -} - -TEST_F(ClpConnectorTest, test1JsonString) { - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder() - .startTableScan() - .outputType( - ROW({"requestId", "__json_string", "method"}, - {VARCHAR(), VARCHAR(), VARCHAR()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_1")) - .assignments({ - {"requestId", - std::make_shared( - "requestId", "requestId", VARCHAR())}, - {"__json_string", - std::make_shared( - "__json_string", "__json_string", VARCHAR())}, - {"method", - std::make_shared( - "method", "method", VARCHAR())}, - }) - .endTableScan() - .filter("method = 'GET'") - .planNode(); - - const auto methodVector = makeFlatVector({ - "GET", - "GET", - "GET", - "GET", - "GET", - }); - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// requestId - makeFlatVector( - {"req-100", "req-105", "req-107", "req-109", "req-102"}), - // __json_string - makeFlatVector({ - R"({"timestamp":"2025-04-30T08:45:00Z","requestId":"req-100","userId":"user201","method":"GET","path":"/api/users/1","responseTimeMs":25,"status":200})", - R"({"timestamp":"2025-04-30T08:45:25Z","requestId":"req-105","userId":"user204","method":"GET","path":"/api/dashboard","responseTimeMs":155,"status":200})", - R"({"timestamp":"2025-04-30T08:45:35Z","requestId":"req-107","userId":"user202","method":"GET","path":"/api/users/2/details","responseTimeMs":41,"status":200})", - R"({"timestamp":"2025-04-30T08:45:45Z","requestId":"req-109","userId":"user203","method":"GET","path":"/api/products?category=books","responseTimeMs":88,"status":200})", - R"({"timestamp":"2025-04-30T08:45:10Z","requestId":"req-102","method":"GET","path":"/public/products","responseTimeMs":18,"status":200,"userId":null})", - }), - // method - methodVector}); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_1_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - auto irExpected = makeRowVector( - {// requestId - makeFlatVector( - {"req-100", "req-102", "req-105", "req-107", "req-109"}), - // __json_string - makeFlatVector({ - R"({"method":"GET","path":"/api/users/1","requestId":"req-100","responseTimeMs":25,"status":200,"timestamp":"2025-04-30T08:45:00Z","userId":"user201"})", - R"({"method":"GET","path":"/public/products","requestId":"req-102","responseTimeMs":18,"status":200,"timestamp":"2025-04-30T08:45:10Z","userId":null})", - R"({"method":"GET","path":"/api/dashboard","requestId":"req-105","responseTimeMs":155,"status":200,"timestamp":"2025-04-30T08:45:25Z","userId":"user204"})", - R"({"method":"GET","path":"/api/users/2/details","requestId":"req-107","responseTimeMs":41,"status":200,"timestamp":"2025-04-30T08:45:35Z","userId":"user202"})", - R"({"method":"GET","path":"/api/products?category=books","requestId":"req-109","responseTimeMs":88,"status":200,"timestamp":"2025-04-30T08:45:45Z","userId":"user203"})", - }), - // method - methodVector}); - test::assertEqualVectors(irExpected, irOutput); -} - -TEST_F(ClpConnectorTest, test2NoPushdown) { - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType( - ROW({"timestamp", "event"}, - {TIMESTAMP(), - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()})})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_2")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"event", - std::make_shared( - "event", - "event", - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()}))}}) - .endTableScan() - .filter( - "event.severity IN ('WARNING', 'ERROR') AND " - "((event.type = 'network' AND event.subtype = 'connection') OR " - "(event.type = 'storage' AND event.subtype LIKE 'disk_usage%'))") - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - // event - makeRowVector({ - // event.type - makeFlatVector({"storage"}), - // event.subtype - makeFlatVector({"disk_usage"}), - // event.severity - makeFlatVector({"WARNING"}), - })}); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - test::assertEqualVectors(expected, irOutput); -} - -TEST_F(ClpConnectorTest, test2Pushdown) { - auto kqlQuery = std::make_shared( - "(event.severity: \"WARNING\" OR event.severity: \"ERROR\") AND " - "((event.type: \"network\" AND event.subtype: \"connection\") OR " - "(event.type: \"storage\" AND event.subtype: \"disk*\"))"); - auto plan = - PlanBuilder() - .startTableScan() - .outputType( - ROW({"timestamp", "event"}, - {TIMESTAMP(), - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()})})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_2")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"event", - std::make_shared( - "event", - "event", - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()}))}}) - .endTableScan() - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - // event - makeRowVector({ - // event.type - makeFlatVector({"storage"}), - // event.subtype - makeFlatVector({"disk_usage"}), - // event.severity - makeFlatVector({"WARNING"}), - })}); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - test::assertEqualVectors(expected, irOutput); -} - -TEST_F(ClpConnectorTest, test2Hybrid) { - auto kqlQuery = std::make_shared( - "((event.type: \"network\" AND event.subtype: \"connection\") OR " - "(event.type: \"storage\" AND event.subtype: \"disk*\"))"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType( - ROW({"timestamp", "event"}, - {TIMESTAMP(), - ROW({"type", "subtype", "severity", "tags"}, - {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())})})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_2")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"event", - std::make_shared( - "event", - "event", - ROW({"type", "subtype", "severity", "tags"}, - {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())}))}}) - .endTableScan() - .filter("upper(event.severity) IN ('WARNING', 'ERROR')") - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - // event - makeRowVector( - {// event.type - makeFlatVector({"storage"}), - // event.subtype - makeFlatVector({"disk_usage"}), - // event.severity - makeFlatVector({"WARNING"}), - // event.tags - makeArrayVector( - {{"\"filesystem\"", "\"monitoring\""}})}) - - }); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - test::assertEqualVectors(expected, irOutput); -} - -TEST_F(ClpConnectorTest, test2JsonString) { - auto kqlQuery = std::make_shared( - "(event.severity: \"WARNING\" OR event.severity: \"ERROR\") AND " - "((event.type: \"network\" AND event.subtype: \"connection\") OR " - "(event.type: \"storage\" AND event.subtype: \"disk*\"))"); - auto plan = - PlanBuilder() - .startTableScan() - .outputType( - ROW({"timestamp", "__json_string"}, {TIMESTAMP(), VARCHAR()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_2")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"__json_string", - std::make_shared( - "__json_string", "__json_string", VARCHAR())}}) - .endTableScan() - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - // __json_string - makeFlatVector( - {R"({"timestamp":"2025-04-30T08:50:05Z","event":{"type":"storage","subtype":"disk_usage","severity":"WARNING","tags":["filesystem", "monitoring"],"details":{"mount":"/var/log","usage":{"percent":92}}}})"})}); - test::assertEqualVectors(expected, output); - - auto irOutput = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - auto irExpected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - // __json_string - makeFlatVector( - {R"({"event":{"details":{"mount":"/var/log","usage":{"percent":92}},"severity":"WARNING","subtype":"disk_usage","tags":["filesystem","monitoring"],"type":"storage"},"timestamp":1746003005})"})}); - - test::assertEqualVectors(irExpected, irOutput); -} - -TEST_F(ClpConnectorTest, test3TimestampMarshalling) { - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp"}, {TIMESTAMP()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_3")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}}) - .endTableScan() - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_3.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector({ - // timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds), - Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds), - Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds), - Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - }); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test4IrTimestampNoPushdown) { - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp"}, {TIMESTAMP()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_4")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}}) - .endTableScan() - .filter("\"timestamp\" < timestamp '2025-08-24 02:36:45'") - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_4_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - auto expected = makeRowVector({ - // timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - }); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test4IrTimestampPushdown) { - // Only the second event meet the condition, the first event is a date string - // which is not supported yet so the value will be NULL. - // This test can not use the `timestamp()` literal, since the integer - // timestamps are in microsecond precision, and we currently assume all IR - // timestamps are millisecond precision when comparing against timestamp - // literals. - const std::shared_ptr kqlQuery = - std::make_shared("(timestamp < 1756003005000000)"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp"}, {TIMESTAMP()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_4")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}}) - .endTableScan() - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_4_ir.clp.zst"), - ClpConnectorSplit::SplitType::kIr, - kqlQuery)}); - auto expected = makeRowVector({ - // timestamp - makeFlatVector( - {Timestamp(kTestTimestampSeconds, kTestTimestampNanoseconds)}), - }); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5FloatTimestampNoPushdown) { - // Test filtering rows with a timestamp parsed from a date string and floats - // in various formats. - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .filter( - "\"timestamp\" < timestamp '2025-04-30 08:51:10' AND \"timestamp\" >= timestamp '2025-04-30 08:50:05.124'") - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003005, 124000000), - Timestamp(1746003005, 124100000), - Timestamp(1746003005, 125000000), - Timestamp(1746003005, 126000000), - Timestamp(1746003005, 127000000), - Timestamp(1746003060, 0), - Timestamp(1746003065, 0)}), - makeFlatVector( - {1.2345678912345E9, - 1E16, - 1.234567891234567E9, - 1.234567891234567E9, - -1.234567891234567E-9, - 1234567891.234567, - -1234567891.234567})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5FloatTimestampPushdown) { - // Test filtering rows with a timestamp parsed from a date string and floats - // in various formats. - const std::shared_ptr kqlQuery = std::make_shared( - R"(timestamp < timestamp("1746003070000", "\L") and timestamp >= timestamp("1746003005124", "\L"))"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003005, 124000000), - Timestamp(1746003005, 124100000), - Timestamp(1746003005, 125000000), - Timestamp(1746003005, 126000000), - Timestamp(1746003005, 127000000), - Timestamp(1746003060, 0), - Timestamp(1746003065, 0)}), - makeFlatVector( - {1.2345678912345E9, - 1E16, - 1.234567891234567E9, - 1.234567891234567E9, - -1.234567891234567E-9, - 1234567891.234567, - -1234567891.234567})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5NewTimestampFormatFloatTimestampPushdown) { - // Test filtering rows with a timestamp parsed from a date string and floats - // in various formats. - const std::shared_ptr kqlQuery = std::make_shared( - R"(timestamp < timestamp("1746003070000", "\L") and timestamp >= timestamp("1746003005124", "\L"))"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.v0.5.0.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003005, 124000000), - Timestamp(1746003005, 124100000), - Timestamp(1746003005, 125000000), - Timestamp(1746003005, 126000000), - Timestamp(1746003005, 127000001), - Timestamp(1746003060, 0), - Timestamp(1746003065, 0)}), - makeFlatVector( - {1.2345678912345E9, - 1E16, - 1.234567891234567E9, - 1.234567891234567E9, - -1.234567891234567E-9, - 1234567891.234567, - -1234567891.234567})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5FormattedFloatNoPushdown) { - // Test floats of only FormattedFloat type - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .filter( - "\"floatValue\" = 0.0 OR \"floatValue\" = 1.2345678912345E-29") - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003005, 123457000), - Timestamp(1746003115, 0), - Timestamp(1746003120, 0), - Timestamp(1746003125, 0), - Timestamp(1746003130, 0), - Timestamp(1746003135, 0), - Timestamp(1746003140, 0), - Timestamp(1746003145, 0), - Timestamp(1746003185, 0), - Timestamp(1746003190, 0)}), - makeFlatVector( - {1.2345678912345E-29, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5FormattedFloatPushdown) { - // Test floats of only FormattedFloat type - const std::shared_ptr kqlQuery = std::make_shared( - "(floatValue: 0.0 or floatValue: 1.2345678912345E-29)"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003005, 123457000), - Timestamp(1746003115, 0), - Timestamp(1746003120, 0), - Timestamp(1746003125, 0), - Timestamp(1746003130, 0), - Timestamp(1746003135, 0), - Timestamp(1746003140, 0), - Timestamp(1746003145, 0), - Timestamp(1746003185, 0), - Timestamp(1746003190, 0)}), - makeFlatVector( - {1.2345678912345E-29, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5DictionaryFloatNoPushdown) { - // Test floats of only DictionaryFloat type - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .filter("\"floatValue\" > 1.999999 AND \"floatValue\" < 2.000001") - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003195, 0), Timestamp(1746003200, 0)}), - makeFlatVector({2, 2})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5DictionaryFloatPushdown) { - // Test floats of only DictionaryFloat type - const std::shared_ptr kqlQuery = std::make_shared( - "(floatValue > 1.999999 and floatValue < 2.000001)"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector( - {Timestamp(1746003195, 0), Timestamp(1746003200, 0)}), - makeFlatVector({2, 2})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5HybridNoPushdown) { - // Test floats of both FormattedFloat and DictionaryFloat types - const std::shared_ptr kqlQuery = nullptr; - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .filter("\"floatValue\" = 1.0") - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector({ - Timestamp(1746003105, 0), - Timestamp(1746003110, 0), - Timestamp(1746003150, 0), - Timestamp(1746003155, 0), - Timestamp(1746003160, 0), - Timestamp(1746003205, 0), - }), - makeFlatVector({1, 1, 1, 1, 1, 1})}); - test::assertEqualVectors(expected, output); -} - -TEST_F(ClpConnectorTest, test5HybridPushdown) { - // Test floats of both FormattedFloat and DictionaryFloat types - const std::shared_ptr kqlQuery = - std::make_shared("(floatValue: 1.0)"); - auto plan = - PlanBuilder(pool_.get()) - .startTableScan() - .outputType(ROW({"timestamp", "floatValue"}, {TIMESTAMP(), DOUBLE()})) - .tableHandle( - std::make_shared(kClpConnectorId, "test_5")) - .assignments( - {{"timestamp", - std::make_shared( - "timestamp", "timestamp", TIMESTAMP())}, - {"floatValue", - std::make_shared( - "floatValue", "floatValue", DOUBLE())}}) - .endTableScan() - .orderBy({"\"timestamp\" ASC"}, false) - .planNode(); - - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_5.clps"), - ClpConnectorSplit::SplitType::kArchive, - kqlQuery)}); - auto expected = makeRowVector( - {// timestamp - makeFlatVector({ - Timestamp(1746003105, 0), - Timestamp(1746003110, 0), - Timestamp(1746003150, 0), - Timestamp(1746003155, 0), - Timestamp(1746003160, 0), - Timestamp(1746003205, 0), - }), - makeFlatVector({1, 1, 1, 1, 1, 1})}); - test::assertEqualVectors(expected, output); -} - -} // namespace - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::Init init{&argc, &argv, false}; - return RUN_ALL_TESTS(); -} diff --git a/velox-connector/src/connector/tests/examples/test_1.clps b/velox-connector/src/connector/tests/examples/test_1.clps deleted file mode 100644 index 9538d88..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_1.clps and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_1.ndjson b/velox-connector/src/connector/tests/examples/test_1.ndjson deleted file mode 100644 index aca627d..0000000 --- a/velox-connector/src/connector/tests/examples/test_1.ndjson +++ /dev/null @@ -1,10 +0,0 @@ -{"timestamp": "2025-04-30T08:45:00Z", "requestId": "req-100", "userId": "user201", "method": "GET", "path": "/api/users/1", "responseTimeMs": 25, "status": 200} -{"timestamp": "2025-04-30T08:45:05Z", "requestId": "req-101", "userId": "user202", "method": "POST", "path": "/api/orders", "responseTimeMs": 110, "status": 201} -{"timestamp": "2025-04-30T08:45:10Z", "requestId": "req-102", "userId": null, "method": "GET", "path": "/public/products", "responseTimeMs": 18, "status": 200} -{"timestamp": "2025-04-30T08:45:15Z", "requestId": "req-103", "userId": "user203", "method": "PUT", "path": "/api/items/5", "responseTimeMs": 95, "status": 200} -{"timestamp": "2025-04-30T08:45:20Z", "requestId": "req-104", "userId": "user201", "method": "DELETE", "path": "/api/notifications/3", "responseTimeMs": 32, "status": 204} -{"timestamp": "2025-04-30T08:45:25Z", "requestId": "req-105", "userId": "user204", "method": "GET", "path": "/api/dashboard", "responseTimeMs": 155, "status": 200} -{"timestamp": "2025-04-30T08:45:30Z", "requestId": "req-106", "userId": null, "method": "POST", "path": "/auth/login", "responseTimeMs": 68, "status": 200} -{"timestamp": "2025-04-30T08:45:35Z", "requestId": "req-107", "userId": "user202", "method": "GET", "path": "/api/users/2/details", "responseTimeMs": 41, "status": 200} -{"timestamp": "2025-04-30T08:45:40Z", "requestId": "req-108", "userId": "user205", "method": "PATCH", "path": "/api/settings", "responseTimeMs": 128, "status": 200} -{"timestamp": "2025-04-30T08:45:45Z", "requestId": "req-109", "userId": "user203", "method": "GET", "path": "/api/products?category=books", "responseTimeMs": 88, "status": 200} diff --git a/velox-connector/src/connector/tests/examples/test_1_ir.clp.zst b/velox-connector/src/connector/tests/examples/test_1_ir.clp.zst deleted file mode 100644 index 6f81659..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_1_ir.clp.zst and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_2.clps b/velox-connector/src/connector/tests/examples/test_2.clps deleted file mode 100644 index 22ddd92..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_2.clps and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_2.ndjson b/velox-connector/src/connector/tests/examples/test_2.ndjson deleted file mode 100644 index 8bccc2d..0000000 --- a/velox-connector/src/connector/tests/examples/test_2.ndjson +++ /dev/null @@ -1,10 +0,0 @@ -{"timestamp": "2025-04-30T08:50:00Z", "event": {"type": "network", "subtype": "connection", "severity": "INFO", "tags": ["inbound", "tcp"], "details": {"source": {"ip": "192.168.3.10"}, "destination": {"ip": "10.0.1.5", "port": 80}}}} -{"timestamp": "2025-04-30T08:50:05Z", "event": {"type": "storage", "subtype": "disk_usage", "severity": "WARNING", "tags": ["filesystem", "monitoring"], "details": {"mount": "/var/log", "usage": {"percent": 92}}}} -{"timestamp": "2025-04-30T08:50:10Z", "event": {"type": "process", "subtype": "start", "severity": "INFO", "tags": ["system", "init"], "details": {"name": "cron", "id": 1234}}} -{"timestamp": "2025-04-30T08:50:15Z", "event": {"type": "authentication", "subtype": "failure", "severity": "ERROR", "tags": ["login", "security"], "user": {"name": "guest", "method": "ssh"}, "details": {"reason": "invalid password", "ip": "203.0.113.20"}}} -{"timestamp": "2025-04-30T08:50:20Z", "event": {"type": "service", "subtype": "status", "severity": "INFO", "tags": ["daemon", "http"], "details": {"name": "httpd", "state": "running", "uptime": {"seconds": 3600}}}} -{"timestamp": "2025-04-30T08:50:25Z", "event": {"type": "security", "subtype": "alert", "severity": "HIGH", "tags": ["threat", "sql"], "details": {"rule": {"id": "SQLI-001"}, "target": "/api/data", "payload": "SELECT * FROM users;"}}} -{"timestamp": "2025-04-30T08:50:30Z", "event": {"type": "memory", "subtype": "usage", "severity": "NORMAL", "tags": ["resources"], "details": {"total": {"gb": 16}, "used": {"gb": 8.5}, "free": {"gb": 7.5}}}} -{"timestamp": "2025-04-30T08:50:35Z", "event": {"type": "configuration", "subtype": "change", "severity": "INFO", "tags": ["admin", "settings"], "user": {"name": "admin"}, "details": {"component": "database", "setting": {"name": "timeout", "oldValue": 30, "newValue": 60}}}} -{"timestamp": "2025-04-30T08:50:40Z", "event": {"type": "file", "subtype": "access", "severity": "WARNING", "tags": ["file", "privileged"], "user": {"name": "user101"}, "details": {"operation": "read", "path": "/etc/shadow", "permissions": "rw-------", "flags": ["sensitive", "audit"]}}} -{"timestamp": "2025-04-30T08:50:45Z", "event": {"type": "backup", "subtype": "status", "severity": "INFO", "tags": ["backup", "daily"], "details": {"job": {"name": "daily_backup", "status": "completed"}, "duration": {"seconds": 1200}, "size": {"gb": 25.3}}}} diff --git a/velox-connector/src/connector/tests/examples/test_2_ir.clp.zst b/velox-connector/src/connector/tests/examples/test_2_ir.clp.zst deleted file mode 100644 index de9aaf3..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_2_ir.clp.zst and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_3.clps b/velox-connector/src/connector/tests/examples/test_3.clps deleted file mode 100644 index 3dd9f37..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_3.clps and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_3.ndjson b/velox-connector/src/connector/tests/examples/test_3.ndjson deleted file mode 100644 index 0839ab1..0000000 --- a/velox-connector/src/connector/tests/examples/test_3.ndjson +++ /dev/null @@ -1,4 +0,0 @@ -{"timestamp": "2025-04-30T08:50:05.000Z"} -{"timestamp": 1746003005.0} -{"timestamp": 1746003005000000} -{"timestamp": 1746003005000000000} diff --git a/velox-connector/src/connector/tests/examples/test_4.ndjson b/velox-connector/src/connector/tests/examples/test_4.ndjson deleted file mode 100644 index 950b7e7..0000000 --- a/velox-connector/src/connector/tests/examples/test_4.ndjson +++ /dev/null @@ -1,3 +0,0 @@ -{"timestamp": "2025-04-30T08:50:05.000Z"} -{"timestamp": 1746003005000000} -{"timestamp": 1766003005000000} diff --git a/velox-connector/src/connector/tests/examples/test_4_ir.clp.zst b/velox-connector/src/connector/tests/examples/test_4_ir.clp.zst deleted file mode 100644 index 37110c5..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_4_ir.clp.zst and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_5.clps b/velox-connector/src/connector/tests/examples/test_5.clps deleted file mode 100644 index 965ef45..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_5.clps and /dev/null differ diff --git a/velox-connector/src/connector/tests/examples/test_5.ndjson b/velox-connector/src/connector/tests/examples/test_5.ndjson deleted file mode 100644 index b81c9e2..0000000 --- a/velox-connector/src/connector/tests/examples/test_5.ndjson +++ /dev/null @@ -1,37 +0,0 @@ -{ "timestamp": 1746003005.123456789, "floatValue": 0.00000000000000000000000000001234567891234500 } -{ "timestamp": 1746003005.124, "floatValue": 1.234567891234500E9 } -{ "timestamp": 1746003005.124100, "floatValue": 1E16 } -{ "timestamp": 1746003005.125, "floatValue": 1.234567891234567E+9 } -{ "timestamp": 1746003005.126, "floatValue": 1.234567891234567E+0009 } -{ "timestamp": 1746003005.127000001, "floatValue": -1.234567891234567E-0009 } -{ "timestamp": "2025-04-30T08:51:00Z", "floatValue": 1.234567891234567e9 } -{ "timestamp": "2025-04-30T08:51:05Z", "floatValue": -1.234567891234567E9 } -{ "timestamp": "2025-04-30T08:51:10Z", "floatValue": -1.234567891234567E-9 } -{ "timestamp": "2025-04-30T08:51:15Z", "floatValue": -1.234567891234500E9 } -{ "timestamp": "2025-04-30T08:51:20Z", "floatValue": -1E16 } -{ "timestamp": "2025-04-30T08:51:25Z", "floatValue": -1.234567891234567E+9 } -{ "timestamp": "2025-04-30T08:51:30Z", "floatValue": -1.234567891234567E+0009 } -{ "timestamp": "2025-04-30T08:51:35Z", "floatValue": -1.234567891234567E-0009 } -{ "timestamp": "2025-04-30T08:51:40Z", "floatValue": -1.234567891234567e9 } -{ "timestamp": "2025-04-30T08:51:45Z", "floatValue": 1.000E0 } -{ "timestamp": "2025-04-30T08:51:50Z", "floatValue": 1E0 } -{ "timestamp": "2025-04-30T08:51:55Z", "floatValue": 0.000E0 } -{ "timestamp": "2025-04-30T08:52:00Z", "floatValue": -0.000E0 } -{ "timestamp": "2025-04-30T08:52:05Z", "floatValue": 0E0 } -{ "timestamp": "2025-04-30T08:52:10Z", "floatValue": 0E-0000 } -{ "timestamp": "2025-04-30T08:52:15Z", "floatValue": -0E+0000 } -{ "timestamp": "2025-04-30T08:52:20Z", "floatValue": 0.000E-0000 } -{ "timestamp": "2025-04-30T08:52:25Z", "floatValue": -0.000E+0000 } -{ "timestamp": "2025-04-30T08:52:30Z", "floatValue": 1.000E+0 } -{ "timestamp": "2025-04-30T08:52:35Z", "floatValue": 1.000E-0 } -{ "timestamp": "2025-04-30T08:52:40Z", "floatValue": 1.000E-00 } -{ "timestamp": "2025-04-30T08:52:45Z", "floatValue": 1.000000000000000E010 } -{ "timestamp": "2025-04-30T08:52:50Z", "floatValue": 1.000000000000000E+010 } -{ "timestamp": "2025-04-30T08:52:55Z", "floatValue": 1.7976931348623157E308} -{ "timestamp": "2025-04-30T08:53:00Z", "floatValue": 4.9406564584124654E-324} -{ "timestamp": "2025-04-30T08:53:05Z", "floatValue": 0.0000000000000000E0000 } -{ "timestamp": "2025-04-30T08:53:10Z", "floatValue": -0.0000000000000000E0000 } - -{ "timestamp": "2025-04-30T08:53:15Z", "floatValue": 1.99999999999999999E0000 } -{ "timestamp": "2025-04-30T08:53:20Z", "floatValue": 2.00000000000000001E0000 } -{ "timestamp": "2025-04-30T08:53:25Z", "floatValue": 1.00000000000000001E0000 } diff --git a/velox-connector/src/connector/tests/examples/test_5.v0.5.0.clps b/velox-connector/src/connector/tests/examples/test_5.v0.5.0.clps deleted file mode 100644 index 169d042..0000000 Binary files a/velox-connector/src/connector/tests/examples/test_5.v0.5.0.clps and /dev/null differ