diff --git a/velox/docs/develop/types.rst b/velox/docs/develop/types.rst index 80c94020f23..b21b6ef3e77 100644 --- a/velox/docs/develop/types.rst +++ b/velox/docs/develop/types.rst @@ -172,6 +172,7 @@ JSON VARCHAR TIMESTAMP WITH TIME ZONE BIGINT UUID HUGEINT IPADDRESS HUGEINT +IPPREFIX ROW(HUGEINT,TINYINT) ======================== ===================== TIMESTAMP WITH TIME ZONE represents a time point in milliseconds precision @@ -182,14 +183,31 @@ Supported range of milliseconds is [0xFFF8000000000000L, 0x7FFFFFFFFFFFF] store timezone ID. Supported range of timezone ID is [1, 1680]. The definition of timezone IDs can be found in ``TimeZoneDatabase.cpp``. -IPADDRESS represents an IPV6 or IPV4 formatted IPV6 address. Its physical -type is HUGEINT. The format that the address is stored in is defined as part of `(RFC 4291#section-2.5.5.2) `_ +IPADDRESS represents an IPv6 or IPv4 formatted IPv6 address. Its physical +type is HUGEINT. The format that the address is stored in is defined as part of `RFC 4291#section-2.5.5.2 `_. As Velox is run on Little Endian systems and the standard is network byte(Big Endian) order, we reverse the bytes to allow for masking and other bit operations used in IPADDRESS/IPPREFIX related functions. This type can be used to create IPPREFIX networks as well as to check IPADDRESS validity within IPPREFIX networks. +IPPREFIX represents an IPv6 or IPv4 formatted IPv6 address along with a one byte +prefix length. Its physical type is ROW(HUGEINT, TINYINT). The IPADDRESS is stored in +the HUGEINT and is in the form defined in `RFC 4291#section-2.5.5.2 `_. +The prefix length is stored in the TINYINT. +The IP address stored is the canonical(smallest) IP address in the +subnet range. This type can be used in IP subnet functions. + +Example: + +In this example the first 32 bits(*FFFF:FFFF*) represents the network prefix. +As a result the IPPREFIX object stores *FFFF:FFFF::* and the length 32 for both of these IPPREFIX objects. + +:: + + IPPREFIX 'FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF/32' -- IPPREFIX 'FFFF:FFFF:0000:0000:0000:0000:0000:0000/32' + IPPREFIX 'FFFF:FFFF:4455:6677:8899:AABB:CCDD:EEFF/32' -- IPPREFIX 'FFFF:FFFF:0000:0000:0000:0000:0000:0000/32' + Spark Types ~~~~~~~~~~~~ The `data types `_ in Spark have some semantic differences compared to those in diff --git a/velox/expression/tests/CustomTypeTest.cpp b/velox/expression/tests/CustomTypeTest.cpp index c55015d8590..97afc1f6c79 100644 --- a/velox/expression/tests/CustomTypeTest.cpp +++ b/velox/expression/tests/CustomTypeTest.cpp @@ -217,6 +217,7 @@ TEST_F(CustomTypeTest, getCustomTypeNames) { "TIMESTAMP WITH TIME ZONE", "UUID", "IPADDRESS", + "IPPREFIX", }), names); @@ -231,6 +232,7 @@ TEST_F(CustomTypeTest, getCustomTypeNames) { "TIMESTAMP WITH TIME ZONE", "UUID", "IPADDRESS", + "IPPREFIX", "FANCY_INT", }), names); diff --git a/velox/functions/prestosql/IPAddressFunctions.h b/velox/functions/prestosql/IPAddressFunctions.h index abd214a9aba..a08ef92de53 100644 --- a/velox/functions/prestosql/IPAddressFunctions.h +++ b/velox/functions/prestosql/IPAddressFunctions.h @@ -16,11 +16,13 @@ #pragma once #include "velox/functions/prestosql/types/IPAddressType.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" namespace facebook::velox::functions { void registerIPAddressFunctions(const std::string& prefix) { registerIPAddressType(); + registerIPPrefixType(); } } // namespace facebook::velox::functions diff --git a/velox/functions/prestosql/TypeOf.cpp b/velox/functions/prestosql/TypeOf.cpp index 622f0133903..85cf19128f7 100644 --- a/velox/functions/prestosql/TypeOf.cpp +++ b/velox/functions/prestosql/TypeOf.cpp @@ -16,6 +16,7 @@ #include "velox/expression/VectorFunction.h" #include "velox/functions/prestosql/types/HyperLogLogType.h" #include "velox/functions/prestosql/types/IPAddressType.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" #include "velox/functions/prestosql/types/JsonType.h" #include "velox/functions/prestosql/types/TimestampWithTimeZoneType.h" #include "velox/functions/prestosql/types/UuidType.h" @@ -78,6 +79,8 @@ std::string typeName(const TypePtr& type) { case TypeKind::VARBINARY: if (isHyperLogLogType(type)) { return "HyperLogLog"; + } else if (isIPPrefixType(type)) { + return "ipprefix"; } return "varbinary"; case TypeKind::TIMESTAMP: diff --git a/velox/functions/prestosql/tests/IPAddressCastTest.cpp b/velox/functions/prestosql/tests/IPAddressCastTest.cpp index 0f62b6b7248..a13ec9114e9 100644 --- a/velox/functions/prestosql/tests/IPAddressCastTest.cpp +++ b/velox/functions/prestosql/tests/IPAddressCastTest.cpp @@ -24,20 +24,20 @@ namespace { class IPAddressCastTest : public functions::test::FunctionBaseTest { protected: std::optional castToVarchar( - const std::optional input) { + const std::optional& input) { auto result = evaluateOnce( "cast(cast(c0 as ipaddress) as varchar)", input); return result; } std::optional castFromVarbinary( - const std::optional input) { + const std::optional& input) { auto result = evaluateOnce("cast(from_hex(c0) as ipaddress)", input); return result; } - std::optional allCasts(const std::optional input) { + std::optional allCasts(const std::optional& input) { auto result = evaluateOnce( "cast(cast(cast(cast(c0 as ipaddress) as varbinary) as ipaddress) as varchar)", input); @@ -45,7 +45,7 @@ class IPAddressCastTest : public functions::test::FunctionBaseTest { } }; -int128_t stringToInt128(std::string value) { +int128_t stringToInt128(const std::string& value) { int128_t res = 0; for (char c : value) { res = res * 10 + c - '0'; diff --git a/velox/functions/prestosql/types/CMakeLists.txt b/velox/functions/prestosql/types/CMakeLists.txt index 0089307a4f0..14c407ba89c 100644 --- a/velox/functions/prestosql/types/CMakeLists.txt +++ b/velox/functions/prestosql/types/CMakeLists.txt @@ -17,7 +17,8 @@ velox_add_library( JsonType.cpp TimestampWithTimeZoneType.cpp UuidType.cpp - IPAddressType.cpp) + IPAddressType.cpp + IPPrefixType.cpp) velox_link_libraries( velox_presto_types diff --git a/velox/functions/prestosql/types/IPPrefixType.cpp b/velox/functions/prestosql/types/IPPrefixType.cpp new file mode 100644 index 00000000000..aad808d7cfb --- /dev/null +++ b/velox/functions/prestosql/types/IPPrefixType.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "velox/expression/CastExpr.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" + +namespace facebook::velox { + +namespace { + +class IPPrefixCastOperator : public exec::CastOperator { + public: + bool isSupportedFromType(const TypePtr& other) const override { + return false; + } + + bool isSupportedToType(const TypePtr& other) const override { + return false; + } + + void castTo( + const BaseVector& input, + exec::EvalCtx& context, + const SelectivityVector& rows, + const TypePtr& resultType, + VectorPtr& result) const override { + context.ensureWritable(rows, resultType, result); + VELOX_NYI( + "Cast from {} to IPPrefix not yet supported", input.type()->toString()); + } + + void castFrom( + const BaseVector& input, + exec::EvalCtx& context, + const SelectivityVector& rows, + const TypePtr& resultType, + VectorPtr& result) const override { + context.ensureWritable(rows, resultType, result); + VELOX_NYI( + "Cast from IPPrefix to {} not yet supported", resultType->toString()); + } +}; + +class IPPrefixTypeFactories : public CustomTypeFactories { + public: + TypePtr getType() const override { + return IPPrefixType::get(); + } + + exec::CastOperatorPtr getCastOperator() const override { + return std::make_shared(); + } +}; + +} // namespace + +void registerIPPrefixType() { + registerCustomType( + "ipprefix", std::make_unique()); +} + +} // namespace facebook::velox diff --git a/velox/functions/prestosql/types/IPPrefixType.h b/velox/functions/prestosql/types/IPPrefixType.h new file mode 100644 index 00000000000..22107bbd483 --- /dev/null +++ b/velox/functions/prestosql/types/IPPrefixType.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "velox/type/SimpleFunctionApi.h" +#include "velox/type/Type.h" + +namespace facebook::velox { + +class IPPrefixType : public RowType { + IPPrefixType() : RowType({"ip", "prefix"}, {HUGEINT(), TINYINT()}) {} + + public: + static const std::shared_ptr& get() { + static const std::shared_ptr instance{ + new IPPrefixType()}; + + return instance; + } + + bool equivalent(const Type& other) const override { + // Pointer comparison works since this type is a singleton. + return this == &other; + } + + const char* name() const override { + return "IPPREFIX"; + } + + std::string toString() const override { + return name(); + } + + folly::dynamic serialize() const override { + folly::dynamic obj = folly::dynamic::object; + obj["name"] = "Type"; + obj["type"] = name(); + return obj; + } + + const std::vector& parameters() const override { + static const std::vector kEmpty = {}; + return kEmpty; + } +}; + +FOLLY_ALWAYS_INLINE bool isIPPrefixType(const TypePtr& type) { + // Pointer comparison works since this type is a singleton. + return IPPrefixType::get() == type; +} + +FOLLY_ALWAYS_INLINE std::shared_ptr IPPREFIX() { + return IPPrefixType::get(); +} + +struct IPPrefixT { + using type = Row; + static constexpr const char* typeName = "ipprefix"; +}; + +using IPPrefix = CustomType; + +void registerIPPrefixType(); + +} // namespace facebook::velox diff --git a/velox/functions/prestosql/types/tests/CMakeLists.txt b/velox/functions/prestosql/types/tests/CMakeLists.txt index e2b8105d268..6dbffad027d 100644 --- a/velox/functions/prestosql/types/tests/CMakeLists.txt +++ b/velox/functions/prestosql/types/tests/CMakeLists.txt @@ -19,7 +19,8 @@ add_executable( TimestampWithTimeZoneTypeTest.cpp TypeTestBase.cpp UuidTypeTest.cpp - IPAddressTypeTest.cpp) + IPAddressTypeTest.cpp + IPPrefixTypeTest.cpp) add_test(velox_presto_types_test velox_presto_types_test) diff --git a/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp b/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp new file mode 100644 index 00000000000..12873470cda --- /dev/null +++ b/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "velox/functions/prestosql/types/IPPrefixType.h" +#include "velox/functions/prestosql/types/tests/TypeTestBase.h" + +namespace facebook::velox::test { + +class IPPrefixTypeTest : public testing::Test, public TypeTestBase { + public: + IPPrefixTypeTest() { + registerIPPrefixType(); + } +}; + +TEST_F(IPPrefixTypeTest, basic) { + ASSERT_STREQ(IPPREFIX()->name(), "IPPREFIX"); + ASSERT_STREQ(IPPREFIX()->kindName(), "ROW"); + ASSERT_EQ(IPPREFIX()->name(), "IPPREFIX"); + ASSERT_TRUE(IPPREFIX()->parameters().empty()); + + ASSERT_TRUE(hasType("IPPREFIX")); + ASSERT_EQ(*getType("IPPREFIX", {}), *IPPREFIX()); +} + +TEST_F(IPPrefixTypeTest, serde) { + testTypeSerde(IPPREFIX()); +} +} // namespace facebook::velox::test diff --git a/velox/functions/tests/FunctionRegistryTest.cpp b/velox/functions/tests/FunctionRegistryTest.cpp index 0456c77c6fd..242ad60479c 100644 --- a/velox/functions/tests/FunctionRegistryTest.cpp +++ b/velox/functions/tests/FunctionRegistryTest.cpp @@ -26,6 +26,7 @@ #include "velox/functions/Registerer.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" #include "velox/functions/prestosql/tests/utils/FunctionBaseTest.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" #include "velox/functions/tests/RegistryTestUtil.h" #include "velox/type/Type.h" @@ -501,4 +502,19 @@ TEST_F(FunctionRegistryOverwriteTest, overwrite) { ASSERT_EQ(signatures.size(), 1); } +TEST_F(FunctionRegistryTest, ipPrefixRegistration) { + registerFunction({"ipprefix_func"}); + + auto& simpleFunctions = exec::simpleFunctions(); + auto signatures = simpleFunctions.getFunctionSignatures("ipprefix_func"); + ASSERT_EQ(signatures.size(), 1); + + auto result = resolveFunctionWithMetadata("ipprefix_func", {IPPREFIX()}); + EXPECT_TRUE(result.has_value()); + EXPECT_EQ(*result->first, *IPPREFIX()); + EXPECT_TRUE(result->second.defaultNullBehavior); + EXPECT_TRUE(result->second.deterministic); + EXPECT_FALSE(result->second.supportsFlattening); +} + } // namespace facebook::velox diff --git a/velox/functions/tests/RegistryTestUtil.h b/velox/functions/tests/RegistryTestUtil.h index 60e1bf5929d..4f1132ede02 100644 --- a/velox/functions/tests/RegistryTestUtil.h +++ b/velox/functions/tests/RegistryTestUtil.h @@ -18,6 +18,7 @@ #include "velox/expression/FunctionSignature.h" #include "velox/expression/VectorFunction.h" #include "velox/functions/Macros.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" namespace facebook::velox { @@ -97,6 +98,17 @@ struct VariadicFunc { } }; +template +struct IPPrefixFunc { + VELOX_DEFINE_FUNCTION_TYPES(T); + + FOLLY_ALWAYS_INLINE bool call( + out_type& /* result */, + const arg_type& /* arg1 */) { + return true; + } +}; + class VectorFuncOne : public velox::exec::VectorFunction { public: void apply(