From 6c7f9e41ca5882f508556275d873a9d64b13ecf2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 22 Jul 2024 16:06:26 +0800 Subject: [PATCH] [function](feature) Implement CRC32 function --- be/src/vec/functions/function_string.cpp | 28 +++++++- .../doris/catalog/BuiltinScalarFunctions.java | 2 + .../expressions/functions/scalar/Crc32.java | 71 +++++++++++++++++++ .../visitor/ScalarFunctionVisitor.java | 5 ++ gensrc/script/doris_builtins_functions.py | 2 + .../test_string_function_like.out | 12 ++++ .../test_string_function_like.groovy | 4 ++ 7 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java diff --git a/be/src/vec/functions/function_string.cpp b/be/src/vec/functions/function_string.cpp index fc4fd83214dd7b..a342005adb9861 100644 --- a/be/src/vec/functions/function_string.cpp +++ b/be/src/vec/functions/function_string.cpp @@ -82,7 +82,7 @@ struct NameQuoteImpl { } }; -struct NameStringLenght { +struct NameStringLength { static constexpr auto name = "length"; }; @@ -104,6 +104,28 @@ struct StringLengthImpl { } }; +struct NameCrc32 { + static constexpr auto name = "crc32"; +}; + +struct Crc32Impl { + using ReturnType = DataTypeInt64; + static constexpr auto TYPE_INDEX = TypeIndex::String; + using Type = String; + using ReturnColumnType = ColumnVector; + + static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, + PaddedPODArray& res) { + auto size = offsets.size(); + res.resize(size); + for (int i = 0; i < size; ++i) { + res[i] = crc32_z(0L, (const unsigned char*)data.data() + offsets[i - 1], + offsets[i] - offsets[i - 1]); + } + return Status::OK(); + } +}; + struct NameStringUtf8Length { static constexpr auto name = "char_length"; }; @@ -941,7 +963,8 @@ using StringFindInSetImpl = StringFunctionImpl; -using FunctionStringLength = FunctionUnaryToType; +using FunctionStringLength = FunctionUnaryToType; +using FunctionCrc32 = FunctionUnaryToType; using FunctionStringUTF8Length = FunctionUnaryToType; using FunctionStringSpace = FunctionUnaryToType; using FunctionStringStartsWith = @@ -976,6 +999,7 @@ using FunctionStringRPad = FunctionStringPad; void register_function_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 0f0dc8b5d2ae63..412ea35f9a8379 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -125,6 +125,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh; import org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; @@ -731,6 +732,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Least.class, "least"), scalar(Left.class, "left"), scalar(Length.class, "length"), + scalar(Crc32.class, "crc32"), scalar(Like.class, "like"), scalar(Ln.class, "ln"), scalar(Locate.class, "locate"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java new file mode 100644 index 00000000000000..036807062faf2c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'crc32'. This class is generated by GenerateFunction. + */ +public class Crc32 extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(BigIntType.INSTANCE).args(StringType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public Crc32(Expression arg) { + super("crc32", arg); + } + + /** + * withChildren. + */ + @Override + public Crc32 withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new Crc32(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitCrc32(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 2f5ef3faa851f9..79d88b10690593 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -132,6 +132,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh; import org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; @@ -1476,6 +1477,10 @@ default R visitLength(Length length, C context) { return visitScalarFunction(length, context); } + default R visitCrc32(Crc32 crc32, C context) { + return visitScalarFunction(crc32, context); + } + default R visitLike(Like like, C context) { return visitStringRegexPredicate(like, context); } diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 00e8127725703b..fea535e509d2bc 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1619,6 +1619,7 @@ [['rpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], 'ALWAYS_NULLABLE'], [['append_trailing_char_if_absent'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'], [['length'], 'INT', ['VARCHAR'], ''], + [['crc32'], 'BIGINT', ['VARCHAR'], ''], [['bit_length'], 'INT', ['VARCHAR'], ''], [['char_length', 'character_length'], 'INT', ['VARCHAR'], ''], @@ -1685,6 +1686,7 @@ [['rpad'], 'STRING', ['STRING', 'INT', 'STRING'], 'ALWAYS_NULLABLE'], [['append_trailing_char_if_absent'], 'STRING', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'], [['length'], 'INT', ['STRING'], ''], + [['crc32'], 'BIGINT', ['STRING'], ''], [['bit_length'], 'INT', ['STRING'], ''], [['char_length', 'character_length'], 'INT', ['STRING'], ''], diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out index 51fbfc68af1adc..9fcfc2d6ee9854 100644 --- a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out @@ -245,3 +245,15 @@ bb -- !sql -- +-- !crc32_1 -- +348606243 + +-- !crc32_2 -- +130583814 + +-- !crc32_3 -- +2707236321 + +-- !crc32_4 -- +\N + diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy index 9c5deed8651f95..e092526b03cfa3 100644 --- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy @@ -90,4 +90,8 @@ suite("test_string_function_like") { qt_sql "SELECT k FROM ${tbName} WHERE NOT LIKE(k, \"%\") ORDER BY k;" // sql "DROP TABLE ${tbName};" + qt_crc32_1 "select crc32(\"DORIS\");" + qt_crc32_2 "select crc32(\"APACHE DORIS\");" + qt_crc32_3 "select crc32(10);" + qt_crc32_4 "select crc32(NULL);" }