From 12769871cd2e72aa4fd8c9da19c426dd955a6c04 Mon Sep 17 00:00:00 2001 From: Jianliang Qi Date: Thu, 27 Jun 2024 17:54:53 +0800 Subject: [PATCH 1/3] [fix](ES Catalog)Add array types support in esquery function --- gensrc/script/doris_builtins_functions.py | 23 +++++++++- .../external_table_p0/es/test_es_query.out | 42 +++++++++++++++++-- .../external_table_p0/es/test_es_query.groovy | 10 ++++- 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 85e44c053b73ef..4e373510b1646a 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1582,7 +1582,28 @@ [['esquery'], 'BOOLEAN', ['DATEV2', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['DATETIMEV2', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['TIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_TINYINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_SMALLINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_INT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_BIGINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_LARGEINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_FLOAT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DOUBLE', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_TIME', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DATE', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DATETIME', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_TIMEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DATEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DATETIMEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_CHAR', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_VARCHAR', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_JSONB', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DECIMALV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL32', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL64', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL128', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY_STRING', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['MAP', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], ''], diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index 605e2f1aa93178..d751719389f665 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -1,9 +1,9 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql01 -- -["2020-01-01", "2020-01-02"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N -- !sql02 -- -["2020-01-01", "2020-01-02"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N -- !sql03 -- 2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 @@ -24,11 +24,17 @@ I'm not null or empty I'm not null or empty +-- !sql07 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] I'm not null or empty +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N + -- !sql20 -- -["2020-01-01", "2020-01-02"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql21 -- -["2020-01-01", "2020-01-02"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql22 -- 2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 @@ -36,6 +42,12 @@ I'm not null or empty 2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 +-- !sql23 -- +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09T12:10:10 text2 ["2020-01-01", "2020-01-02"] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string4 [1, 2, 3, 4] 2022-08-08 2022-08-11T12:10:10 text3_4*5 ["2020-01-01", "2020-01-02"] 6.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + -- !sql_5_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -98,6 +110,11 @@ I'm not null or empty -- !sql_5_18 -- I'm not null or empty +-- !sql_5_19 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -160,6 +177,11 @@ I'm not null or empty -- !sql_6_18 -- I'm not null or empty +-- !sql_6_19 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2222 2022-08-08T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] I'm not null or empty string3 text3_4*5 5.0 2022-08-08T00:00 3333 2022-08-08T20:10:10 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -245,6 +267,12 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 +-- !sql_7_24 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + -- !sql_7_24 -- value1 value2 @@ -330,3 +358,9 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 +-- !sql_8_22 -- +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N \N string2 2022-08-08T12:10:10 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 +[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 + diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index 38556c32812551..9fb3512e3216c9 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -89,7 +89,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { // test external table for datetime sql """ CREATE TABLE `test_v1` ( - `c_datetime` array NULL, + `c_datetime` array NULL, `c_long` array NULL, `c_unsigned_long` array NULL, `c_text` array NULL, @@ -130,9 +130,10 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql04 """select message from test_v1 where message != ''""" order_qt_sql05 """select message from test_v1 where message is not null""" order_qt_sql06 """select message from test_v1 where not_null_or_empty(message)""" + order_qt_sql07 """select * from test_v1 where esquery(c_datetime, '{"term":{"c_datetime":"2020-01-01 12:00:00"}}');""" sql """ CREATE TABLE `test_v2` ( - `c_datetime` array NULL, + `c_datetime` array NULL, `c_long` array NULL, `c_unsigned_long` array NULL, `c_text` array NULL, @@ -169,6 +170,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql20 """select * from test_v2 where test2='text#1'""" order_qt_sql21 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')""" order_qt_sql22 """select test4,test5,test6,test7,test8 from test_v2 order by test8""" + order_qt_sql23 """select * from test_v2 where esquery(c_long, '{"term":{"c_long":"-1"}}');""" sql """switch test_es_query_es5""" order_qt_sql_5_02 """select * from test1 where test2='text#1'""" @@ -188,6 +190,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_5_16 """select message from test1 where message != ''""" order_qt_sql_5_17 """select message from test1 where message is not null""" order_qt_sql_5_18 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_5_19 """select * from test1 where esquery(c_unsigned_long, '{"match":{"c_unsigned_long":0}}')""" sql """switch test_es_query_es6""" // order_qt_sql_6_01 """show tables""" @@ -208,6 +211,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_6_16 """select message from test1 where message != ''""" order_qt_sql_6_17 """select message from test1 where message is not null""" order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)""" + order_qt_sql_6_19 """select * from test1 where esquery(c_person, '{"match":{"c_person.name":"Andy"}}')""" List> tables6N = sql """show tables""" boolean notContainHide = true @@ -252,6 +256,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_7_21 """select message from test1 where not_null_or_empty(message)""" order_qt_sql_7_22 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" order_qt_sql_7_23 """select * from test1 where level = 'debug'""" + order_qt_sql_7_24 """select * from test1 where esquery(c_float, '{"match":{"c_float":1.1}}')""" List> tables7N = sql """show tables""" boolean notContainHide7 = true @@ -296,5 +301,6 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_8_19 """select message from test1 where not_null_or_empty(message)""" order_qt_sql_8_20 """select * from test1 where esquery(my_wildcard, '{ "wildcard": { "my_wildcard": { "value":"*quite*lengthy" } } }');""" order_qt_sql_8_21 """select * from test1 where level = 'debug'""" + order_qt_sql_8_22 """select * from test1 where esquery(c_ip, '{"match":{"c_ip":"192.168.0.1"}}')""" } } From 1c6ef37b34aaaaa394d6a38ac87a56cdbc071b5e Mon Sep 17 00:00:00 2001 From: Jianliang Qi Date: Thu, 27 Jun 2024 21:04:18 +0800 Subject: [PATCH 2/3] fix signature --- .../nereids/trees/expressions/functions/scalar/EsQuery.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EsQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EsQuery.java index a5fbd339c9fa88..28a6988bca49f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EsQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EsQuery.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.types.coercion.AnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -38,7 +39,8 @@ public class EsQuery extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) + FunctionSignature.ret(BooleanType.INSTANCE).args(AnyDataType.INSTANCE_WITHOUT_INDEX, + VarcharType.SYSTEM_DEFAULT) ); /** From 054d273ecc59170cfe0c6b490a13814c37d169cd Mon Sep 17 00:00:00 2001 From: Jianliang Qi Date: Fri, 28 Jun 2024 18:01:14 +0800 Subject: [PATCH 3/3] fix comments --- gensrc/script/doris_builtins_functions.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 4e373510b1646a..420403b7a77494 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1582,28 +1582,7 @@ [['esquery'], 'BOOLEAN', ['DATEV2', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['DATETIMEV2', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['TIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_TINYINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_SMALLINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_INT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_BIGINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_LARGEINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_FLOAT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DOUBLE', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_TIME', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DATE', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DATETIME', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_TIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DATEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DATETIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_CHAR', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_VARCHAR', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_JSONB', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DECIMALV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL32', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL64', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_DECIMAL128', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY_STRING', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY', 'VARCHAR'], '', ['T']], [['esquery'], 'BOOLEAN', ['MAP', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], ''], [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], ''],