[enhencement](hive-catalog) add hive.recursive_directories_table and hive.ignore_absent_partitions configs for HiveCatalog

docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/create_table.hql

@@ -0,0 +1,31 @@
+create database if not exists default;
+use default;
+
+CREATE TABLE `hive_recursive_directories_table`(
+  `id` int,
+  `name` string)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+LOCATION
+  '/user/doris/suites/default/hive_recursive_directories_table';
+
+
+CREATE TABLE `hive_ignore_absent_partitions_table`(
+  `id` int,
+  `name` string)
+PARTITIONED BY (country STRING, city STRING)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+LOCATION
+  '/user/doris/suites/default/hive_ignore_absent_partitions_table';
+
+ALTER TABLE hive_ignore_absent_partitions_table ADD PARTITION (country='USA', city='NewYork');
+ALTER TABLE hive_ignore_absent_partitions_table ADD PARTITION (country='India', city='Delhi');

docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -x
+
+CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+## mkdir and put data to hdfs
+hadoop fs -mkdir -p /user/doris/suites/default/hive_recursive_directories_table
+hadoop fs -mkdir -p /user/doris/suites/default/hive_ignore_absent_partitions_table
+
+# create table
+hive -f "${CUR_DIR}"/create_table.hql
+
+hadoop fs -rm -r /user/doris/suites/default/hive_ignore_absent_partitions_table/country=India
+

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java

@@ -364,7 +364,9 @@ private FileCacheValue getFileCache(String location, String inputFormat,
         // So we need to recursively list data location.
         // https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
         List<RemoteFile> remoteFiles = new ArrayList<>();
-        Status status = fs.listFiles(location, true, remoteFiles);
+        boolean isRecursiveDirectories = Boolean.valueOf(
+                catalog.getProperties().getOrDefault("hive.recursive_directories", "false"));
+        Status status = fs.listFiles(location, isRecursiveDirectories, remoteFiles);
         if (status.ok()) {
             for (RemoteFile remoteFile : remoteFiles) {
                 String srcPath = remoteFile.getPath().toString();
@@ -376,6 +378,10 @@ private FileCacheValue getFileCache(String location, String inputFormat,
             // Hive doesn't aware that the removed partition is missing.
             // Here is to support this case without throw an exception.
             LOG.warn(String.format("File %s not exist.", location));
+            if (!Boolean.valueOf(catalog.getProperties()
+                    .getOrDefault("hive.ignore_absent_partitions", "true"))) {
+                throw new UserException("Partition location does not exist: " + location);
+            }
         } else {
             throw new RuntimeException(status.getErrMsg());
         }

regression-test/data/external_table_p0/hive/hive_config_test.out

@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_outfile --
+1	doris
+2	nereids
+
+-- !check_outfile --
+1	doris
+2	nereids
+
+-- !check_outfile --
+1	doris
+2	nereids
+
+-- !1 --
+1	doris
+2	nereids
+
+-- !2 --
+1	doris
+1	doris
+1	doris
+2	nereids
+2	nereids
+2	nereids
+
+-- !check_outfile --
+1	doris
+2	nereids
+
+-- !3 --
+1	doris	USA	NewYork
+2	nereids	USA	NewYork
+

regression-test/suites/external_table_p0/hive/hive_config_test.groovy

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("hive_config_test", "p0,external,hive,external_docker,external_docker_hive") {
+    String db_name = "regression_test_external_table_p0_hive"
+    String internal_table = "hive_config_test"
+    String catalog_name = "docker_hive"
+
+    // create table and insert
+    sql """ DROP TABLE IF EXISTS ${internal_table} """
+    sql """
+    CREATE TABLE IF NOT EXISTS ${internal_table} (
+        `id` INT NOT NULL,
+        `name` STRING NOT NULL
+        )
+        DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
+    """
+    // insert data into interal table
+    sql """ INSERT INTO ${internal_table} VALUES (1, 'doris'), (2, 'nereids'); """
+
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("diable Hive test.")
+        return;
+    }
+
+    for (String hivePrefix : ["hive2"]) {
+        String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
+        String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+        String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+        def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}"
+        // It's okay to use random `hdfsUser`, but can not be empty.
+        def hdfsUserName = "doris"
+
+
+        def test_outfile = {format, uri ->
+            def res = sql """
+                SELECT * FROM internal.${db_name}.${internal_table} t ORDER BY id
+                INTO OUTFILE "${defaultFS}${uri}"
+                FORMAT AS ${format}
+                PROPERTIES (
+                    "fs.defaultFS"="${defaultFS}",
+                    "hadoop.username" = "${hdfsUserName}"
+                );
+            """
+
+            def outfile_url = res[0][3]
+            // check data correctness
+            order_qt_check_outfile """ select * from hdfs(
+                    "uri" = "${outfile_url}.${format}",
+                    "hadoop.username" = "${hdfsUserName}",
+                    "format" = "${format}");
+                """
+        }
+
+
+        // 1. test hive.recursive_directories table config
+        test_outfile("orc", "/user/doris/suites/default/hive_recursive_directories_table/exp_")
+        test_outfile("orc", "/user/doris/suites/default/hive_recursive_directories_table/1/exp_")
+        test_outfile("orc", "/user/doris/suites/default/hive_recursive_directories_table/2/exp_")
+
+        // test hive.recursive_directories_table = false
+        sql """drop catalog if exists ${catalog_name}"""
+        sql """create catalog if not exists ${catalog_name} properties (
+            "type"="hms",
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+            'hive.recursive_directories' = 'false'
+        );"""
+        sql """use `${catalog_name}`.`default`"""
+        order_qt_1 """ select * from hive_recursive_directories_table order by id;"""
+
+        // test hive.recursive_directories_table = true
+        sql """drop catalog if exists ${catalog_name}"""
+        sql """create catalog if not exists ${catalog_name} properties (
+            "type"="hms",
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+            'hive.recursive_directories' = 'true'
+        );"""
+        sql """ use `${catalog_name}`.`default` """
+        order_qt_2 """ select * from hive_recursive_directories_table order by id; """
+
+        // 2. test hive.ignore_absent_partitions-table
+        test_outfile("orc", "/user/doris/suites/default/hive_ignore_absent_partitions_table/country=USA/city=NewYork/exp_")
+
+        // test 'hive.ignore_absent_partitions' = 'true'
+        sql """drop catalog if exists ${catalog_name}"""
+        sql """create catalog if not exists ${catalog_name} properties (
+            "type"="hms",
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+            'hive.ignore_absent_partitions' = 'true'
+        );"""
+        sql """use `${catalog_name}`.`default`"""
+        order_qt_3 """ select * from hive_ignore_absent_partitions_table order by id;"""
+
+
+        // 'hive.ignore_absent_partitions' = 'false'
+        sql """drop catalog if exists ${catalog_name}"""
+        sql """create catalog if not exists ${catalog_name} properties (
+            "type"="hms",
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+            'hive.ignore_absent_partitions' = 'false'
+        );"""
+        sql """use `${catalog_name}`.`default`"""
+        test {
+            sql """ select * from hive_ignore_absent_partitions_table order by id;"""
+
+            exception "Partition location does not exist"
+        }
+    }
+}