Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[improve](nestedtype) support schema change for nested type #39210

Merged
merged 5 commits into from
Aug 14, 2024
Merged

[improve](nestedtype) support schema change for nested type #39210

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e247305
improve schema change for nested type
amorynan Aug 12, 2024
099b717
fixed
amorynan Aug 12, 2024
794a996
fixed
amorynan Aug 12, 2024
d4d32ff
fix
amorynan Aug 13, 2024
124ffd2
fix cases
amorynan Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,11 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException {
throw new DdlException("Dest column name is empty");
}

// now nested type can only support change order
if (type.isComplexType() && !type.equals(other.type)) {
throw new DdlException("Can not change " + type + " to " + other);
}

if (!ColumnType.isSchemaChangeAllowed(type, other.type)) {
throw new DdlException("Can not change " + getDataType() + " to " + other.getDataType());
}
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,9 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true;

// Currently, we do not support schema change between complex types with subtypes.
schemaChangeMatrix[PrimitiveType.ARRAY.ordinal()][PrimitiveType.ARRAY.ordinal()] = false;
schemaChangeMatrix[PrimitiveType.STRUCT.ordinal()][PrimitiveType.STRUCT.ordinal()] = false;
schemaChangeMatrix[PrimitiveType.MAP.ordinal()][PrimitiveType.MAP.ordinal()] = false;
schemaChangeMatrix[PrimitiveType.ARRAY.ordinal()][PrimitiveType.ARRAY.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.STRUCT.ordinal()][PrimitiveType.STRUCT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.MAP.ordinal()][PrimitiveType.MAP.ordinal()] = true;
}

static boolean isSchemaChangeAllowed(Type lhs, Type rhs) {
Expand Down
46 changes: 46 additions & 0 deletions regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
col0 bigint No true \N
col_array array<text> Yes false \N NONE
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

-- !sql --
col0 bigint No true \N
col_map map<char(32),text> Yes false \N NONE
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

-- !sql --
col0 bigint No true \N
col_struct struct<f1:varchar(1)> Yes false \N NONE
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE

65 changes: 65 additions & 0 deletions ...sion-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("create_nestedtypes_with_schemachange", "p0") {

def create_nested_table_and_schema_change = {testTablex, nested_type, column_name, error ->
// create basic type
sql "DROP TABLE IF EXISTS $testTablex"
sql """ CREATE TABLE $testTablex (
col0 BIGINT NOT NULL, col2 int NOT NULL, col3 array<int> NULL, col4 map<int, int> NULL, col5 struct<f1: int> NULL
)
/* mow */
UNIQUE KEY(col0) DISTRIBUTED BY HASH(col0) BUCKETS 4 PROPERTIES (
"enable_unique_key_merge_on_write" = "true",
"replication_num" = "1"
); """
// alter table add nested type
if (error != '') {
// check nested type do not support other type
test {
sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type AFTER col0"
exception (error)
}
} else {
// check nested type can only support change order
sql "ALTER TABLE $testTablex ADD COLUMN $column_name $nested_type"
sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type AFTER col0"
waitForSchemaChangeDone {
sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='$testTablex' ORDER BY createtime DESC LIMIT 1 """
time 600
}
}
// desc table
qt_sql "DESC $testTablex"
}

// array
create_nested_table_and_schema_change.call("test_array_schemachange", "ARRAY<STRING>", "col_array", '')
// map
create_nested_table_and_schema_change.call("test_map_schemachange", "MAP<char(32), string>", "col_map", '')
// struct
create_nested_table_and_schema_change.call("test_struct_schemachange", "STRUCT<f1: varchar(1)>", "col_struct", '')

// array with other type
create_nested_table_and_schema_change.call("test_array_schemachange_1", "ARRAY<STRING>", "col3", "errCode = 2");
// map with other type
create_nested_table_and_schema_change.call("test_map_schemachange_1", "MAP<char(32), string>", "col4", "errCode = 2");
// struct with other type
create_nested_table_and_schema_change.call("test_struct_schemachange_1", "STRUCT<f1: varchar(1)>", "col5", "errCode = 2");

}
60 changes: 30 additions & 30 deletions regression-test/suites/schema_change_p0/test_dup_schema_value_modify3.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ suite("test_dup_schema_value_modify3", "p0") {
" (789012345, 'Grace', 2.19656, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00', {'a': 700, 'b': 200}, '[\"abc\", \"def\"]');"

//TODO Test the dup model by modify a value type from MAP to BOOLEAN
errorMessage = "errCode = 2, detailMessage = Can not change MAP to BOOLEAN"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m BOOLEAN """
Expand All @@ -102,8 +102,8 @@ suite("test_dup_schema_value_modify3", "p0") {


// TODO Test the dup model by modify a value type from MAP to TINYINT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to TINYINT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m TINYINT """
Expand All @@ -116,8 +116,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//Test the dup model by modify a value type from MAP to SMALLINT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to SMALLINT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m SMALLINT """
Expand All @@ -130,8 +130,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//Test the dup model by modify a value type from MAP to INT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to INT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m INT """
Expand All @@ -145,8 +145,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//Test the dup model by modify a value type from MAP to BIGINT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to BIGINT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m BIGINT """
Expand All @@ -159,8 +159,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//Test the dup model by modify a value type from MAP to LARGEINT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to LARGEINT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m LARGEINT """
Expand All @@ -173,8 +173,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//Test the dup model by modify a value type from MAP to FLOAT
errorMessage = "errCode = 2, detailMessage = Can not change MAP to FLOAT"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m FLOAT """
Expand All @@ -187,8 +187,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//TODO Test the dup model by modify a value type from MAP to DECIMAL
errorMessage = "errCode = 2, detailMessage = Can not change MAP to DECIMAL128"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m DECIMAL(38,0) """
Expand All @@ -202,8 +202,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//TODO Test the dup model by modify a value type from MAP to DATE
errorMessage = "errCode = 2, detailMessage = Can not change MAP to DATEV2"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m DATE """
Expand All @@ -216,8 +216,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//TODO Test the dup model by modify a value type from MAP to DATEV2
errorMessage = "errCode = 2, detailMessage = Can not change MAP to DATEV2"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m DATEV2 """
Expand All @@ -231,8 +231,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//TODO Test the dup model by modify a value type from MAP to DATETIME
errorMessage = "errCode = 2, detailMessage = Can not change MAP to DATETIMEV2"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m DATETIME """
Expand All @@ -245,8 +245,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//TODO Test the dup model by modify a value type from MAP to DATETIME
errorMessage = "errCode = 2, detailMessage = Can not change MAP to DATETIMEV2"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m DATETIMEV2 """
Expand All @@ -260,8 +260,8 @@ suite("test_dup_schema_value_modify3", "p0") {


//Test the dup model by modify a value type from MAP to VARCHAR
errorMessage = "errCode = 2, detailMessage = Can not change MAP to VARCHAR"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m VARCHAR(100) """
Expand All @@ -273,8 +273,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//Test the dup model by modify a value type from MAP to STRING
errorMessage = "errCode = 2, detailMessage = Can not change MAP to STRING"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m STRING """
Expand All @@ -286,8 +286,8 @@ suite("test_dup_schema_value_modify3", "p0") {
}, errorMessage)

//Test the dup model by modify a value type from MAP to JSON
errorMessage = "errCode = 2, detailMessage = Can not change MAP to JSON"
expectException({
errorMessage = "errCode = 2, detailMessage = Can not change"
expectExceptionLike({
sql initTable
sql initTableData
sql """ alter table ${tbName1} MODIFY column m JSON """
Expand Down
Loading
Loading