This is an automated email from the ASF dual-hosted git repository. dbecker pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3247cc683935f078a424a567389fa53764b300d9 Author: Peter Rozsa <[email protected]> AuthorDate: Fri Mar 10 09:07:42 2023 +0100 IMPALA-10173: Allow implicit casts between numeric and string types when inserting into table This patch adds an expiremental query option called ALLOW_UNSAFE_CASTS which allows implicit casting between some numeric types and string types. A new type of compatibility is introduced for this purpose, and the compatibility rule handling is refactored also. The new approach uses an enum to differentiate the compatibility levels, and to make it easier to pass them through methods. The unsafe compatibility is used only in two cases: for set operations and for insert statements. The insert statements and set operations accept unsafe implicitly casted expressions only when the source expressions are constant. The following implicit type casts are enabled in unsafe mode: - String -> Float, Double - String -> Tinyint, Smallint, Int, Bigint - Float, Double -> String - Tinyint, Smallint, Int, Bigint -> String The patch also covers IMPALA-3217, and adds two more rules to handle implicit casting in set operations and insert statements between string types: - String -> Char(n) - String -> Varchar(n) The unsafe implicit casting requires that the source expression must be constant in this case as well. Tests: - tests added to AnalyzeExprsTest.java - new test class added to test_insert.py Change-Id: Iee5db2301216c2e088b4b3e4f6cb5a1fd10600f7 Reviewed-on: http://gerrit.cloudera.org:8080/19881 Reviewed-by: Daniel Becker <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/service/query-options.cc | 3 + be/src/service/query-options.h | 7 +- common/thrift/ImpalaService.thrift | 4 + common/thrift/Query.thrift | 4 + .../org/apache/impala/analysis/AnalyticExpr.java | 6 +- .../java/org/apache/impala/analysis/Analyzer.java | 115 ++++++++-- .../org/apache/impala/analysis/ArithmeticExpr.java | 5 +- .../apache/impala/analysis/BinaryPredicate.java | 2 +- .../java/org/apache/impala/analysis/CaseExpr.java | 2 +- .../java/org/apache/impala/analysis/ColumnDef.java | 7 +- .../apache/impala/analysis/CompoundPredicate.java | 2 +- .../main/java/org/apache/impala/analysis/Expr.java | 76 +++++-- .../apache/impala/analysis/FunctionCallExpr.java | 2 +- .../org/apache/impala/analysis/InPredicate.java | 2 +- .../org/apache/impala/analysis/InsertStmt.java | 9 +- .../org/apache/impala/analysis/LikePredicate.java | 2 +- .../org/apache/impala/analysis/ModifyStmt.java | 2 +- .../org/apache/impala/analysis/NumericLiteral.java | 5 +- .../org/apache/impala/analysis/PartitionSpec.java | 4 +- .../org/apache/impala/analysis/RangePartition.java | 4 +- .../org/apache/impala/analysis/StatementBase.java | 36 +++- .../impala/analysis/TimestampArithmeticExpr.java | 2 +- .../java/org/apache/impala/analysis/TypesUtil.java | 7 +- .../apache/impala/catalog/BinaryCompatibility.java | 38 ++++ .../impala/catalog/CheckEmptyCompatibility.java | 41 ++++ .../apache/impala/catalog/CompatibilityRule.java | 28 +++ .../impala/catalog/DefaultCompatibility.java | 165 ++++++++++++++ .../impala/catalog/DiagonalCompatibility.java | 30 +++ .../impala/catalog/FixedUdaCompatibility.java | 33 +++ .../java/org/apache/impala/catalog/Function.java | 10 +- .../java/org/apache/impala/catalog/ScalarType.java | 91 +++----- .../catalog/StrictOverrideCompatibility.java | 50 +++++ .../main/java/org/apache/impala/catalog/Type.java | 239 ++++----------------- .../apache/impala/catalog/TypeCompatibility.java | 73 +++++++ .../apache/impala/catalog/UnsafeCompatibility.java | 85 ++++++++ .../org/apache/impala/planner/HashJoinNode.java | 2 +- .../apache/impala/analysis/AnalyzeExprsTest.java | 51 ++++- .../apache/impala/analysis/AnalyzeStmtsTest.java | 4 +- .../org/apache/impala/analysis/TypesUtilTest.java | 119 +++++----- .../queries/QueryTest/insert-unsafe.test | 213 ++++++++++++++++++ tests/query_test/test_insert.py | 23 ++ 41 files changed, 1219 insertions(+), 384 deletions(-) diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc index af0a74463..09715f0d4 100644 --- a/be/src/service/query-options.cc +++ b/be/src/service/query-options.cc @@ -1106,6 +1106,9 @@ Status impala::SetQueryOption(const string& key, const string& value, RETURN_IF_ERROR( QueryOptionValidator<int32_t>::NotEquals(option, int32_t_val, 1)); query_options->__set_max_sort_run_size(int32_t_val); + } + case TImpalaQueryOptions::ALLOW_UNSAFE_CASTS: { + query_options->__set_allow_unsafe_casts(IsTrue(value)); break; } default: diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h index f88e5fc74..0a424301e 100644 --- a/be/src/service/query-options.h +++ b/be/src/service/query-options.h @@ -50,7 +50,7 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type> // time we add or remove a query option to/from the enum TImpalaQueryOptions. #define QUERY_OPTS_TABLE \ DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(), \ - TImpalaQueryOptions::MAX_SORT_RUN_SIZE + 1); \ + TImpalaQueryOptions::ALLOW_UNSAFE_CASTS + 1); \ REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded, ABORT_ON_DEFAULT_LIMIT_EXCEEDED) \ QUERY_OPT_FN(abort_on_error, ABORT_ON_ERROR, TQueryOptionLevel::REGULAR) \ REMOVED_QUERY_OPT_FN(allow_unsupported_formats, ALLOW_UNSUPPORTED_FORMATS) \ @@ -291,8 +291,9 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type> QUERY_OPT_FN(join_selectivity_correlation_factor, JOIN_SELECTIVITY_CORRELATION_FACTOR, \ TQueryOptionLevel::ADVANCED) \ QUERY_OPT_FN(max_fragment_instances_per_node, MAX_FRAGMENT_INSTANCES_PER_NODE, \ - TQueryOptionLevel::ADVANCED); \ - QUERY_OPT_FN(max_sort_run_size, MAX_SORT_RUN_SIZE, TQueryOptionLevel::DEVELOPMENT) ; + TQueryOptionLevel::ADVANCED) \ + QUERY_OPT_FN(max_sort_run_size, MAX_SORT_RUN_SIZE, TQueryOptionLevel::DEVELOPMENT) \ + QUERY_OPT_FN(allow_unsafe_casts, ALLOW_UNSAFE_CASTS, TQueryOptionLevel::DEVELOPMENT); /// Enforce practical limits on some query options to avoid undesired query state. static const int64_t SPILLABLE_BUFFER_LIMIT = 1LL << 40; // 1 TB diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 26651a00f..20f83bf44 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -795,6 +795,10 @@ enum TImpalaQueryOptions { // high fragmentation of variable length data. MAX_SORT_RUN_SIZE = 157; + // Allowing implicit casts with loss of precision, adds the capability to use + // implicit casts between numeric and string types in set operations and insert + // statements. + ALLOW_UNSAFE_CASTS = 158; } // The summary of a DML statement. diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift index e57608587..36e3a827d 100644 --- a/common/thrift/Query.thrift +++ b/common/thrift/Query.thrift @@ -638,6 +638,10 @@ struct TQueryOptions { // Configures the in-memory sort algorithm used in the sorter. // See comment in ImpalaService.thrift 158: optional i32 max_sort_run_size = 0; + + // See comment in ImpalaService.thrift + 159: optional bool allow_unsafe_casts = false; + } // Impala currently has three types of sessions: Beeswax, HiveServer2 and external diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java index 22a452070..eff2d1989 100644 --- a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java +++ b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java @@ -26,6 +26,7 @@ import org.apache.impala.catalog.AggregateFunction; import org.apache.impala.catalog.Function; import org.apache.impala.catalog.ScalarType; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.InternalException; import org.apache.impala.common.TreeNode; @@ -380,8 +381,9 @@ public class AnalyticExpr extends Expr { + "a RANGE window with PRECEDING/FOLLOWING: " + toSql()); } Expr rangeExpr = boundary.getExpr(); - if (!Type.isImplicitlyCastable( - rangeExpr.getType(), orderByElements_.get(0).getExpr().getType(), false, true)) { + if (!Type.isImplicitlyCastable(rangeExpr.getType(), + orderByElements_.get(0).getExpr().getType(), + TypeCompatibility.STRICT_DECIMAL)) { throw new AnalysisException( "The value expression of a PRECEDING/FOLLOWING clause of a RANGE window must " + "be implicitly convertable to the ORDER BY expression's type: " diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index ef4dd77cc..a5bba3f27 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -32,6 +32,7 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -66,6 +67,7 @@ import org.apache.impala.catalog.StructField; import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.catalog.VirtualColumn; import org.apache.impala.catalog.VirtualTable; import org.apache.impala.catalog.iceberg.IcebergMetadataTable; @@ -3277,22 +3279,54 @@ public class Analyzer { public Type getCompatibleType(Type lastCompatibleType, Expr lastCompatibleExpr, Expr expr) throws AnalysisException { + return getCompatibleType( + lastCompatibleType, lastCompatibleExpr, expr, getRegularCompatibilityLevel()); + } + + public static Type getCompatibleType(Type leftType, Expr leftExpr, Type rightType, + Expr rightExpr, TypeCompatibility compatibility) throws AnalysisException { + Type compatibleType = + Type.getAssignmentCompatibleType(leftType, rightType, compatibility); + checkCompatibility( + leftType, leftExpr, rightType, rightExpr, compatibleType, compatibility); + return compatibleType; + } + + public static Type getCompatibleType(Type lastCompatibleType, Expr lastCompatibleExpr, + Expr expr, TypeCompatibility compatibility) throws AnalysisException { Type newCompatibleType; if (lastCompatibleType == null) { newCompatibleType = expr.getType(); } else { newCompatibleType = Type.getAssignmentCompatibleType( - lastCompatibleType, expr.getType(), false, isDecimalV2()); - } - if (!newCompatibleType.isValid()) { - throw new AnalysisException(String.format( - "Incompatible return types '%s' and '%s' of exprs '%s' and '%s'.", - lastCompatibleType.toSql(), expr.getType().toSql(), - lastCompatibleExpr.toSql(), expr.toSql())); + lastCompatibleType, expr.getType(), compatibility); } + checkCompatibility(lastCompatibleType, lastCompatibleExpr, expr.getType(), expr, + newCompatibleType, compatibility); return newCompatibleType; } + private static void checkCompatibility(Type leftType, Expr leftExpr, Type rightType, + Expr rightExpr, Type compatibleType, TypeCompatibility compatibility) + throws AnalysisException { + if (compatibility.isUnsafe()) { + Optional<Expr> nonConstExpr = leftExpr.getFirstNonConstSourceExpr(); + if (!nonConstExpr.isPresent()) { + nonConstExpr = rightExpr.getFirstNonConstSourceExpr(); + } + if (nonConstExpr.isPresent()) { + throw new AnalysisException(String.format( + "Unsafe implicit cast is prohibited for non-const expression: %s", + nonConstExpr.get().toSql())); + } + } + if (!compatibleType.isValid()) { + throw new AnalysisException( + String.format("Incompatible return types '%s' and '%s' of exprs '%s' and '%s'.", + leftType.toSql(), rightType.toSql(), leftExpr.toSql(), rightExpr.toSql())); + } + } + /** * Determines compatible type for given exprs, and casts them to compatible type. * Calls analyze() on each of the exprs. @@ -3346,6 +3380,10 @@ public class Analyzer { if (exprLists == null || exprLists.size() == 0) return null; if (exprLists.size() == 1) return exprLists.get(0); + TypeCompatibility regularCompatibility = this.getRegularCompatibilityLevel(); + TypeCompatibility permissiveCompatibility = this.getPermissiveCompatibilityLevel(); + TypeCompatibility compatibilityLevel = regularCompatibility; + // Determine compatible types for exprs, position by position. List<Expr> firstList = exprLists.get(0); List<Expr> widestExprs = new ArrayList<>(firstList.size()); @@ -3360,22 +3398,37 @@ public class Analyzer { firstList.get(i).toSql())); } widestExprs.add(firstList.get(i)); + for (int j = 1; j < exprLists.size(); ++j) { Preconditions.checkState(exprLists.get(j).size() == firstList.size()); Type preType = compatibleType; - compatibleType = getCompatibleType( - compatibleType, widestExprs.get(i), exprLists.get(j).get(i)); + Expr expr = exprLists.get(j).get(i); + + try { + compatibleType = getCompatibleType( + compatibleType, widestExprs.get(i), expr, compatibilityLevel); + } catch (AnalysisException e) { + compatibilityLevel = permissiveCompatibility; + if (permissiveCompatibility.isUnsafe()) { + compatibleType = getCompatibleType( + compatibleType, widestExprs.get(i), expr, compatibilityLevel); + } else { + throw e; + } + } + // compatibleType will be updated if a new wider type is encountered - if (preType != compatibleType) widestExprs.set(i, exprLists.get(j).get(i)); + if (preType != compatibleType) widestExprs.set(i, expr); } // Now that we've found a compatible type, add implicit casts if necessary. for (int j = 0; j < exprLists.size(); ++j) { // Checking compatibility with every expression - Type commonType = getCompatibleType(compatibleType, - widestExprs.get(i), exprLists.get(j).get(i)); + Expr expr = exprLists.get(j).get(i); + Type commonType = getCompatibleType( + expr.getType(), expr, compatibleType, widestExprs.get(i), compatibilityLevel); Preconditions.checkState(commonType.equals(compatibleType)); - if (!exprLists.get(j).get(i).getType().equals(compatibleType)) { - Expr castExpr = exprLists.get(j).get(i).castTo(compatibleType); + if (!expr.getType().equals(compatibleType)) { + Expr castExpr = expr.castTo(compatibleType, compatibilityLevel); exprLists.get(j).set(i, castExpr); } } @@ -3407,6 +3460,40 @@ public class Analyzer { return globalState_.queryCtx.client_request.getQuery_options(); } public boolean isDecimalV2() { return getQueryOptions().isDecimal_v2(); } + + /** + * This method returns the compatibility level depending on Decimal V2 query option. The + * unsafe compatibility level is not considered in this method. IMPALA-10173 introduces + * UNSAFE type compatibility just for set operations and insert statements, therefore + * any other unaffected parts must use this method to decide the compatibility level. + * The method applies a transformation to 'compatibility', returns a compatibility level + * that will carry the strict decimal meaning (isStrictDecimal() returns true for it) + * considering the Decimal V2 query option. + */ + public TypeCompatibility getRegularCompatibilityLevel(TypeCompatibility compatibility) { + TQueryOptions queryOptions = getQueryOptions(); + if (queryOptions.isDecimal_v2()) { + return TypeCompatibility.applyStrictDecimal(compatibility); + } + return compatibility; + } + + public TypeCompatibility getRegularCompatibilityLevel() { + return getRegularCompatibilityLevel(TypeCompatibility.DEFAULT); + } + + /** + * This method returns compatibility level depending on both Decimal V2 and unsafe casts + * query option. It allows UNSAFE type compatibility for set operation and insert + * statements. + */ + public TypeCompatibility getPermissiveCompatibilityLevel() { + TQueryOptions queryOptions = getQueryOptions(); + if (queryOptions.isAllow_unsafe_casts()) { + return TypeCompatibility.UNSAFE; + } + return getRegularCompatibilityLevel(TypeCompatibility.DEFAULT); + } public AuthorizationFactory getAuthzFactory() { return globalState_.authzFactory; } public AuthorizationConfig getAuthzConfig() { return getAuthzFactory().getAuthorizationConfig(); diff --git a/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java b/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java index 0edd21339..57fec5ccd 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java +++ b/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java @@ -23,6 +23,7 @@ import org.apache.impala.catalog.Function.CompareMode; import org.apache.impala.catalog.ScalarFunction; import org.apache.impala.catalog.ScalarType; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.thrift.TExprNode; import org.apache.impala.thrift.TExprNodeType; @@ -219,7 +220,7 @@ public class ArithmeticExpr extends Expr { throw new AnalysisException("Invalid non-integer argument to operation '" + op_.toString() + "': " + this.toSql()); } - type_ = Type.getAssignmentCompatibleType(t0, t1, false, false); + type_ = Type.getAssignmentCompatibleType(t0, t1, TypeCompatibility.DEFAULT); // If both of the children are null, we'll default to the INT version of the // operator. This prevents the BE from seeing NULL_TYPE. if (type_.isNull()) type_ = Type.INT; @@ -241,7 +242,7 @@ public class ArithmeticExpr extends Expr { fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), CompareMode.IS_SUPERTYPE_OF); Preconditions.checkNotNull(fn_); - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); type_ = fn_.getReturnType(); return; default: diff --git a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java index 8e5b147ff..41cbc0cba 100644 --- a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java +++ b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java @@ -235,7 +235,7 @@ public class BinaryPredicate extends Predicate { if (!contains(Subquery.class)) { // Don't perform any casting for predicates with subqueries here. Any casting // required will be performed when the subquery is unnested. - castForFunctionCall(true, analyzer.isDecimalV2()); + castForFunctionCall(true, analyzer.getRegularCompatibilityLevel()); } // Determine selectivity diff --git a/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java b/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java index 1483b5a90..3ae00a62e 100644 --- a/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java +++ b/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java @@ -294,7 +294,7 @@ public class CaseExpr extends Expr { // If no case expr was given, then the when exprs should always return // boolean or be castable to boolean. if (!Type.isImplicitlyCastable(whenExpr.getType(), Type.BOOLEAN, - false, analyzer.isDecimalV2())) { + analyzer.getRegularCompatibilityLevel())) { Preconditions.checkState(isCase()); throw new AnalysisException("When expr '" + whenExpr.toSql() + "'" + " is not of type boolean and not castable to type boolean."); diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java index f491e2607..1e14595e6 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java +++ b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java @@ -28,10 +28,10 @@ import java.util.Map; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.Pair; import org.apache.impala.compat.MetastoreShim; -import org.apache.impala.service.FeSupport; import org.apache.impala.thrift.TColumn; import org.apache.impala.util.ExprUtil; import org.apache.impala.util.KuduUtil; @@ -291,8 +291,9 @@ public class ColumnDef { } } - if (!Type.isImplicitlyCastable(defaultValLiteral.getType(), type_, - true, analyzer.isDecimalV2())) { + TypeCompatibility compatibility = + analyzer.getRegularCompatibilityLevel(TypeCompatibility.STRICT); + if (!Type.isImplicitlyCastable(defaultValLiteral.getType(), type_, compatibility)) { throw new AnalysisException(String.format("Default value %s (type: %s) " + "is not compatible with column '%s' (type: %s).", defaultValue_.toSql(), defaultValue_.getType().toSql(), colName_, type_.toSql())); diff --git a/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java b/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java index 3e3715be7..cc169a183 100644 --- a/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java +++ b/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java @@ -141,7 +141,7 @@ public class CompoundPredicate extends Predicate { CompareMode.IS_NONSTRICT_SUPERTYPE_OF); Preconditions.checkState(fn_ != null); Preconditions.checkState(fn_.getReturnType().isBoolean()); - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); computeSelectivity(analyzer); } diff --git a/fe/src/main/java/org/apache/impala/analysis/Expr.java b/fe/src/main/java/org/apache/impala/analysis/Expr.java index 5b2ed1463..0453e01c8 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Expr.java +++ b/fe/src/main/java/org/apache/impala/analysis/Expr.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.Optional; import java.util.Set; import org.apache.impala.analysis.BinaryPredicate.Operator; @@ -34,6 +35,7 @@ import org.apache.impala.catalog.Function.CompareMode; import org.apache.impala.catalog.PrimitiveType; import org.apache.impala.catalog.ScalarType; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.InternalException; import org.apache.impala.common.SqlCastException; @@ -653,15 +655,16 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl * called with fn(decimal(10,2), decimal(5,3)) * both children will be cast to (11, 3). * - * If strictDecimal is true, we will only consider casts between decimal types that - * result in no loss of information. If it is not possible to come with such casts, - * we will throw an exception. + * 'compatibility' defines the mode of wildcard type resolution; + * if TypeCompatibility.isStrictDecimal() is true, it only considers casts that result + * in no loss of information, if 'compatibility' is DEFAULT it does not guarantee + * decimal cast without information loss. */ - protected void castForFunctionCall( - boolean ignoreWildcardDecimals, boolean strictDecimal) throws AnalysisException { + protected void castForFunctionCall(boolean ignoreWildcardDecimals, + TypeCompatibility compatibility) throws AnalysisException { Preconditions.checkState(fn_ != null); Type[] fnArgs = fn_.getArgs(); - Type resolvedWildcardType = getResolvedWildCardType(strictDecimal); + Type resolvedWildcardType = getResolvedWildCardType(compatibility); if (resolvedWildcardType != null) { if (resolvedWildcardType.isNull()) { throw new SqlCastException(String.format( @@ -709,11 +712,13 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl /** * Returns the max resolution type of all the wild card decimal types. - * Returns null if there are no wild card types. If strictDecimal is enabled, will - * return an invalid type if it is not possible to come up with a decimal type that - * is guaranteed to not lose information. + * Returns null if there are no wild card types. If compatibility.isStrictDecimal() is + * true, it will return an invalid type if it is not possible to come up with a decimal + * type that is guaranteed to not lose information. */ - Type getResolvedWildCardType(boolean strictDecimal) { + Type getResolvedWildCardType(TypeCompatibility compatibility) { + Preconditions.checkState(compatibility.equals(TypeCompatibility.DEFAULT) + || compatibility.equals(TypeCompatibility.STRICT_DECIMAL)); Type result = null; Type[] fnArgs = fn_.getArgs(); for (int i = 0; i < children_.size(); ++i) { @@ -730,8 +735,7 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl ScalarType decimalType = (ScalarType) childType; result = decimalType.getMinResolutionDecimal(); } else { - result = Type.getAssignmentCompatibleType( - result, childType, false, strictDecimal); + result = Type.getAssignmentCompatibleType(result, childType, compatibility); } } if (result != null && !result.isNull()) { @@ -1538,18 +1542,22 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl } /** - * Casts this expr to a specific target type. It checks the validity of the cast and - * calls uncheckedCastTo(). + * Casts this expr to a specific target type. It checks the validity of the cast + * according to 'compatibility' and calls uncheckedCastTo(). * @param targetType * type to be cast to + * @param compatibility + * compatibility level that defines the relation between 'targetType' and the + * type of the expression * @return cast expression, or converted literal, * should never return null * @throws AnalysisException * when an invalid cast is asked for, for example, * failure to convert a string literal to a date literal */ - public final Expr castTo(Type targetType) throws AnalysisException { - Type type = Type.getAssignmentCompatibleType(this.type_, targetType, false, false); + public final Expr castTo(Type targetType, TypeCompatibility compatibility) + throws AnalysisException { + Type type = Type.getAssignmentCompatibleType(this.type_, targetType, compatibility); Preconditions.checkState(type.isValid(), "cast %s to %s", this.type_, targetType); // If the targetType is NULL_TYPE then ignore the cast because NULL_TYPE // is compatible with all types and no cast is necessary. @@ -1565,6 +1573,10 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl return uncheckedCastTo(targetType); } + public final Expr castTo(Type targetType) throws AnalysisException { + return castTo(targetType, TypeCompatibility.DEFAULT); + } + /** * Create an expression equivalent to 'this' but returning targetType; * possibly by inserting an implicit cast, @@ -1925,6 +1937,38 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl return null; } + /** + * Returns the first non-const expression on the following path: + * - checking whether the expression itself is constant or not + * If there's an underlying slot ref: + * - checking the slot desc's source expressions constness + */ + public Optional<Expr> getFirstNonConstSourceExpr() { + SlotRef slotRef = unwrapSlotRef(false); + + if (slotRef == null) { + Preconditions.checkState(isAnalyzed()); + return isConstant() ? Optional.empty() : Optional.of(this); + } + + SlotDescriptor slotDesc = slotRef.getDesc(); + Preconditions.checkNotNull(slotDesc); + + if (slotDesc.getSourceExprs().isEmpty()) { + return Optional.of(this); + } + + Optional<Expr> nonConstSourceExpr = Optional.empty(); + for (Expr expr : slotDesc.getSourceExprs()) { + Preconditions.checkState(expr.isAnalyzed()); + if (!expr.isConstant()) { + nonConstSourceExpr = Optional.of(expr); + break; + } + } + return nonConstSourceExpr; + } + /** * Returns true if 'this' is a constant. */ diff --git a/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java index 90dde1450..a3bb1d0b5 100644 --- a/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java @@ -758,7 +758,7 @@ public class FunctionCallExpr extends Expr { "Analytic function requires an OVER clause: " + toSql()); } - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); type_ = fn_.getReturnType(); if (type_.isDecimal() && type_.isWildcardDecimal()) { type_ = resolveDecimalReturnType(analyzer); diff --git a/fe/src/main/java/org/apache/impala/analysis/InPredicate.java b/fe/src/main/java/org/apache/impala/analysis/InPredicate.java index e576b754d..ca175f84d 100644 --- a/fe/src/main/java/org/apache/impala/analysis/InPredicate.java +++ b/fe/src/main/java/org/apache/impala/analysis/InPredicate.java @@ -167,7 +167,7 @@ public class InPredicate extends Predicate { } Preconditions.checkNotNull(fn_); Preconditions.checkState(fn_.getReturnType().isBoolean()); - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); } computeSelectivity(); } diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java index 79f029f52..ec9dc8946 100644 --- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java @@ -869,8 +869,7 @@ public class InsertStmt extends DmlStatementBase { Expr widestTypeExpr = (widestTypeExprList != null) ? widestTypeExprList.get(i) : null; Expr compatibleExpr = checkTypeCompatibility(targetTableName_.toString(), - targetColumn, selectListExprs.get(i), analyzer.getQueryOptions().isDecimal_v2(), - widestTypeExpr); + targetColumn, selectListExprs.get(i), analyzer, widestTypeExpr); if (targetColumn.getPosition() < numClusteringCols) { // This is a dynamic clustering column tmpPartitionKeyExprs.add(compatibleExpr); @@ -892,7 +891,7 @@ public class InsertStmt extends DmlStatementBase { // tableColumns is guaranteed to exist after the earlier analysis checks Column tableColumn = table_.getColumn(pkv.getColName()); Expr compatibleExpr = checkTypeCompatibility(targetTableName_.toString(), - tableColumn, pkv.getLiteralValue(), analyzer.isDecimalV2(), null); + tableColumn, pkv.getLiteralValue(), analyzer, null); tmpPartitionKeyExprs.add(compatibleExpr); tmpPartitionKeyNames.add(pkv.getColName()); } @@ -1048,9 +1047,7 @@ public class InsertStmt extends DmlStatementBase { Expr widestTypeExpr = (widestTypeExprList != null) ? widestTypeExprList.get(i) : null; Expr compatibleExpr = checkTypeCompatibility(targetTableName_.toString(), - targetColumn, selectListExprs.get(i), - analyzer.getQueryOptions().isDecimal_v2(), - widestTypeExpr); + targetColumn, selectListExprs.get(i), analyzer, widestTypeExpr); Expr icebergPartitionTransformExpr = getIcebergPartitionTransformExpr(partField, compatibleExpr); partitionKeyExprs_.add(icebergPartitionTransformExpr); diff --git a/fe/src/main/java/org/apache/impala/analysis/LikePredicate.java b/fe/src/main/java/org/apache/impala/analysis/LikePredicate.java index cdaf22ed7..4f32e88cd 100644 --- a/fe/src/main/java/org/apache/impala/analysis/LikePredicate.java +++ b/fe/src/main/java/org/apache/impala/analysis/LikePredicate.java @@ -146,7 +146,7 @@ public class LikePredicate extends Predicate { "invalid regular expression in '" + this.toSql() + "'"); } } - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); } @Override diff --git a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java index 483c95fdd..197f72bad 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java @@ -315,7 +315,7 @@ public abstract class ModifyStmt extends DmlStatementBase { } rhsExpr = checkTypeCompatibility(targetTableRef_.getDesc().getTable().getFullName(), - c, rhsExpr, analyzer.isDecimalV2(), null /*widestTypeSrcExpr*/); + c, rhsExpr, analyzer, null /*widestTypeSrcExpr*/); uniqueSlots.add(lhsSlotRef.getSlotId()); selectList.add(new SelectListItem(rhsExpr, null)); referencedColumns_.add(colIndexMap.get(c.getName())); diff --git a/fe/src/main/java/org/apache/impala/analysis/NumericLiteral.java b/fe/src/main/java/org/apache/impala/analysis/NumericLiteral.java index 59daf977c..e9b4b12c6 100644 --- a/fe/src/main/java/org/apache/impala/analysis/NumericLiteral.java +++ b/fe/src/main/java/org/apache/impala/analysis/NumericLiteral.java @@ -463,7 +463,10 @@ public class NumericLiteral extends LiteralExpr { */ @Override protected Expr uncheckedCastTo(Type targetType) throws SqlCastException { - Preconditions.checkState(targetType.isNumericType()); + Preconditions.checkState(targetType.isNumericType() || targetType.isStringType()); + if (targetType.isStringType()) { + return new CastExpr(targetType, this); + } if (type_ == targetType) return this; try { BigDecimal converted = convertValue(value_, targetType); diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java index 3d4e477a5..2ce541c3a 100644 --- a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java +++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java @@ -117,8 +117,8 @@ public class PartitionSpec extends PartitionSpecBase { Type colType = c.getType(); Type literalType = pk.getValue().getType(); - Type compatibleType = Type.getAssignmentCompatibleType(colType, literalType, - false, analyzer.isDecimalV2()); + Type compatibleType = Type.getAssignmentCompatibleType( + colType, literalType, analyzer.getRegularCompatibilityLevel()); if (!compatibleType.isValid()) { throw new AnalysisException(String.format("Value of partition spec (column=%s) " + "has incompatible type: '%s'. Expected type: '%s'.", diff --git a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java index fe303ba9a..f8b98f773 100644 --- a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java +++ b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java @@ -24,10 +24,10 @@ import java.util.List; import java.util.Map; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.InternalException; import org.apache.impala.common.Pair; -import org.apache.impala.service.FeSupport; import org.apache.impala.thrift.TRangePartition; import org.apache.impala.util.ExprUtil; import org.apache.impala.util.KuduUtil; @@ -213,7 +213,7 @@ public class RangePartition extends StmtNode { org.apache.impala.catalog.Type literalType = literal.getType(); if (!org.apache.impala.catalog.Type.isImplicitlyCastable(literalType, colType, - true, analyzer.isDecimalV2())) { + analyzer.getRegularCompatibilityLevel(TypeCompatibility.STRICT))) { throw new AnalysisException(String.format("Range partition value %s " + "(type: %s) is not type compatible with partitioning column '%s' (type: %s).", literal.toSql(), literalType, pkColumn.getColName(), colType.toSql())); diff --git a/fe/src/main/java/org/apache/impala/analysis/StatementBase.java b/fe/src/main/java/org/apache/impala/analysis/StatementBase.java index 4fb2d4394..4210f2958 100644 --- a/fe/src/main/java/org/apache/impala/analysis/StatementBase.java +++ b/fe/src/main/java/org/apache/impala/analysis/StatementBase.java @@ -20,9 +20,11 @@ package org.apache.impala.analysis; import java.util.Collections; import java.util.List; +import java.util.Optional; import org.apache.commons.lang.NotImplementedException; import org.apache.impala.catalog.Column; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import org.apache.impala.rewrite.ExprRewriter; @@ -197,12 +199,11 @@ public abstract class StatementBase extends StmtNode { * This is only used when constructing an AnalysisException message to make sure the * right expression is blamed in the error message. * - * If strictDecimal is true, only consider casts that result in no loss of information - * when casting between decimal types. + * If compatibility is unsafe and the source expression is not constant, compatibility + * ignores the unsafe option. */ public static Expr checkTypeCompatibility(String dstTableName, Column dstCol, - Expr srcExpr, boolean strictDecimal, Expr widestTypeSrcExpr) - throws AnalysisException { + Expr srcExpr, Analyzer analyzer, Expr widestTypeSrcExpr) throws AnalysisException { Type dstColType = dstCol.getType(); Type srcExprType = srcExpr.getType(); @@ -210,8 +211,25 @@ public abstract class StatementBase extends StmtNode { // Trivially compatible, unless the type is complex. if (dstColType.equals(srcExprType) && !dstColType.isComplexType()) return srcExpr; - Type compatType = Type.getAssignmentCompatibleType( - dstColType, srcExprType, false, strictDecimal); + TypeCompatibility permissiveCompatibility = + analyzer.getPermissiveCompatibilityLevel(); + TypeCompatibility compatibilityLevel = analyzer.getRegularCompatibilityLevel(); + + Type compatType = + Type.getAssignmentCompatibleType(srcExprType, dstColType, compatibilityLevel); + + if (compatType.isInvalid() && permissiveCompatibility.isUnsafe()) { + Optional<Expr> expr = srcExpr.getFirstNonConstSourceExpr(); + if (expr.isPresent()) { + throw new AnalysisException(String.format( + "Unsafe implicit cast is prohibited for non-const expression: %s ", + expr.get().toSql())); + } + compatType = Type.getAssignmentCompatibleType( + srcExprType, dstColType, permissiveCompatibility); + compatibilityLevel = permissiveCompatibility; + } + if (!compatType.isValid()) { throw new AnalysisException(String.format( "Target table '%s' is incompatible with source expressions.\nExpression '%s' " + @@ -219,13 +237,15 @@ public abstract class StatementBase extends StmtNode { dstTableName, srcExpr.toSql(), srcExprType.toSql(), dstCol.getName(), dstColType.toSql())); } - if (!compatType.equals(dstColType) && !compatType.isNull()) { + + if (!compatType.equals(dstColType) && !compatType.isNull() + && !permissiveCompatibility.isUnsafe()) { throw new AnalysisException(String.format( "Possible loss of precision for target table '%s'.\nExpression '%s' (type: " + "%s) would need to be cast to %s for column '%s'", dstTableName, widestTypeSrcExpr.toSql(), srcExprType.toSql(), dstColType.toSql(), dstCol.getName())); } - return srcExpr.castTo(compatType); + return srcExpr.castTo(compatType, compatibilityLevel); } } diff --git a/fe/src/main/java/org/apache/impala/analysis/TimestampArithmeticExpr.java b/fe/src/main/java/org/apache/impala/analysis/TimestampArithmeticExpr.java index 49995cd5c..1071937b4 100644 --- a/fe/src/main/java/org/apache/impala/analysis/TimestampArithmeticExpr.java +++ b/fe/src/main/java/org/apache/impala/analysis/TimestampArithmeticExpr.java @@ -169,7 +169,7 @@ public class TimestampArithmeticExpr extends Expr { fn_ = getBuiltinFunction(analyzer, funcOpName.toLowerCase(), collectChildReturnTypes(), CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - castForFunctionCall(false, analyzer.isDecimalV2()); + castForFunctionCall(false, analyzer.getRegularCompatibilityLevel()); Preconditions.checkNotNull(fn_); Preconditions.checkState(fn_.getReturnType().isTimestamp() diff --git a/fe/src/main/java/org/apache/impala/analysis/TypesUtil.java b/fe/src/main/java/org/apache/impala/analysis/TypesUtil.java index 211a296a5..cda5e0533 100644 --- a/fe/src/main/java/org/apache/impala/analysis/TypesUtil.java +++ b/fe/src/main/java/org/apache/impala/analysis/TypesUtil.java @@ -21,6 +21,7 @@ import java.math.BigDecimal; import org.apache.impala.catalog.ScalarType; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; import com.google.common.base.Preconditions; @@ -116,14 +117,14 @@ public class TypesUtil { case SUBTRACT: // If one of the types is null, use the compatible type without promotion. // Otherwise, promote the compatible type to the next higher resolution type, - // to ensure that that a <op> b won't overflow/underflow. + // to ensure that a <op> b won't overflow/underflow. Type compatibleType = - ScalarType.getAssignmentCompatibleType(t1, t2, false, false); + ScalarType.getAssignmentCompatibleType(t1, t2, TypeCompatibility.DEFAULT); Preconditions.checkState(compatibleType.isScalarType()); type = ((ScalarType) compatibleType).getNextResolutionType(); break; case MOD: - type = ScalarType.getAssignmentCompatibleType(t1, t2, false, false); + type = ScalarType.getAssignmentCompatibleType(t1, t2, TypeCompatibility.DEFAULT); break; case DIVIDE: type = Type.DOUBLE; diff --git a/fe/src/main/java/org/apache/impala/catalog/BinaryCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/BinaryCompatibility.java new file mode 100644 index 000000000..94096d477 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/BinaryCompatibility.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.apache.impala.catalog.Type.BINARY; +import static org.apache.impala.catalog.Type.STRING; +import static org.apache.impala.catalog.Type.VARCHAR; + +// BINARY can be only cast to STRING. +public class BinaryCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + for (int i = 0; i < PrimitiveType.values().length; ++i) { + if (i != BINARY.ordinal() && i != STRING.ordinal()) { + matrix[BINARY.ordinal()][i] = PrimitiveType.INVALID_TYPE; + matrix[i][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE; + } + } + // STRING -> BINARY conversion is not lossy, but implicit cast is not allowed to + // enforce exact type match in function calls. + matrix[STRING.ordinal()][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE; + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/CheckEmptyCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/CheckEmptyCompatibility.java new file mode 100644 index 000000000..91ab50fd9 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/CheckEmptyCompatibility.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import com.google.common.base.Preconditions; + +public class CheckEmptyCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + // Check all the necessary entries that should be filled. + for (int i = 0; i < PrimitiveType.values().length; ++i) { + for (int j = i; j < PrimitiveType.values().length; ++j) { + PrimitiveType t1 = PrimitiveType.values()[i]; + PrimitiveType t2 = PrimitiveType.values()[j]; + // INVALID_TYPE and NULL_TYPE are handled separately. + if (t1 == PrimitiveType.INVALID_TYPE || t2 == PrimitiveType.INVALID_TYPE) { + continue; + } + if (t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE) { + continue; + } + Preconditions.checkNotNull(matrix[i][j]); + } + } + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/CompatibilityRule.java b/fe/src/main/java/org/apache/impala/catalog/CompatibilityRule.java new file mode 100644 index 000000000..a718e2f6b --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/CompatibilityRule.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +/** + * Compatibility rules are separate operations to compose a compatibility matrix. Matrices + * start with empty values, and rules fill them by applying the rules defined in the + * implementations. + * <p> + * Example: 'DiagonalCompatibility' sets the matrix's main diagonal equal to the + * corresponding row and column values. + */ +public interface CompatibilityRule { void apply(PrimitiveType[][] matrix); } diff --git a/fe/src/main/java/org/apache/impala/catalog/DefaultCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/DefaultCompatibility.java new file mode 100644 index 000000000..cb8c5c9c6 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/DefaultCompatibility.java @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.apache.impala.catalog.Type.BIGINT; +import static org.apache.impala.catalog.Type.BOOLEAN; +import static org.apache.impala.catalog.Type.CHAR; +import static org.apache.impala.catalog.Type.DATE; +import static org.apache.impala.catalog.Type.DATETIME; +import static org.apache.impala.catalog.Type.DECIMAL; +import static org.apache.impala.catalog.Type.DOUBLE; +import static org.apache.impala.catalog.Type.FLOAT; +import static org.apache.impala.catalog.Type.INT; +import static org.apache.impala.catalog.Type.SMALLINT; +import static org.apache.impala.catalog.Type.STRING; +import static org.apache.impala.catalog.Type.TIMESTAMP; +import static org.apache.impala.catalog.Type.TINYINT; +import static org.apache.impala.catalog.Type.VARCHAR; + +public class DefaultCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + matrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; + matrix[BOOLEAN.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; + matrix[BOOLEAN.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[BOOLEAN.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + matrix[BOOLEAN.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[BOOLEAN.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BOOLEAN.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + matrix[TINYINT.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; + matrix[TINYINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[TINYINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + // 8 bit integer fits in mantissa of both float and double. + matrix[TINYINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[TINYINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TINYINT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.DECIMAL; + + matrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[SMALLINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + // 16 bit integer fits in mantissa of both float and double. + matrix[SMALLINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[SMALLINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[SMALLINT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.DECIMAL; + + matrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + // 32 bit integer fits only in the mantissa of double. + // TODO: arguably we should promote INT + FLOAT to DOUBLE to avoid loss of precision, + // but we depend on it remaining FLOAT for some use cases, e.g. + // "insert into tbl (float_col) select int_col + float_col from ..." + matrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[INT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[INT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[INT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.DECIMAL; + + // 64 bit integer does not fit in the mantissa of double or float. + // TODO: arguably we should always promote BIGINT + FLOAT to double here to keep as + // much precision as possible, but we depend on this implicit cast for some use + // cases, similarly to INT + FLOAT. + matrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + // TODO: we're breaking the definition of strict compatibility for BIGINT + DOUBLE, + // but this forces function overloading to consider the DOUBLE overload first. + matrix[BIGINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[BIGINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[BIGINT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.DECIMAL; + + matrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + matrix[FLOAT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.FLOAT; + + matrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DOUBLE.ordinal()][DECIMAL.ordinal()] = PrimitiveType.DOUBLE; + + // We can convert some but not all string values to date. + matrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.DATE; + matrix[DATE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DATE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DATE.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + matrix[DATETIME.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.TIMESTAMP; + matrix[DATETIME.ordinal()][DATE.ordinal()] = PrimitiveType.DATETIME; + matrix[DATETIME.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DATETIME.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DATETIME.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DATETIME.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + // We can convert some but not all date values to timestamps. + matrix[TIMESTAMP.ordinal()][DATE.ordinal()] = PrimitiveType.TIMESTAMP; + // We can convert some but not all string values to timestamps. + matrix[TIMESTAMP.ordinal()][STRING.ordinal()] = PrimitiveType.TIMESTAMP; + matrix[TIMESTAMP.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TIMESTAMP.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TIMESTAMP.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + matrix[STRING.ordinal()][VARCHAR.ordinal()] = PrimitiveType.STRING; + matrix[STRING.ordinal()][CHAR.ordinal()] = PrimitiveType.STRING; + matrix[STRING.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + matrix[VARCHAR.ordinal()][CHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[VARCHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[VARCHAR.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + + matrix[CHAR.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + + matrix[DECIMAL.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DECIMAL.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DECIMAL.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DECIMAL.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DECIMAL.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[DECIMAL.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/DiagonalCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/DiagonalCompatibility.java new file mode 100644 index 000000000..eda9e1a33 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/DiagonalCompatibility.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +// Each type is compatible with itself. +public class DiagonalCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + PrimitiveType[] primitiveTypes = PrimitiveType.values(); + for (int i = 0; i < primitiveTypes.length; ++i) { + // Each type is compatible with itself. + matrix[i][i] = primitiveTypes[i]; + } + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/FixedUdaCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/FixedUdaCompatibility.java new file mode 100644 index 000000000..7acb977a2 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/FixedUdaCompatibility.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.apache.impala.catalog.Type.FIXED_UDA_INTERMEDIATE; + +// FIXED_UDA_INTERMEDIATE cannot be cast to/from another type +public class FixedUdaCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + for (int i = 0; i < PrimitiveType.values().length; ++i) { + if (i != FIXED_UDA_INTERMEDIATE.ordinal()) { + matrix[FIXED_UDA_INTERMEDIATE.ordinal()][i] = PrimitiveType.INVALID_TYPE; + matrix[i][FIXED_UDA_INTERMEDIATE.ordinal()] = PrimitiveType.INVALID_TYPE; + } + } + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/Function.java b/fe/src/main/java/org/apache/impala/catalog/Function.java index b718442fe..e7f1ad924 100644 --- a/fe/src/main/java/org/apache/impala/catalog/Function.java +++ b/fe/src/main/java/org/apache/impala/catalog/Function.java @@ -210,8 +210,10 @@ public class Function extends CatalogObjectImpl { switch (mode) { case IS_IDENTICAL: return calcIdenticalMatchScore(other); case IS_INDISTINGUISHABLE: return calcIndistinguishableMatchScore(other); - case IS_SUPERTYPE_OF: return calcSuperTypeOfMatchScore(other, true); - case IS_NONSTRICT_SUPERTYPE_OF: return calcSuperTypeOfMatchScore(other, false); + case IS_SUPERTYPE_OF: + return calcSuperTypeOfMatchScore(other, TypeCompatibility.ALL_STRICT); + case IS_NONSTRICT_SUPERTYPE_OF: + return calcSuperTypeOfMatchScore(other, TypeCompatibility.DEFAULT); default: Preconditions.checkState(false); return -1; @@ -241,7 +243,7 @@ public class Function extends CatalogObjectImpl { * Otherwise returns the number of arguments whose types are an exact match or a * wildcard variant. */ - private int calcSuperTypeOfMatchScore(Function other, boolean strict) { + private int calcSuperTypeOfMatchScore(Function other, TypeCompatibility compatibility) { if (!other.name_.equals(name_)) return -1; if (!this.hasVarArgs_ && other.argTypes_.length != this.argTypes_.length) { return -1; @@ -256,7 +258,7 @@ public class Function extends CatalogObjectImpl { continue; } if (!Type.isImplicitlyCastable( - other.argTypes_[i], extendedArgTypes[i], strict, strict)) { + other.argTypes_[i], extendedArgTypes[i], compatibility)) { return -1; } } diff --git a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java index 48b1d5755..12f7a63f0 100644 --- a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java +++ b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java @@ -453,54 +453,42 @@ public class ScalarType extends Type { * Returns INVALID_TYPE if there is no such type or if any of t1 and t2 * is INVALID_TYPE. * - * If strictDecimal is true, only return types that result in no loss of information - * when both inputs are decimal. - * If strict is true, only return types that result in no loss of information - * when at least one of the inputs is not decimal. + * 'compatibility' sets the mode of the type compatibility lookup, unsafe + * compatibility makes difference in t1 -> t2 and t2-> t1 lookup. */ - public static ScalarType getAssignmentCompatibleType(ScalarType t1, - ScalarType t2, boolean strict, boolean strictDecimal) { + public static ScalarType getAssignmentCompatibleType( + ScalarType t1, ScalarType t2, TypeCompatibility compatibility) { if (!t1.isValid() || !t2.isValid()) return INVALID; if (t1.equals(t2)) return t1; if (t1.isNull()) return t2; if (t2.isNull()) return t1; - if (t1.type_ == PrimitiveType.VARCHAR || t2.type_ == PrimitiveType.VARCHAR) { - if (t1.type_ == PrimitiveType.STRING || t2.type_ == PrimitiveType.STRING) { - return STRING; - } - if (t1.isStringType() && t2.isStringType()) { - return createVarcharType(Math.max(t1.len_, t2.len_)); - } - return INVALID; + PrimitiveType smallerType = + (t1.type_.ordinal() < t2.type_.ordinal() ? t1.type_ : t2.type_); + PrimitiveType largerType = + (t1.type_.ordinal() > t2.type_.ordinal() ? t1.type_ : t2.type_); + PrimitiveType result = null; + + if (compatibility.isUnsafe()) { + // Unsafe compatibility differentiates t1 -> t2 and t2 -> t1 compatibility + result = unsafeCompatibilityMatrix[t1.type_.ordinal()][t2.type_.ordinal()]; + } else if (compatibility.isStrict()) { + result = strictCompatibilityMatrix[smallerType.ordinal()][largerType.ordinal()]; } - if (t1.type_ == PrimitiveType.CHAR || t2.type_ == PrimitiveType.CHAR) { - Preconditions.checkState(t1.type_ != PrimitiveType.VARCHAR); - Preconditions.checkState(t2.type_ != PrimitiveType.VARCHAR); - if (t1.type_ == PrimitiveType.STRING || t2.type_ == PrimitiveType.STRING) { - return STRING; - } - if (t1.type_ == PrimitiveType.CHAR && t2.type_ == PrimitiveType.CHAR) { - return createCharType(Math.max(t1.len_, t2.len_)); - } - return INVALID; + if (result == null) { + result = compatibilityMatrix[smallerType.ordinal()][largerType.ordinal()]; } - if (t1.isDecimal() || t2.isDecimal()) { - // The case of decimal and float/double must be handled carefully. There are two - // modes: strict and non-strict. In non-strict mode, we convert to the floating - // point type, since it can contain a larger range of values than any decimal (but - // has lower precision in some parts of its range), so it is generally better. - // In strict mode, we avoid conversion in either direction because there are also - // decimal values (e.g. 0.1) that cannot be exactly represented in binary - // floating point. - // TODO: it might make sense to promote to double in many cases, but this would - // require more work elsewhere to avoid breaking things, e.g. inserting decimal - // literals into float columns. - if (t1.isFloatingPointType()) return strict ? INVALID : t1; - if (t2.isFloatingPointType()) return strict ? INVALID : t2; + if (result == PrimitiveType.VARCHAR) { + return createVarcharType(Math.max(t1.len_, t2.len_)); + } + if (result == PrimitiveType.CHAR) { + return createCharType(Math.max(t1.len_, t2.len_)); + } + + if (result == PrimitiveType.DECIMAL) { // Allow casts between decimal and numeric types by converting // numeric types to the containing decimal type. ScalarType t1Decimal = t1.getMinResolutionDecimal(); @@ -517,36 +505,23 @@ public class ScalarType extends Type { } if (t1Decimal.isSupertypeOf(t2Decimal)) return t1; if (t2Decimal.isSupertypeOf(t1Decimal)) return t2; + return TypesUtil.getDecimalAssignmentCompatibleType( - t1Decimal, t2Decimal, strictDecimal); + t1Decimal, t2Decimal, compatibility.isStrictDecimal()); } - PrimitiveType smallerType = - (t1.type_.ordinal() < t2.type_.ordinal() ? t1.type_ : t2.type_); - PrimitiveType largerType = - (t1.type_.ordinal() > t2.type_.ordinal() ? t1.type_ : t2.type_); - PrimitiveType result = null; - if (strict) { - result = strictCompatibilityMatrix[smallerType.ordinal()][largerType.ordinal()]; - } - if (result == null) { - result = compatibilityMatrix[smallerType.ordinal()][largerType.ordinal()]; - } Preconditions.checkNotNull(result); return createType(result); } /** - * Returns true t1 can be implicitly cast to t2, false otherwise. + * According to 'compatibility', returns true if t1 can be implicitly cast to t2, false + * otherwise. * - * If strictDecimal is true, only consider casts that result in no loss of information - * when casting between decimal types. - * If strict is true, only consider casts that result in no loss of information when - * casting between any two types other than both decimals. */ - public static boolean isImplicitlyCastable(ScalarType t1, ScalarType t2, - boolean strict, boolean strictDecimal) { - return getAssignmentCompatibleType(t1, t2, strict, strictDecimal).matchesType(t2); + public static boolean isImplicitlyCastable( + ScalarType t1, ScalarType t2, TypeCompatibility compatibility) { + return getAssignmentCompatibleType(t1, t2, compatibility).matchesType(t2); } /** @@ -554,6 +529,6 @@ public class ScalarType extends Type { * wide as dest.) */ public static boolean isAssignable(ScalarType dest, ScalarType source) { - return isImplicitlyCastable(source, dest, false, false); + return isImplicitlyCastable(source, dest, TypeCompatibility.DEFAULT); } } diff --git a/fe/src/main/java/org/apache/impala/catalog/StrictOverrideCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/StrictOverrideCompatibility.java new file mode 100644 index 000000000..57125e427 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/StrictOverrideCompatibility.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.apache.impala.catalog.Type.BIGINT; +import static org.apache.impala.catalog.Type.DATE; +import static org.apache.impala.catalog.Type.DECIMAL; +import static org.apache.impala.catalog.Type.DOUBLE; +import static org.apache.impala.catalog.Type.FLOAT; +import static org.apache.impala.catalog.Type.INT; +import static org.apache.impala.catalog.Type.STRING; +import static org.apache.impala.catalog.Type.TIMESTAMP; + +public class StrictOverrideCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + matrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; + matrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; + matrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TIMESTAMP.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[TIMESTAMP.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + // The case of decimal and float/double must be handled carefully. There are two + // modes: strict and non-strict. In non-strict mode, we convert to the floating + // point type, since it can contain a larger range of values than any decimal (but + // has lower precision in some parts of its range), so it is generally better. + // In strict mode, we avoid conversion in either direction because there are also + // decimal values (e.g. 0.1) that cannot be exactly represented in binary + // floating point. + // TODO: it might make sense to promote to double in many cases, but this would + // require more work elsewhere to avoid breaking things, e.g. inserting decimal + // literals into float columns. + matrix[DOUBLE.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + matrix[FLOAT.ordinal()][DECIMAL.ordinal()] = PrimitiveType.INVALID_TYPE; + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java b/fe/src/main/java/org/apache/impala/catalog/Type.java index c3935efd5..b0952f0da 100644 --- a/fe/src/main/java/org/apache/impala/catalog/Type.java +++ b/fe/src/main/java/org/apache/impala/catalog/Type.java @@ -339,43 +339,33 @@ public abstract class Type { * Implicit casts are always allowed when no loss of information would result (i.e. * every value of t1 can be represented exactly by a value of t2). Implicit casts are * allowed in certain other cases such as casting numeric types to floating point types - * and converting strings to timestamps. - * - * If strictDecimal is true, only consider casts that result in no loss of information - * when casting between decimal types. - * If strict is true, only consider casts that result in no loss of information when - * casting between any two types other than both decimals. + * and converting strings to timestamps. 'compatibility' defines the mode for type + * compatibility calculation. * * TODO: Support casting of non-scalar types. */ public static boolean isImplicitlyCastable( - Type t1, Type t2, boolean strict, boolean strictDecimal) { + Type t1, Type t2, TypeCompatibility compatibility) { if (t1.isScalarType() && t2.isScalarType()) { return ScalarType.isImplicitlyCastable( - (ScalarType) t1, (ScalarType) t2, strict, strictDecimal); + (ScalarType) t1, (ScalarType) t2, compatibility); } return false; } /** * Return type t such that values from both t1 and t2 can be assigned to t without an - * explicit cast. If strict, does not consider conversions that would result in loss - * of precision (e.g. converting decimal to float). Returns INVALID_TYPE if there is - * no such type or if any of t1 and t2 is INVALID_TYPE. - * - * If strictDecimal is true, only consider casts that result in no loss of information - * when casting between decimal types. - * If strict is true, only consider casts that result in no loss of information when - * casting between any two types other than both decimals. - * + * explicit cast. 'compatibility' defines the mode for type compatibility calculation + * and strictness. Returns INVALID_TYPE if there is no such type or if any of t1 and t2 + * is INVALID_TYPE. * * TODO: Support struct types. */ public static Type getAssignmentCompatibleType( - Type t1, Type t2, boolean strict, boolean strictDecimal) { + Type t1, Type t2, TypeCompatibility compatibility) { if (t1.isScalarType() && t2.isScalarType()) { return ScalarType.getAssignmentCompatibleType( - (ScalarType) t1, (ScalarType) t2, strict, strictDecimal); + (ScalarType) t1, (ScalarType) t2, compatibility); } else if (t1.isArrayType() && t2.isArrayType()) { // Only support exact match for array types. if (t1.equals(t2)) return t2; @@ -683,6 +673,8 @@ public abstract class Type { * conservative casting in arithmetic exprs: TINYINT + TINYINT -> BIGINT * comparison of many types as double: INT < FLOAT -> comparison as DOUBLE * special cases when dealing with dates and timestamps. + * Types are looked up in specific order ("smaller" first) in + * 'getAssignmentCompatibleType()'. */ protected static PrimitiveType[][] compatibilityMatrix; @@ -692,185 +684,42 @@ public abstract class Type { * is valid. */ protected static PrimitiveType[][] strictCompatibilityMatrix; + /** + * If unsafe mode is enabled, 'unsafeCompatibilityMatrix' is used for lookup, deciding + * whether a conversion in the given direction is allowed. This matrix is not + * symmetrical, lookups with different orders could yield different results, for + * example: STRING, TINYINT would result in TINYINT, but TINYINT, STRING will result in + * STRING. This is required for enabling implicit casting from numeric types to string + * types and vice-versa. + */ + protected static PrimitiveType[][] unsafeCompatibilityMatrix; static { - compatibilityMatrix = new - PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length]; - strictCompatibilityMatrix = new - PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length]; - - for (int i = 0; i < PrimitiveType.values().length; ++i) { - // Each type is compatible with itself. - compatibilityMatrix[i][i] = PrimitiveType.values()[i]; - - if (i != BINARY.ordinal() && i != STRING.ordinal()) { - // BINARY can be only cast to / from STRING. - compatibilityMatrix[BINARY.ordinal()][i] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[i][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE; - } - - // FIXED_UDA_INTERMEDIATE cannot be cast to/from another type - if (i != FIXED_UDA_INTERMEDIATE.ordinal()) { - compatibilityMatrix[FIXED_UDA_INTERMEDIATE.ordinal()][i] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[i][FIXED_UDA_INTERMEDIATE.ordinal()] = - PrimitiveType.INVALID_TYPE; - } - } + List<CompatibilityRule> defaultCompatibilityRules = new ArrayList<>(); + defaultCompatibilityRules.add(new DiagonalCompatibility()); + defaultCompatibilityRules.add(new BinaryCompatibility()); + defaultCompatibilityRules.add(new FixedUdaCompatibility()); + defaultCompatibilityRules.add(new DefaultCompatibility()); + defaultCompatibilityRules.add(new CheckEmptyCompatibility()); + compatibilityMatrix = getCompatibilityMatrix(defaultCompatibilityRules); + + List<CompatibilityRule> strictCompatibilityRules = new ArrayList<>(); + strictCompatibilityRules.add(new StrictOverrideCompatibility()); + strictCompatibilityMatrix = getCompatibilityMatrix(strictCompatibilityRules); + + List<CompatibilityRule> unsafeCompatibilityRules = + new ArrayList<>(defaultCompatibilityRules); + unsafeCompatibilityRules.add( + unsafeCompatibilityRules.size() - 1, new UnsafeCompatibility()); + unsafeCompatibilityMatrix = getCompatibilityMatrix(unsafeCompatibilityRules); + } - compatibilityMatrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; - compatibilityMatrix[BOOLEAN.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; - compatibilityMatrix[BOOLEAN.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[BOOLEAN.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - compatibilityMatrix[BOOLEAN.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; - compatibilityMatrix[TINYINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[TINYINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 8 bit integer fits in mantissa of both float and double. - compatibilityMatrix[TINYINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT; - compatibilityMatrix[SMALLINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 16 bit integer fits in mantissa of both float and double. - compatibilityMatrix[SMALLINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; - // 32 bit integer fits only mantissa of double. - // TODO: arguably we should promote INT + FLOAT to DOUBLE to avoid loss of precision, - // but we depend on it remaining FLOAT for some use cases, e.g. - // "insert into tbl (float_col) select int_col + float_col from ..." - compatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - strictCompatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - // 64 bit integer does not fit in mantissa of double or float. - // TODO: arguably we should always promote BIGINT + FLOAT to double here to keep as - // much precision as possible, but we depend on this implicit cast for some use - // cases, similarly to INT + FLOAT. - compatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; - strictCompatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE; - // TODO: we're breaking the definition of strict compatibility for BIGINT + DOUBLE, - // but this forces function overloading to consider the DOUBLE overload first. - compatibilityMatrix[BIGINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[BIGINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; - compatibilityMatrix[FLOAT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - // We can convert some but not all string values to date. - compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.DATE; - strictCompatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[DATETIME.ordinal()][TIMESTAMP.ordinal()] = - PrimitiveType.TIMESTAMP; - compatibilityMatrix[DATETIME.ordinal()][DATE.ordinal()] = PrimitiveType.DATETIME; - compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - - // We can convert some but not all date values to timestamps. - compatibilityMatrix[TIMESTAMP.ordinal()][DATE.ordinal()] = PrimitiveType.TIMESTAMP; - strictCompatibilityMatrix[TIMESTAMP.ordinal()][DATE.ordinal()] = - PrimitiveType.INVALID_TYPE; - // We can convert some but not all string values to timestamps. - compatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] = - PrimitiveType.TIMESTAMP; - strictCompatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TIMESTAMP.ordinal()][VARCHAR.ordinal()] = - PrimitiveType.INVALID_TYPE; - compatibilityMatrix[TIMESTAMP.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[STRING.ordinal()][VARCHAR.ordinal()] = PrimitiveType.STRING; - compatibilityMatrix[STRING.ordinal()][CHAR.ordinal()] = PrimitiveType.STRING; - // STRING <-> BINARY conversion is not lossy, but implicit cast is not allowed to - // enforce exact type match in function calls. - compatibilityMatrix[STRING.ordinal()][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE; - strictCompatibilityMatrix[STRING.ordinal()][BINARY.ordinal()] - = PrimitiveType.INVALID_TYPE; - - compatibilityMatrix[VARCHAR.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; - - // Check all of the necessary entries that should be filled. - for (int i = 0; i < PrimitiveType.values().length; ++i) { - for (int j = i; j < PrimitiveType.values().length; ++j) { - PrimitiveType t1 = PrimitiveType.values()[i]; - PrimitiveType t2 = PrimitiveType.values()[j]; - // DECIMAL, NULL, and INVALID_TYPE are handled separately. - if (t1 == PrimitiveType.INVALID_TYPE || - t2 == PrimitiveType.INVALID_TYPE) continue; - if (t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE) continue; - if (t1 == PrimitiveType.DECIMAL || t2 == PrimitiveType.DECIMAL) continue; - Preconditions.checkNotNull(compatibilityMatrix[i][j]); - } + public static PrimitiveType[][] getCompatibilityMatrix(List<CompatibilityRule> rules) { + PrimitiveType[][] compatibilityMatrix = + new PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length]; + for (CompatibilityRule rule : rules) { + rule.apply(compatibilityMatrix); } + return compatibilityMatrix; } } diff --git a/fe/src/main/java/org/apache/impala/catalog/TypeCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/TypeCompatibility.java new file mode 100644 index 000000000..3f5fbd955 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/TypeCompatibility.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import com.google.common.base.Preconditions; +/** + Enumeration to decide the mode of the compatibility lookup for type comparisons. + */ +public enum TypeCompatibility { + /** + * Allow implicit casts between string types (STRING, VARCHAR, CHAR) and between numeric + * types and string types. From STRING to [VAR]CHAR types truncation may happen. + */ + UNSAFE, + /** + * Baseline compatibility between types + */ + DEFAULT, + /** + * Only consider casts that result in no loss of information when casting between + * decimal types. + */ + STRICT_DECIMAL, + /** + * Only consider casts that result in no loss of information when casting between any + * two types other than both decimals + */ + STRICT, + /** + * only consider casts that result in no loss of information when casting between any + * types + */ + ALL_STRICT; + + public static TypeCompatibility applyStrictDecimal(TypeCompatibility compatibility) { + switch (compatibility) { + case DEFAULT: + case STRICT_DECIMAL: return STRICT_DECIMAL; + case STRICT: + case ALL_STRICT: return ALL_STRICT; + case UNSAFE: return UNSAFE; + default: + // Unreachable state + Preconditions.checkState(false); + return compatibility; + } + } + + public boolean isDefault() { return this.equals(DEFAULT); } + + public boolean isUnsafe() { return this.equals(UNSAFE); } + + public boolean isStrict() { return this.equals(STRICT) || this.equals(ALL_STRICT); } + + public boolean isStrictDecimal() { + return this.equals(STRICT_DECIMAL) || this.equals(ALL_STRICT); + } +} diff --git a/fe/src/main/java/org/apache/impala/catalog/UnsafeCompatibility.java b/fe/src/main/java/org/apache/impala/catalog/UnsafeCompatibility.java new file mode 100644 index 000000000..d8544d72a --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/UnsafeCompatibility.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.apache.impala.catalog.Type.BIGINT; +import static org.apache.impala.catalog.Type.CHAR; +import static org.apache.impala.catalog.Type.DOUBLE; +import static org.apache.impala.catalog.Type.FLOAT; +import static org.apache.impala.catalog.Type.INT; +import static org.apache.impala.catalog.Type.SMALLINT; +import static org.apache.impala.catalog.Type.STRING; +import static org.apache.impala.catalog.Type.TINYINT; +import static org.apache.impala.catalog.Type.VARCHAR; + +public class UnsafeCompatibility implements CompatibilityRule { + @Override + public void apply(PrimitiveType[][] matrix) { + // Numeric types to STRING + matrix[TINYINT.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[SMALLINT.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + + // STRING to numeric types + matrix[STRING.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; + matrix[STRING.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; + matrix[STRING.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[STRING.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + matrix[STRING.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[STRING.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + + // Numeric types to VARCHAR + matrix[TINYINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + matrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + + // VARCHAR to numeric types + matrix[VARCHAR.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; + matrix[VARCHAR.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; + matrix[VARCHAR.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[VARCHAR.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + matrix[VARCHAR.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[VARCHAR.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + + // Numeric types to CHAR + matrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + matrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + matrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + matrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + matrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + matrix[DOUBLE.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; + + // CHAR to numeric types + matrix[CHAR.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT; + matrix[CHAR.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT; + matrix[CHAR.ordinal()][INT.ordinal()] = PrimitiveType.INT; + matrix[CHAR.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT; + matrix[CHAR.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT; + matrix[CHAR.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; + + // CHAR, VARCHAR to STRING + matrix[CHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + matrix[VARCHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; + } +} diff --git a/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java b/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java index d521bc49c..de04ce023 100644 --- a/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java +++ b/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java @@ -85,7 +85,7 @@ public class HashJoinNode extends JoinNode { t0.toSql() + " to " + t1.toSql() + " in join predicate."); } Type compatibleType = Type.getAssignmentCompatibleType( - t0, t1, false, analyzer.isDecimalV2()); + t0, t1, analyzer.getRegularCompatibilityLevel()); if (compatibleType.isInvalid()) { throw new InternalException(String.format( "Unable create a hash join with equi-join predicate %s " + diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java index 568ae3582..8d74d380b 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java @@ -42,7 +42,9 @@ import org.apache.impala.catalog.ScalarType; import org.apache.impala.catalog.Table; import org.apache.impala.catalog.TestSchemaUtils; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.TypeCompatibility; import org.apache.impala.common.AnalysisException; +import org.apache.impala.common.Pair; import org.apache.impala.thrift.TExpr; import org.apache.impala.thrift.TFunction; import org.apache.impala.thrift.TFunctionBinaryType; @@ -1315,7 +1317,8 @@ public class AnalyzeExprsTest extends AnalyzerTest { Type.BIGINT, Type.FLOAT, Type.DOUBLE , Type.NULL }; for (Type type1: numericTypes) { for (Type type2: numericTypes) { - Type t = Type.getAssignmentCompatibleType(type1, type2, false, false); + Type t = + Type.getAssignmentCompatibleType(type1, type2, TypeCompatibility.DEFAULT); assertTrue(t.isScalarType()); ScalarType compatibleType = (ScalarType) t; Type promotedType = compatibleType.getNextResolutionType(); @@ -1398,10 +1401,10 @@ public class AnalyzeExprsTest extends AnalyzerTest { for (Type type2: types) { // Prefer strict matching. Type compatibleType = Type.getAssignmentCompatibleType( - type1, type2, true, true); + type1, type2, TypeCompatibility.ALL_STRICT); if (compatibleType.isInvalid()) { - compatibleType = Type.getAssignmentCompatibleType( - type1, type2, false, false); + compatibleType = + Type.getAssignmentCompatibleType(type1, type2, TypeCompatibility.DEFAULT); } typeCastTest(type1, type2, false, null, cmpOp, compatibleType); typeCastTest(type1, type2, true, null, cmpOp, compatibleType); @@ -3356,4 +3359,44 @@ public class AnalyzeExprsTest extends AnalyzerTest { Assert.assertEquals(expected_str, select.getResultExprs().get(0).toSqlImpl()); } + @Test + public void testUnsafeCasts() { + AnalysisContext unsafeCtx = createAnalysisCtx(); + unsafeCtx.getQueryOptions().setAllow_unsafe_casts(true); + + String[] numericTypes = {"tinyint", "smallint", "int", "bigint", "float", "double"}; + Pair<String, String>[] stringTypes = + new Pair[] {Pair.create("string", "alltypesnopart"), + Pair.create("varchar", "chars_medium"), Pair.create("char", "chars_medium")}; + + for (String numericType : numericTypes) { + for (Pair<String, String> stringType : stringTypes) { + String numericToStringStatement = + String.format("insert into functional.%s(%s_col) values(cast(100 as %s))", + stringType.second, stringType.first, numericType); + String stringToNumericStatement = String.format( + "insert into functional.alltypesnopart(%s_col) values(\"100\")", numericType); + String nonConstStatement = String.format( + "insert into functional.alltypesnopart(%s_col) select string_col " + + "from functional.alltypes", + numericType); + + // Constant values are allowed + AnalyzesOk(numericToStringStatement, unsafeCtx); + AnalyzesOk(stringToNumericStatement, unsafeCtx); + // Non-constant values are not allowed + AnalysisError(nonConstStatement, unsafeCtx, + "Unsafe implicit cast is prohibited for non-const expression: string_col"); + } + } + + // Decimal is not allowed + AnalysisError( + "insert into functional.alltypesnopart(string_col) values (cast(100 as decimal))", + unsafeCtx, + "Target table 'functional.alltypesnopart' is incompatible with " + + "source expressions.\nExpression 'cast(100 as decimal(9,0))' " + + "(type: DECIMAL(9,0)) is not compatible with column " + + "'string_col' (type: STRING)"); + } } diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java index bc06cb19e..c453c0aef 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java @@ -3561,8 +3561,8 @@ public class AnalyzeStmtsTest extends AnalyzerTest { // Regression test for IMPALA-12042: Transitive compatibility is not // allowed (boolean -> tinyint -> decimal(4,1)) AnalysisError("values (true), (123), (111.0)", - "Incompatible return types 'DECIMAL(4,1)' and 'BOOLEAN'" - + " of exprs '111.0' and 'TRUE'"); + "Incompatible return types 'BOOLEAN' and 'DECIMAL(4,1)'" + + " of exprs 'TRUE' and '111.0'"); } @Test diff --git a/fe/src/test/java/org/apache/impala/analysis/TypesUtilTest.java b/fe/src/test/java/org/apache/impala/analysis/TypesUtilTest.java index 872323ed7..7f0da73bd 100644 --- a/fe/src/test/java/org/apache/impala/analysis/TypesUtilTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/TypesUtilTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.List; +import org.apache.impala.catalog.TypeCompatibility; import org.junit.Assert; import org.junit.Test; @@ -129,56 +130,56 @@ public class TypesUtilTest extends AnalyzerTest { public void TestNumericImplicitCast() { // Decimals can be cast to integers if there is no loss of precision. Assert.assertTrue(Type.isImplicitlyCastable( - ScalarType.createDecimalType(2, 0), Type.TINYINT, false, false)); + ScalarType.createDecimalType(2, 0), Type.TINYINT, TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - ScalarType.createDecimalType(4, 0), Type.SMALLINT, false, false)); + ScalarType.createDecimalType(4, 0), Type.SMALLINT, TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - ScalarType.createDecimalType(9, 0), Type.INT, false, false)); + ScalarType.createDecimalType(9, 0), Type.INT, TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - ScalarType.createDecimalType(18, 0), Type.BIGINT, false, false)); + ScalarType.createDecimalType(18, 0), Type.BIGINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(3, 0), Type.TINYINT, false, false)); + ScalarType.createDecimalType(3, 0), Type.TINYINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(5, 0), Type.SMALLINT, false, false)); + ScalarType.createDecimalType(5, 0), Type.SMALLINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(10, 0), Type.INT, false, false)); + ScalarType.createDecimalType(10, 0), Type.INT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(19, 0), Type.BIGINT, false, false)); + ScalarType.createDecimalType(19, 0), Type.BIGINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(2, 1), Type.TINYINT, false, false)); + ScalarType.createDecimalType(2, 1), Type.TINYINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(4, 1), Type.SMALLINT, false, false)); + ScalarType.createDecimalType(4, 1), Type.SMALLINT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(2, 1), Type.INT, false, false)); + ScalarType.createDecimalType(2, 1), Type.INT, TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - ScalarType.createDecimalType(18, 5), Type.BIGINT, false, false)); + ScalarType.createDecimalType(18, 5), Type.BIGINT, TypeCompatibility.DEFAULT)); // Integers are only converted to decimal when all values of the source type can be // represented in the destination type. Assert.assertFalse(Type.isImplicitlyCastable( - Type.TINYINT, ScalarType.createDecimalType(2, 0), false, false)); + Type.TINYINT, ScalarType.createDecimalType(2, 0), TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - Type.SMALLINT, ScalarType.createDecimalType(4, 0), false, false)); + Type.SMALLINT, ScalarType.createDecimalType(4, 0), TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - Type.INT, ScalarType.createDecimalType(9, 0), false, false)); + Type.INT, ScalarType.createDecimalType(9, 0), TypeCompatibility.DEFAULT)); Assert.assertFalse(Type.isImplicitlyCastable( - Type.BIGINT, ScalarType.createDecimalType(18, 0), false, false)); + Type.BIGINT, ScalarType.createDecimalType(18, 0), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.TINYINT, ScalarType.createDecimalType(3, 0), false, false)); + Type.TINYINT, ScalarType.createDecimalType(3, 0), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.SMALLINT, ScalarType.createDecimalType(5, 0), false, false)); + Type.SMALLINT, ScalarType.createDecimalType(5, 0), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.INT, ScalarType.createDecimalType(10, 0), false, false)); + Type.INT, ScalarType.createDecimalType(10, 0), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.BIGINT, ScalarType.createDecimalType(19, 0), false, false)); + Type.BIGINT, ScalarType.createDecimalType(19, 0), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.TINYINT, ScalarType.createDecimalType(4, 1), false, false)); + Type.TINYINT, ScalarType.createDecimalType(4, 1), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.SMALLINT, ScalarType.createDecimalType(6, 1), false, false)); + Type.SMALLINT, ScalarType.createDecimalType(6, 1), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.INT, ScalarType.createDecimalType(11, 1), false, false)); + Type.INT, ScalarType.createDecimalType(11, 1), TypeCompatibility.DEFAULT)); Assert.assertTrue(Type.isImplicitlyCastable( - Type.BIGINT, ScalarType.createDecimalType(20, 1), false, false)); + Type.BIGINT, ScalarType.createDecimalType(20, 1), TypeCompatibility.DEFAULT)); // Only promotions are allowed for integer types. List<Type> intTypes = Arrays.<Type>asList(Type.TINYINT, Type.SMALLINT, Type.INT, @@ -186,35 +187,45 @@ public class TypesUtilTest extends AnalyzerTest { for (Type t1: intTypes) { for (Type t2: intTypes) { if (t1.getSlotSize() == t2.getSlotSize()) { - Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, true, false)); - Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, false, false)); + Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, TypeCompatibility.STRICT)); + Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, TypeCompatibility.DEFAULT)); } else if (t1.getSlotSize() < t2.getSlotSize()) { - Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, true, false)); - Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(t2, t1, true, false)); - Assert.assertFalse(Type.isImplicitlyCastable(t2, t1, false, false)); + Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, TypeCompatibility.STRICT)); + Assert.assertTrue(Type.isImplicitlyCastable(t1, t2, TypeCompatibility.DEFAULT)); + Assert.assertFalse(Type.isImplicitlyCastable(t2, t1, TypeCompatibility.STRICT)); + Assert.assertFalse( + Type.isImplicitlyCastable(t2, t1, TypeCompatibility.DEFAULT)); } else { - Assert.assertFalse(Type.isImplicitlyCastable(t1, t2, true, false)); - Assert.assertFalse(Type.isImplicitlyCastable(t1, t2, false, false)); - Assert.assertTrue(Type.isImplicitlyCastable(t2, t1, true, false)); - Assert.assertTrue(Type.isImplicitlyCastable(t2, t1, false, false)); + Assert.assertFalse(Type.isImplicitlyCastable(t1, t2, TypeCompatibility.STRICT)); + Assert.assertFalse( + Type.isImplicitlyCastable(t1, t2, TypeCompatibility.DEFAULT)); + Assert.assertTrue(Type.isImplicitlyCastable(t2, t1, TypeCompatibility.STRICT)); + Assert.assertTrue(Type.isImplicitlyCastable(t2, t1, TypeCompatibility.DEFAULT)); } } } // Only promotions are allowed for floating point types. - Assert.assertTrue(Type.isImplicitlyCastable(Type.FLOAT, Type.FLOAT, true, false)); - Assert.assertFalse(Type.isImplicitlyCastable(Type.DOUBLE, Type.FLOAT, false, false)); - Assert.assertTrue(Type.isImplicitlyCastable(Type.FLOAT, Type.DOUBLE, false, false)); - Assert.assertTrue(Type.isImplicitlyCastable(Type.FLOAT, Type.DOUBLE, true, false)); + Assert.assertTrue( + Type.isImplicitlyCastable(Type.FLOAT, Type.FLOAT, TypeCompatibility.STRICT)); + Assert.assertFalse( + Type.isImplicitlyCastable(Type.DOUBLE, Type.FLOAT, TypeCompatibility.DEFAULT)); + Assert.assertTrue( + Type.isImplicitlyCastable(Type.FLOAT, Type.DOUBLE, TypeCompatibility.DEFAULT)); + Assert.assertTrue( + Type.isImplicitlyCastable(Type.FLOAT, Type.DOUBLE, TypeCompatibility.STRICT)); // Decimal is convertible to a floating point types only in non-strict mode. List<ScalarType> dts = Arrays.asList(ScalarType.createDecimalType(30, 10), ScalarType.createDecimalType(2, 0)); for (Type dt: dts) { - Assert.assertFalse(Type.isImplicitlyCastable(dt, Type.FLOAT, true, false)); - Assert.assertTrue(Type.isImplicitlyCastable(dt, Type.FLOAT, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(dt, Type.DOUBLE, true, false)); - Assert.assertTrue(Type.isImplicitlyCastable(dt, Type.DOUBLE, false, false)); + Assert.assertFalse( + Type.isImplicitlyCastable(dt, Type.FLOAT, TypeCompatibility.STRICT)); + Assert.assertTrue( + Type.isImplicitlyCastable(dt, Type.FLOAT, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(dt, Type.DOUBLE, TypeCompatibility.STRICT)); + Assert.assertTrue( + Type.isImplicitlyCastable(dt, Type.DOUBLE, TypeCompatibility.DEFAULT)); } } @@ -222,16 +233,24 @@ public class TypesUtilTest extends AnalyzerTest { // Test that we don't allow casting to/from complex types. public void TestComplexImplicitCast() { ArrayType arrayType = new ArrayType(Type.INT); - Assert.assertFalse(Type.isImplicitlyCastable(Type.INT, arrayType, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(arrayType, Type.INT, false, false)); + Assert.assertFalse( + Type.isImplicitlyCastable(Type.INT, arrayType, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(arrayType, Type.INT, TypeCompatibility.DEFAULT)); MapType mapType = new MapType(Type.STRING, Type.INT); - Assert.assertFalse(Type.isImplicitlyCastable(Type.INT, mapType, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(mapType, Type.INT, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(mapType, arrayType, false, false)); + Assert.assertFalse( + Type.isImplicitlyCastable(Type.INT, mapType, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(mapType, Type.INT, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(mapType, arrayType, TypeCompatibility.DEFAULT)); StructType structType = new StructType(Lists.newArrayList( new StructField("foo", Type.FLOAT, ""), new StructField("bar", Type.FLOAT, ""))); - Assert.assertFalse(Type.isImplicitlyCastable(structType, Type.INT, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(Type.INT, structType, false, false)); - Assert.assertFalse(Type.isImplicitlyCastable(arrayType, structType, false, false)); + Assert.assertFalse( + Type.isImplicitlyCastable(structType, Type.INT, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(Type.INT, structType, TypeCompatibility.DEFAULT)); + Assert.assertFalse( + Type.isImplicitlyCastable(arrayType, structType, TypeCompatibility.DEFAULT)); } } diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert-unsafe.test b/testdata/workloads/functional-query/queries/QueryTest/insert-unsafe.test new file mode 100644 index 000000000..05e588632 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/insert-unsafe.test @@ -0,0 +1,213 @@ +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select 1.0 union select int_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: int_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select 1 union select int_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: int_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select int_col from unsafe_insert union select 1; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: int_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select 1 union select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select 1 union select 2; +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select 1; +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (10.0); +---- CATCH +AnalysisException: Target table '$DATABASE.unsafe_insert' is incompatible with source expressions. +Expression '10.0' (type: DECIMAL(3,1)) is not compatible with column 'string_col' (type: STRING) +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (100), (1000), (1000.0); +---- CATCH +AnalysisException: Target table '$DATABASE.unsafe_insert' is incompatible with source expressions. +Expression '100' (type: DECIMAL(6,1)) is not compatible with column 'string_col' (type: STRING) +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as TINYINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as SMALLINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as FLOAT)); +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as DOUBLE)); +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as INT)); +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) values (cast(100 as BIGINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as TINYINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as SMALLINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as FLOAT)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as DOUBLE)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as INT)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast(100 as BIGINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as TINYINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as SMALLINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as FLOAT)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as DOUBLE)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as INT)); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast(100 as BIGINT)); +==== +---- QUERY +INSERT INTO unsafe_insert(float_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(bigint_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(smallint_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(tinyint_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(int_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(double_col) values ("100"); +==== +---- QUERY +INSERT INTO unsafe_insert(float_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(bigint_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(smallint_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(tinyint_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(int_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(double_col) values (cast("100" as VARCHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(float_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(bigint_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(smallint_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(tinyint_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(int_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(double_col) values (cast("100" as CHAR(10))); +==== +---- QUERY +INSERT INTO unsafe_insert(char_col) values (cast("100" as STRING)); +==== +---- QUERY +INSERT INTO unsafe_insert(varchar_col) values (cast("100" as STRING)); +==== +---- QUERY +INSERT INTO unsafe_insert(bigint_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select smallint_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: smallint_col +==== +---- QUERY +INSERT INTO unsafe_insert(tinyint_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select tinyint_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: tinyint_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select int_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: int_col +==== +---- QUERY +INSERT INTO unsafe_insert(int_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(smallint_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(float_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select double_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: double_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select bigint_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: bigint_col +==== +---- QUERY +INSERT INTO unsafe_insert(string_col) select float_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: float_col +==== +---- QUERY +INSERT INTO unsafe_insert(double_col) select string_col from unsafe_insert; +---- CATCH +AnalysisException: Unsafe implicit cast is prohibited for non-const expression: string_col + diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py index d70af89da..c3cb270b2 100644 --- a/tests/query_test/test_insert.py +++ b/tests/query_test/test_insert.py @@ -535,3 +535,26 @@ class TestInsertNonPartitionedTable(ImpalaTestSuite): .format(table_name), exec_options) result = self.client.execute("select f0 from {0}".format(table_name)) assert result.data == ["2"] + + +class TestUnsafeImplicitCasts(ImpalaTestSuite): + """Test to check 'allow_unsafe_casts' query-option behaviour on insert statements.""" + @classmethod + def get_workload(cls): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestUnsafeImplicitCasts, cls).add_test_dimensions() + cls.ImpalaTestMatrix.add_constraint(lambda v: + (v.get_value('table_format').file_format == 'parquet')) + + def test_unsafe_insert(self, vector, unique_database): + create_stmt = """create table {0}.unsafe_insert(tinyint_col tinyint, + smallint_col smallint, int_col int, bigint_col bigint, float_col float, + double_col double, decimal_col decimal, timestamp_col timestamp, date_col date, + string_col string, varchar_col varchar(100), char_col char(100), + bool_col boolean, binary_col binary)""".format(unique_database) + self.client.execute(create_stmt) + vector.get_value('exec_option')['allow_unsafe_casts'] = "true" + self.run_test_case('QueryTest/insert-unsafe', vector, unique_database)
