This is an automated email from the ASF dual-hosted git repository. csringhofer pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a6bb0c7c45a4b4788cb8ca737da6b115f86558ca Author: Steve Carlin <[email protected]> AuthorDate: Fri Sep 5 08:22:04 2025 -0700 IMPALA-14408: Use regular path for Calcite planner instead of CalciteJniFrontend When the --use_calcite_planner=true option is set at the server level, the queries will no longer go through CalciteJniFrontend. Instead, they will go through the regular JniFrontend, which is the path that is used when the query option for "use_calcite_planner" is set. The CalciteJniFrontend will be removed in a later commit. This commit also enables fallback to the original planner when an unsupported feature exception is thrown. This needed to be added to allow the tests to run properly. During initial database load, there are queries that access complex columns which throws the unsupported exception. Change-Id: I732516ca8f7ea64f73484efd67071910c9b62c8f Reviewed-on: http://gerrit.cloudera.org:8080/23523 Reviewed-by: Steve Carlin <[email protected]> Tested-by: Steve Carlin <[email protected]> --- be/src/service/impala-server.cc | 3 + bin/start-impala-cluster.py | 4 +- .../java/org/apache/impala/service/Frontend.java | 10 +- .../calcite/service/CalciteAnalysisDriver.java | 10 ++ .../impala/calcite/service/CalciteQueryParser.java | 9 ++ .../impala/calcite/service/UnsupportedChecker.java | 101 +++++++++++++++++++++ 6 files changed, 131 insertions(+), 6 deletions(-) diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index 7b9b639ab..40409a252 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -278,6 +278,8 @@ DEFINE_validator(ssl_minimum_version, [](const char* flagname, const string& val return false; }); +DEFINE_bool(use_calcite_planner, false, "By default this flag is false. If true, " + "the Calcite planner will be used to compile queries."); DEFINE_int32(idle_session_timeout, 0, "The time, in seconds, that a session may be idle" " for before it is closed (and all running queries cancelled) by Impala. If 0, idle" " sessions are never expired. It can be overridden by the query option" @@ -2018,6 +2020,7 @@ void ImpalaServer::InitializeConfigVariables() { // Set idle_session_timeout here to let the SET command return the value of // the command line option FLAGS_idle_session_timeout default_query_options_.__set_idle_session_timeout(FLAGS_idle_session_timeout); + default_query_options_.__set_use_calcite_planner(FLAGS_use_calcite_planner); // The next query options used to be set with flags. Setting them in // default_query_options_ here in order to make default_query_options // take precedence over the legacy flags. diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py index 6dad0c923..e6042417f 100755 --- a/bin/start-impala-cluster.py +++ b/bin/start-impala-cluster.py @@ -703,9 +703,7 @@ def build_impalad_arg_lists(cluster_size, num_coordinators, use_exclusive_coordi args = "-allow_tuple_caching=true {args}".format(args=args) if options.use_calcite_planner.lower() == 'true': - args = "-jni_frontend_class={jni_frontend_class} {args}".format( - jni_frontend_class="org/apache/impala/calcite/service/CalciteJniFrontend", - args=args) + args = "-use_calcite_planner=true {args}".format(args=args) os.environ["USE_CALCITE_PLANNER"] = "true" if options.enable_ranger_authz: diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java index 781fba6bb..ee209c487 100644 --- a/fe/src/main/java/org/apache/impala/service/Frontend.java +++ b/fe/src/main/java/org/apache/impala/service/Frontend.java @@ -149,6 +149,7 @@ import org.apache.impala.catalog.iceberg.IcebergMetadataTable; import org.apache.impala.catalog.paimon.FePaimonTable; import org.apache.impala.catalog.paimon.FeShowFileStmtSupport; import org.apache.impala.common.AnalysisException; +import org.apache.impala.common.UnsupportedFeatureException; import org.apache.impala.common.UserCancelledException; import org.apache.impala.common.FileSystemUtil; import org.apache.impala.common.ImpalaException; @@ -2399,7 +2400,7 @@ public class Frontend { try { request = getTExecRequest(compilerFactory, planCtx, timeline); } catch (Exception e) { - if (!shouldFallbackToRegularPlanner(planCtx)) { + if (!shouldFallbackToRegularPlanner(planCtx, e)) { throw e; } LOG.info("Calcite planner failed: ", e); @@ -2417,17 +2418,20 @@ public class Frontend { return request; } - private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx) { + private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx, Exception e) { // TODO: Need a fallback flag for various modes. In production, we will most // likely want to fallback to the original planner, but in testing, we might want // the query to fail. // There are some cases where we will always want to fallback, e.g. if the statement // fails at parse time because it is not a select statement. + if (e instanceof UnsupportedFeatureException) { + return true; + } TQueryCtx queryCtx = planCtx.getQueryContext(); try { return !(Parser.parse(queryCtx.client_request.stmt, queryCtx.client_request.query_options) instanceof QueryStmt); - } catch (Exception e) { + } catch (Exception f) { return false; } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java index 05cb65722..d2c19b7f8 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java @@ -156,8 +156,18 @@ public class CalciteAnalysisDriver implements AnalysisDriver { validatedNode_ = sqlValidator_.validate(parsedStmt_.getParsedSqlNode()); return new CalciteAnalysisResult(this); } catch (ImpalaException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_); + } catch (ImpalaException u) { + e = u; + } return new CalciteAnalysisResult(this, e); } catch (CalciteContextException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_); + } catch (ImpalaException u) { + return new CalciteAnalysisResult(this, u); + } return new CalciteAnalysisResult(this, new AnalysisException(e.getMessage(), e.getCause())); } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java index 7edf6def1..a72b7c9c3 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java @@ -23,7 +23,11 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.impala.calcite.parser.ImpalaSqlParserImpl; import org.apache.impala.calcite.validate.ImpalaConformance; +import org.apache.impala.common.ImpalaException; import org.apache.impala.common.ParseException; +import org.apache.impala.common.UnsupportedFeatureException; + +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +63,11 @@ public class CalciteQueryParser implements CompilerStep { SqlNode sqlNode = parser.parseQuery(); return sqlNode; } catch (SqlParseException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e); + } catch (ImpalaException u) { + throw new ParseException(u.getMessage()); + } throw new ParseException(e.getMessage()); } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java new file mode 100644 index 000000000..b9b7e3455 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.calcite.service; + +import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.ParseException; +import org.apache.impala.common.UnsupportedFeatureException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * UnsupportedChecker containts methods which determine if a feature + * is unsupported for the Calcite planner. + */ +public class UnsupportedChecker { + + private static Pattern LEFT_SEMI = Pattern.compile(".*\\bleft\\ssemi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern RIGHT_SEMI = Pattern.compile(".*\\bright\\ssemi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern LEFT_ANTI = Pattern.compile(".*\\bleft\\santi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern RIGHT_ANTI = Pattern.compile(".*\\bright\\santi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern INPUT_FILE_NAME = Pattern.compile(".*\\binput__file__name\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern FILE_POSITION = Pattern.compile(".*\\bfile__position\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern TABLE_NOT_FOUND = + Pattern.compile(".*\\bTable '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); + + private static Pattern COLUMN_NOT_FOUND = + Pattern.compile(".*\\bColumn '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); + + public static void throwUnsupportedIfKnownException(Exception e) + throws ImpalaException { + String s = e.toString().replace("\n"," "); + if (LEFT_ANTI.matcher(s).matches() || RIGHT_ANTI.matcher(s).matches()) { + throw new UnsupportedFeatureException("Anti joins not supported."); + } + if (LEFT_SEMI.matcher(s).matches() || RIGHT_SEMI.matcher(s).matches()) { + throw new UnsupportedFeatureException("Semi joins not supported."); + } + if (INPUT_FILE_NAME.matcher(s).matches() || FILE_POSITION.matcher(s).matches()) { + throw new UnsupportedFeatureException("Virtual columns not supported."); + } + } + + public static void throwUnsupportedIfKnownException(Exception e, + StmtTableCache stmtTableCache) throws ImpalaException { + throwUnsupportedIfKnownException(e); + String s = e.toString().replace("\n"," "); + Matcher m = TABLE_NOT_FOUND.matcher(s); + + // If the error given is "table/column not found", it is possible that the message + // was generated by a complex column that looks like a table + // (e.g. mytbl.my_complex_column) which is currently not supported. We check for + // this possibility by seeing if the 'table not found' is identified as a column + // within one of the tables in the query. This check isn't fool-proof in that + // it might actually be a table that also is a column name in another table. + // However, that case should be extremely rare, and the result would be that + // the wrong error message will show up. + if (m.matches()) { + if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { + throw new UnsupportedFeatureException( + "Complex column " + m.group(1) + " not supported."); + } + } + + m = COLUMN_NOT_FOUND.matcher(s); + if (m.matches()) { + if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { + throw new UnsupportedFeatureException( + "Complex column " + m.group(1) + " not supported."); + } + } + } +}
