[
https://issues.apache.org/jira/browse/HIVE-23814?focusedWorklogId=457513&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-457513
]
ASF GitHub Bot logged work on HIVE-23814:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Jul/20 04:40
Start Date: 11/Jul/20 04:40
Worklog Time Spent: 10m
Work Description: pvary commented on a change in pull request #1222:
URL: https://github.com/apache/hive/pull/1222#discussion_r453154673
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/Driver.java
##########
@@ -410,260 +386,304 @@ private void compileInternal(String command, boolean
deferClose) throws CommandP
}
}
//Save compile-time PerfLogging for WebUI.
- //Execution-time Perf logs are done by either another thread's PerfLogger
- //or a reset PerfLogger.
+ //Execution-time Perf logs are done by either another thread's PerfLogger
or a reset PerfLogger.
driverContext.getQueryDisplay().setPerfLogStarts(QueryDisplay.Phase.COMPILATION,
perfLogger.getStartTimes());
driverContext.getQueryDisplay().setPerfLogEnds(QueryDisplay.Phase.COMPILATION,
perfLogger.getEndTimes());
}
- private void runInternal(String command, boolean alreadyCompiled) throws
CommandProcessorException {
+ /**
+ * Compile a new query, but potentially reset taskID counter. Not resetting
task counter
+ * is useful for generating re-entrant QL queries.
+ * @param command The HiveQL query to compile
+ * @param resetTaskIds Resets taskID counter if true.
+ * @return 0 for ok
+ */
+ public int compile(String command, boolean resetTaskIds) {
+ try {
+ compile(command, resetTaskIds, false);
+ return 0;
+ } catch (CommandProcessorException cpr) {
+ return cpr.getErrorCode();
+ }
+ }
+
+ // deferClose indicates if the close/destroy should be deferred when the
process has been
+ // interrupted, it should be set to true if the compile is called within
another method like
+ // runInternal, which defers the close to the called in that method.
+ @VisibleForTesting
+ public void compile(String command, boolean resetTaskIds, boolean
deferClose) throws CommandProcessorException {
+ preparForCompile(resetTaskIds);
+
+ Compiler compiler = new Compiler(context, driverContext, driverState);
+ QueryPlan plan = compiler.compile(command, deferClose);
+ driverContext.setPlan(plan);
+
+ compileFinished(deferClose);
+ }
+
+ private void preparForCompile(boolean resetTaskIds) throws
CommandProcessorException {
+ driverTxnHandler.createTxnManager();
DriverState.setDriverState(driverState);
+ prepareContext();
+ setQueryId();
- driverState.lock();
- try {
- if (alreadyCompiled) {
- if (driverState.isCompiled()) {
- driverState.executing();
- } else {
- String errorMessage = "FAILED: Precompiled query has been cancelled
or closed.";
- CONSOLE.printError(errorMessage);
- throw DriverUtils.createProcessorException(driverContext, 12,
errorMessage, null, null);
- }
- } else {
- driverState.compiling();
- }
- } finally {
- driverState.unlock();
+ if (resetTaskIds) {
+ TaskFactory.resetId();
+ }
+ }
+
+ private void prepareContext() throws CommandProcessorException {
+ if (context != null && context.getExplainAnalyze() !=
AnalyzeState.RUNNING) {
+ // close the existing ctx etc before compiling a new query, but does not
destroy driver
+ closeInProcess(false);
}
- // a flag that helps to set the correct driver state in finally block by
tracking if
- // the method has been returned by an error or not.
- boolean isFinishedWithError = true;
try {
- HiveDriverRunHookContext hookContext = new
HiveDriverRunHookContextImpl(driverContext.getConf(),
- alreadyCompiled ? context.getCmd() : command);
- // Get all the driver run hooks and pre-execute them.
- try {
- driverContext.getHookRunner().runPreDriverHooks(hookContext);
- } catch (Exception e) {
- String errorMessage = "FAILED: Hive Internal Error: " +
Utilities.getNameMessage(e);
- CONSOLE.printError(errorMessage + "\n" +
StringUtils.stringifyException(e));
- throw DriverUtils.createProcessorException(driverContext, 12,
errorMessage,
- ErrorMsg.findSQLState(e.getMessage()), e);
+ if (context == null) {
+ context = new Context(driverContext.getConf());
}
+ } catch (IOException e) {
+ throw new CommandProcessorException(e);
+ }
- if (!alreadyCompiled) {
- // compile internal will automatically reset the perf logger
- compileInternal(command, true);
- } else {
- // Since we're reusing the compiled plan, we need to update its start
time for current run
-
driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
- }
+ context.setHiveTxnManager(driverContext.getTxnManager());
+ context.setStatsSource(driverContext.getStatsSource());
+ context.setHDFSCleanup(true);
+
+ driverTxnHandler.setContext(context);
+ }
+
+ private void setQueryId() {
+ String queryId =
Strings.isNullOrEmpty(driverContext.getQueryState().getQueryId()) ?
+ QueryPlan.makeQueryId() : driverContext.getQueryState().getQueryId();
+
+ SparkSession ss = SessionState.get().getSparkSession();
+ if (ss != null) {
+ ss.onQuerySubmission(queryId);
+ }
+ driverContext.getQueryDisplay().setQueryId(queryId);
+
+ setTriggerContext(queryId);
+ }
+
+ private void setTriggerContext(String queryId) {
+ long queryStartTime;
+ // query info is created by SQLOperation which will have start time of the
operation. When JDBC Statement is not
+ // used queryInfo will be null, in which case we take creation of Driver
instance as query start time (which is also
+ // the time when query display object is created)
+ if (driverContext.getQueryInfo() != null) {
+ queryStartTime = driverContext.getQueryInfo().getBeginTime();
+ } else {
+ queryStartTime = driverContext.getQueryDisplay().getQueryStartTime();
+ }
+ WmContext wmContext = new WmContext(queryStartTime, queryId);
+ context.setWmContext(wmContext);
+ }
- //Reset the PerfLogger so that it doesn't retain any previous values.
- // Any value from compilation phase can be obtained through the map set
in queryDisplay during compilation.
- PerfLogger perfLogger = SessionState.getPerfLogger(true);
+ private void compileFinished(boolean deferClose) {
+ if (DriverState.getDriverState().isAborted() && !deferClose) {
+ closeInProcess(true);
+ }
+ }
- // the reason that we set the txn manager for the cxt here is because
each
- // query has its own ctx object. The txn mgr is shared across the
- // same instance of Driver, which can run multiple queries.
- context.setHiveTxnManager(driverContext.getTxnManager());
+ /**
+ * @return The current query plan associated with this Driver, if any.
+ */
+ @Override
+ public QueryPlan getPlan() {
+ return driverContext.getPlan();
+ }
- DriverUtils.checkInterrupted(driverState, driverContext, "at acquiring
the lock.", null, null);
+ /**
+ * @return The current FetchTask associated with the Driver's plan, if any.
+ */
+ @Override
+ public FetchTask getFetchTask() {
+ return driverContext.getFetchTask();
+ }
- lockAndRespond();
+ public void releaseLocksAndCommitOrRollback(boolean commit) throws
LockException {
+ releaseLocksAndCommitOrRollback(commit, driverContext.getTxnManager());
Review comment:
We still do not want to get rid of this method here and call the
txnmanager directly?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 457513)
Time Spent: 1h 10m (was: 1h)
> Clean up Driver
> ---------------
>
> Key: HIVE-23814
> URL: https://issues.apache.org/jira/browse/HIVE-23814
> Project: Hive
> Issue Type: Sub-task
> Components: Hive
> Reporter: Miklos Gergely
> Assignee: Miklos Gergely
> Priority: Major
> Labels: pull-request-available
> Time Spent: 1h 10m
> Remaining Estimate: 0h
>
> Driver is now cut down to it's minimal size by extracting all of it's sub
> tasks to separate classes. The rest should be cleaned up by
> * moving out some smaller parts of the code to sub task and utility classes
> wherever it is still possible
> * cut large functions to meaningful and manageable parts
> * re-order the functions to follow the order of processing
> * fix checkstyle issues
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
