This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new e82cf3b3a IMPALA-13643: Add scan_multithread_constraint in
test_scanners.py
e82cf3b3a is described below
commit e82cf3b3a1c2860c722d74e33cf9816d6abfe7af
Author: Riza Suminto <[email protected]>
AuthorDate: Sat Jan 4 15:03:05 2025 -0800
IMPALA-13643: Add scan_multithread_constraint in test_scanners.py
Debug action HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT exist inside
HdfsScanNode (hdfs-scan-node.cc) code path. MT_DOP > 0 executes using
HdfsScanNodeMt (hdfs-scan-node-mt.cc) rather than HdfsScanNode, and
always start single scanner thread per ScanNode. Thus, there is no need
to exercise HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT and MT_DOP > 0
combination.
This patch adds scan_multithread_constraint in test_scanners.py where
'mt_dop' exec option dimension is declared. This reduce core test vector
combination from 1254 to 1138 and exhaustive test vector combination
from 7530 to 6774.
Testing:
- Run and pass test_scanners.py in core exploration.
Change-Id: I77c2e6f9bbd4bc1825fa1f006a22ee1a6ea5a606
Reviewed-on: http://gerrit.cloudera.org:8080/22300
Reviewed-by: Impala Public Jenkins <[email protected]>
Reviewed-by: Jason Fehr <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
tests/query_test/test_scanners.py | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/tests/query_test/test_scanners.py
b/tests/query_test/test_scanners.py
index 7da670de4..4da8977da 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -67,12 +67,25 @@ DEBUG_ACTION_DIMS = [None,
'-1:OPEN:[email protected]']
# Trigger injected soft limit failures when scanner threads check memory limit.
-DEBUG_ACTION_DIMS.append('HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT:[email protected]')
+DEBUG_ACTION_SCAN_MULTITHREAD =
['HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT:[email protected]']
+DEBUG_ACTION_DIMS.extend(DEBUG_ACTION_SCAN_MULTITHREAD)
MT_DOP_VALUES = [0, 1, 4]
BATCH_SIZES = [0, 1, 16]
+def scan_multithread_constraint(vector):
+ opts = vector.get_value('exec_option')
+ if vector.get_value('mt_dop') and 'debug_action' in opts:
+ # DEBUG_ACTION_SCAN_MULTITHREAD exist inside HdfsScanNode
(hdfs-scan-node.cc)
+ # code path. MT_DOP > 0 executes using HdfsScanNodeMt
(hdfs-scan-node-mt.cc) rather
+ # than HdfsScanNode, and always start single scanner thread per ScanNode.
+ # Thus, no need to exercise DEBUG_ACTION_SCAN_MULTITHREAD.
+ return (vector.get_value('mt_dop') == 0
+ or opts['debug_action'] not in DEBUG_ACTION_SCAN_MULTITHREAD)
+ return True
+
+
class TestScannersAllTableFormats(ImpalaTestSuite):
@classmethod
@@ -89,6 +102,7 @@ class TestScannersAllTableFormats(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
batch_sizes=BATCH_SIZES, debug_action_options=DEBUG_ACTION_DIMS))
add_exec_option_dimension(cls, 'mt_dop', MT_DOP_VALUES)
+ cls.ImpalaTestMatrix.add_constraint(scan_multithread_constraint)
def test_scanners(self, vector):
self.run_test_case('QueryTest/scanners', vector)
@@ -227,6 +241,7 @@ class TestScannersAllTableFormatsWithLimit(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(batch_sizes=[100]))
add_exec_option_dimension(cls, 'mt_dop', MT_DOP_VALUES)
+ cls.ImpalaTestMatrix.add_constraint(scan_multithread_constraint)
def test_limit(self, vector):
vector.get_value('exec_option')['abort_on_error'] = 1
@@ -266,6 +281,7 @@ class TestScannersMixedTableFormats(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
batch_sizes=BATCH_SIZES, debug_action_options=DEBUG_ACTION_DIMS))
add_exec_option_dimension(cls, 'mt_dop', MT_DOP_VALUES)
+ cls.ImpalaTestMatrix.add_constraint(scan_multithread_constraint)
def test_mixed_format(self, vector):
self.run_test_case('QueryTest/mixed-format', vector)
