This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9019a71770e [Fix](multi-catalog) Filter _temporary dir which is
generated by spark. (#26194)
9019a71770e is described below
commit 9019a71770ea158b035159bb523a4501aee87b55
Author: Xiangyu Wang <[email protected]>
AuthorDate: Fri Nov 3 13:42:29 2023 +0800
[Fix](multi-catalog) Filter _temporary dir which is generated by spark.
(#26194)
Sometimes there are some temporary files which are generated by some spark
jobs, we need filter these files.
```
errCode = 2, detailMessage = (xxx.xxx.xxx.xxx.xxx)
[CANCELLED][INTERNAL_ERROR]failed to init reader for file
hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc,
err: [INTERNAL_ERROR]Init OrcReader failed. reason = Failed to parse the
postscript from
hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc
0. @ 0x000000000bdbd89f
doris::Status doris::Status::Error<6, true,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>&>(std::basic_string_view<char, std::char_traits<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>&)
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173
in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-36
[...]
doris::vectorized::OrcReader::_create_file_reader()
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187
in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
2. @ 0x000000000ecceff9
doris::vectorized::OrcReader::init_reader(std::vector<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > > > const*,
std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >,
std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>,
doris::ColumnValueRange<(doris::PrimitiveType)4>,
doris::ColumnValueRange<(doris: [...]
doris::vectorized::VFileScanner::_get_next_reader()
/root/doris-master/be/src/vec/exec/scan/vfile_scanner.cpp:800 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
4. @ 0x000000000ff4256f
doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
/root/doris-master/be/src/common/status.h:448 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
5. @ 0x000000000ffdb1aa
doris::vectorized::VScanner::get_block(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
/root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
6. @ 0x000000000ff3b379
doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*,
doris::vectorized::ScannerContext*,
std::shared_ptr<doris::vectorized::VScanner>)
/root/doris-master/be/src/common/status.h:346 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
7. @ 0x000000000ff3ca11 std::_Function_handler<void (),
doris::vectorized::ScannerSch [...]
doris::ThreadPool::dispatch_thread()
/root/doris-master/be/src/util/threadpool.cpp:0 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
9. @ 0x000000000c97c2ac
doris::Thread::supervise_thread(void*)
/var/local/ldb-toolchain/bin/../usr/include/pthread.h:562 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
10. @ 0x0000000000007ea5 start_thread in /usr/lib64/libpthread-2.17.so 11.
@ 0x00000000000feb0d __clone in /usr/lib64/libc-2.17.so 0. @
0x000000000c647366
doris::Status doris::Status::Error<6, true,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >(std::basic_string_view<char, std::char_traits<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >&&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_6 [...]
doris::vectorized::VFileScanner::_get_next_reader()
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187
in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
2. @ 0x000000000ff4256f
doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
/root/doris-master/be/src/common/status.h:448 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
3. @ 0x000000000ffdb1aa
doris::vectorized::VScanner::get_block(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
/root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in
/hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be
4. @ 0x000000000ff3b379
doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*,
doris::vectorized::ScannerContext*,
std::shared_ptr<doris::vectorized::VScanner>)
/root/doris-master/be/src/common/status.h:346 in
/hdpData10/yarn/localcache/usercache/hive/appcache/a…
```
Co-authored-by: wangxiangyu <[email protected]>
---
.../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 480edd60f2c..bac891eb920 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -1058,9 +1058,14 @@ public class HiveMetaStoreCache {
return false;
}
for (String name : path.toString().split("/")) {
+ // generated by hive
if (name.startsWith(".hive-staging")) {
return false;
}
+ // generated by spark
+ if ("_temporary".equals(name)) {
+ return false;
+ }
}
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
