[kylin] branch master updated: KYLIN-4896 Optimize the process of writing big resource files into HDFS.

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new 985ff83  KYLIN-4896 Optimize the process of writing big resource files 
into HDFS.
985ff83 is described below

commit 985ff834a2eefc536a3bb2e516ab36d8f5667893
Author: helenzeng0503 <linghui_helen_z...@163.com>
AuthorDate: Fri Feb 5 09:40:30 2021 +0800

    KYLIN-4896 Optimize the process of writing big resource files into HDFS.
---
 .../main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index fc00ec2..230ee73 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -88,8 +88,10 @@ public class HBaseResourceStore extends 
PushdownResourceStore {
         tableName = metadataUrl.getIdentifier();
         createHTableIfNeeded(tableName);
 
-        kvSizeLimit = Integer
+        int kvSizeLimitActual = Integer
                 
.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize",
 "10485760"));
+        kvSizeLimit = kvSizeLimitActual > 10485760 ? kvSizeLimitActual : 
10485760;
+        logger.debug("hbase.client.keyvalue.maxsize is {}, kvSizeLimit is set 
to {}", kvSizeLimitActual, kvSizeLimit);
     }
 
     protected Connection getConnection() throws IOException {
@@ -328,6 +330,7 @@ public class HBaseResourceStore extends 
PushdownResourceStore {
             byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
 
             if (content.length > kvSizeLimit) {
+                logger.info("Length of content exceeds the limit of {} bytes, 
push down {} to HDFS", kvSizeLimit, resPath);
                 pushdown = writePushdown(resPath, 
ContentWriter.create(content));
                 content = BytesUtil.EMPTY_BYTE_ARRAY;
             }