(doris) branch branch-2.1 updated: [opt](catalog) cache the Configuration object (#45433)(#45756) (#45758)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9fa94bbde96 [opt](catalog) cache the Configuration object
(#45433)(#45756) (#45758)
9fa94bbde96 is described below
commit 9fa94bbde96073150e3d43b7d09f1ce20f946616
Author: Mingyu Chen (Rayner)
AuthorDate: Sun Dec 22 08:01:00 2024 +0800
[opt](catalog) cache the Configuration object (#45433)(#45756) (#45758)
---
.../apache/doris/datasource/ExternalCatalog.java | 22 +++
.../datasource/hive/HiveMetaStoreClientHelper.java | 7 +---
.../doris/datasource/ExternalCatalogTest.java | 46 --
3 files changed, 65 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index ddbf7e5e4e6..e1d37b009e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -146,6 +146,9 @@ public abstract class ExternalCatalog
protected MetaCache> metaCache;
protected PreExecutionAuthenticator preExecutionAuthenticator;
+private volatile Configuration cachedConf = null;
+private byte[] confLock = new byte[0];
+
public ExternalCatalog() {
}
@@ -157,6 +160,20 @@ public abstract class ExternalCatalog
}
public Configuration getConfiguration() {
+// build configuration is costly, so we cache it.
+if (cachedConf != null) {
+return cachedConf;
+}
+synchronized (confLock) {
+if (cachedConf != null) {
+return cachedConf;
+}
+cachedConf = buildConf();
+return cachedConf;
+}
+}
+
+private Configuration buildConf() {
Configuration conf =
DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth());
Map catalogProperties =
catalogProperty.getHadoopProperties();
for (Map.Entry entry : catalogProperties.entrySet()) {
@@ -402,6 +419,10 @@ public abstract class ExternalCatalog
this.convertedProperties = null;
}
+synchronized (this.confLock) {
+this.cachedConf = null;
+}
+
refreshOnlyCatalogCache(invalidCache);
}
@@ -752,6 +773,7 @@ public abstract class ExternalCatalog
}
}
this.propLock = new byte[0];
+this.confLock = new byte[0];
this.initialized = false;
setDefaultPropsIfMissing(true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index f6b6af2d2d2..f407284bf44 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -42,7 +42,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopAuthenticator;
import org.apache.doris.datasource.ExternalCatalog;
-import org.apache.doris.fs.remote.dfs.DFSFileSystem;
import org.apache.doris.thrift.TExprOpcode;
import com.google.common.base.Strings;
@@ -840,11 +839,7 @@ public class HiveMetaStoreClientHelper {
}
public static Configuration getConfiguration(HMSExternalTable table) {
-Configuration conf =
DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth());
-for (Map.Entry entry :
table.getHadoopProperties().entrySet()) {
-conf.set(entry.getKey(), entry.getValue());
-}
-return conf;
+return table.getCatalog().getConfiguration();
}
public static Optional getSerdeProperty(Table table, String key) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
index 43348ca8a0e..70e5e5f37af 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
@@ -22,9 +22,12 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
-import org.apache.doris.mysql.privilege.Auth;
+import org.apac
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman merged PR #45758: URL: https://github.com/apache/doris/pull/45758 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [Optimize](Variant) optimize schema update performance #45480 [doris]
doris-robot commented on PR #45730: URL: https://github.com/apache/doris/pull/45730#issuecomment-2558176864 TPC-DS: Total hot run time: 195053 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 28aa84658149fef5266a5f03c3f02a6c32267942, data reload: false query1 1222934 913 913 query2 6246213621072107 query3 11015 422342004200 query4 66440 29123 23480 23480 query5 5347470 447 447 query6 461 175 172 172 query7 5686317 305 305 query8 314 229 231 229 query9 9151267726642664 query10 514 267 259 259 query11 17860 15144 15658 15144 query12 154 107 102 102 query13 1549437 427 427 query14 10114 648075126480 query15 210 186 184 184 query16 7308526 533 526 query17 1092543 554 543 query18 1786307 311 307 query19 216 147 142 142 query20 120 113 107 107 query21 66 41 44 41 query22 4530451444534453 query23 34739 33893 33982 33893 query24 6178287129142871 query25 516 385 385 385 query26 691 160 158 158 query27 1939291 289 289 query28 4717256325252525 query29 690 446 418 418 query30 253 166 158 158 query31 1007839 771 771 query32 67 52 54 52 query33 447 284 285 284 query34 927 505 509 505 query35 844 744 737 737 query36 1088944 931 931 query37 117 74 73 73 query38 4086399839783978 query39 1512143114721431 query40 149 84 82 82 query41 49 49 49 49 query42 113 96 99 96 query43 539 496 505 496 query44 1174803 797 797 query45 196 173 179 173 query46 1163751 696 696 query47 1975189518811881 query48 472 375 366 366 query49 744 399 387 387 query50 846 414 414 414 query51 7323737571237123 query52 105 90 92 90 query53 252 189 184 184 query54 562 452 461 452 query55 76 76 75 75 query56 262 245 238 238 query57 1211108610791079 query58 220 206 207 206 query59 3246308729532953 query60 289 285 263 263 query61 106 103 103 103 query62 793 653 662 653 query63 208 179 184 179 query64 1803653 602 602 query65 3226313131243124 query66 733 303 311 303 query67 15941 15272 15313 15272 query68 4574560 545 545 query69 457 249 247 247 query70 1119105410701054 query71 435 255 247 247 query72 6604397638873887 query73 778 332 332 332 query74 10440 898389728972 query75 3304261326322613 query76 2549109911381099 query77 518 280 260 260 query78 10697 990594479447 query79 7317588 598 588 query80 1727414 402 402 query81 545 241 243 241 query82 1290120 115 115 query83 249 144 143 143 query84 295 80 78 78 query85 1622312 288 288 query86 469 303 294 294 query87 4440425642064206 query88 5689235524042355 query89 417 282 283 282 query90 2043188 185 185 query91 196 142 146 142 query92 65 47 47 47 query93 6227530 519 519 query94 841 294 280 280 query95 351 243 248 243 query96 645 274 280 274 query97 3411315231633152 query98 216 214 203 203 query99 1746130712861286 Total cold run time: 336271 ms Total hot run time: 195053 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558162617 TeamCity be ut coverage result: Function Coverage: 38.88% (10133/26064) Line Coverage: 29.80% (85264/286145) Region Coverage: 28.93% (43551/150525) Branch Coverage: 25.44% (22179/87174) Coverage Report: http://coverage.selectdb-in.cc/coverage/edac294809a0eb6aba895ef82e1c2a71e94cf118_edac294809a0eb6aba895ef82e1c2a71e94cf118/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Improve](Variant) pick random backend as coordinator [doris]
doris-robot commented on PR #45754: URL: https://github.com/apache/doris/pull/45754#issuecomment-2558166601 TPC-DS: Total hot run time: 196272 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit bcd1c7a2499102a6679da420744c4c56f694960c, data reload: false query1 1281988 966 966 query2 6243235223202320 query3 11023 478446304630 query4 33047 23712 23138 23138 query5 4331462 474 462 query6 285 197 189 189 query7 3987314 309 309 query8 314 228 227 227 query9 9529270827142708 query10 473 255 248 248 query11 17813 15113 15197 15113 query12 159 108 104 104 query13 1600438 430 430 query14 9529714969316931 query15 256 217 198 198 query16 7922445 464 445 query17 1576593 626 593 query18 2112328 347 328 query19 272 171 174 171 query20 143 115 113 113 query21 209 111 108 108 query22 47394612 query23 34576 34184 33903 33903 query24 10397 257425002500 query25 655 396 387 387 query26 1148153 158 153 query27 3068328 332 328 query28 7739247124932471 query29 857 427 415 415 query30 222 146 149 146 query31 1078869 833 833 query32 88 55 57 55 query33 770 292 285 285 query34 1002519 519 519 query35 915 774 738 738 query36 1090965 968 965 query37 137 77 80 77 query38 4248426044524260 query39 1503146014991460 query40 210 105 100 100 query41 44 47 47 47 query42 114 99 104 99 query43 541 498 500 498 query44 1468840 847 840 query45 188 173 169 169 query46 1200745 712 712 query47 2058193319701933 query48 429 334 332 332 query49 1005391 408 391 query50 850 390 391 390 query51 7158728673017286 query52 99 93 92 92 query53 260 188 192 188 query54 1132409 404 404 query55 89 85 85 85 query56 275 249 236 236 query57 1274115911841159 query58 235 231 241 231 query59 3431310531913105 query60 280 261 249 249 query61 110 109 110 109 query62 864 686 659 659 query63 218 197 182 182 query64 3986681 665 665 query65 3237315731963157 query66 839 305 304 304 query67 16004 15572 15524 15524 query68 5077559 563 559 query69 479 251 250 250 query70 1214111210511051 query71 436 256 271 256 query72 7066404840284028 query73 783 359 356 356 query74 10298 882489228824 query75 3646264226862642 query76 3592108911251089 query77 585 279 303 279 query78 10144 929593709295 query79 1701614 609 609 query80 763 426 425 425 query81 509 281 236 236 query82 1073119 120 119 query83 180 141 142 141 query84 236 71 73 71 query85 1602302 314 302 query86 479 282 303 282 query87 4708434745144347 query88 3919221321482148 query89 436 286 287 286 query90 2057195 190 190 query91 141 117 104 104 query92 66 50 50 50 query93 2544570 553 553 query94 869 292 280 280 query95 351 258 253 253 query96 624 283 274 274 query97 2828266026912660 query98 226 196 195 195 query99 1585130213231302 Total cold run time: 304151 ms Total hot run time: 196272 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go t
Re: [PR] [Improve](Variant) pick random backend as coordinator [doris]
doris-robot commented on PR #45754: URL: https://github.com/apache/doris/pull/45754#issuecomment-2558167928 ClickBench: Total hot run time: 33.04 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit bcd1c7a2499102a6679da420744c4c56f694960c, data reload: false query1 0.030.030.03 query2 0.060.040.03 query3 0.230.070.08 query4 1.600.110.11 query5 0.410.410.41 query6 1.170.650.65 query7 0.020.010.02 query8 0.040.030.03 query9 0.590.500.51 query10 0.550.570.55 query11 0.150.110.11 query12 0.130.110.11 query13 0.610.610.60 query14 2.722.752.73 query15 0.910.820.82 query16 0.380.390.38 query17 0.980.981.03 query18 0.230.210.20 query19 1.971.842.00 query20 0.020.010.01 query21 15.36 0.600.60 query22 2.512.552.27 query23 16.81 0.960.80 query24 3.580.971.68 query25 0.270.180.18 query26 0.420.130.14 query27 0.030.040.04 query28 9.971.141.07 query29 12.57 3.213.22 query30 0.240.060.06 query31 2.860.380.37 query32 3.270.460.45 query33 3.103.073.16 query34 16.64 4.424.43 query35 4.484.484.46 query36 0.690.470.47 query37 0.090.060.06 query38 0.050.040.03 query39 0.040.030.03 query40 0.170.120.12 query41 0.080.020.03 query42 0.040.020.02 query43 0.030.020.03 Total cold run time: 106.1 s Total hot run time: 33.04 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558183524 TPC-DS: Total hot run time: 196269 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit a184e2d3e023d61fb58e80a1fcad25ae5287197f, data reload: false query1 1327947 951 947 query2 6242235623362336 query3 5 476948164769 query4 33205 23460 23250 23250 query5 4269436 465 436 query6 283 180 173 173 query7 3994301 291 291 query8 306 236 248 236 query9 9452267826812678 query10 472 242 244 242 query11 17877 15092 15150 15092 query12 147 101 99 99 query13 1580413 412 412 query14 9859687468866874 query15 253 225 199 199 query16 8319490 481 481 query17 1590600 583 583 query18 2171344 328 328 query19 242 169 159 159 query20 123 118 118 118 query21 212 112 118 112 query22 5060478845464546 query23 34483 33747 33780 33747 query24 11480 252124802480 query25 686 425 397 397 query26 1744151 151 151 query27 2916328 334 328 query28 8043246824622462 query29 1034416 426 416 query30 233 146 150 146 query31 1088855 841 841 query32 95 54 56 54 query33 804 275 300 275 query34 982 528 497 497 query35 909 749 774 749 query36 1120948 954 948 query37 272 75 78 75 query38 4234419743644197 query39 1518164314971497 query40 261 102 101 101 query41 49 43 43 43 query42 111 104 100 100 query43 556 490 510 490 query44 1335832 830 830 query45 190 164 170 164 query46 1192728 719 719 query47 2067197119261926 query48 423 328 320 320 query49 1075386 393 386 query50 826 383 394 383 query51 7338719071907190 query52 104 97 93 93 query53 261 189 190 189 query54 1080421 401 401 query55 84 81 82 81 query56 267 261 243 243 query57 1328117911731173 query58 242 228 223 223 query59 3408321032293210 query60 276 240 246 240 query61 106 104 111 104 query62 879 676 693 676 query63 223 185 185 185 query64 4949674 622 622 query65 3244317931653165 query66 1369302 307 302 query67 15911 15610 15597 15597 query68 4783549 556 549 query69 438 259 252 252 query70 1162111511331115 query71 453 246 256 246 query72 6268415739863986 query73 770 359 370 359 query74 9874884889788848 query75 3429265825972597 query76 2711105411391054 query77 425 283 281 281 query78 10272 974394369436 query79 1547607 584 584 query80 955 423 424 423 query81 539 229 231 229 query82 739 116 124 116 query83 257 155 145 145 query84 229 67 68 67 query85 1425314 303 303 query86 391 306 283 283 query87 4448443043134313 query88 3099224421922192 query89 425 294 297 294 query90 2047185 184 184 query91 136 105 104 104 query92 59 51 54 51 query93 1571542 542 542 query94 1042329 280 280 query95 355 248 245 245 query96 616 268 278 268 query97 2854269126422642 query98 215 204 195 195 query99 1566133013091309 Total cold run time: 303993 ms Total hot run time: 196269 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go t
Re: [PR] [Improve](Variant) pick random backend as coordinator [doris]
doris-robot commented on PR #45754: URL: https://github.com/apache/doris/pull/45754#issuecomment-2558163401 TPC-H: Total hot run time: 41789 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit bcd1c7a2499102a6679da420744c4c56f694960c, data reload: false -- Round 1 -- q1 17574 824781108110 q2 2043181 166 166 q3 10641 117112491171 q4 10570 789 842 789 q5 7644295428742874 q6 263 154 147 147 q7 1078621 597 597 q8 9467196319871963 q9 7299697268886888 q10 7059232923202320 q11 486 260 259 259 q12 483 220 224 220 q13 17755 290029202900 q14 262 207 219 207 q15 606 492 492 492 q16 657 575 576 575 q17 1015619 487 487 q18 7280659866656598 q19 1366112910231023 q20 497 188 185 185 q21 4299360935043504 q22 377 314 315 314 Total cold run time: 108721 ms Total hot run time: 41789 ms - Round 2, with runtime_filter_mode=off - q1 8103800680608006 q2 351 250 237 237 q3 3168304531663045 q4 2209187417871787 q5 5765584858625848 q6 252 148 140 140 q7 2347177718261777 q8 3633375337893753 q9 9082902290649022 q10 3754376036963696 q11 626 514 501 501 q12 860 602 596 596 q13 11289 314731203120 q14 319 275 294 275 q15 595 499 509 499 q16 719 653 643 643 q17 2006175217191719 q18 8340772374487448 q19 1914180417271727 q20 2155189618551855 q21 5798562755255525 q22 676 578 608 578 Total cold run time: 73961 ms Total hot run time: 61797 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558180405 TeamCity be ut coverage result: Function Coverage: 38.87% (10132/26064) Line Coverage: 29.80% (85269/286145) Region Coverage: 28.94% (43561/150525) Branch Coverage: 25.45% (22182/87174) Coverage Report: http://coverage.selectdb-in.cc/coverage/a184e2d3e023d61fb58e80a1fcad25ae5287197f_a184e2d3e023d61fb58e80a1fcad25ae5287197f/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558180580 TPC-H: Total hot run time: 39627 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit a184e2d3e023d61fb58e80a1fcad25ae5287197f, data reload: false -- Round 1 -- q1 17589 751572407240 q2 2053181 178 178 q3 10595 108112231081 q4 10548 710 720 710 q5 7607268526942685 q6 235 150 147 147 q7 974 627 599 599 q8 9255182219261822 q9 6532637363876373 q10 6993231223322312 q11 471 267 263 263 q12 436 226 228 226 q13 18171 299228812881 q14 237 216 222 216 q15 553 521 498 498 q16 671 607 591 591 q17 963 497 581 497 q18 7207673666716671 q19 1337103510141014 q20 468 186 182 182 q21 4040327131293129 q22 384 323 312 312 Total cold run time: 107319 ms Total hot run time: 39627 ms - Round 2, with runtime_filter_mode=off - q1 7229723673357236 q2 340 233 238 233 q3 2889282229542822 q4 2050185318511851 q5 5665568656305630 q6 231 141 142 141 q7 2261181117961796 q8 3326354634793479 q9 8967895190908951 q10 3625355335463546 q11 583 486 518 486 q12 817 619 584 584 q13 13743 314031293129 q14 306 290 288 288 q15 555 516 502 502 q16 702 636 656 636 q17 1867161315921592 q18 8278774676277627 q19 1708150715021502 q20 2102184818791848 q21 5664549155035491 q22 635 596 627 596 Total cold run time: 73543 ms Total hot run time: 59966 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558184929 ClickBench: Total hot run time: 32.47 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit a184e2d3e023d61fb58e80a1fcad25ae5287197f, data reload: false query1 0.040.030.04 query2 0.080.030.03 query3 0.240.070.07 query4 1.610.100.10 query5 0.430.410.42 query6 1.140.650.64 query7 0.030.020.02 query8 0.040.040.03 query9 0.580.490.48 query10 0.550.560.56 query11 0.150.110.10 query12 0.130.100.11 query13 0.610.600.60 query14 2.832.862.77 query15 0.890.840.83 query16 0.370.370.36 query17 1.071.041.02 query18 0.230.220.21 query19 1.871.851.99 query20 0.010.020.01 query21 15.38 0.570.57 query22 2.563.072.02 query23 16.91 0.950.98 query24 2.940.361.30 query25 0.130.100.06 query26 0.600.150.15 query27 0.050.050.06 query28 11.30 1.101.07 query29 12.59 3.263.22 query30 0.250.070.07 query31 2.850.370.38 query32 3.270.460.47 query33 3.043.113.07 query34 16.97 4.524.52 query35 4.514.524.54 query36 0.670.480.47 query37 0.090.060.06 query38 0.050.040.04 query39 0.040.030.02 query40 0.160.140.13 query41 0.080.030.02 query42 0.040.020.03 query43 0.040.030.03 Total cold run time: 107.42 s Total hot run time: 32.47 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](recycler) Fix CountdownEvent error [doris]
Thearas commented on PR #45760: URL: https://github.com/apache/doris/pull/45760#issuecomment-2558154972 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [Optimize](Variant) optimize schema update performance (#45480) (#45731)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 39c69c766e9 [Optimize](Variant) optimize schema update performance
(#45480) (#45731)
39c69c766e9 is described below
commit 39c69c766e9b17bf3bccd89410acb6be3931ecf3
Author: lihangyu
AuthorDate: Sat Dec 21 23:41:03 2024 +0800
[Optimize](Variant) optimize schema update performance (#45480) (#45731)
(#45480)
---
be/src/olap/rowset_builder.cpp| 24 +---
be/src/olap/tablet_schema.cpp | 15 +++
be/src/olap/tablet_schema.h | 3 +++
be/src/vec/common/schema_util.cpp | 5 ++---
4 files changed, 33 insertions(+), 14 deletions(-)
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 1929ffbb78e..c668df4bd33 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -327,21 +327,22 @@ Status RowsetBuilder::commit_txn() {
SCOPED_TIMER(_commit_txn_timer);
const RowsetWriterContext& rw_ctx = _rowset_writer->context();
-if (rw_ctx.tablet_schema->num_variant_columns() > 0) {
+if (rw_ctx.tablet_schema->num_variant_columns() > 0 && _rowset->num_rows()
> 0) {
// Need to merge schema with `rw_ctx.merged_tablet_schema` in prior,
// merged schema keeps the newest merged schema for the rowset, which
is updated and merged
// during flushing segments.
if (rw_ctx.merged_tablet_schema != nullptr) {
RETURN_IF_ERROR(tablet()->update_by_least_common_schema(rw_ctx.merged_tablet_schema));
+} else {
+// We should merge rowset schema further, in case that the
merged_tablet_schema maybe null
+// when enable_memtable_on_sink_node is true, the
merged_tablet_schema will not be passed to
+// the destination backend.
+// update tablet schema when meet variant columns, before
commit_txn
+// Eg. rowset schema: A(int),B(float), C(int), D(int)
+// _tabelt->tablet_schema: A(bigint), B(double)
+// => update_schema: A(bigint), B(double), C(int), D(int)
+
RETURN_IF_ERROR(tablet()->update_by_least_common_schema(rw_ctx.tablet_schema));
}
-// We should merge rowset schema further, in case that the
merged_tablet_schema maybe null
-// when enable_memtable_on_sink_node is true, the merged_tablet_schema
will not be passed to
-// the destination backend.
-// update tablet schema when meet variant columns, before commit_txn
-// Eg. rowset schema: A(int),B(float), C(int), D(int)
-// _tabelt->tablet_schema: A(bigint), B(double)
-// => update_schema: A(bigint), B(double), C(int), D(int)
-
RETURN_IF_ERROR(tablet()->update_by_least_common_schema(rw_ctx.tablet_schema));
}
// Transfer ownership of `PendingRowsetGuard` to `TxnManager`
@@ -379,7 +380,6 @@ Status BaseRowsetBuilder::cancel() {
void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
const
OlapTableSchemaParam* table_schema_param,
const TabletSchema&
ori_tablet_schema) {
-_tablet_schema->copy_from(ori_tablet_schema);
// find the right index id
int i = 0;
auto indexes = table_schema_param->indexes();
@@ -388,11 +388,13 @@ void
BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
break;
}
}
-
if (!indexes.empty() && !indexes[i]->columns.empty() &&
indexes[i]->columns[0]->unique_id() >= 0) {
+_tablet_schema->shawdow_copy_without_columns(ori_tablet_schema);
_tablet_schema->build_current_tablet_schema(index_id,
table_schema_param->version(),
indexes[i],
ori_tablet_schema);
+} else {
+_tablet_schema->copy_from(ori_tablet_schema);
}
if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version())
{
// After schema change, should include extracted column
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 7da0f99537a..488e9755b23 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1047,6 +1047,21 @@ void TabletSchema::copy_from(const TabletSchema&
tablet_schema) {
_table_id = tablet_schema.table_id();
}
+void TabletSchema::shawdow_copy_without_columns(const TabletSchema&
tablet_schema) {
+*this = tablet_schema;
+_field_path_to_index.clear();
+_field_name_to_index.clear();
+_field_id_to_index.clear();
+_num_columns = 0;
+_num_variant_columns = 0;
+_num_null_columns = 0;
+_num_key_columns = 0;
+_cols.clear();
+_vl_field_mem_size =
Re: [PR] [Optimize](Variant) optimize schema update performance (#45480) [doris]
eldenmoon merged PR #45731: URL: https://github.com/apache/doris/pull/45731 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](recycler) Fix CountdownEvent error [doris]
Yukang-Lian commented on PR #45760: URL: https://github.com/apache/doris/pull/45760#issuecomment-2558154984 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558157194 TPC-H: Total hot run time: 39622 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit edac294809a0eb6aba895ef82e1c2a71e94cf118, data reload: false -- Round 1 -- q1 17641 739072527252 q2 2042181 172 172 q3 10573 111311891113 q4 10206 757 729 729 q5 7582268527002685 q6 235 145 145 145 q7 965 632 612 612 q8 9230179418911794 q9 6611636263726362 q10 7002229822842284 q11 467 258 257 257 q12 404 220 230 220 q13 17781 295029202920 q14 245 211 208 208 q15 555 510 499 499 q16 652 578 591 578 q17 969 550 548 548 q18 7341661766096609 q19 1341994 1061994 q20 482 191 181 181 q21 4020314631933146 q22 376 314 316 314 Total cold run time: 106720 ms Total hot run time: 39622 ms - Round 2, with runtime_filter_mode=off - q1 7229719072427190 q2 329 223 231 223 q3 2932279729042797 q4 2126187618061806 q5 5556566856855668 q6 223 142 142 142 q7 2213185217771777 q8 3334353434763476 q9 8903899190218991 q10 3597352635523526 q11 610 496 504 496 q12 812 617 628 617 q13 10839 315631123112 q14 307 292 276 276 q15 563 522 503 503 q16 701 651 641 641 q17 1827163115761576 q18 7883747474567456 q19 1694148915041489 q20 2088181818431818 q21 5476524952565249 q22 649 564 567 564 Total cold run time: 69891 ms Total hot run time: 59393 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558161727 ClickBench: Total hot run time: 33.19 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit edac294809a0eb6aba895ef82e1c2a71e94cf118, data reload: false query1 0.030.030.03 query2 0.070.030.04 query3 0.240.080.08 query4 1.590.110.11 query5 0.410.430.41 query6 1.170.650.65 query7 0.030.020.01 query8 0.040.040.02 query9 0.580.490.51 query10 0.550.570.55 query11 0.140.110.11 query12 0.130.120.11 query13 0.610.620.59 query14 2.732.862.75 query15 0.890.840.84 query16 0.370.370.39 query17 1.061.001.08 query18 0.220.200.21 query19 1.971.842.04 query20 0.010.010.01 query21 15.36 0.570.58 query22 2.652.902.37 query23 16.78 1.140.88 query24 3.260.841.71 query25 0.140.210.12 query26 0.430.140.13 query27 0.050.040.04 query28 9.981.111.08 query29 12.59 3.263.22 query30 0.250.070.06 query31 2.850.380.39 query32 3.240.470.47 query33 3.163.023.10 query34 17.21 4.524.54 query35 4.544.504.54 query36 0.680.500.48 query37 0.090.060.06 query38 0.050.040.04 query39 0.040.030.02 query40 0.170.130.13 query41 0.080.020.02 query42 0.040.030.02 query43 0.040.030.02 Total cold run time: 106.52 s Total hot run time: 33.19 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
sollhui commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558171056 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris-website) branch asf-site updated (309d3418f77 -> af5f4dd9c04)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard 309d3418f77 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 new af5f4dd9c04 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (309d3418f77) \ N -- N -- N refs/heads/asf-site (af5f4dd9c04) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [Optimize](Variant) optimize schema update performance #45480 [doris]
eldenmoon commented on PR #45730: URL: https://github.com/apache/doris/pull/45730#issuecomment-2558155629 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558160172 TPC-DS: Total hot run time: 190899 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit edac294809a0eb6aba895ef82e1c2a71e94cf118, data reload: false query1 981 399 388 388 query2 6517244623662366 query3 6703226 213 213 query4 34294 23213 23380 23213 query5 4330461 454 454 query6 287 190 192 190 query7 4632313 316 313 query8 306 244 233 233 query9 9770273327032703 query10 489 256 254 254 query11 18362 14989 15084 14989 query12 159 109 104 104 query13 1681441 422 422 query14 10595 721373067213 query15 289 195 178 178 query16 8294481 407 407 query17 1824585 580 580 query18 2158330 296 296 query19 370 151 152 151 query20 116 112 110 110 query21 209 101 100 100 query22 4394416741904167 query23 34494 33848 33866 33848 query24 11452 247824962478 query25 675 383 388 383 query26 1834151 147 147 query27 2907323 327 323 query28 8107243924482439 query29 1069400 412 400 query30 311 145 150 145 query31 1055814 811 811 query32 102 57 59 57 query33 788 307 293 293 query34 990 514 520 514 query35 873 742 739 739 query36 1124911 953 911 query37 277 72 76 72 query38 4177420441444144 query39 1489142614461426 query40 283 102 101 101 query41 47 51 45 45 query42 113 98 101 98 query43 532 496 495 495 query44 1245806 820 806 query45 178 162 165 162 query46 1140691 730 691 query47 1957189018821882 query48 404 326 344 326 query49 1283403 415 403 query50 778 374 380 374 query51 7237714670537053 query52 103 102 89 89 query53 265 189 190 189 query54 1183397 408 397 query55 86 87 84 84 query56 262 252 255 252 query57 1243114811271127 query58 247 219 233 219 query59 3261304530923045 query60 283 244 250 244 query61 114 111 108 108 query62 895 685 673 673 query63 215 190 190 190 query64 5104703 643 643 query65 3257315532383155 query66 1450316 314 314 query67 15844 15493 15388 15388 query68 6315576 560 560 query69 465 256 257 256 query70 1146113810841084 query71 464 253 293 253 query72 6753409740744074 query73 787 350 356 350 query74 9928888290288882 query75 3823266926452645 query76 4184111710371037 query77 746 274 270 270 query78 11839 10130 97009700 query79 1971603 605 603 query80 766 438 455 438 query81 527 233 232 232 query82 659 118 119 118 query83 176 147 147 147 query84 257 74 77 74 query85 783 305 310 305 query86 467 303 306 303 query87 4568448047034480 query88 3727221721832183 query89 414 296 303 296 query90 2306193 186 186 query91 149 106 110 106 query92 65 54 52 52 query93 1138537 547 537 query94 966 284 295 284 query95 359 242 251 242 query96 615 284 284 284 query97 2881266827032668 query98 227 207 231 207 query99 1631136113161316 Total cold run time: 308595 ms Total hot run time: 190899 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
[PR] [fix](recycler) Fix CountdownEvent error [doris]
Yukang-Lian opened a new pull request, #45760: URL: https://github.com/apache/doris/pull/45760 ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: Fix CountdownEvent error "Invoking add_count() after wait() was invoked" ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [x] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [x] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch auto-pick-45480-branch-3.0 updated (ebb89c0801b -> 28aa8465814)
This is an automated email from the ASF dual-hosted git repository. eldenmoon pushed a change to branch auto-pick-45480-branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git from ebb89c0801b [Optimize](Variant) optimize schema update performance (#45480) add 28aa8465814 fix compile No new revisions were added by this update. Summary of changes: be/src/olap/tablet_schema.cpp | 1 - 1 file changed, 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [Optimize](Variant) optimize schema update performance #45480 [doris]
doris-robot commented on PR #45730: URL: https://github.com/apache/doris/pull/45730#issuecomment-2558173866 TPC-H: Total hot run time: 40443 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 28aa84658149fef5266a5f03c3f02a6c32267942, data reload: false -- Round 1 -- q1 17578 747972967296 q2 2046181 168 168 q3 10676 110111361101 q4 10567 705 730 705 q5 7753283627642764 q6 236 150 149 149 q7 964 609 603 603 q8 9575191220171912 q9 8093643164476431 q10 7010233223102310 q11 452 258 253 253 q12 398 216 207 207 q13 17765 294029772940 q14 264 204 218 204 q15 555 513 526 513 q16 669 614 617 614 q17 976 663 527 527 q18 7270648765816487 q19 2517109510081008 q20 464 200 194 194 q21 4057315330843084 q22 10811014973 973 Total cold run time: 110966 ms Total hot run time: 40443 ms - Round 2, with runtime_filter_mode=off - q1 7457734772287228 q2 325 229 227 227 q3 3008286728952867 q4 2068180017841784 q5 5621578857295729 q6 228 146 142 142 q7 2197178917731773 q8 3352362635003500 q9 8976896389438943 q10 3571347934993479 q11 602 493 509 493 q12 809 596 631 596 q13 16564 314531523145 q14 291 284 275 275 q15 580 525 523 523 q16 721 671 677 671 q17 1881164816071607 q18 8248802674007400 q19 4093167015431543 q20 2101185718341834 q21 5565539253865386 q22 1137103410231023 Total cold run time: 79395 ms Total hot run time: 60168 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
sollhui commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558132355 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement DropStoragePolicyCommand in nereids [doris]
msridhar78 commented on PR #44825: URL: https://github.com/apache/doris/pull/44825#issuecomment-2558135030 > we can add some cases to make sure implement is right done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement DropStoragePolicyCommand in nereids [doris]
msridhar78 commented on PR #44825: URL: https://github.com/apache/doris/pull/44825#issuecomment-2558135054 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement showStoragePolicyCommand in nereids [doris]
msridhar78 commented on code in PR #44393:
URL: https://github.com/apache/doris/pull/44393#discussion_r1894636022
##
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##
@@ -5031,6 +5033,11 @@ public LogicalPlan visitShowDelete(ShowDeleteContext
ctx) {
return new ShowDeleteCommand(dbName);
}
+@Override
+public LogicalPlan visitShowStoragePolicy(ShowStoragePolicyContext ctx) {
+return new ShowStoragePolicyCommand(ctx.identifierOrText().getText());
Review Comment:
done - please review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement showStoragePolicyCommand in nereids [doris]
msridhar78 commented on PR #44393: URL: https://github.com/apache/doris/pull/44393#issuecomment-2558134009 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Refactor](join) some refactor of join process [doris]
BiteThet commented on PR #44346: URL: https://github.com/apache/doris/pull/44346#issuecomment-2558048716 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement adminSetPartitionVersionCommand in nereids [doris]
rijeshkp commented on PR #45426: URL: https://github.com/apache/doris/pull/45426#issuecomment-2558048585 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
hello-stephen commented on PR #45759: URL: https://github.com/apache/doris/pull/45759#issuecomment-2558061158 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](catalog) fix npe after replaying the external catalog [doris]
github-actions[bot] commented on PR #45756: URL: https://github.com/apache/doris/pull/45756#issuecomment-2558062840 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](catalog) fix npe after replaying the external catalog [doris]
github-actions[bot] commented on PR #45756: URL: https://github.com/apache/doris/pull/45756#issuecomment-2558062852 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] memtable flush reserve process mem and improve logs [doris]
yiguolei commented on code in PR #45743:
URL: https://github.com/apache/doris/pull/45743#discussion_r1894592503
##
be/src/runtime/memory/thread_mem_tracker_mgr.h:
##
@@ -278,34 +278,44 @@ inline void ThreadMemTrackerMgr::flush_untracked_mem() {
_stop_consume = false;
}
-inline doris::Status ThreadMemTrackerMgr::try_reserve(int64_t size) {
+inline doris::Status ThreadMemTrackerMgr::try_reserve(int64_t size,
Review Comment:
把这个API 的单测补充一下
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] memtable flush reserve process mem and improve logs [doris]
yiguolei commented on code in PR #45743:
URL: https://github.com/apache/doris/pull/45743#discussion_r1894592567
##
be/src/runtime/thread_context.h:
##
@@ -249,13 +249,22 @@ class ThreadContext {
thread_mem_tracker_mgr->consume(size, skip_large_memory_check);
}
+doris::Status reserve_memory(const int64_t size) const {
Review Comment:
xxx 和 try_xxx 区分不清楚是否要检查query和wg的limit。
我们换个名字，增加一个新的函数，try_reserve_process_memory
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] memtable flush reserve process mem and improve logs [doris]
yiguolei commented on code in PR #45743:
URL: https://github.com/apache/doris/pull/45743#discussion_r1894593201
##
be/src/olap/memtable_flush_executor.cpp:
##
@@ -140,6 +141,36 @@ Status FlushToken::wait() {
return Status::OK();
}
+Status FlushToken::_try_reserve_memory(int64_t size) {
+auto* thread_context = doris::thread_context();
+auto* memtable_flush_executor =
+ExecEnv::GetInstance()->storage_engine().memtable_flush_executor();
+Status st;
+do {
+// only try to reserve process memory
+st = thread_context->reserve_memory(size);
+if (st.ok()) {
+memtable_flush_executor->inc_flushing_task();
+break;
+}
+if (_is_shutdown()) {
+st = Status::Cancelled("flush memtable already cancelled");
+break;
+}
+// Make sure at least one memtable is flushing even reserve memory
failed.
+if (memtable_flush_executor->check_and_inc_has_any_flushing_task()) {
+// If there are already any flushing task, Wait for some time and
retry.
+LOG(INFO) << fmt::format(
Review Comment:
这个每1min 打印一次吧，否则内存满的时候刷屏了
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](profile) add index page profile for io #45675 [doris]
doris-robot commented on PR #45757: URL: https://github.com/apache/doris/pull/45757#issuecomment-2558067107 TPC-H: Total hot run time: 40691 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit f26107d6acc738c98da0f916c53394d70fd5d604, data reload: false -- Round 1 -- q1 17575 742973007300 q2 2061166 173 166 q3 10692 109511361095 q4 10561 734 857 734 q5 7740283927762776 q6 237 148 152 148 q7 991 614 606 606 q8 9583197120471971 q9 8039641164006400 q10 6990231022992299 q11 466 261 251 251 q12 411 213 212 212 q13 17766 296329952963 q14 239 221 208 208 q15 558 522 527 522 q16 673 604 600 600 q17 988 573 505 505 q18 7209671365736573 q19 2411103010621030 q20 468 221 213 213 q21 3905324631473147 q22 1094973 972 972 Total cold run time: 110657 ms Total hot run time: 40691 ms - Round 2, with runtime_filter_mode=off - q1 7339726172407240 q2 329 235 243 235 q3 3000289628612861 q4 2068178717571757 q5 5682571357115711 q6 231 145 147 145 q7 2209179017381738 q8 3385357634363436 q9 8967897488508850 q10 3601349035193490 q11 612 503 516 503 q12 823 628 588 588 q13 16373 319231383138 q14 311 269 272 269 q15 579 529 529 529 q16 696 666 671 666 q17 1864164615981598 q18 8311766075147514 q19 5544154415021502 q20 2085186018901860 q21 5396534152435243 q22 1119102910171017 Total cold run time: 80524 ms Total hot run time: 59890 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
doris-robot commented on PR #45759: URL: https://github.com/apache/doris/pull/45759#issuecomment-2558067843 TPC-H: Total hot run time: 40834 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit d128ed2927d4dd426bae01b20365bb226d235747, data reload: false -- Round 1 -- q1 17897 756673377337 q2 2827172 173 172 q3 11287 114211241124 q4 11005 743 840 743 q5 7783290028912891 q6 234 149 151 149 q7 974 632 611 611 q8 9930196019801960 q9 6847641564386415 q10 7049227923492279 q11 466 259 267 259 q12 406 218 212 212 q13 17776 298829992988 q14 237 212 204 204 q15 557 505 524 505 q16 687 623 631 623 q17 958 551 658 551 q18 7387656665426542 q19 1478103810211021 q20 468 200 192 192 q21 3917322830863086 q22 1081985 970 970 Total cold run time: 111251 ms Total hot run time: 40834 ms - Round 2, with runtime_filter_mode=off - q1 7299722172337221 q2 331 241 234 234 q3 3162302928752875 q4 2035181318131813 q5 5633573856095609 q6 222 140 141 140 q7 2182178017231723 q8 3319354234453445 q9 8856888688838883 q10 3564348735443487 q11 592 512 511 511 q12 781 580 579 579 q13 17123 309229872987 q14 283 280 270 270 q15 554 509 502 502 q16 703 652 672 652 q17 1820159415541554 q18 7655748272187218 q19 2239158115971581 q20 1990180717521752 q21 5251510651545106 q22 1102948 990 948 Total cold run time: 76696 ms Total hot run time: 59090 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] memtable flush reserve process mem and improve logs [doris]
yiguolei commented on code in PR #45743:
URL: https://github.com/apache/doris/pull/45743#discussion_r1894593784
##
be/src/olap/memtable_flush_executor.cpp:
##
@@ -140,6 +141,36 @@ Status FlushToken::wait() {
return Status::OK();
}
+Status FlushToken::_try_reserve_memory(int64_t size) {
+auto* thread_context = doris::thread_context();
+auto* memtable_flush_executor =
+ExecEnv::GetInstance()->storage_engine().memtable_flush_executor();
+Status st;
+do {
+// only try to reserve process memory
+st = thread_context->reserve_memory(size);
+if (st.ok()) {
+memtable_flush_executor->inc_flushing_task();
+break;
+}
+if (_is_shutdown()) {
+st = Status::Cancelled("flush memtable already cancelled");
Review Comment:
如果在等待期间，要flush的这个memtable 关联的导入任务呗cancel了，怎么处理？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Fix](ccr) Wrong result of tosql for add column with comment #45319 [doris]
wyxxxcat commented on PR #45727: URL: https://github.com/apache/doris/pull/45727#issuecomment-2558141247 run p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558142181 TPC-H: Total hot run time: 39643 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 67b043fcf584d979fdd14c12b2f01414c5543ef8, data reload: false -- Round 1 -- q1 17581 741672477247 q2 2050181 175 175 q3 10643 108411341084 q4 10564 701 661 661 q5 7572264926622649 q6 238 149 148 148 q7 981 625 612 612 q8 9367185219131852 q9 6614636164296361 q10 7043231123282311 q11 477 262 267 262 q12 408 224 222 222 q13 17794 304629212921 q14 252 215 208 208 q15 545 519 491 491 q16 662 570 605 570 q17 986 566 594 566 q18 7201671866496649 q19 1350980 1025980 q20 461 199 187 187 q21 4188322631773177 q22 368 317 310 310 Total cold run time: 107345 ms Total hot run time: 39643 ms - Round 2, with runtime_filter_mode=off - q1 7193809771827182 q2 329 232 234 232 q3 2916285629362856 q4 2062183518311831 q5 5720566756335633 q6 223 144 138 138 q7 2269180618021802 q8 3348350334513451 q9 8894892189388921 q10 3594351334963496 q11 599 501 519 501 q12 822 582 579 579 q13 11908 313731533137 q14 321 285 290 285 q15 558 509 512 509 q16 708 635 671 635 q17 1867162815841584 q18 8275765876927658 q19 1702161415451545 q20 2126192519141914 q21 5631534355455343 q22 649 617 582 582 Total cold run time: 71714 ms Total hot run time: 59814 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement showStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44393: URL: https://github.com/apache/doris/pull/44393#issuecomment-2558144342 TPC-H: Total hot run time: 39791 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 9f787c9a01e38bae7fc61587442dc7699c7a2eed, data reload: false -- Round 1 -- q1 17597 747072877287 q2 2044185 169 169 q3 10596 106712011067 q4 10581 792 741 741 q5 7623267226472647 q6 242 146 142 142 q7 988 624 610 610 q8 9254188119561881 q9 6631642665766426 q10 7029231323132313 q11 476 268 261 261 q12 433 224 228 224 q13 17808 285629722856 q14 253 214 210 210 q15 569 512 508 508 q16 672 577 567 567 q17 975 594 544 544 q18 7261671866886688 q19 1342101810621018 q20 469 183 188 183 q21 4045322431373137 q22 376 318 312 312 Total cold run time: 107264 ms Total hot run time: 39791 ms - Round 2, with runtime_filter_mode=off - q1 7307725772877257 q2 334 233 229 229 q3 2920280029452800 q4 2070180317701770 q5 5643566256415641 q6 227 143 143 143 q7 2277184018261826 q8 3356358635313531 q9 8857887488598859 q10 3586354535203520 q11 603 516 494 494 q12 849 621 598 598 q13 11983 304731573047 q14 304 270 297 270 q15 557 510 511 510 q16 697 671 663 663 q17 1851163916111611 q18 8267793877457745 q19 1756159314761476 q20 2138181719181817 q21 5677548754825482 q22 630 577 576 576 Total cold run time: 71889 ms Total hot run time: 59865 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558142162 TeamCity be ut coverage result: Function Coverage: 38.88% (10133/26064) Line Coverage: 29.81% (85304/286145) Region Coverage: 28.94% (43556/150525) Branch Coverage: 25.44% (22178/87174) Coverage Report: http://coverage.selectdb-in.cc/coverage/67b043fcf584d979fdd14c12b2f01414c5543ef8_67b043fcf584d979fdd14c12b2f01414c5543ef8/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558144907 TPC-DS: Total hot run time: 196259 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 67b043fcf584d979fdd14c12b2f01414c5543ef8, data reload: false query1 13261006898 898 query2 6281242024642420 query3 10980 464647114646 query4 33477 23346 23294 23294 query5 4734482 445 445 query6 294 224 184 184 query7 3994308 301 301 query8 307 242 241 241 query9 9462268326752675 query10 480 251 251 251 query11 17989 15057 15317 15057 query12 163 103 105 103 query13 1584425 426 425 query14 9512751273707370 query15 291 194 187 187 query16 8156479 498 479 query17 1764595 567 567 query18 2188306 323 306 query19 379 158 151 151 query20 122 114 115 114 query21 209 116 116 116 query22 4847471244654465 query23 35961 33995 33544 33544 query24 10791 254624912491 query25 518 401 391 391 query26 754 166 157 157 query27 2305326 329 326 query28 7526245724492449 query29 673 422 426 422 query30 228 150 150 150 query31 1053848 865 848 query32 95 55 59 55 query33 767 294 288 288 query34 992 529 518 518 query35 911 757 774 757 query36 1107951 965 951 query37 125 72 78 72 query38 4208435641494149 query39 1515145914851459 query40 211 104 102 102 query41 45 42 45 42 query42 116 111 103 103 query43 536 510 502 502 query44 1306841 833 833 query45 194 168 175 168 query46 1206722 719 719 query47 2047192919631929 query48 421 318 341 318 query49 883 391 410 391 query50 863 397 390 390 query51 7417731772157215 query52 107 98 93 93 query53 259 188 182 182 query54 1218442 418 418 query55 82 79 74 74 query56 270 236 240 236 query57 1277117011741170 query58 235 218 219 218 query59 3466324030063006 query60 280 264 247 247 query61 107 105 107 105 query62 874 711 686 686 query63 213 184 190 184 query64 3837687 627 627 query65 3258320331813181 query66 796 304 296 296 query67 15973 15659 15420 15420 query68 5229554 546 546 query69 483 311 242 242 query70 1222112711561127 query71 467 254 252 252 query72 6735416040254025 query73 791 357 382 357 query74 10480 892088618861 query75 3422274526512651 query76 3778113710221022 query77 580 276 268 268 query78 10261 933495849334 query79 2239611 634 611 query80 956 418 429 418 query81 552 226 232 226 query82 1132117 123 117 query83 266 145 145 145 query84 233 70 78 70 query85 1601314 302 302 query86 476 303 300 300 query87 4520462645304530 query88 3894221021852185 query89 425 287 287 287 query90 2081186 191 186 query91 138 106 107 106 query92 70 53 52 52 query93 2442554 542 542 query94 941 280 267 267 query95 349 248 250 248 query96 630 281 281 281 query97 2892270726502650 query98 230 200 190 190 query99 1555133212841284 Total cold run time: 306635 ms Total hot run time: 196259 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go
Re: [PR] [Enhancement] (nereids)implement DropStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44825: URL: https://github.com/apache/doris/pull/44825#issuecomment-2558146744 TPC-H: Total hot run time: 40584 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit a82b7baf4b2df3ff2267b77d3db1d0816aa6df33, data reload: false -- Round 1 -- q1 18835 746873917391 q2 2050175 165 165 q3 10973 117112361171 q4 10819 710 780 710 q5 7747269127572691 q6 244 148 148 148 q7 1015631 617 617 q8 9909186519701865 q9 6940658166796581 q10 7126231223652312 q11 480 273 276 273 q12 477 235 231 231 q13 18267 304030363036 q14 262 223 214 214 q15 563 508 498 498 q16 691 587 589 587 q17 987 614 567 567 q18 7292689369006893 q19 1390991 836 836 q20 482 189 194 189 q21 4065338832863286 q22 382 329 323 323 Total cold run time: 110996 ms Total hot run time: 40584 ms - Round 2, with runtime_filter_mode=off - q1 7408729273127292 q2 325 229 233 229 q3 3069295329582953 q4 2141182018271820 q5 5609568056485648 q6 235 146 143 143 q7 2232182518021802 q8 3363350133893389 q9 8952897388998899 q10 3620359435693569 q11 597 492 494 492 q12 822 662 578 578 q13 11264 308630853085 q14 311 283 268 268 q15 561 511 511 511 q16 680 669 645 645 q17 1844160615841584 q18 8419771374467446 q19 1727164015631563 q20 2129195719381938 q21 5655552854145414 q22 652 567 562 562 Total cold run time: 71615 ms Total hot run time: 59830 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
doris-robot commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558146481 ClickBench: Total hot run time: 33.77 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit 67b043fcf584d979fdd14c12b2f01414c5543ef8, data reload: false query1 0.030.030.03 query2 0.080.040.03 query3 0.240.070.07 query4 1.600.110.11 query5 0.410.430.42 query6 1.150.650.66 query7 0.020.020.01 query8 0.040.030.03 query9 0.580.500.50 query10 0.550.560.57 query11 0.140.110.11 query12 0.160.110.10 query13 0.610.620.59 query14 2.722.822.86 query15 0.900.830.83 query16 0.390.370.37 query17 1.031.071.07 query18 0.230.200.21 query19 1.871.882.02 query20 0.010.010.02 query21 15.36 0.590.58 query22 2.852.402.47 query23 16.83 0.950.80 query24 3.641.281.59 query25 0.200.170.10 query26 0.540.140.14 query27 0.040.040.05 query28 9.621.101.07 query29 12.59 3.223.25 query30 0.250.060.07 query31 2.840.390.37 query32 3.270.460.46 query33 3.043.343.16 query34 16.90 4.464.48 query35 4.514.544.46 query36 0.640.470.52 query37 0.100.060.06 query38 0.050.040.04 query39 0.040.020.02 query40 0.170.120.13 query41 0.080.030.02 query42 0.030.020.02 query43 0.030.040.03 Total cold run time: 106.38 s Total hot run time: 33.77 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement showStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44393: URL: https://github.com/apache/doris/pull/44393#issuecomment-2558147515 TPC-DS: Total hot run time: 195757 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 9f787c9a01e38bae7fc61587442dc7699c7a2eed, data reload: false query1 13141003937 937 query2 6262232522702270 query3 10973 464045674567 query4 33526 23404 23248 23248 query5 4097460 460 460 query6 292 189 178 178 query7 4006302 306 302 query8 305 255 239 239 query9 9345270127012701 query10 469 260 248 248 query11 17747 15104 15307 15104 query12 155 105 104 104 query13 1590426 431 426 query14 10091 716369246924 query15 264 228 194 194 query16 8193469 455 455 query17 1676589 568 568 query18 2098312 308 308 query19 374 155 164 155 query20 125 109 112 109 query21 216 109 109 109 query22 4958452346334523 query23 34632 33736 33738 33736 query24 10484 240324762403 query25 623 394 393 393 query26 1263159 156 156 query27 2635325 345 325 query28 8115243024692430 query29 849 416 422 416 query30 236 153 153 153 query31 1034817 832 817 query32 97 60 58 58 query33 748 304 286 286 query34 1180532 532 532 query35 909 749 754 749 query36 1134948 968 948 query37 131 77 71 71 query38 4323414742314147 query39 1506145314961453 query40 197 101 106 101 query41 47 42 43 42 query42 111 105 102 102 query43 521 490 503 490 query44 1301859 851 851 query45 196 171 165 165 query46 1183757 753 753 query47 2070194119291929 query48 415 328 318 318 query49 965 395 411 395 query50 842 426 401 401 query51 7302725970457045 query52 101 92 90 90 query53 266 182 185 182 query54 1129410 450 410 query55 86 79 82 79 query56 266 253 254 253 query57 1300119311391139 query58 241 225 231 225 query59 3369320530813081 query60 283 253 252 252 query61 108 106 112 106 query62 881 705 714 705 query63 218 189 188 188 query64 3975731 669 669 query65 3326331432583258 query66 796 306 311 306 query67 16527 15517 15429 15429 query68 5591567 538 538 query69 481 250 253 250 query70 1232113910661066 query71 509 263 262 262 query72 7068415940434043 query73 801 358 366 358 query74 10038 882888788828 query75 4075264226432642 query76 3737111811831118 query77 672 260 267 260 query78 10249 938993769376 query79 1130589 595 589 query80 710 451 432 432 query81 544 229 231 229 query82 595 129 119 119 query83 174 152 149 149 query84 246 84 72 72 query85 1306325 297 297 query86 388 293 270 270 query87 4529434646024346 query88 3374223921992199 query89 451 287 290 287 query90 2112191 188 188 query91 142 104 114 104 query92 64 50 49 49 query93 1817554 547 547 query94 893 284 266 266 query95 351 254 247 247 query96 612 278 278 278 query97 2874267226862672 query98 214 211 198 198 query99 1558132713121312 Total cold run time: 304534 ms Total hot run time: 195757 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go
Re: [PR] [Enhancement] (nereids)implement showStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44393: URL: https://github.com/apache/doris/pull/44393#issuecomment-2558148956 ClickBench: Total hot run time: 32.08 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit 9f787c9a01e38bae7fc61587442dc7699c7a2eed, data reload: false query1 0.040.040.03 query2 0.070.030.04 query3 0.240.070.07 query4 1.600.100.10 query5 0.450.410.41 query6 1.160.650.65 query7 0.020.020.02 query8 0.040.030.03 query9 0.560.520.50 query10 0.560.580.55 query11 0.150.110.10 query12 0.140.110.11 query13 0.610.610.60 query14 2.802.842.72 query15 0.910.820.82 query16 0.380.360.42 query17 1.061.061.07 query18 0.230.200.21 query19 1.911.912.04 query20 0.010.010.01 query21 15.35 0.600.60 query22 2.292.371.66 query23 17.16 1.010.92 query24 2.941.250.38 query25 0.320.210.20 query26 0.290.130.14 query27 0.050.040.03 query28 11.26 1.091.07 query29 12.58 3.263.19 query30 0.250.060.07 query31 2.870.390.37 query32 3.270.450.45 query33 3.163.093.15 query34 17.02 4.434.47 query35 4.504.444.45 query36 0.690.470.49 query37 0.090.060.06 query38 0.050.040.03 query39 0.030.030.02 query40 0.170.130.13 query41 0.080.030.02 query42 0.030.020.02 query43 0.040.030.03 Total cold run time: 107.43 s Total hot run time: 32.08 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement DropStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44825: URL: https://github.com/apache/doris/pull/44825#issuecomment-2558149882 TPC-DS: Total hot run time: 196482 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit a82b7baf4b2df3ff2267b77d3db1d0816aa6df33, data reload: false query1 1382972 1025972 query2 6238232122962296 query3 11085 467647924676 query4 33441 23297 23568 23297 query5 4849473 454 454 query6 285 190 186 186 query7 4005306 306 306 query8 304 240 251 240 query9 9523271126862686 query10 492 240 260 240 query11 17959 15010 15049 15010 query12 153 105 99 99 query13 1572419 413 413 query14 9320763574707470 query15 276 193 194 193 query16 8015505 475 475 query17 1592605 616 605 query18 2174314 328 314 query19 374 169 170 169 query20 133 123 117 117 query21 211 108 110 108 query22 49694555 query23 34557 33946 33661 33661 query24 10115 254025092509 query25 678 379 409 379 query26 725 159 155 155 query27 2145342 342 342 query28 6867248924772477 query29 749 434 431 431 query30 227 151 147 147 query31 1050869 860 860 query32 80 56 54 54 query33 683 287 286 286 query34 968 507 527 507 query35 914 763 768 763 query36 1109972 993 972 query37 123 73 76 73 query38 4287411741784117 query39 1499145214381438 query40 207 105 102 102 query41 53 43 47 43 query42 119 102 110 102 query43 534 511 497 497 query44 1218853 848 848 query45 183 173 172 172 query46 1188711 727 711 query47 2027192719331927 query48 421 320 319 319 query49 898 385 407 385 query50 836 400 403 400 query51 7388714473497144 query52 100 94 92 92 query53 267 187 186 186 query54 1128410 421 410 query55 82 81 85 81 query56 265 244 251 244 query57 1279113211581132 query58 229 221 240 221 query59 3474312431893124 query60 289 248 251 248 query61 113 104 106 104 query62 894 686 670 670 query63 213 184 194 184 query64 3682681 629 629 query65 3299322131853185 query66 805 319 327 319 query67 15987 15601 15438 15438 query68 5172564 551 551 query69 484 245 246 245 query70 1225114010971097 query71 476 249 248 248 query72 6879418940274027 query73 788 355 373 355 query74 10079 888390028883 query75 3753261126272611 query76 3717102611171026 query77 600 288 304 288 query78 10240 949393879387 query79 2509623 625 623 query80 1397421 422 421 query81 537 248 222 222 query82 1341116 116 116 query83 240 144 139 139 query84 287 67 71 67 query85 1408302 300 300 query86 456 298 303 298 query87 4722439444144394 query88 3783221721732173 query89 430 285 288 285 query90 2045190 185 185 query91 137 103 107 103 query92 63 49 53 49 query93 2849541 542 541 query94 889 293 311 293 query95 355 243 270 243 query96 629 276 276 276 query97 2869268027222680 query98 227 200 190 190 query99 1666133213081308 Total cold run time: 304558 ms Total hot run time: 196482 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go t
Re: [PR] [opt](profile) add index page profile for io #45675 [doris]
airborne12 commented on PR #45757: URL: https://github.com/apache/doris/pull/45757#issuecomment-2558052251 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](nereids) use binary search to prune partitions [doris]
doris-robot commented on PR #44586: URL: https://github.com/apache/doris/pull/44586#issuecomment-2558077792 TPC-DS: Total hot run time: 190276 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 9bb2e041435b28a6b0bd9b7532295f2fd4d98fa8, data reload: false query1 961 390 375 375 query2 6526241423322332 query3 6717218 232 218 query4 33746 23316 23720 23316 query5 4409476 468 468 query6 305 213 190 190 query7 4631304 310 304 query8 308 247 240 240 query9 9341272026902690 query10 446 245 238 238 query11 17981 15138 15228 15138 query12 172 100 100 100 query13 1671406 406 406 query14 11643 728068736873 query15 315 180 192 180 query16 8059417 443 417 query17 1641577 556 556 query18 2049294 290 290 query19 370 147 146 146 query20 119 108 110 108 query21 209 103 108 103 query22 4633439242874287 query23 34486 33619 33507 33507 query24 11197 251824832483 query25 734 386 411 386 query26 1801149 153 149 query27 2954328 336 328 query28 8115242324322423 query29 1031403 447 403 query30 296 148 150 148 query31 1049821 819 819 query32 98 56 61 56 query33 775 298 291 291 query34 994 519 498 498 query35 900 734 733 733 query36 1110943 934 934 query37 271 78 81 78 query38 4286414042024140 query39 1511143214941432 query40 286 100 100 100 query41 48 43 48 43 query42 114 104 97 97 query43 539 495 504 495 query44 1255837 818 818 query45 191 173 166 166 query46 1237697 702 697 query47 1947183318451833 query48 427 318 326 318 query49 1282410 398 398 query50 832 385 385 385 query51 7217702969316931 query52 109 96 98 96 query53 266 186 194 186 query54 1318413 429 413 query55 82 79 80 79 query56 262 236 231 231 query57 1275112511181118 query58 247 220 224 220 query59 3196297530032975 query60 285 260 245 245 query61 114 108 110 108 query62 908 666 682 666 query63 219 191 194 191 query64 5186664 654 654 query65 3301319132703191 query66 1386313 316 313 query67 15905 15652 15592 15592 query68 5172558 566 558 query69 453 259 263 259 query70 1214113010961096 query71 454 256 252 252 query72 6322421840644064 query73 821 356 368 356 query74 10470 880288758802 query75 3632269126892689 query76 3003115510841084 query77 534 279 281 279 query78 10254 991694239423 query79 2316616 614 614 query80 882 434 443 434 query81 536 245 242 242 query82 674 117 122 117 query83 241 149 157 149 query84 241 77 82 77 query85 1324377 388 377 query86 434 302 294 294 query87 4647475843944394 query88 4105221321892189 query89 420 291 304 291 query90 2202191 189 189 query91 137 105 107 105 query92 68 54 52 52 query93 1301545 550 545 query94 1053282 282 282 query95 355 251 256 251 query96 667 280 287 280 query97 2834270326702670 query98 221 199 220 199 query99 1589140213211321 Total cold run time: 305870 ms Total hot run time: 190276 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
(doris-website) branch asf-site updated (402f957e362 -> 309d3418f77)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard 402f957e362 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 new 309d3418f77 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (402f957e362) \ N -- N -- N refs/heads/asf-site (309d3418f77) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Refactor](join) some refactor of join process [doris]
BiteThet commented on PR #44346: URL: https://github.com/apache/doris/pull/44346#issuecomment-2558077476 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](nereids) use binary search to prune partitions [doris]
doris-robot commented on PR #44586: URL: https://github.com/apache/doris/pull/44586#issuecomment-2558079159 ClickBench: Total hot run time: 33.04 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit 9bb2e041435b28a6b0bd9b7532295f2fd4d98fa8, data reload: false query1 0.030.030.03 query2 0.070.040.04 query3 0.230.080.07 query4 1.610.100.10 query5 0.430.410.42 query6 1.140.650.65 query7 0.020.020.02 query8 0.050.030.03 query9 0.570.510.51 query10 0.540.600.55 query11 0.150.110.10 query12 0.130.110.12 query13 0.610.600.60 query14 2.732.722.81 query15 0.890.830.82 query16 0.400.410.40 query17 1.061.071.03 query18 0.230.210.22 query19 1.981.812.03 query20 0.010.010.01 query21 15.36 0.610.56 query22 2.871.741.85 query23 17.11 0.860.85 query24 3.221.441.42 query25 0.250.070.19 query26 0.520.130.13 query27 0.050.040.03 query28 9.801.101.07 query29 12.56 3.233.23 query30 0.250.060.05 query31 2.860.390.39 query32 3.260.460.47 query33 3.083.113.15 query34 16.77 4.454.48 query35 4.564.494.52 query36 0.680.500.48 query37 0.100.060.05 query38 0.050.040.03 query39 0.030.020.03 query40 0.180.130.12 query41 0.080.030.02 query42 0.040.020.02 query43 0.030.030.03 Total cold run time: 106.59 s Total hot run time: 33.04 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](catalog) fix npe after replaying the external catalog [doris]
doris-robot commented on PR #45756: URL: https://github.com/apache/doris/pull/45756#issuecomment-2558044928 TPC-DS: Total hot run time: 196356 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 30c8cf385a51ca9c078bcddfad10adeeeb8bc519, data reload: false query1 1323964 916 916 query2 6227230623262306 query3 11366 491846624662 query4 33154 23261 23274 23261 query5 5005459 438 438 query6 287 185 187 185 query7 3989317 324 317 query8 329 260 242 242 query9 9386269626902690 query10 540 252 247 247 query11 17764 15154 15216 15154 query12 160 104 101 101 query13 1664423 414 414 query14 10567 745771107110 query15 275 212 199 199 query16 7968507 473 473 query17 1717598 590 590 query18 2102292 309 292 query19 341 154 157 154 query20 130 112 113 112 query21 207 139 120 120 query22 4767444544434443 query23 34664 33890 33665 33665 query24 10445 242124692421 query25 623 398 408 398 query26 1149158 158 158 query27 2681340 337 337 query28 8135242824252425 query29 818 415 414 414 query30 229 147 154 147 query31 1015850 848 848 query32 92 59 61 59 query33 749 305 310 305 query34 1028535 541 535 query35 863 765 791 765 query36 1142968 978 968 query37 152 81 78 78 query38 4569412644644126 query39 1541151015051505 query40 206 101 99 99 query41 48 47 44 44 query42 127 105 106 105 query43 526 493 487 487 query44 1311854 838 838 query45 188 174 174 174 query46 1175722 758 722 query47 2040194119681941 query48 429 323 332 323 query49 852 408 397 397 query50 851 402 399 399 query51 7386716971947169 query52 99 91 97 91 query53 268 196 187 187 query54 1061431 431 431 query55 84 83 80 80 query56 263 237 243 237 query57 1306118411431143 query58 225 214 224 214 query59 3335305731483057 query60 265 260 260 260 query61 111 111 121 111 query62 852 687 702 687 query63 221 198 194 194 query64 3945701 693 693 query65 3309331532803280 query66 762 310 300 300 query67 16352 15589 15447 15447 query68 5383557 567 557 query69 481 263 247 247 query70 1168107410961074 query71 447 268 267 267 query72 7041411842184118 query73 783 368 361 361 query74 10394 885388778853 query75 3431265426662654 query76 3723106911291069 query77 665 292 281 281 query78 10533 941792929292 query79 1584613 592 592 query80 867 435 427 427 query81 511 244 233 233 query82 685 121 116 116 query83 170 141 142 141 query84 251 78 75 75 query85 1325313 300 300 query86 396 311 304 304 query87 4515436343774363 query88 381621832183 query89 428 290 299 290 query90 2167191 186 186 query91 138 102 110 102 query92 69 52 50 50 query93 2253551 545 545 query94 871 331 289 289 query95 350 256 249 249 query96 622 283 274 274 query97 2829268126782678 query98 231 214 218 214 query99 1619131612821282 Total cold run time: 306506 ms Total hot run time: 196356 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go t
Re: [PR] [opt](profile) add index page profile for io #45675 [doris]
airborne12 commented on PR #45757: URL: https://github.com/apache/doris/pull/45757#issuecomment-2558051336 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] [opt](profile) add index page profile for io #45675 [doris]
airborne12 opened a new pull request, #45757: URL: https://github.com/apache/doris/pull/45757 cherry pick from #45675 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](profile) add index page profile for io #45675 [doris]
hello-stephen commented on PR #45757: URL: https://github.com/apache/doris/pull/45757#issuecomment-2558051347 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman opened a new pull request, #45758: URL: https://github.com/apache/doris/pull/45758 cherry-pick (#45433)(#45756) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
Thearas commented on PR #45758: URL: https://github.com/apache/doris/pull/45758#issuecomment-2558060083 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](catalog) fix npe after replaying the external catalog [doris]
doris-robot commented on PR #45756: URL: https://github.com/apache/doris/pull/45756#issuecomment-2558041642 TPC-H: Total hot run time: 39437 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 30c8cf385a51ca9c078bcddfad10adeeeb8bc519, data reload: false -- Round 1 -- q1 17597 738372987298 q2 2044180 170 170 q3 10725 106211411062 q4 10568 815 649 649 q5 7600262926752629 q6 243 153 143 143 q7 963 620 595 595 q8 9222180119491801 q9 6541636763906367 q10 7045236223102310 q11 477 259 254 254 q12 418 229 218 218 q13 17775 299629952995 q14 245 223 213 213 q15 578 504 497 497 q16 654 593 603 593 q17 971 554 502 502 q18 7145675766036603 q19 1380980 989 980 q20 469 186 180 180 q21 4104326630643064 q22 367 314 320 314 Total cold run time: 107131 ms Total hot run time: 39437 ms - Round 2, with runtime_filter_mode=off - q1 7256725672437243 q2 334 232 237 232 q3 2909282229622822 q4 2095186918071807 q5 5645563955985598 q6 229 142 141 141 q7 2232182218461822 q8 3344356234683468 q9 8956905789788978 q10 3648350335603503 q11 586 497 506 497 q12 854 603 600 600 q13 13371 308331173083 q14 309 274 278 274 q15 565 508 491 491 q16 680 640 631 631 q17 1854161615961596 q18 8315774877487748 q19 1705162715461546 q20 2082186018701860 q21 5554559155185518 q22 648 569 580 569 Total cold run time: 73171 ms Total hot run time: 60027 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement adminSetPartitionVersionCommand in nereids [doris]
doris-robot commented on PR #45426: URL: https://github.com/apache/doris/pull/45426#issuecomment-2558056926 ClickBench: Total hot run time: 32 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit 4279fbac107bec713989b3de7ed6c6e4dc417665, data reload: false query1 0.030.030.04 query2 0.060.040.03 query3 0.220.070.07 query4 1.610.110.11 query5 0.430.410.41 query6 1.350.650.65 query7 0.020.020.02 query8 0.040.030.03 query9 0.580.510.51 query10 0.550.570.57 query11 0.140.100.11 query12 0.140.110.12 query13 0.610.610.59 query14 2.772.742.73 query15 0.900.820.82 query16 0.380.390.38 query17 1.061.051.07 query18 0.220.210.20 query19 1.891.881.96 query20 0.010.010.02 query21 15.37 0.590.61 query22 2.672.441.41 query23 16.89 1.020.71 query24 2.940.761.26 query25 0.140.160.09 query26 0.580.150.14 query27 0.040.040.05 query28 10.90 1.101.06 query29 12.58 3.213.20 query30 0.250.060.06 query31 2.860.400.37 query32 3.270.460.45 query33 3.073.073.08 query34 17.16 4.484.50 query35 4.534.534.48 query36 0.690.510.48 query37 0.090.060.06 query38 0.050.040.04 query39 0.030.020.02 query40 0.170.120.12 query41 0.080.020.02 query42 0.040.020.02 query43 0.030.030.03 Total cold run time: 107.44 s Total hot run time: 32 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman commented on PR #45758: URL: https://github.com/apache/doris/pull/45758#issuecomment-2558120244 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) push down encode slot [doris]
doris-robot commented on PR #44748: URL: https://github.com/apache/doris/pull/44748#issuecomment-2558317741 TPC-H: Total hot run time: 39537 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 3abcc39b807b930bd3937466779318ce0fcd5773, data reload: false -- Round 1 -- q1 17593 716670657065 q2 2043172 162 162 q3 10573 109511451095 q4 10448 718 787 718 q5 7616268226332633 q6 238 148 145 145 q7 962 621 613 613 q8 9266184719161847 q9 6708644263866386 q10 6986229722912291 q11 465 261 248 248 q12 424 226 231 226 q13 17799 294129982941 q14 261 220 208 208 q15 551 494 495 494 q16 652 610 621 610 q17 981 532 475 475 q18 7447715768666866 q19 1364972 909 909 q20 468 186 184 184 q21 4067316331123112 q22 376 309 314 309 Total cold run time: 107288 ms Total hot run time: 39537 ms - Round 2, with runtime_filter_mode=off - q1 7021698370306983 q2 326 235 231 231 q3 2934279328962793 q4 2046181118151811 q5 5683567456385638 q6 229 147 152 147 q7 181217971797 q8 3337352934993499 q9 8950902089458945 q10 3598350935663509 q11 601 496 535 496 q12 832 637 617 617 q13 11529 314430913091 q14 316 279 297 279 q15 565 529 512 512 q16 713 656 641 641 q17 1826163516261626 q18 8267781077497749 q19 1722156216501562 q20 2045186618981866 q21 5566541554895415 q22 631 573 585 573 Total cold run time: 70959 ms Total hot run time: 59780 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [test](test) is master has core? [doris]
doris-robot commented on PR #45663: URL: https://github.com/apache/doris/pull/45663#issuecomment-2558316987 TPC-DS: Total hot run time: 195570 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit e79fb22b33b744c803c8c05e0083dd77a4afa971, data reload: false query1 1312960 933 933 query2 6236236823052305 query3 10991 453646184536 query4 33695 23373 23326 23326 query5 4392471 451 451 query6 282 197 204 197 query7 4006303 311 303 query8 314 267 241 241 query9 9316271627182716 query10 483 258 248 248 query11 17826 15085 15074 15074 query12 171 107 102 102 query13 1581438 426 426 query14 9559664075646640 query15 289 200 196 196 query16 8235478 495 478 query17 1827580 578 578 query18 2166297 318 297 query19 362 153 154 153 query20 129 121 120 120 query21 210 104 105 104 query22 4721440044654400 query23 34654 33979 33593 33593 query24 10175 250524232423 query25 592 415 394 394 query26 715 170 163 163 query27 2188340 328 328 query28 6276247424332433 query29 768 424 431 424 query30 225 146 162 146 query31 1054811 825 811 query32 105 59 58 58 query33 674 292 300 292 query34 964 515 521 515 query35 898 774 795 774 query36 1137983 964 964 query37 122 72 75 72 query38 4439433541264126 query39 1546143814811438 query40 207 101 104 101 query41 44 41 45 41 query42 119 101 104 101 query43 533 492 481 481 query44 1191866 845 845 query45 192 186 177 177 query46 1198732 733 732 query47 2053190719481907 query48 411 337 330 330 query49 889 392 378 378 query50 836 397 395 395 query51 7229715571647155 query52 107 90 123 90 query53 258 182 185 182 query54 1190414 412 412 query55 85 78 81 78 query56 265 247 252 247 query57 1302114311611143 query58 233 211 236 211 query59 3289322831103110 query60 281 254 257 254 query61 110 119 109 109 query62 865 700 687 687 query63 240 193 191 191 query64 3807671 666 666 query65 3331328732893287 query66 839 287 319 287 query67 16799 15761 15533 15533 query68 4983563 543 543 query69 487 253 257 253 query70 1201113411491134 query71 490 252 247 247 query72 7136411341914113 query73 792 368 354 354 query74 9961893288398839 query75 3803262626452626 query76 3724107410881074 query77 583 288 281 281 query78 10199 944493879387 query79 1438600 596 596 query80 1086435 425 425 query81 517 250 221 221 query82 387 119 112 112 query83 197 141 140 140 query84 284 76 70 70 query85 907 299 303 299 query86 370 304 307 304 query87 4458438643204320 query88 3544221021792179 query89 409 289 289 289 query90 2029183 182 182 query91 145 102 104 102 query92 61 49 49 49 query93 2118545 540 540 query94 800 282 282 282 query95 351 245 242 242 query96 623 277 277 277 query97 2872268726812681 query98 219 189 195 189 query99 1559132413121312 Total cold run time: 300271 ms Total hot run time: 195570 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go
Re: [PR] [test](test) is master has core? [doris]
doris-robot commented on PR #45663: URL: https://github.com/apache/doris/pull/45663#issuecomment-2558317955 ClickBench: Total hot run time: 32.58 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit e79fb22b33b744c803c8c05e0083dd77a4afa971, data reload: false query1 0.040.030.05 query2 0.070.030.03 query3 0.240.070.08 query4 1.590.110.11 query5 0.410.420.41 query6 1.170.660.65 query7 0.020.020.01 query8 0.040.040.03 query9 0.580.520.52 query10 0.560.570.57 query11 0.140.100.10 query12 0.140.110.10 query13 0.610.600.60 query14 2.742.762.83 query15 0.910.830.82 query16 0.390.370.38 query17 1.040.970.97 query18 0.220.200.21 query19 1.881.811.98 query20 0.010.010.01 query21 15.36 0.590.57 query22 3.142.191.68 query23 17.02 0.980.83 query24 2.701.561.16 query25 0.150.090.21 query26 0.420.140.13 query27 0.050.040.05 query28 10.25 1.111.07 query29 12.53 3.203.20 query30 0.250.060.06 query31 2.850.400.39 query32 3.240.470.45 query33 3.123.113.08 query34 16.93 4.464.51 query35 4.494.434.48 query36 0.660.480.48 query37 0.100.060.06 query38 0.040.030.04 query39 0.040.020.02 query40 0.170.120.12 query41 0.080.020.02 query42 0.040.020.02 query43 0.040.020.03 Total cold run time: 106.47 s Total hot run time: 32.58 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [test](test) is master has core? [doris]
924060929 commented on PR #45663: URL: https://github.com/apache/doris/pull/45663#issuecomment-2558306432 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) push down encode slot [doris]
englefly commented on PR #44748: URL: https://github.com/apache/doris/pull/44748#issuecomment-2558359622 run cloud_p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) push down encode slot [doris]
englefly commented on PR #44748: URL: https://github.com/apache/doris/pull/44748#issuecomment-2558363409 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) when check shape failed, print all plan with memo [doris]
doris-robot commented on PR #45627: URL: https://github.com/apache/doris/pull/45627#issuecomment-2558365393 TPC-H: Total hot run time: 39946 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 4dc24cddb00bfc0146974133e473528d337e9091, data reload: false -- Round 1 -- q1 17594 833973017301 q2 2055189 181 181 q3 10553 107411711074 q4 10529 728 748 728 q5 7643270526962696 q6 241 147 144 144 q7 979 617 605 605 q8 9252189019301890 q9 6639641264446412 q10 7056233223182318 q11 461 268 261 261 q12 422 220 219 219 q13 17951 287831042878 q14 249 217 218 217 q15 557 535 510 510 q16 666 589 603 589 q17 992 553 611 553 q18 7359673366566656 q19 13551080939 939 q20 471 184 186 184 q21 4021331232793279 q22 380 312 313 312 Total cold run time: 107425 ms Total hot run time: 39946 ms - Round 2, with runtime_filter_mode=off - q1 7283722472817224 q2 332 234 229 229 q3 2938291429792914 q4 2105185218321832 q5 5632564056855640 q6 230 139 141 139 q7 2294182918621829 q8 3383353635483536 q9 8973901989938993 q10 3625354035813540 q11 597 523 507 507 q12 811 591 628 591 q13 13786 310030963096 q14 312 273 284 273 q15 554 504 505 504 q16 699 651 652 651 q17 1875166616571657 q18 8346773175737573 q19 1755159216751592 q20 2112182818711828 q21 5653547354325432 q22 671 611 572 572 Total cold run time: 73966 ms Total hot run time: 60152 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Fix](ccr) Wrong result of tosql for add column with comment #45319 [doris]
wyxxxcat commented on PR #45727: URL: https://github.com/apache/doris/pull/45727#issuecomment-2558327781 run p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris-website) branch asf-site updated (af5f4dd9c04 -> 3dfe869a469)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard af5f4dd9c04 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 new 3dfe869a469 Automated deployment with doris branch @ 0099afc4772236e765a789913c886527663abc76 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (af5f4dd9c04) \ N -- N -- N refs/heads/asf-site (3dfe869a469) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) push down encode slot [doris]
englefly commented on PR #44748: URL: https://github.com/apache/doris/pull/44748#issuecomment-2558310294 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [test](test) is master has core? [doris]
doris-robot commented on PR #45663: URL: https://github.com/apache/doris/pull/45663#issuecomment-2558313612 TeamCity be ut coverage result: Function Coverage: 38.87% (10132/26064) Line Coverage: 29.81% (85295/286144) Region Coverage: 28.94% (43556/150525) Branch Coverage: 25.44% (22174/87174) Coverage Report: http://coverage.selectdb-in.cc/coverage/e79fb22b33b744c803c8c05e0083dd77a4afa971_e79fb22b33b744c803c8c05e0083dd77a4afa971/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [feat](nereids) adjust min/max for partition key #41729 branch-2.1 (#45713)
This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 7d9a7512971 [feat](nereids) adjust min/max for partition key #41729
branch-2.1 (#45713)
7d9a7512971 is described below
commit 7d9a7512971b3ec9dc0dee68835999b123c8650a
Author: minghong
AuthorDate: Sun Dec 22 11:20:01 2024 +0800
[feat](nereids) adjust min/max for partition key #41729 branch-2.1 (#45713)
### What problem does this PR solve?
pick #41729
---
.../doris/nereids/stats/StatsCalculator.java | 174 +
1 file changed, 174 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index d4b4022fbe7..bfe5ec4be48 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -18,9 +18,15 @@
package org.apache.doris.nereids.stats;
import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ListPartitionItem;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
@@ -136,11 +142,13 @@ import org.apache.doris.statistics.StatisticRange;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.statistics.StatisticsBuilder;
import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -899,6 +907,155 @@ public class StatsCalculator extends
DefaultPlanVisitor {
}
}
+/**
+ * Determine whether it is a partition key inside the function.
+ */
+private ColumnStatistic updateMinMaxForPartitionKey(OlapTable olapTable,
+List selectedPartitionNames,
+SlotReference slot, ColumnStatistic cache) {
+if (olapTable.getPartitionType() == PartitionType.LIST) {
+cache = updateMinMaxForListPartitionKey(olapTable,
selectedPartitionNames, slot, cache);
+} else if (olapTable.getPartitionType() == PartitionType.RANGE) {
+cache = updateMinMaxForTheFirstRangePartitionKey(olapTable,
selectedPartitionNames, slot, cache);
+}
+return cache;
+}
+
+private double convertLegacyLiteralToDouble(LiteralExpr literal) throws
org.apache.doris.common.AnalysisException {
+return StatisticsUtil.convertToDouble(literal.getType(),
literal.getStringValue());
+}
+
+private ColumnStatistic updateMinMaxForListPartitionKey(OlapTable
olapTable,
+List selectedPartitionNames,
+SlotReference slot, ColumnStatistic cache) {
+int partitionColumnIdx =
olapTable.getPartitionColumns().indexOf(slot.getColumn().get());
+if (partitionColumnIdx != -1) {
+try {
+LiteralExpr minExpr = null;
+LiteralExpr maxExpr = null;
+double minValue = 0;
+double maxValue = 0;
+for (String selectedPartitionName : selectedPartitionNames) {
+PartitionItem item =
olapTable.getPartitionItemOrAnalysisException(
+selectedPartitionName);
+if (item instanceof ListPartitionItem) {
+ListPartitionItem lp = (ListPartitionItem) item;
+for (PartitionKey key : lp.getItems()) {
+if (minExpr == null) {
+minExpr =
key.getKeys().get(partitionColumnIdx);
+minValue =
convertLegacyLiteralToDouble(minExpr);
+maxExpr =
key.getKeys().get(partitionColumnIdx);
+maxValue =
convertLegacyLiteralToDouble(maxExpr);
+} else {
+double current =
convertLegacyLiteralToDouble(key.getKeys().get(partitionColumnIdx));
+if (current > maxValue) {
+maxValue
Re: [PR] [feat](nereids) adjust min/max for partition key #41729 branch-2.1 [doris]
englefly merged PR #45713: URL: https://github.com/apache/doris/pull/45713 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [test](test) is master has core? [doris]
doris-robot commented on PR #45663: URL: https://github.com/apache/doris/pull/45663#issuecomment-2558314864 TPC-H: Total hot run time: 39726 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit e79fb22b33b744c803c8c05e0083dd77a4afa971, data reload: false -- Round 1 -- q1 17620 738482147384 q2 2063180 165 165 q3 10673 111411481114 q4 10536 740 784 740 q5 7609268025802580 q6 239 153 144 144 q7 969 616 604 604 q8 9248177918461779 q9 6479642764006400 q10 7048229323262293 q11 470 266 257 257 q12 440 228 228 228 q13 17781 297129852971 q14 251 223 212 212 q15 557 515 493 493 q16 675 579 593 579 q17 962 604 521 521 q18 7294679866976697 q19 14261068971 971 q20 464 188 177 177 q21 4011321731093109 q22 381 308 322 308 Total cold run time: 107196 ms Total hot run time: 39726 ms - Round 2, with runtime_filter_mode=off - q1 7237719772357197 q2 327 236 234 234 q3 2896276129732761 q4 2067179717801780 q5 5695570056475647 q6 228 142 144 142 q7 2238179918191799 q8 3392352835133513 q9 9022901789828982 q10 3572354935583549 q11 605 501 509 501 q12 853 601 613 601 q13 12805 316631023102 q14 307 296 286 286 q15 563 508 518 508 q16 677 652 667 652 q17 1854166116331633 q18 8305782778227822 q19 1729160415071507 q20 2053185418991854 q21 5599539753875387 q22 650 614 583 583 Total cold run time: 72674 ms Total hot run time: 60040 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[I] [Enhancement] 编译包排除spark 中已经有的jar [doris-spark-connector]
melin opened a new issue, #241: URL: https://github.com/apache/doris-spark-connector/issues/241 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Description 建议编译包排除spark 中已经有的jar，例如jackson，org.apache.commons等 ### Solution _No response_ ### Are you willing to submit PR? - [ ] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) push down encode slot [doris]
englefly commented on PR #44748: URL: https://github.com/apache/doris/pull/44748#issuecomment-2558359516 run p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [improvement](memory) Storage page cache use LRU-K cache, K=2 [doris]
xinyiZzz commented on PR #45719: URL: https://github.com/apache/doris/pull/45719#issuecomment-2558354015 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](nereids) when check shape failed, print all plan with memo [doris]
englefly commented on PR #45627: URL: https://github.com/apache/doris/pull/45627#issuecomment-2558355800 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](Nereids) set correct sort key for aggregate #45369 branch-3.0 [doris]
englefly commented on PR #45706: URL: https://github.com/apache/doris/pull/45706#issuecomment-2558356773 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](profile) add index page profile for io #45675 [doris]
doris-robot commented on PR #45757: URL: https://github.com/apache/doris/pull/45757#issuecomment-2558069811 TPC-DS: Total hot run time: 195806 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit f26107d6acc738c98da0f916c53394d70fd5d604, data reload: false query1 1299913 910 910 query2 6290209420552055 query3 11013 420341494149 query4 66936 29443 23429 23429 query5 5454461 443 443 query6 473 180 183 180 query7 5679309 310 309 query8 315 227 231 227 query9 9102269026532653 query10 540 264 253 253 query11 18017 15300 15872 15300 query12 161 99 103 99 query13 1572423 434 423 query14 10036 760074297429 query15 218 183 179 179 query16 7476534 515 515 query17 1085582 598 582 query18 1907318 317 317 query19 232 157 154 154 query20 117 114 112 112 query21 71 47 47 47 query22 4597452144054405 query23 34374 34226 33869 33869 query24 6045293228612861 query25 543 397 417 397 query26 694 167 164 164 query27 1851296 295 295 query28 4678253825132513 query29 711 454 431 431 query30 243 164 163 163 query31 1038778 813 778 query32 65 53 56 53 query33 479 268 275 268 query34 931 533 511 511 query35 848 755 743 743 query36 1088975 975 975 query37 122 68 70 68 query38 4113399439043904 query39 1546146714881467 query40 144 85 84 84 query41 46 46 44 44 query42 111 100 96 96 query43 540 501 497 497 query44 1157796 806 796 query45 189 171 171 171 query46 1176726 730 726 query47 1969186419291864 query48 468 377 365 365 query49 737 368 363 363 query50 830 417 418 417 query51 7136715972607159 query52 97 93 86 86 query53 250 188 184 184 query54 562 456 440 440 query55 79 73 73 73 query56 250 232 239 232 query57 1152109511181095 query58 211 201 204 201 query59 3106283029302830 query60 279 248 251 248 query61 108 108 108 108 query62 784 640 682 640 query63 208 194 187 187 query64 1914640 614 614 query65 3236324631723172 query66 756 299 297 297 query67 15605 15436 15317 15317 query68 4559540 555 540 query69 412 246 250 246 query70 1158111610411041 query71 351 247 246 246 query72 6564411138953895 query73 761 343 351 343 query74 10163 896288258825 query75 3331263726372637 query76 2261944 1071944 query77 491 274 260 260 query78 10608 971195069506 query79 3661596 593 593 query80 2005420 449 420 query81 559 242 248 242 query82 756 115 125 115 query83 256 152 152 152 query84 295 88 86 86 query85 1631375 354 354 query86 488 272 309 272 query87 4424423742014201 query88 4816241623492349 query89 411 287 294 287 query90 2024185 188 185 query91 181 143 143 143 query92 70 49 49 49 query93 5390532 533 532 query94 901 286 286 286 query95 356 253 246 246 query96 628 279 279 279 query97 3336312531573125 query98 225 197 200 197 query99 1814128312741274 Total cold run time: 329945 ms Total hot run time: 195806 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to th
Re: [PR] [enhance](nereids) add, sub, diff, floor, ceil date function implement Monotonic [doris]
924060929 commented on code in PR #44943:
URL: https://github.com/apache/doris/pull/44943#discussion_r1894594377
##
regression-test/suites/nereids_rules_p0/partition_prune/test_convert_tz.groovy:
##
@@ -93,7 +93,7 @@ suite("test_convert_tz") {
}
explain {
sql "SELECT * FROM test_convert_tz WHERE not convert_tz(timestamp,
'Asia/Shanghai', 'Europe/Paris') <= '2021-01-01';";
-contains("partitions=2/3 (p2,p3)")
+contains("partitions=3/3 (p1,p2,p3)")
Review Comment:
this test has rollback, we should improve it later
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [enhance](nereids) date_add, date_sub, date_diff, date_floor, date_ceil function implement Monotonic [doris]
924060929 merged PR #44943: URL: https://github.com/apache/doris/pull/44943 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [enhance](nereids) date_add, date_sub, date_diff, date_floor, date_ceil function implement Monotonic (#44943)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f6071a3833b [enhance](nereids) date_add, date_sub, date_diff,
date_floor, date_ceil function implement Monotonic (#44943)
f6071a3833b is described below
commit f6071a3833bb095f01065a9d10bc828f59cac883
Author: feiniaofeiafei
AuthorDate: Sat Dec 21 18:00:07 2024 +0800
[enhance](nereids) date_add, date_sub, date_diff, date_floor, date_ceil
function implement Monotonic (#44943)
date_add, date_sub, date_diff, date_floor, date_ceil function implement
Monotonic, so we can do prune range partition for this functions, for example
`date_add(dt, 1) = '2024-01-01'`
---
.../rules/OneRangePartitionEvaluator.java | 15 +-
.../{Monotonic.java => DateAddSubMonotonic.java} | 27 +-
...{Monotonic.java => DateCeilFloorMonotonic.java} | 36 +-
.../{Monotonic.java => DateDiffMonotonic.java} | 28 +-
.../trees/expressions/functions/Monotonic.java | 5 +
.../expressions/functions/scalar/DayCeil.java | 17 +-
.../expressions/functions/scalar/DayFloor.java | 17 +-
.../expressions/functions/scalar/DaysAdd.java | 8 +-
.../expressions/functions/scalar/DaysDiff.java | 13 +-
.../expressions/functions/scalar/DaysSub.java | 8 +-
.../expressions/functions/scalar/FromDays.java | 18 +-
.../expressions/functions/scalar/HourCeil.java | 17 +-
.../expressions/functions/scalar/HourFloor.java| 17 +-
.../expressions/functions/scalar/HoursAdd.java | 9 +-
.../expressions/functions/scalar/HoursDiff.java| 13 +-
.../expressions/functions/scalar/HoursSub.java | 9 +-
.../functions/scalar/MicroSecondsAdd.java | 9 +-
.../functions/scalar/MicroSecondsDiff.java | 13 +-
.../functions/scalar/MicroSecondsSub.java | 9 +-
.../functions/scalar/MilliSecondsAdd.java | 9 +-
.../functions/scalar/MilliSecondsDiff.java | 13 +-
.../functions/scalar/MilliSecondsSub.java | 9 +-
.../expressions/functions/scalar/MinuteCeil.java | 17 +-
.../expressions/functions/scalar/MinuteFloor.java | 17 +-
.../expressions/functions/scalar/MinutesAdd.java | 9 +-
.../expressions/functions/scalar/MinutesDiff.java | 13 +-
.../expressions/functions/scalar/MinutesSub.java | 9 +-
.../expressions/functions/scalar/MonthCeil.java| 17 +-
.../expressions/functions/scalar/MonthFloor.java | 17 +-
.../expressions/functions/scalar/MonthsAdd.java| 8 +-
.../expressions/functions/scalar/MonthsDiff.java | 13 +-
.../expressions/functions/scalar/MonthsSub.java| 8 +-
.../expressions/functions/scalar/SecondCeil.java | 17 +-
.../expressions/functions/scalar/SecondFloor.java | 17 +-
.../expressions/functions/scalar/SecondsAdd.java | 9 +-
.../expressions/functions/scalar/SecondsDiff.java | 13 +-
.../expressions/functions/scalar/SecondsSub.java | 9 +-
.../functions/scalar/UnixTimestamp.java| 39 +-
.../expressions/functions/scalar/YearCeil.java | 17 +-
.../expressions/functions/scalar/YearFloor.java| 17 +-
.../expressions/functions/scalar/YearsAdd.java | 8 +-
.../expressions/functions/scalar/YearsDiff.java| 13 +-
.../expressions/functions/scalar/YearsSub.java | 8 +-
.../trees/expressions/literal/DateTimeLiteral.java | 5 +-
.../test_add_sub_diff_ceil_floor.groovy| 407 +
.../partition_prune/test_convert_tz.groovy | 6 +-
46 files changed, 950 insertions(+), 82 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 7e91d5502f7..eb9fd6e1491 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -48,6 +48,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.MaxLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -807,22 +808,29 @@ public class OneRangePartitionEvaluator
: new NonNullable(funcChild));
partitionSlotContainsNull.put((Expression) func,
withNullable.nullable());
-if (!result.childre
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
doris-robot commented on PR #45759: URL: https://github.com/apache/doris/pull/45759#issuecomment-2558070242 TPC-DS: Total hot run time: 188607 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit d128ed2927d4dd426bae01b20365bb226d235747, data reload: false query1 962 367 366 366 query2 6518215121522151 query3 6711220 226 220 query4 33528 23332 23392 23332 query5 4360448 444 444 query6 275 185 187 185 query7 4627316 313 313 query8 303 229 227 227 query9 9600266426652664 query10 468 256 257 256 query11 18014 15201 15156 15156 query12 150 100 105 100 query13 1637425 410 410 query14 9874720567196719 query15 240 170 172 170 query16 8095454 413 413 query17 1610565 535 535 query18 2123303 300 300 query19 256 168 142 142 query20 110 105 110 105 query21 60 50 50 50 query22 4532427442864274 query23 34494 33407 35769 33407 query24 13150 292028702870 query25 556 402 406 402 query26 1166173 174 173 query27 2857295 305 295 query28 8343245324552453 query29 681 454 444 444 query30 337 166 163 163 query31 1015795 808 795 query32 94 63 63 63 query33 785 293 287 287 query34 956 508 507 507 query35 863 723 720 720 query36 1101971 928 928 query37 200 72 76 72 query38 4066385437933793 query39 1460141814401418 query40 230 88 86 86 query41 53 53 49 49 query42 113 100 102 100 query43 529 504 497 497 query44 1318790 780 780 query45 184 165 169 165 query46 1149721 709 709 query47 1881180917771777 query48 455 362 396 362 query49 1287393 397 393 query50 797 407 410 407 query51 7255695371016953 query52 105 91 92 91 query53 266 190 190 190 query54 1128461 461 461 query55 79 77 79 77 query56 284 252 246 246 query57 1190109310811081 query58 237 208 237 208 query59 3388305931243059 query60 286 257 261 257 query61 150 127 128 127 query62 862 646 692 646 query63 209 193 190 190 query64 5355755 729 729 query65 3294315331483148 query66 1293309 329 309 query67 16042 15400 15512 15400 query68 4720559 552 552 query69 428 258 265 258 query70 1202111011321110 query71 422 255 258 255 query72 6138249938322499 query73 761 338 342 338 query74 10357 890889418908 query75 3384261825832583 query76 2908107410671067 query77 409 273 291 273 query78 10521 982295729572 query79 7153583 590 583 query80 1858445 427 427 query81 594 242 240 240 query82 920 119 121 119 query83 270 148 146 146 query84 296 80 81 80 query85 2212325 309 309 query86 486 283 279 279 query87 4396427343074273 query88 5364236423922364 query89 454 289 294 289 query90 2032186 183 183 query91 184 164 144 144 query92 65 48 53 48 query93 5312558 530 530 query94 916 296 292 292 query95 343 252 253 252 query96 636 279 286 279 query97 3309306631363066 query98 210 204 196 196 query99 1657127412991274 Total cold run time: 315513 ms Total hot run time: 188607 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
Re: [PR] [opt](nereids) use binary search to prune partitions [doris]
924060929 commented on PR #44586: URL: https://github.com/apache/doris/pull/44586#issuecomment-2558070726 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](nereids) use binary search to prune partitions [doris]
doris-robot commented on PR #44586: URL: https://github.com/apache/doris/pull/44586#issuecomment-2558075254 TPC-H: Total hot run time: 40369 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 9bb2e041435b28a6b0bd9b7532295f2fd4d98fa8, data reload: false -- Round 1 -- q1 17630 744073377337 q2 2061179 169 169 q3 10545 115411811154 q4 10236 702 747 702 q5 7622274826962696 q6 242 156 152 152 q7 1009641 613 613 q8 9265191019961910 q9 6633645864806458 q10 6991231623252316 q11 473 264 271 264 q12 425 225 229 225 q13 17783 296829582958 q14 246 223 229 223 q15 552 508 495 495 q16 661 592 609 592 q17 991 598 527 527 q18 7555679568276795 q19 13581015973 973 q20 481 178 181 178 q21 4398337833243324 q22 387 308 319 308 Total cold run time: 107544 ms Total hot run time: 40369 ms - Round 2, with runtime_filter_mode=off - q1 7590723772057205 q2 329 235 231 231 q3 2941283328132813 q4 1975173716861686 q5 5374541454095409 q6 216 137 140 137 q7 2158169016691669 q8 3250340534223405 q9 8646868686808680 q10 3502349634503450 q11 602 500 514 500 q12 798 568 591 568 q13 6 298029782978 q14 308 260 265 260 q15 560 496 497 496 q16 669 656 631 631 q17 1827158716161587 q18 7886750574317431 q19 1719157814921492 q20 2074178818401788 q21 5547535852435243 q22 624 587 605 587 Total cold run time: 69711 ms Total hot run time: 58246 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement adminSetPartitionVersionCommand in nereids [doris]
doris-robot commented on PR #45426: URL: https://github.com/apache/doris/pull/45426#issuecomment-2558052906 TPC-H: Total hot run time: 39791 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit 4279fbac107bec713989b3de7ed6c6e4dc417665, data reload: false -- Round 1 -- q1 16798 745072607260 q2 1121169 174 169 q3 1372117411651165 q4 1265770 764 764 q5 2673272226242624 q6 239 151 151 151 q7 977 611 618 611 q8 9255182419041824 q9 6643636064076360 q10 7057230823042304 q11 457 263 261 261 q12 431 232 224 224 q13 17748 298729782978 q14 242 209 207 207 q15 570 514 519 514 q16 636 580 613 580 q17 976 591 596 591 q18 7247675966256625 q19 1360102710391027 q20 475 190 185 185 q21 402230503050 q22 381 317 319 317 Total cold run time: 81945 ms Total hot run time: 39791 ms - Round 2, with runtime_filter_mode=off - q1 7253726072547254 q2 324 226 235 226 q3 2898282827812781 q4 1973171917631719 q5 5360534253985342 q6 219 138 141 138 q7 2107174017011701 q8 3227339533863386 q9 8542859786638597 q10 3504346034143414 q11 577 482 493 482 q12 792 562 591 562 q13 6600298929632963 q14 290 271 270 270 q15 550 496 508 496 q16 659 633 642 633 q17 1786159315801580 q18 7911750775267507 q19 1679148715751487 q20 2007183818591838 q21 5415521852675218 q22 619 584 551 551 Total cold run time: 64292 ms Total hot run time: 58145 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman merged PR #45759: URL: https://github.com/apache/doris/pull/45759 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: [opt](catalog) cache the Configuration object (#45433)(#45756) (#45759)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8861e67136c [opt](catalog) cache the Configuration object
(#45433)(#45756) (#45759)
8861e67136c is described below
commit 8861e67136ca5baee846dbdcc2049cbb69663872
Author: Mingyu Chen (Rayner)
AuthorDate: Sat Dec 21 20:00:16 2024 +0800
[opt](catalog) cache the Configuration object (#45433)(#45756) (#45759)
cherry-pick (#45433)(#45756)
---
.../apache/doris/datasource/ExternalCatalog.java | 22
.../datasource/hive/HiveMetaStoreClientHelper.java | 7 +---
.../doris/datasource/ExternalCatalogTest.java | 40 +++---
3 files changed, 59 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index cde08113373..c8ca21e88ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -152,6 +152,9 @@ public abstract class ExternalCatalog
protected MetaCache> metaCache;
protected PreExecutionAuthenticator preExecutionAuthenticator;
+private volatile Configuration cachedConf = null;
+private byte[] confLock = new byte[0];
+
public ExternalCatalog() {
}
@@ -163,6 +166,20 @@ public abstract class ExternalCatalog
}
public Configuration getConfiguration() {
+// build configuration is costly, so we cache it.
+if (cachedConf != null) {
+return cachedConf;
+}
+synchronized (confLock) {
+if (cachedConf != null) {
+return cachedConf;
+}
+cachedConf = buildConf();
+return cachedConf;
+}
+}
+
+private Configuration buildConf() {
Configuration conf =
DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth());
Map catalogProperties =
catalogProperty.getHadoopProperties();
for (Map.Entry entry : catalogProperties.entrySet()) {
@@ -408,6 +425,10 @@ public abstract class ExternalCatalog
this.convertedProperties = null;
}
+synchronized (this.confLock) {
+this.cachedConf = null;
+}
+
refreshOnlyCatalogCache(invalidCache);
}
@@ -762,6 +783,7 @@ public abstract class ExternalCatalog
}
}
this.propLock = new byte[0];
+this.confLock = new byte[0];
this.initialized = false;
setDefaultPropsIfMissing(true);
if (tableAutoAnalyzePolicy == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 884cfbee45b..706bd653a85 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -42,7 +42,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopAuthenticator;
import org.apache.doris.datasource.ExternalCatalog;
-import org.apache.doris.fs.remote.dfs.DFSFileSystem;
import org.apache.doris.thrift.TExprOpcode;
import com.google.common.base.Strings;
@@ -843,11 +842,7 @@ public class HiveMetaStoreClientHelper {
}
public static Configuration getConfiguration(HMSExternalTable table) {
-Configuration conf =
DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth());
-for (Map.Entry entry :
table.getHadoopProperties().entrySet()) {
-conf.set(entry.getKey(), entry.getValue());
-}
-return conf;
+return table.getCatalog().getConfiguration();
}
public static Optional getSerdeProperty(Table table, String key) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
index 43348ca8a0e..f8e72c366b5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
@@ -22,9 +22,10 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
-import org.apache.doris.
Re: [PR] [Enhancement] (nereids)implement adminSetPartitionVersionCommand in nereids [doris]
doris-robot commented on PR #45426: URL: https://github.com/apache/doris/pull/45426#issuecomment-2558055324 TPC-DS: Total hot run time: 189857 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 4279fbac107bec713989b3de7ed6c6e4dc417665, data reload: false query1 998 370 386 370 query2 6528227522882275 query3 6697226 219 219 query4 33788 23465 23233 23233 query5 4302467 485 467 query6 276 198 183 183 query7 4633307 316 307 query8 306 237 232 232 query9 9655270126912691 query10 473 249 241 241 query11 18012 15260 15141 15141 query12 161 98 102 98 query13 1656432 397 397 query14 10984 720375757203 query15 321 181 186 181 query16 8183449 453 449 query17 1754570 544 544 query18 2082299 290 290 query19 364 155 157 155 query20 115 106 116 106 query21 207 105 111 105 query22 4591417842314178 query23 34365 33692 33654 33654 query24 11223 245224492449 query25 670 389 376 376 query26 1813149 148 148 query27 2772328 329 328 query28 8020241024482410 query29 1041411 415 411 query30 300 149 152 149 query31 1078813 822 813 query32 100 62 56 56 query33 776 301 304 301 query34 939 518 515 515 query35 899 738 722 722 query36 1100919 927 919 query37 177 70 75 70 query38 4338402842104028 query39 1528144414091409 query40 289 104 99 99 query41 48 43 44 43 query42 120 102 103 102 query43 544 482 483 482 query44 1249834 815 815 query45 187 169 169 169 query46 1144727 711 711 query47 1949185118721851 query48 404 325 326 325 query49 1252400 391 391 query50 801 381 392 381 query51 7317713469766976 query52 106 96 96 96 query53 263 187 186 186 query54 1276413 416 413 query55 89 81 78 78 query56 268 241 244 241 query57 1271114111101110 query58 246 217 239 217 query59 3069304928632863 query60 285 256 262 256 query61 137 130 132 130 query62 876 685 680 680 query63 227 196 193 193 query64 5169793 649 649 query65 3247316132683161 query66 1253314 332 314 query67 15831 15486 15492 15486 query68 5072576 553 553 query69 445 253 251 251 query70 1246106811441068 query71 449 256 247 247 query72 6431412141344121 query73 789 364 365 364 query74 10605 878687948786 query75 3678261526392615 query76 3014106510991065 query77 540 287 283 283 query78 10322 945693069306 query79 2230627 611 611 query80 1045441 427 427 query81 528 237 231 231 query82 953 125 124 124 query83 252 147 142 142 query84 241 68 72 68 query85 1345331 302 302 query86 450 302 290 290 query87 4590433743754337 query88 3948224722592247 query89 409 301 298 298 query90 2114195 195 195 query91 138 104 110 104 query92 70 51 52 51 query93 1614551 544 544 query94 1034258 305 258 query95 353 253 255 253 query96 628 282 279 279 query97 2887268526902685 query98 217 193 202 193 query99 1548131813331318 Total cold run time: 305327 ms Total hot run time: 189857 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
Re: [PR] [fix](catalog) fix npe after replaying the external catalog [doris]
doris-robot commented on PR #45756: URL: https://github.com/apache/doris/pull/45756#issuecomment-2558046238 ClickBench: Total hot run time: 33.17 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit 30c8cf385a51ca9c078bcddfad10adeeeb8bc519, data reload: false query1 0.030.030.05 query2 0.070.030.03 query3 0.240.070.07 query4 1.600.110.10 query5 0.420.420.40 query6 1.180.640.64 query7 0.020.020.02 query8 0.040.040.02 query9 0.570.510.50 query10 0.560.570.56 query11 0.150.110.11 query12 0.140.110.10 query13 0.600.610.58 query14 2.742.762.84 query15 0.910.830.82 query16 0.380.380.38 query17 0.981.021.00 query18 0.220.210.21 query19 1.961.831.97 query20 0.010.010.01 query21 15.36 0.600.56 query22 2.832.721.61 query23 17.03 0.950.86 query24 2.951.662.10 query25 0.360.260.09 query26 0.380.140.15 query27 0.050.030.04 query28 9.301.111.07 query29 12.58 3.243.21 query30 0.250.060.06 query31 2.880.410.37 query32 3.230.460.46 query33 3.083.133.13 query34 17.05 4.494.52 query35 4.504.524.48 query36 0.670.480.48 query37 0.090.060.06 query38 0.040.030.04 query39 0.040.020.03 query40 0.180.120.12 query41 0.080.020.03 query42 0.040.020.02 query43 0.040.040.03 Total cold run time: 105.83 s Total hot run time: 33.17 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman commented on PR #45759: URL: https://github.com/apache/doris/pull/45759#issuecomment-2558061166 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman opened a new pull request, #45759: URL: https://github.com/apache/doris/pull/45759 cherry-pick (#45433)(#45756) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](catalog) cache the Configuration object (#45433)(#45756) [doris]
morningman commented on PR #45758: URL: https://github.com/apache/doris/pull/45758#issuecomment-2558060152 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement DropStoragePolicyCommand in nereids [doris]
doris-robot commented on PR #44825: URL: https://github.com/apache/doris/pull/44825#issuecomment-2558151191 ClickBench: Total hot run time: 32.15 s ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools ClickBench test result on commit a82b7baf4b2df3ff2267b77d3db1d0816aa6df33, data reload: false query1 0.040.030.03 query2 0.080.030.04 query3 0.230.060.08 query4 1.610.100.10 query5 0.430.400.40 query6 1.220.650.65 query7 0.020.010.02 query8 0.040.030.03 query9 0.580.500.51 query10 0.540.580.56 query11 0.140.110.10 query12 0.140.110.11 query13 0.610.610.61 query14 2.832.752.73 query15 0.900.820.82 query16 0.380.390.38 query17 1.021.041.05 query18 0.220.210.20 query19 1.941.852.01 query20 0.020.010.01 query21 15.37 0.580.59 query22 2.971.791.79 query23 16.95 0.990.81 query24 3.151.350.55 query25 0.250.090.23 query26 0.370.140.14 query27 0.060.040.04 query28 10.67 1.101.07 query29 12.63 3.213.21 query30 0.250.060.07 query31 2.850.380.38 query32 3.260.460.45 query33 3.083.073.10 query34 16.88 4.474.53 query35 4.544.474.49 query36 0.680.490.47 query37 0.090.050.06 query38 0.040.040.03 query39 0.030.020.02 query40 0.150.120.13 query41 0.080.030.02 query42 0.030.020.03 query43 0.040.030.02 Total cold run time: 107.41 s Total hot run time: 32.15 s ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](csv reader) fix core dump when parsing csv with enclose [doris]
sollhui commented on PR #45485: URL: https://github.com/apache/doris/pull/45485#issuecomment-2558152563 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
