This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 0213350f958 branch-4.2: [improvement](parser) Optimize string literal
lexing (#66902) (#69109)
0213350f958 is described below
commit 0213350f95887a4ed4776556f9b622efcad8cd49
Author: morrySnow <[email protected]>
AuthorDate: Fri Oct 16 14:13:48 2026 +0800
branch-4.2: [improvement](parser) Optimize string literal lexing (#66902)
(#69109)
### What problem does this PR solve?
Issue Number: N/A
Related PR: https://github.com/apache/doris/pull/66902
Problem Summary: String literal lexing evaluated SQL-mode predicates for
every character. Select the mode-specific loop after the opening quote
while preserving token semantics. Map the grammar and tests into
branch-4.2 fe-core, use Java 8-compatible APIs, and adapt optional JMH
tools to the existing syntax parser and benchmark profile.
### Release note
Reduce string literal lexing overhead while preserving SQL-mode
behavior.
### Backport validation
- Test: Unit Test: DorisLexerStringLiteralTest, NereidsParserTest;
optional parser benchmark sources compiled by run-fe-ut.sh; FE
Checkstyle (0 violations).
- Combined eight-backport validation: 283 tests across 13 classes, 0
failures/errors after correcting identity-cast assertions. This
standalone backport has not had a separate unit-test run.
- An isolated combined eight-backport ASAN cluster is deployed, but no
targeted regression suite for this PR or full regression suite was run.
- Full build.sh --fe and ASAN build.sh --be succeeded on the combined
eight-backport tree; deployment smoke validation passed.
- Behavior changed: No (SQL semantics are unchanged)
- Does this need documentation: No
(cherry picked from commit 5e83a92277132f1c4533ec14b2752d160e282403)
### Scope
This is an independent backport of #66902 onto branch-4.2.
The branch contains one commit and does not include the other seven
picks.
The 283-test result above was obtained from the combined backport tree;
this standalone branch has not had a separate unit-test run.
### Check List (For Author)
- Test:
- [x] Unit Test: 283 targeted tests passed on the combined backport
tree.
- [x] Manual test: standalone patch matches the validated pick; FE
Checkstyle passed on the combined tree.
- [ ] Regression test: no targeted regression suite for this PR was run;
the full regression suite was not run.
- Behavior changed: see the backport validation details above.
- Does this need documentation: No.
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 10 +-
.../parser/DorisLexerStringLiteralTest.java | 235 +++++++++++++++++++++
fe/fe-sql-parser-benchmark/README.md | 37 ++++
fe/fe-sql-parser-benchmark/pom.xml | 94 +++++++++
.../benchmark/ParserBenchmarkSupport.java | 44 ++++
.../benchmark/ParserControlBenchmark.java | 50 +++++
.../benchmark/StringLiteralBenchmark.java | 110 ++++++++++
.../benchmark/StringLiteralTokenCorpus.java | 121 +++++++++++
fe/pom.xml | 1 +
9 files changed, 700 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 29f4e1bdb23..2963c21d324 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -667,8 +667,14 @@ ATSIGN: '@';
DOUBLEATSIGN: '@@';
STRING_LITERAL
- : '\'' ( {!isNoBackslashEscapes}? '\\'. | '\'\'' |
{!isNoBackslashEscapes}? ~('\'' | '\\') | {isNoBackslashEscapes}? ~('\''))* '\''
- | '"' ( {!isNoBackslashEscapes}? '\\'. | '""' | {!isNoBackslashEscapes}?
~('"'| '\\') | {isNoBackslashEscapes}? ~('"'))* '"'
+ : '\'' (
+ {!isNoBackslashEscapes}? ('\\' . | '\'\'' | ~('\'' | '\\'))*
+ | {isNoBackslashEscapes}? ('\'\'' | ~('\''))*
+ ) '\''
+ | '"' (
+ {!isNoBackslashEscapes}? ('\\' . | '""' | ~('"' | '\\'))*
+ | {isNoBackslashEscapes}? ('""' | ~('"'))*
+ ) '"'
;
VARBINARY_LITERAL
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/DorisLexerStringLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/DorisLexerStringLiteralTest.java
new file mode 100644
index 00000000000..b34e47cd03e
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/DorisLexerStringLiteralTest.java
@@ -0,0 +1,235 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser;
+
+import org.apache.doris.nereids.DorisLexer;
+
+import com.google.common.base.Strings;
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.Token;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+class DorisLexerStringLiteralTest {
+ @Test
+ void lexesValidStringsAsOneTokenInBothSqlModes() {
+ List<String> literals = Arrays.asList(
+ singleQuoted(""),
+ doubleQuoted(""),
+ singleQuoted("plain ASCII"),
+ doubleQuoted("plain ASCII"),
+ singleQuoted("δΈζπ"),
+ doubleQuoted("δΈζπ"),
+ singleQuoted("it''s"),
+ doubleQuoted("a\"\"b"),
+ singleQuoted("line\nbreak"),
+ singleQuoted("a\\b"),
+ singleQuoted("a\\\\"));
+
+ for (boolean noBackslashEscapes : new boolean[] {false, true}) {
+ for (String literal : literals) {
+ assertSingleStringToken(literal, noBackslashEscapes);
+ }
+ }
+
+ assertSingleStringToken(singleQuoted("a\\'b"), false);
+ assertSingleStringToken(doubleQuoted("a\\\"b"), false);
+ assertSingleStringToken(singleQuoted("a\\"), true);
+ assertSingleStringToken(doubleQuoted("a\\"), true);
+ }
+
+ @Test
+ void preservesModeSensitiveTokenBoundariesAndPositions() {
+ String escapedQuote = singleQuoted("a\\'b");
+ Assertions.assertEquals(Arrays.asList(
+ "STRING_LITERAL|'a\\'b'|0|0|5|1|0",
+ "EOF|<EOF>|0|6|5|1|6"),
+ snapshot(escapedQuote, false));
+ Assertions.assertEquals(Arrays.asList(
+ "STRING_LITERAL|'a\\'|0|0|3|1|0",
+ "IDENTIFIER|b|0|4|4|1|4",
+ "UNRECOGNIZED|'|0|5|5|1|5",
+ "EOF|<EOF>|0|6|5|1|6"),
+ snapshot(escapedQuote, true));
+
+ String trailingBackslash = singleQuoted("a\\");
+ Assertions.assertEquals(Arrays.asList(
+ "UNRECOGNIZED|'|0|0|0|1|0",
+ "IDENTIFIER|a|0|1|1|1|1",
+ "UNRECOGNIZED|\\|0|2|2|1|2",
+ "UNRECOGNIZED|'|0|3|3|1|3",
+ "EOF|<EOF>|0|4|3|1|4"),
+ snapshot(trailingBackslash, false));
+ Assertions.assertEquals(Arrays.asList(
+ "STRING_LITERAL|'a\\'|0|0|3|1|0",
+ "EOF|<EOF>|0|4|3|1|4"),
+ snapshot(trailingBackslash, true));
+ }
+
+ @Test
+ void lexesLongStringsWithoutChangingTheirSourceInterval() {
+ for (int length : new int[] {0, 1, 16, 256, 4096, 65536}) {
+ for (boolean noBackslashEscapes : new boolean[] {false, true}) {
+ assertSingleStringToken(singleQuoted(Strings.repeat("a",
length)), noBackslashEscapes);
+ assertSingleStringToken(singleQuoted(Strings.repeat("''",
length)), noBackslashEscapes);
+ assertSingleStringToken(singleQuoted(Strings.repeat("\\n",
length)), noBackslashEscapes);
+ }
+ }
+ }
+
+ @Test
+ void semanticPredicateCallsDoNotScaleWithStringLength() {
+ int shortCount = predicateCalls(singleQuoted(Strings.repeat("a", 16)),
false);
+ int longCount = predicateCalls(singleQuoted(Strings.repeat("a",
4096)), false);
+ int noBackslashShortCount =
predicateCalls(singleQuoted(Strings.repeat("a", 16)), true);
+ int noBackslashLongCount =
predicateCalls(singleQuoted(Strings.repeat("a", 4096)), true);
+
+ Assertions.assertTrue(longCount <= shortCount + 2,
+ () -> "default mode predicate calls scaled from " + shortCount
+ " to " + longCount);
+ Assertions.assertTrue(noBackslashLongCount <= noBackslashShortCount +
2,
+ () -> "NO_BACKSLASH_ESCAPES predicate calls scaled from "
+ + noBackslashShortCount + " to " +
noBackslashLongCount);
+ }
+
+ @Test
+ void lexesDeterministicallyWithSharedStaticDfa() throws Exception {
+ List<String> inputs = Arrays.asList(
+ singleQuoted("plain"),
+ doubleQuoted("δΈζπ"),
+ singleQuoted("a\\'b"),
+ doubleQuoted("a\\\"b"),
+ singleQuoted("''\\n"),
+ doubleQuoted("\"\"\\n"),
+ "'unterminated",
+ "\"unterminated");
+ List<List<String>> expected = new ArrayList<>();
+ for (boolean noBackslashEscapes : new boolean[] {false, true}) {
+ for (String input : inputs) {
+ expected.add(snapshot(input, noBackslashEscapes));
+ }
+ }
+
+ ExecutorService executor = Executors.newFixedThreadPool(8);
+ try {
+ List<Callable<Void>> tasks = new ArrayList<>();
+ for (int thread = 0; thread < 8; thread++) {
+ tasks.add(() -> {
+ for (int repetition = 0; repetition < 100; repetition++) {
+ int caseIndex = 0;
+ for (boolean noBackslashEscapes : new boolean[]
{false, true}) {
+ for (String input : inputs) {
+
Assertions.assertEquals(expected.get(caseIndex++),
+ snapshot(input, noBackslashEscapes));
+ }
+ }
+ }
+ return null;
+ });
+ }
+ for (Future<Void> result : executor.invokeAll(tasks)) {
+ result.get();
+ }
+ } finally {
+ executor.shutdownNow();
+ Assertions.assertTrue(executor.awaitTermination(10,
TimeUnit.SECONDS));
+ }
+ }
+
+ private static int predicateCalls(String sql, boolean noBackslashEscapes) {
+ CountingLexer lexer = new CountingLexer(sql);
+ lexer.isNoBackslashEscapes = noBackslashEscapes;
+ while (lexer.nextToken().getType() != Token.EOF) {
+ // Consume all tokens.
+ }
+ return lexer.predicateCalls;
+ }
+
+ private static void assertSingleStringToken(String literal, boolean
noBackslashEscapes) {
+ List<Token> tokens = lex(literal, noBackslashEscapes);
+ Assertions.assertEquals(2, tokens.size(), () -> snapshot(literal,
noBackslashEscapes).toString());
+ int codePointLength = literal.codePointCount(0, literal.length());
+ Token string = tokens.get(0);
+ Assertions.assertEquals(DorisLexer.STRING_LITERAL, string.getType());
+ Assertions.assertEquals(literal, string.getText());
+ Assertions.assertEquals(Token.DEFAULT_CHANNEL, string.getChannel());
+ Assertions.assertEquals(0, string.getStartIndex());
+ Assertions.assertEquals(codePointLength - 1, string.getStopIndex());
+ Assertions.assertEquals(1, string.getLine());
+ Assertions.assertEquals(0, string.getCharPositionInLine());
+
+ Token eof = tokens.get(1);
+ Assertions.assertEquals(Token.EOF, eof.getType());
+ Assertions.assertEquals(codePointLength, eof.getStartIndex());
+ Assertions.assertEquals(codePointLength - 1, eof.getStopIndex());
+ }
+
+ private static List<String> snapshot(String sql, boolean
noBackslashEscapes) {
+ List<String> snapshot = new ArrayList<>();
+ for (Token token : lex(sql, noBackslashEscapes)) {
+ String type =
DorisLexer.VOCABULARY.getSymbolicName(token.getType());
+ snapshot.add(type + "|" + token.getText().replace("\n", "\\n") +
"|"
+ + token.getChannel() + "|" + token.getStartIndex() + "|" +
token.getStopIndex()
+ + "|" + token.getLine() + "|" +
token.getCharPositionInLine());
+ }
+ return snapshot;
+ }
+
+ private static List<Token> lex(String sql, boolean noBackslashEscapes) {
+ DorisLexer lexer = new DorisLexer(new
CaseInsensitiveStream(CharStreams.fromString(sql)));
+ lexer.isNoBackslashEscapes = noBackslashEscapes;
+ List<Token> tokens = new ArrayList<>();
+ Token token;
+ do {
+ token = lexer.nextToken();
+ tokens.add(token);
+ } while (token.getType() != Token.EOF);
+ return tokens;
+ }
+
+ private static String singleQuoted(String payload) {
+ return "'" + payload + "'";
+ }
+
+ private static String doubleQuoted(String payload) {
+ return "\"" + payload + "\"";
+ }
+
+ private static class CountingLexer extends DorisLexer {
+ private int predicateCalls;
+
+ CountingLexer(String sql) {
+ super(new CaseInsensitiveStream(CharStreams.fromString(sql)));
+ }
+
+ @Override
+ public boolean sempred(RuleContext localctx, int ruleIndex, int
predIndex) {
+ predicateCalls++;
+ return super.sempred(localctx, ruleIndex, predIndex);
+ }
+ }
+}
diff --git a/fe/fe-sql-parser-benchmark/README.md
b/fe/fe-sql-parser-benchmark/README.md
new file mode 100644
index 00000000000..f45494ba9fb
--- /dev/null
+++ b/fe/fe-sql-parser-benchmark/README.md
@@ -0,0 +1,37 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Doris FE parser benchmarks
+
+The optional benchmark Maven profile builds JMH benchmarks against the lexer
and
+syntax parser in fe-core. JMH is excluded from the default FE build and its
runtime
+artifacts. This branch does not have a standalone fe-sql-parser module.
+
+From the fe/ directory:
+
+```bash
+mvn -Pbenchmark -pl fe-sql-parser-benchmark -am package -DskipTests
+java -jar fe-sql-parser-benchmark/target/doris-fe-sql-parser-benchmarks.jar \
+ '.*StringLiteralBenchmark.*' -prof gc -rf json -rff
/tmp/string-literal-benchmark.json
+```
+
+Use the same JDK, corpus, JMH arguments, and machine state for baseline and
candidate
+runs. Run the baseline twice first, and retain raw JSON results and artifact
hashes.
+StringLiteralTokenCorpus provides deterministic token and error snapshots for a
+baseline/candidate comparison.
diff --git a/fe/fe-sql-parser-benchmark/pom.xml
b/fe/fe-sql-parser-benchmark/pom.xml
new file mode 100644
index 00000000000..887522ff735
--- /dev/null
+++ b/fe/fe-sql-parser-benchmark/pom.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns="http://maven.apache.org/POM/4.0.0";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.doris</groupId>
+ <version>${revision}</version>
+ <artifactId>fe</artifactId>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>fe-sql-parser-benchmark</artifactId>
+ <packaging>jar</packaging>
+ <name>Doris FE SQL Parser Benchmarks</name>
+
+ <properties>
+ <jmh.version>1.37</jmh.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.doris</groupId>
+ <artifactId>fe-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>doris-fe-sql-parser-benchmarks</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>build-benchmark-jar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>module-info.class</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserBenchmarkSupport.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserBenchmarkSupport.java
new file mode 100644
index 00000000000..d8c4ea8028e
--- /dev/null
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserBenchmarkSupport.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.sqlparser.benchmark;
+
+import org.apache.doris.nereids.DorisLexer;
+import org.apache.doris.nereids.DorisParser;
+import org.apache.doris.nereids.parser.CaseInsensitiveStream;
+import org.apache.doris.nereids.parser.NereidsParser;
+
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+
+/** Uses the branch-4.2 FE syntax parser without constructing logical plans. */
+final class ParserBenchmarkSupport {
+ private ParserBenchmarkSupport() {
+ }
+
+ static DorisLexer newLexer(String sql, boolean noBackslashEscapes) {
+ DorisLexer lexer = new DorisLexer(new
CaseInsensitiveStream(CharStreams.fromString(sql)));
+ lexer.isNoBackslashEscapes = noBackslashEscapes;
+ return lexer;
+ }
+
+ static ParserRuleContext parseStatement(String sql, boolean
noBackslashEscapes) {
+ return NereidsParser.toAst(new CommonTokenStream(newLexer(sql,
noBackslashEscapes)),
+ DorisParser::singleStatement);
+ }
+}
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserControlBenchmark.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserControlBenchmark.java
new file mode 100644
index 00000000000..7cecf45791f
--- /dev/null
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/ParserControlBenchmark.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.sqlparser.benchmark;
+
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/** Guards against fixed-cost parser regressions on SQL without string
literals. */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(3)
+@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+public class ParserControlBenchmark {
+ @Param({"SELECT 1", "SELECT a FROM t WHERE a > 1"})
+ public String sql;
+
+
+ @Benchmark
+ public Object parseStatementWithoutStringLiteral() {
+ return ParserBenchmarkSupport.parseStatement(sql, false);
+ }
+}
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralBenchmark.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralBenchmark.java
new file mode 100644
index 00000000000..ff5c1681386
--- /dev/null
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralBenchmark.java
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.sqlparser.benchmark;
+
+import org.apache.doris.nereids.DorisLexer;
+
+import com.google.common.base.Strings;
+import org.antlr.v4.runtime.Token;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/** Benchmarks the lexer hotspot and its impact on the public end-to-end
parser entry point. */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(3)
+@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+public class StringLiteralBenchmark {
+ @Param({"false", "true"})
+ public boolean noBackslashEscapes;
+
+ @Param({"plain", "backslash", "doubledQuote", "mixed"})
+ public String pattern;
+
+ @Param({"single", "double"})
+ public String quote;
+
+ @Param({"16", "256", "4096", "65536"})
+ public int payloadLength;
+
+ private String literal;
+ private String statement;
+
+ @Setup(Level.Trial)
+ public void setUp() {
+ String unit;
+ switch (pattern) {
+ case "plain":
+ unit = "a";
+ break;
+ case "backslash":
+ unit = "\\n";
+ break;
+ case "doubledQuote":
+ unit = quote.equals("single") ? "''" : "\"\"";
+ break;
+ case "mixed":
+ unit = quote.equals("single") ? "abc\\n''" : "abc\\n\"\"";
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown pattern: " +
pattern);
+ }
+
+ StringBuilder payload = new StringBuilder(payloadLength);
+ while (payload.length() + unit.length() <= payloadLength) {
+ payload.append(unit);
+ }
+ payload.append(Strings.repeat("a", payloadLength - payload.length()));
+ String quoteCharacter = quote.equals("single") ? "'" : "\"";
+ literal = quoteCharacter + payload + quoteCharacter;
+ statement = "SELECT " + literal;
+ }
+
+ @Benchmark
+ public int lexStringLiteral() {
+ DorisLexer lexer = ParserBenchmarkSupport.newLexer(literal,
noBackslashEscapes);
+ int checksum = 1;
+ Token token;
+ do {
+ token = lexer.nextToken();
+ checksum = 31 * checksum + token.getType();
+ checksum = 31 * checksum + token.getStartIndex();
+ checksum = 31 * checksum + token.getStopIndex();
+ } while (token.getType() != Token.EOF);
+ return checksum;
+ }
+
+ @Benchmark
+ public Object parseSelectStringLiteral() {
+ return ParserBenchmarkSupport.parseStatement(statement,
noBackslashEscapes);
+ }
+}
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralTokenCorpus.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralTokenCorpus.java
new file mode 100644
index 00000000000..5f1f92779ff
--- /dev/null
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/StringLiteralTokenCorpus.java
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.sqlparser.benchmark;
+
+import org.apache.doris.nereids.DorisLexer;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.Token;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+/** Emits a deterministic token snapshot for baseline/candidate differential
testing. */
+public final class StringLiteralTokenCorpus {
+ private static final String[] ALPHABET = {"a", "'", "\"", "\\", "\n"};
+
+ private StringLiteralTokenCorpus() {
+ }
+
+ public static void main(String[] args) throws IOException {
+ List<String> payloads = new ArrayList<>();
+ for (int length = 0; length <= 5; length++) {
+ enumeratePayloads(new StringBuilder(), length, payloads);
+ }
+ payloads.add("δΈζπ");
+ payloads.add("\\\r\n");
+ payloads.add("'\"\\\nδΈπ");
+
+ try (PrintWriter output = new PrintWriter(Files.newBufferedWriter(
+ Paths.get(args[0]), StandardCharsets.UTF_8))) {
+ int caseId = 0;
+ for (boolean noBackslashEscapes : new boolean[] {false, true}) {
+ for (String quote : new String[] {"'", "\""}) {
+ for (String payload : payloads) {
+ emit(output, caseId++, noBackslashEscapes, quote +
payload + quote);
+ emit(output, caseId++, noBackslashEscapes, quote +
payload);
+ emit(output, caseId++, noBackslashEscapes,
+ quote + payload + quote + " next /* comment
*/");
+ }
+ }
+ }
+ }
+ }
+
+ private static void enumeratePayloads(StringBuilder payload, int
remaining, List<String> output) {
+ if (remaining == 0) {
+ output.add(payload.toString());
+ return;
+ }
+ for (String character : ALPHABET) {
+ int start = payload.length();
+ payload.append(character);
+ enumeratePayloads(payload, remaining - 1, output);
+ payload.setLength(start);
+ }
+ }
+
+ private static void emit(PrintWriter output, int caseId, boolean
noBackslashEscapes, String sql) {
+ DorisLexer lexer = ParserBenchmarkSupport.newLexer(sql,
noBackslashEscapes);
+ ErrorCollector errors = new ErrorCollector();
+ lexer.removeErrorListeners();
+ lexer.addErrorListener(errors);
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ tokens.fill();
+
+ StringBuilder snapshot = new StringBuilder();
+
snapshot.append(caseId).append('|').append(noBackslashEscapes).append('|').append(encode(sql));
+ for (Token token : tokens.getTokens()) {
+
snapshot.append('|').append(DorisLexer.VOCABULARY.getSymbolicName(token.getType()))
+ .append(',').append(token.getChannel())
+ .append(',').append(token.getStartIndex())
+ .append(',').append(token.getStopIndex())
+ .append(',').append(token.getLine())
+ .append(',').append(token.getCharPositionInLine())
+ .append(',').append(token.getTokenIndex())
+ .append(',').append(encode(token.getText()));
+ }
+ for (String error : errors.errors) {
+ snapshot.append("|ERROR,").append(error);
+ }
+ output.println(snapshot);
+ }
+
+ private static String encode(String value) {
+ return
Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static class ErrorCollector extends BaseErrorListener {
+ private final List<String> errors = new ArrayList<>();
+
+ @Override
+ public void syntaxError(Recognizer<?, ?> recognizer, Object
offendingSymbol, int line,
+ int charPositionInLine, String message, RecognitionException
exception) {
+ errors.add(line + "," + charPositionInLine + "," +
encode(message));
+ }
+ }
+}
diff --git a/fe/pom.xml b/fe/pom.xml
index 556c47639a6..1969ca5b3c0 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -479,6 +479,7 @@ under the License.
<id>benchmark</id>
<modules>
<module>fe-benchmark</module>
+ <module>fe-sql-parser-benchmark</module>
</modules>
</profile>
</profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
