github-actions[bot] commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4063719188
##########
be/src/storage/index/inverted/char_filter/icu_normalizer_char_filter.cpp:
##########
@@ -59,33 +63,97 @@ void ICUNormalizerCharFilter::fill() {
input.resize(_reader->size());
_reader->readCopy(input.data(), 0, static_cast<int32_t>(input.size()));
normalize_text(input, _buf);
+ build_source_byte_offset_runs();
_transformed_input.init(_buf.data(), static_cast<int32_t>(_buf.size()),
false);
}
void ICUNormalizerCharFilter::normalize_text(const std::string& input,
std::string& output) {
output.clear();
+ _edits.reset();
if (input.empty()) {
return;
}
UErrorCode status = U_ZERO_ERROR;
- icu::UnicodeString src16 = icu::UnicodeString::fromUTF8(input);
- UNormalizationCheckResult quick_result = _normalizer->quickCheck(src16,
status);
- if (U_SUCCESS(status) && quick_result == UNORM_YES) {
+ icu::StringByteSink<std::string> sink(&output);
+ _normalizer->normalizeUTF8(0, icu::StringPiece(input), sink, &_edits,
status);
+ if (U_FAILURE(status)) {
+ LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
output = input;
+ _edits.reset();
+ _edits.addUnchanged(static_cast<int32_t>(input.size()));
return;
}
+}
- icu::UnicodeString result16;
- status = U_ZERO_ERROR;
- _normalizer->normalize(src16, result16, status);
+void ICUNormalizerCharFilter::build_source_byte_offset_runs() {
+ _offset_correction_runs.clear();
+ UErrorCode status = U_ZERO_ERROR;
+ auto iterator = _edits.getFineChangesIterator();
+ while (iterator.next(status)) {
+ if (U_FAILURE(status)) {
+ _offset_correction_runs.clear();
+ return;
+ }
+
+ const int32_t source_start = iterator.sourceIndex();
+ const int32_t destination_start = iterator.destinationIndex();
+ const int32_t source_length = iterator.oldLength();
+ const int32_t destination_length = iterator.newLength();
+ if (!_offset_correction_runs.empty()) {
+ auto& previous = _offset_correction_runs.back();
+ const int64_t previous_source_end =
+ static_cast<int64_t>(previous.source_start) +
+ static_cast<int64_t>(previous.source_length) *
previous.repeat_count;
+ const int64_t previous_destination_end =
+ static_cast<int64_t>(previous.destination_start) +
+ static_cast<int64_t>(previous.destination_length) *
previous.repeat_count;
+ if (previous.source_length == source_length &&
+ previous.destination_length == destination_length &&
+ previous_source_end == source_start &&
+ previous_destination_end == destination_start) {
+ ++previous.repeat_count;
+ continue;
+ }
+ }
+ _offset_correction_runs.push_back(
+ {source_start, destination_start, source_length,
destination_length, 1});
+ }
if (U_FAILURE(status)) {
- LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
- output = input;
- return;
+ _offset_correction_runs.clear();
}
+}
- result16.toUTF8String(output);
+int32_t ICUNormalizerCharFilter::correct_offset(int32_t current_offset) const {
+ if (current_offset < 0 || _offset_correction_runs.empty()) {
+ return DorisCharFilter::correct_offset(current_offset);
+ }
+
+ const auto next_run = std::ranges::upper_bound(_offset_correction_runs,
current_offset, {},
+
&OffsetCorrectionRun::destination_start);
+ if (next_run == _offset_correction_runs.begin()) {
+ return DorisCharFilter::correct_offset(current_offset);
+ }
+
+ const auto& run = *std::prev(next_run);
+ const int64_t source_end = static_cast<int64_t>(run.source_start) +
+ static_cast<int64_t>(run.source_length) *
run.repeat_count;
+ const int64_t destination_end =
static_cast<int64_t>(run.destination_start) +
+
static_cast<int64_t>(run.destination_length) * run.repeat_count;
+ if (run.destination_length == 0 && current_offset ==
run.destination_start) {
+ return
DorisCharFilter::correct_offset(static_cast<int32_t>(source_end));
+ }
+
+ if (current_offset <= destination_end) {
Review Comment:
[P1] Preserve a source interval for every rune produced by an ICU expansion.
For default `nfkc_cf`, U+FB01 (three source bytes) becomes `fi`, and this
formula maps destination boundaries `[0,1,2]` to `[0,3,3]`. The tokenizer
provenance path accepts that as an exact two-rune map, so NGram or offset-aware
Pinyin can publish the second normalized rune at the zero-width source span
`[3,3)`. This is not exercised by the current one-to-one full-width test.
Please carry conservative start/end intervals for changed edit spans and cover
ligature expansion through an offset-aware downstream tokenizer/filter.
##########
be/src/storage/index/inverted/tokenizer/keyword/keyword_tokenizer.h:
##########
@@ -45,6 +45,9 @@ class KeywordTokenizer : public DorisTokenizer {
int32_t length = std::min(_char_length, MAX_TOKEN_LENGTH_LIMIT);
std::string_view term(_char_buffer, length);
set(token, term);
+ set_source_byte_offsets(term, 0);
Review Comment:
[P1] Choose this capped prefix on a UTF-8 scalar boundary before publishing
provenance. With 8,191 ASCII bytes followed by a three-byte CJK rune, the
8,192-byte term contains only the rune's first byte; `set_source_byte_offsets`
then rejects the invalid prefix and leaves no map, while the new end offset
points inside source span `[8191,8194)`. Offset-aware Pinyin/WordDelimiter
consumers consequently cannot project valid spans. Reuse a rune-safe prefix
boundary (as the fixed IK path does) for the term, map, and token end, with a
cap-straddling multibyte/reset test.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -150,47 +204,207 @@ private static String
buildIdentityFromPolicyProperties(IndexPolicyTypeEnum type
* Resolve a component (tokenizer) to its identity.
*/
private static String resolveComponentIdentity(String name,
IndexPolicyTypeEnum expectedType) {
+ return resolveComponentIdentity(name, expectedType, false);
+ }
+
+ private static String resolveComponentIdentity(
+ String name, IndexPolicyTypeEnum expectedType, boolean
lowercaseDownstream) {
if (Strings.isNullOrEmpty(name)) {
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
- }
-
- // For custom component, get its properties
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
try {
Env env = Env.getCurrentEnv();
- if (env == null || env.getIndexPolicyMgr() == null) {
- return name;
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ if (policy.isInvalid()) {
+ return "invalid-policy:" + policy.getId() + ":" +
policy.getName();
+ }
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ String type = sortedProps.get(IndexPolicy.PROP_TYPE);
+ String normalizedType =
normalizeBuiltinComponentName(type, expectedType);
+ if (normalizedType != null) {
+ if ("empty".equals(normalizedType)) {
+ return "";
+ }
+ sortedProps.put(IndexPolicy.PROP_TYPE,
normalizedType);
+ canonicalizeEffectiveComponentProperties(
+ sortedProps, normalizedType, expectedType);
+ if (sortedProps.size() == 1) {
+ return normalizedType;
+ }
+ }
+ if (expectedType == IndexPolicyTypeEnum.TOKENIZER
+ &&
"ngram".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ // This setting only limits policy creation; it
does not change emitted tokens.
+ sortedProps.remove(PROP_MAX_NGRAM_DIFF);
+ }
+ if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER
+ &&
"char_replace".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ String replacement =
sortedProps.getOrDefault("replacement", " ");
+ String pattern = canonicalizeCharReplacePattern(
+ sortedProps.get("pattern"), replacement,
lowercaseDownstream);
+ if (pattern.isEmpty()) {
+ return "";
+ }
+ sortedProps.put("pattern", pattern);
+ sortedProps.put("replacement", replacement);
+ }
+ if (normalizedType != null && sortedProps.size() == 1)
{
+ return normalizedType;
+ }
+ return sortedProps.toString();
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution or the original name.
+ }
+
+ String normalizedName = normalizeBuiltinComponentName(name,
expectedType);
+ return "empty".equals(normalizedName) ? "" : normalizedName == null ?
name : normalizedName;
+ }
+
+ private static void canonicalizeEffectiveComponentProperties(
+ TreeMap<String, String> properties, String type,
IndexPolicyTypeEnum expectedType) {
+ if ("pinyin".equals(type)) {
+ removeBooleanDefaults(properties, true,
+ "keep_first_letter", "keep_full_pinyin",
"keep_none_chinese",
+ "keep_none_chinese_together",
"keep_none_chinese_in_first_letter",
+ "lowercase", "trim_whitespace", "ignore_pinyin_offset",
+ "none_chinese_pinyin_tokenize");
+ removeBooleanDefaults(properties, false,
Review Comment:
[P1] Canonicalize settings after applying their dependencies. With the
effective default `ignore_pinyin_offset=true`, both Pinyin implementations
publish the whole input/token span and never consume the candidate-relative
offsets changed by `fixed_pinyin_offset`; therefore a type-only policy and one
with `fixed_pinyin_offset=true` emit identical terms, positions, and offsets.
This code nevertheless keeps the non-default flag, so analyzer aliases can
evade CREATE/ALTER duplicate-index rejection. Remove gated values when their
consumer is disabled (and audit the other dependent Pinyin options), with
identity plus duplicate-validation coverage.
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -214,29 +236,97 @@ bool PinyinFilter::readTerm(Token* token) {
return false;
}
-bool PinyinFilter::processCurrentToken() {
- processed_candidate_ = true;
+bool PinyinFilter::prepareCurrentSource(std::vector<UChar32>&
source_codepoints) {
+ size_t source_start = 0;
+ size_t source_end = current_token_text_.size();
+ if (config_->trimWhitespace) {
+ source_start = current_token_text_.find_first_not_of(" \t\n\r");
+ if (source_start == std::string::npos) {
+ return false;
+ }
+ source_end = current_token_text_.find_last_not_of(" \t\n\r") + 1;
+ }
+ current_source_ = current_token_text_.substr(source_start, source_end -
source_start);
- if (!has_current_token_) {
+ if (current_source_.empty()) {
return false;
}
- current_source_ = current_token_text_;
+ if (config_->ignorePinyinOffset) {
+ convertToCodepoints(current_source_, source_codepoints);
+ current_start_offset_ += static_cast<int32_t>(source_start);
Review Comment:
[P1] Do not apply post-filter trim bytes directly to source offsets in the
default mode. With `icu_normalizer -> keyword -> pinyin`, source U+3000
followed by a three-byte CJK rune is transformed from six bytes to `" <rune>"`;
Keyword correctly supplies source `[0,6)`, but this branch adds the transformed
one-byte trim and emits `[1,4)`, whose endpoints are inside the original UTF-8
scalars. The exact-map trim fix is bypassed because `ignore_pinyin_offset`
defaults to true. Preserve the upstream whole-token source span (or project
trim through compact correction data), with default-mode width-changing
whitespace coverage.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -150,47 +204,207 @@ private static String
buildIdentityFromPolicyProperties(IndexPolicyTypeEnum type
* Resolve a component (tokenizer) to its identity.
*/
private static String resolveComponentIdentity(String name,
IndexPolicyTypeEnum expectedType) {
+ return resolveComponentIdentity(name, expectedType, false);
+ }
+
+ private static String resolveComponentIdentity(
+ String name, IndexPolicyTypeEnum expectedType, boolean
lowercaseDownstream) {
if (Strings.isNullOrEmpty(name)) {
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
- }
-
- // For custom component, get its properties
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
try {
Env env = Env.getCurrentEnv();
- if (env == null || env.getIndexPolicyMgr() == null) {
- return name;
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ if (policy.isInvalid()) {
+ return "invalid-policy:" + policy.getId() + ":" +
policy.getName();
+ }
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ String type = sortedProps.get(IndexPolicy.PROP_TYPE);
+ String normalizedType =
normalizeBuiltinComponentName(type, expectedType);
+ if (normalizedType != null) {
+ if ("empty".equals(normalizedType)) {
+ return "";
+ }
+ sortedProps.put(IndexPolicy.PROP_TYPE,
normalizedType);
+ canonicalizeEffectiveComponentProperties(
+ sortedProps, normalizedType, expectedType);
+ if (sortedProps.size() == 1) {
+ return normalizedType;
+ }
+ }
+ if (expectedType == IndexPolicyTypeEnum.TOKENIZER
+ &&
"ngram".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ // This setting only limits policy creation; it
does not change emitted tokens.
+ sortedProps.remove(PROP_MAX_NGRAM_DIFF);
+ }
+ if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER
+ &&
"char_replace".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ String replacement =
sortedProps.getOrDefault("replacement", " ");
+ String pattern = canonicalizeCharReplacePattern(
+ sortedProps.get("pattern"), replacement,
lowercaseDownstream);
+ if (pattern.isEmpty()) {
+ return "";
+ }
+ sortedProps.put("pattern", pattern);
+ sortedProps.put("replacement", replacement);
+ }
+ if (normalizedType != null && sortedProps.size() == 1)
{
+ return normalizedType;
+ }
+ return sortedProps.toString();
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution or the original name.
+ }
+
+ String normalizedName = normalizeBuiltinComponentName(name,
expectedType);
+ return "empty".equals(normalizedName) ? "" : normalizedName == null ?
name : normalizedName;
+ }
+
+ private static void canonicalizeEffectiveComponentProperties(
+ TreeMap<String, String> properties, String type,
IndexPolicyTypeEnum expectedType) {
+ if ("pinyin".equals(type)) {
+ removeBooleanDefaults(properties, true,
+ "keep_first_letter", "keep_full_pinyin",
"keep_none_chinese",
+ "keep_none_chinese_together",
"keep_none_chinese_in_first_letter",
+ "lowercase", "trim_whitespace", "ignore_pinyin_offset",
+ "none_chinese_pinyin_tokenize");
+ removeBooleanDefaults(properties, false,
+ "keep_separate_first_letter", "keep_joined_full_pinyin",
"keep_original",
+ "keep_none_chinese_in_joined_full_pinyin",
"remove_duplicated_term",
+ "fixed_pinyin_offset", "keep_separate_chinese");
+ removeIntegerDefault(properties, "limit_first_letter_length", 16);
+ return;
+ }
+
+ if (expectedType == IndexPolicyTypeEnum.TOKEN_FILTER) {
+ if ("asciifolding".equals(type)) {
+ removeBooleanDefaults(properties, false, "preserve_original");
+ } else if ("word_delimiter".equals(type)) {
+ removeBooleanDefaults(properties, true, "generate_word_parts",
"generate_number_parts",
+ "split_on_case_change", "split_on_numerics",
"stem_english_possessive");
+ removeBooleanDefaults(properties, false, "catenate_words",
"catenate_numbers",
+ "catenate_all", "preserve_original");
+ } else if ("icu_normalizer".equals(type)) {
+ canonicalizeIcuNormalizerDefaults(properties, false);
}
+ return;
+ }
- IndexPolicy policy = env.getIndexPolicyMgr().getPolicyByName(name);
- if (policy == null || policy.getType() != expectedType) {
- return name;
+ if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER) {
+ if ("icu_normalizer".equals(type)) {
+ canonicalizeIcuNormalizerDefaults(properties, true);
}
- if (policy.isInvalid()) {
- return "invalid-policy:" + policy.getId() + ":" +
policy.getName();
+ return;
+ }
+
+ if (expectedType != IndexPolicyTypeEnum.TOKENIZER) {
+ return;
+ }
+ switch (type) {
+ case "ngram":
+ removeIntegerDefault(properties, "min_gram", 1);
+ removeIntegerDefault(properties, "max_gram", 2);
+ break;
+ case "edge_ngram":
+ removeIntegerDefault(properties, "min_gram", 1);
+ removeIntegerDefault(properties, "max_gram", 2);
+ break;
+ case "standard":
+ case "char_group":
+ removeIntegerDefault(properties, "max_token_length", 255);
+ break;
+ case "keyword":
+ removeIntegerDefault(properties, "buffer_size", 256);
+ break;
+ case "basic":
+ removeStringDefault(properties, "extra_chars", "");
+ break;
+ default:
+ break;
+ }
+ }
+
+ private static void removeBooleanDefaults(
+ TreeMap<String, String> properties, boolean defaultValue,
String... keys) {
+ for (String key : keys) {
+ String value = properties.get(key);
+ if (value == null || !("true".equalsIgnoreCase(value) ||
"false".equalsIgnoreCase(value))) {
+ continue;
}
+ boolean parsed = Boolean.parseBoolean(value);
+ if (parsed == defaultValue) {
+ properties.remove(key);
+ } else {
+ properties.put(key, Boolean.toString(parsed));
+ }
+ }
+ }
- Map<String, String> props = policy.getProperties();
- if (props == null || props.isEmpty()) {
- return name;
+ private static void removeIntegerDefault(
+ TreeMap<String, String> properties, String key, int defaultValue) {
+ String value = properties.get(key);
+ if (value == null) {
+ return;
+ }
+ try {
+ int parsed = Integer.parseInt(value);
+ if (parsed == defaultValue) {
+ properties.remove(key);
+ } else {
+ properties.put(key, Integer.toString(parsed));
}
+ } catch (NumberFormatException e) {
+ // Invalid policies keep their original identity.
+ }
+ }
- // Build identity from sorted properties
- TreeMap<String, String> sortedProps = new TreeMap<>(props);
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && "ngram".equals(sortedProps.get(IndexPolicy.PROP_TYPE)))
{
- // This setting only limits policy creation; it does not
change emitted tokens.
- sortedProps.remove(PROP_MAX_NGRAM_DIFF);
+ private static void canonicalizeIcuNormalizerDefaults(
+ TreeMap<String, String> properties, boolean hasMode) {
+ String name = properties.get("name");
+ if (name != null) {
+ String normalizedName = name.trim().toLowerCase(Locale.ROOT);
+ if ("nfkc_cf".equals(normalizedName)) {
+ properties.remove("name");
+ } else {
+ properties.put("name", normalizedName);
}
- return sortedProps.toString();
- } catch (RuntimeException e) {
- return name;
}
+ removeStringDefault(properties, "unicode_set_filter", "");
Review Comment:
[P1] Canonicalize the effective UnicodeSet, not only the literal empty
default. A valid policy with `unicode_set_filter=[]` reaches the parsed-empty
branch in both BE ICU factories and installs the same unfiltered normalizer as
an omitted filter, but this line retains `[]`; likewise `[ab]` and `[ba]`
produce the same filtered normalizer with different identities. Analyzer
aliases referencing these policies can therefore evade CREATE/ALTER
duplicate-index rejection even though they emit the same stream. Please derive
this identity field from the parsed set (including parsed-empty), with identity
and duplicate-validation coverage.
##########
be/src/storage/index/inverted/tokenizer/icu/icu_tokenizer.cpp:
##########
@@ -55,13 +55,23 @@ Token* ICUTokenizer::next(Token* token) {
utf8Str_.clear();
int32_t length = std::min(end - start, LUCENE_MAX_WORD_LEN);
auto subString = buffer_.tempSubString(start, length);
+ sourceUtf8Str_.clear();
+ subString.toUTF8String(sourceUtf8Str_);
if (this->lowercase) {
subString.toLower().toUTF8String(utf8Str_);
} else {
subString.toUTF8String(utf8Str_);
}
token->setNoCopy(utf8Str_.data(), 0,
static_cast<int32_t>(utf8Str_.size()));
+ int32_t source_start = 0;
+ int32_t source_end = 0;
+ if (start >= 0 && length >= 0 && advance_source_offset(start,
source_start) &&
Review Comment:
[P1] Do not return a token with stale offsets when this raw UTF-8 cursor
fails. `UnicodeString::fromUTF8` substitutes malformed bytes, so input such as
`alpha <0xff> beta` still yields word tokens: `alpha` gets `[0,5)`, then
advancing to `beta` hits the invalid raw byte, latches
`sourceOffsetsValid_=false`, and skips these setters. Callers reuse one
`Token`, so `beta` and every later token retain `alpha`'s span. Reject
malformed input or define replacement provenance, but always initialize the
returned token's offsets/map; cover invalid bytes between valid words and
downstream reset/reuse.
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -107,6 +118,11 @@ void PinyinFilter::reset() {
done_ = true;
resetVariables();
has_current_token_ = false;
+ std::vector<RuneInfo>().swap(current_runes_);
Review Comment:
[P2] Avoid freeing every reusable scratch buffer on every document reset.
The column writer intentionally reuses this analyzer chain across values, but
these swaps release non-SSO token/source strings plus enabled provenance
vectors and force them to allocate again for each ordinary row. The large-input
retention fix therefore converts steady-state reuse into allocator churn, while
its test only asserts zero capacity after one reset. Retain normal capacity and
evict only above a deliberate high-water mark, with repeated ordinary-row
allocation coverage alongside the large-row reclamation case.
##########
be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer.cpp:
##########
@@ -74,6 +74,10 @@ Token* NGramTokenizer::next(Token* token) {
to_chars(_buffer, _buffer_start, _gram_size);
set(token, _utf8_buffer);
+ set_source_byte_offsets(_utf8_buffer, _offset);
Review Comment:
[P1] Keep this new source coordinate in the raw input's byte space.
`to_code_points()` advances past malformed bytes but drops them from the
buffer, while `_offset` advances only when retained code points are consumed.
Thus raw `a<0xff>b` emits `b` at `[1,2)` instead of `[2,3)`, and a two-rune
`ab` gram erases the source gap as `[0,2)` instead of conservatively covering
`[0,3)`; downstream Pinyin inherits the compressed map. Reject malformed input
or retain raw start/end metadata for buffered runes, with leading/interior
invalid-byte and reset coverage.
##########
be/src/storage/index/inverted/tokenizer/icu/icu_tokenizer.cpp:
##########
@@ -55,13 +55,23 @@ Token* ICUTokenizer::next(Token* token) {
utf8Str_.clear();
int32_t length = std::min(end - start, LUCENE_MAX_WORD_LEN);
auto subString = buffer_.tempSubString(start, length);
+ sourceUtf8Str_.clear();
+ subString.toUTF8String(sourceUtf8Str_);
if (this->lowercase) {
subString.toLower().toUTF8String(utf8Str_);
} else {
subString.toUTF8String(utf8Str_);
}
token->setNoCopy(utf8Str_.data(), 0,
static_cast<int32_t>(utf8Str_.size()));
+ int32_t source_start = 0;
+ int32_t source_end = 0;
+ if (start >= 0 && length >= 0 && advance_source_offset(start,
source_start) &&
+ advance_source_offset(start + length, source_end)) {
Review Comment:
[P1] Do not publish an interior UTF-16 target as a valid source endpoint. If
a supplementary letter straddles `LUCENE_MAX_WORD_LEN`, the capped substring
contains only its high surrogate and converts it to replacement UTF-8, while
`advance_source_offset(start + length)` returns the original scalar's byte
start. This new map therefore includes a replacement rune but
`Token::endOffset` ends before its contributing source scalar; offset-aware
Pinyin/WordDelimiter chains can emit a term/span mismatch or an endpoint inside
UTF-8. Clip the cap to a scalar boundary (or include the scalar consistently)
and add a cap-boundary supplementary-character test through those consumers.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
