This is an automated email from the ASF dual-hosted git repository.
jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git
The following commit(s) were added to refs/heads/master by this push:
new 143993ad0c fix(marshall): YAML parser buffer-underflow on large
OpenAPI 3.1 docs — ParserReader unread-lookback 10 → 256 (TODO-88)
143993ad0c is described below
commit 143993ad0cd76bb2acb1a4cfe89bb19a06779852
Author: James Bognar <[email protected]>
AuthorDate: Tue May 26 16:45:12 2026 -0400
fix(marshall): YAML parser buffer-underflow on large OpenAPI 3.1 docs —
ParserReader unread-lookback 10 → 256 (TODO-88)
---
.../org/apache/juneau/parser/ParserReader.java | 20 ++-
.../rest/openapi/OpenApiYamlRoundTrip_Test.java | 12 +-
.../juneau/yaml/YamlBufferUnderflow_Test.java | 187 +++++++++++++++++++++
3 files changed, 212 insertions(+), 7 deletions(-)
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parser/ParserReader.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parser/ParserReader.java
index 216d87a4c4..fafb70990a 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parser/ParserReader.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parser/ParserReader.java
@@ -51,6 +51,19 @@ public class ParserReader extends Reader implements
Positionable {
// Error message constants
private static final String MSG_bufferUnderflow = "Buffer underflow.";
+ /**
+ * Maximum number of previously-read characters preserved as unread
lookback when the buffer is
+ * refilled <em>without</em> an active {@link #mark()}. Originally 10,
which was sufficient for
+ * JSON's short tokens but too small for YAML's indent-aware parser: a
deeply-nested OpenAPI 3.1
+ * document can have indent depth well past 10 chars on a single line,
and the YAML parser's
+ * {@code skipBlanksAndCountIndent} / {@code unreadSpaces} pair (and
similar peek-then-unread
+ * patterns in block-scalar handling) tries to unread the entire indent
at once. When the indent
+ * read crossed the buffer boundary, the rest underflowed. 256 chars
covers any realistic YAML
+ * document indent depth (128 levels at 2-space indent) while still
leaving 768 chars per
+ * 1024-char refill for forward-progress reads.
+ */
+ private static final int UNMARKED_LOOKBACK_CHARS = 256;
+
/** Wrapped reader */
protected final Reader r;
@@ -427,8 +440,11 @@ public class ParserReader extends Reader implements
Positionable {
}
iEnd = iCurrent + x;
} else {
- // Copy the last 10 chars in the buffer
to the beginning of the buffer.
- int copyBuff = Math.min(iCurrent, 10);
+ // Copy the last
UNMARKED_LOOKBACK_CHARS chars in the buffer to the beginning so
+ // callers can still unread() that far
across a buffer-boundary refill. See the
+ // constant's javadoc for why 256 (not
10) — widened to handle YAML's deep-indent
+ // peek-then-unread patterns where a
single line's indent can exceed 10 chars.
+ int copyBuff = Math.min(iCurrent,
UNMARKED_LOOKBACK_CHARS);
System.arraycopy(buff, iCurrent -
copyBuff, buff, 0, copyBuff);
// Number of characters we expect to
copy on the next read.
diff --git
a/juneau-utest/src/test/java/org/apache/juneau/rest/openapi/OpenApiYamlRoundTrip_Test.java
b/juneau-utest/src/test/java/org/apache/juneau/rest/openapi/OpenApiYamlRoundTrip_Test.java
index 97172aa107..fe2097a955 100644
---
a/juneau-utest/src/test/java/org/apache/juneau/rest/openapi/OpenApiYamlRoundTrip_Test.java
+++
b/juneau-utest/src/test/java/org/apache/juneau/rest/openapi/OpenApiYamlRoundTrip_Test.java
@@ -87,11 +87,13 @@ class OpenApiYamlRoundTrip_Test extends TestBase {
@RestPost(path="/pet") public Pet createPet(@Content Pet pet) {
return pet; }
}
- // noInherit applied here too so the /openapi endpoint document (served
via the docs mixins) stays
- // small enough to round-trip cleanly through the YAML parser. The full
mixin surface produces a
- // larger document that exposes a separate YAML parser limitation
(deeply nested/complex docs hit a
- // buffer-underflow on the round-trip path) — tracked separately from
this TODO.
- @Rest(noInherit={"mixins"},
mixins={org.apache.juneau.rest.docs.BasicOpenApiResource.class})
+ // Inherits the FULL BasicRestServlet api-docs mixin surface — six
api-docs URLs, full schema
+ // set, the works. Earlier this resource shipped with
noInherit={"mixins"} + mixins={
+ // BasicOpenApiResource} because round-tripping the full doc through
the YAML parser threw
+ // IOException: Buffer underflow; that latent ParserReader limitation
was fixed by widening the
+ // reader's no-mark unread-lookback window (see
ParserReader.UNMARKED_LOOKBACK_CHARS), so c01
+ // now asserts against the FULL mixin-pack mount surface.
+ @Rest
public static class B extends BasicRestServlet {
private static final long serialVersionUID = 1L;
@RestGet(path="/pet") public Pet getPet() { return new Pet(); }
diff --git
a/juneau-utest/src/test/java/org/apache/juneau/yaml/YamlBufferUnderflow_Test.java
b/juneau-utest/src/test/java/org/apache/juneau/yaml/YamlBufferUnderflow_Test.java
new file mode 100644
index 0000000000..3f920e782f
--- /dev/null
+++
b/juneau-utest/src/test/java/org/apache/juneau/yaml/YamlBufferUnderflow_Test.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.juneau.yaml;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.*;
+import java.util.*;
+
+import org.apache.juneau.*;
+import org.apache.juneau.bean.openapi3.*;
+import org.apache.juneau.json.*;
+import org.apache.juneau.rest.*;
+import org.apache.juneau.rest.annotation.*;
+import org.apache.juneau.rest.mock.MockServletRequest;
+import org.apache.juneau.rest.mock.MockServletResponse;
+import org.apache.juneau.rest.servlet.*;
+import org.junit.jupiter.api.*;
+
+/**
+ * Focused, parser-only regression coverage for the YAML buffer-underflow fix:
the YAML
+ * parser used to throw {@code IOException: Buffer underflow} when
round-tripping documents
+ * that crossed the underlying {@code ParserReader}'s 1024-char buffer
boundary while a single
+ * block-mapping line had more than ~10 chars of leading indent.
+ *
+ * <p>Root cause: {@code ParserReader.readFromBuff()} retained only 10 chars
of unread
+ * lookback when refilling the buffer without an active mark, but {@code
YamlParserSession}
+ * routinely reads N indent spaces (where N can be 12+ for deeply nested
OpenAPI 3.1 docs)
+ * and then unreads all of them via {@code unreadSpaces(r, N)}. Once the read
crossed a
+ * buffer boundary, only 10 of those unreads were honored — the rest threw
underflow. The
+ * fix widens the unmarked-refill lookback window (see
+ * {@code ParserReader.UNMARKED_LOOKBACK_CHARS}).
+ *
+ * <p>Symptom (pre-fix): the live OpenAPI 3.1 doc emitted by {@code
BasicRestServlet} with
+ * the full api-docs mixin pack (six api-docs URLs + components.schemas)
reliably tripped
+ * the underflow on YAML round-trip; this test reproduces the underflow at the
parser layer
+ * with no REST stack, so a future regression of the buffer-management bug
surfaces here
+ * before it reaches the live-doc test.
+ */
+class YamlBufferUnderflow_Test extends TestBase {
+
+
//------------------------------------------------------------------------------------------------------------------
+ // Synthesized minimal repro: many top-level keys whose values are
deeply-nested maps,
+ // emitted with 2-space indents. Total document size deliberately
exceeds the 1024-char
+ // ParserReader buffer; nested indent depth exceeds the 10-char unread
lookback.
+
//------------------------------------------------------------------------------------------------------------------
+
+ private static Map<String,Object> nestedMap(int depth) {
+ Map<String,Object> head = new LinkedHashMap<>();
+ Map<String,Object> cur = head;
+ for (int i = 0; i < depth; i++) {
+ Map<String,Object> next = new LinkedHashMap<>();
+ next.put("leaf", "value-" + i);
+ cur.put("level" + i, next);
+ cur = next;
+ }
+ return head;
+ }
+
+ @Test void a01_largeNestedMap_jsonYamlJsonRoundTrip() throws Exception {
+ // Build a document that's > 1024 chars when serialized to YAML
and has nested
+ // indent depth > 10 spaces (to trip the unread-lookback
underflow).
+ var top = new LinkedHashMap<String,Object>();
+ for (int i = 0; i < 30; i++)
+ top.put("entry" + i, nestedMap(8));
+
+ String yaml = YamlSerializer.DEFAULT_READABLE.toString(top);
+ assertTrue(yaml.length() > 1024, () -> "YAML too small for test
(got " + yaml.length() + " chars)");
+ assertTrue(yaml.contains(" "), () -> "YAML must
contain >10-space indent line for repro");
+
+ Map<?,?> parsed = YamlParser.DEFAULT.parse(yaml, Map.class);
+
+ assertEquals(top.size(), parsed.size(), "Top-level entry count
must match");
+ assertEquals(top.keySet(), parsed.keySet(), "Top-level keys
must match");
+ }
+
+ @Test void a02_jsonToYamlToJson_structuralEquality() throws Exception {
+ // Same shape but driven through JSON to YAML to JSON to verify
the round-trip
+ // path the OpenApiYamlRoundTrip_Test#c01 workaround was
avoiding.
+ var top = new LinkedHashMap<String,Object>();
+ for (int i = 0; i < 20; i++)
+ top.put("k" + i, nestedMap(7));
+
+ String json = JsonSerializer.DEFAULT.toString(top);
+ Map<?,?> fromJson = JsonParser.DEFAULT.parse(json, Map.class);
+ String yaml =
YamlSerializer.DEFAULT_READABLE.toString(fromJson);
+ assertTrue(yaml.length() > 1024);
+
+ // This is the line that used to throw IOException: Buffer
underflow.
+ Map<?,?> fromYaml = YamlParser.DEFAULT.parse(yaml, Map.class);
+ String json2 = JsonSerializer.DEFAULT.toString(fromYaml);
+
+ // Sanity: the second JSON has all the top-level keys.
+ Map<?,?> reparsed = JsonParser.DEFAULT.parse(json2, Map.class);
+ assertEquals(top.keySet(), reparsed.keySet());
+ }
+
+
//------------------------------------------------------------------------------------------------------------------
+ // Pathological cases that exercise the same code paths but with fewer
moving parts.
+
//------------------------------------------------------------------------------------------------------------------
+
+ @Test void b01_deepIndentAcrossBufferBoundary() throws Exception {
+ // Construct a YAML doc whose first 1000+ chars are a single
long string value,
+ // followed by a nested block-mapping that requires >10-char
indent unreads at
+ // the boundary.
+ var top = new LinkedHashMap<String,Object>();
+ var bigStr = new StringBuilder();
+ for (int i = 0; i < 500; i++)
+ bigStr.append('x');
+ top.put("filler", bigStr.toString());
+ top.put("nested", nestedMap(10));
+
+ String yaml = YamlSerializer.DEFAULT_READABLE.toString(top);
+ Map<?,?> parsed = YamlParser.DEFAULT.parse(yaml, Map.class);
+
+ assertEquals(top.keySet(), parsed.keySet());
+ assertEquals(bigStr.toString(), parsed.get("filler"));
+ }
+
+ @Test void b02_manyTopLevelKeysWithDeepIndent() throws Exception {
+ // Many shallow keys whose values are a deep nested map, so
that the sequence of
+ // {top-level-key newline + indent + key}-pairs accumulates
past the buffer
+ // boundary and the indent length flips between 0 (top-level)
and >10 (nested).
+ var top = new LinkedHashMap<String,Object>();
+ for (int i = 0; i < 40; i++)
+ top.put("key_" + i, nestedMap(6));
+
+ String yaml = YamlSerializer.DEFAULT_READABLE.toString(top);
+ Map<?,?> parsed = YamlParser.DEFAULT.parse(yaml, Map.class);
+
+ assertEquals(40, parsed.size());
+ }
+
+
//------------------------------------------------------------------------------------------------------------------
+ // Live-resource repro: the FULL BasicRestServlet OpenAPI doc — six
api-docs URLs and the
+ // full components.schemas set — is the document shape that originally
tripped the
+ // underflow. We re-use the same RestContext-driven path that
OpenApiYamlRoundTrip_Test
+ // uses.
+
//------------------------------------------------------------------------------------------------------------------
+
+ public static class Pet {
+ public int id;
+ public String name;
+ }
+
+ @Rest
+ public static class FullSurface extends BasicRestServlet {
+ private static final long serialVersionUID = 1L;
+ @RestGet(path="/pet") public Pet getPet() { return new Pet(); }
+ }
+
+ public void testMethod() { /* no-op */ }
+
+ private OpenApi getOpenApi(Object resource) throws Exception {
+ var rc = new RestContext(new
RestContext.Args(resource.getClass(), null, null, () -> resource, "", null,
null, null, false));
+ var roc = new
RestOpContext(YamlBufferUnderflow_Test.class.getMethod("testMethod"), rc);
+ var call = RestSession.create(rc).resource(resource).req(new
MockServletRequest()).res(new MockServletResponse()).build();
+ var req = roc.createRequest(call);
+ return rc.getOpenApiProvider().getOpenApi(rc, req.getLocale());
+ }
+
+ @Test void c01_basicRestServletFullMixinSurface_yamlRoundTrip() throws
Exception {
+ var doc = getOpenApi(new FullSurface());
+ String yaml = YamlSerializer.DEFAULT_READABLE.toString(doc);
+ // The full mixin surface produces a multi-KB YAML doc that
crosses the
+ // ParserReader buffer boundary; deep schema nesting puts
indent depth well past the
+ // reader's no-mark unread-lookback limit. Pre-fix this throws
IOException: Buffer
+ // underflow.
+ var parsed = YamlParser.DEFAULT.parse(yaml, OpenApi.class);
+ assertEquals(doc.getOpenapi(), parsed.getOpenapi());
+ assertNotNull(parsed.getPaths());
+ }
+}
