This is an automated email from the ASF dual-hosted git repository.
jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git
The following commit(s) were added to refs/heads/master by this push:
new c59fc14fac TODO-135 - HOCON peekNoSkip/skipWhitespaceAndComments
invariant tightening
c59fc14fac is described below
commit c59fc14fac5e52e75bad268792c71b820476c35e
Author: James Bognar <[email protected]>
AuthorDate: Fri May 29 06:46:51 2026 -0400
TODO-135 - HOCON peekNoSkip/skipWhitespaceAndComments invariant tightening
---
.../apache/juneau/hocon/HoconParserSession.java | 27 +++++-----
.../org/apache/juneau/hocon/HoconTokenizer.java | 12 +++++
.../org/apache/juneau/hocon/HoconParser_Test.java | 63 ++++++++++++++++++++++
.../apache/juneau/hocon/HoconTokenizer_Test.java | 28 ++++++++++
juneau-utest/test-run-history.tsv | 1 +
5 files changed, 116 insertions(+), 15 deletions(-)
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconParserSession.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconParserSession.java
index 9d8eaff24d..bb91419746 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconParserSession.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconParserSession.java
@@ -352,11 +352,10 @@ public class HoconParserSession extends
ReaderParserSession {
} else {
throw new ParseException(this, "Expected =, :
or brace at line {0}", t.getLine());
}
- // Mirror the parseArray guard: peek-before-skip so
that a `}` cached as `peeked` by
- // parseValueOrConcat's concat loop doesn't cause
skipWhitespaceAndComments below to
- // chew through a newline that lives OUTSIDE this
object's scope.
- if (t.peek().type() == HoconTokenizer.TokenType.RBRACE
|| t.peek().type() == HoconTokenizer.TokenType.EOF)
- break;
+ // skipWhitespaceAndComments is a no-op while a token
is cached (see HoconTokenizer), so a
+ // closing `}` already peeked by parseValueOrConcat's
concat loop is preserved here and the
+ // loop terminates naturally via the while-condition
above rather than chewing through a
+ // newline that belongs to an enclosing scope.
t.skipWhitespaceAndComments();
if (t.peek().type() == HoconTokenizer.TokenType.COMMA
|| t.peek().type() == HoconTokenizer.TokenType.NEWLINE)
t.read();
@@ -378,16 +377,14 @@ public class HoconParserSession extends
ReaderParserSession {
// that point. Here we must add the element as-is so
nested arrays like
// `[[1,2,3], [4,5,6]]` (with separators) stay nested
rather than flattening.
arr.getElements().add(parseValueOrConcat(t));
- // Check for closing ] / EOF BEFORE calling
skipWhitespaceAndComments. parseValueOrConcat's
- // internal concat loop calls peekNoSkip(), which
eagerly consumes the closing-bracket char
- // from the underlying reader and stashes it as
peeked=RBRACKET. If we then called
- // skipWhitespaceAndComments here, it would read PAST
the cached `]` and eat any newline
- // that follows — but that newline is the in-array
separator for the NEXT element (when
- // we're nested inside an outer parseArray), or a
meaningful boundary for outer scopes.
- // Eating it would cause adjacent newline-separated
inner arrays like `[[1,2]\n[3,4]]` to
- // be silently re-merged via HOCON array-concatenation
in parseValueOrConcat above us.
- if (t.peek().type() ==
HoconTokenizer.TokenType.RBRACKET || t.peek().type() ==
HoconTokenizer.TokenType.EOF)
- break;
+ // parseValueOrConcat's internal concat loop calls
peekNoSkip(), which eagerly consumes the
+ // closing-bracket char from the underlying reader and
stashes it as peeked=RBRACKET.
+ // skipWhitespaceAndComments is a no-op while a token
is cached (see HoconTokenizer), so it
+ // will NOT read past the cached `]` and eat a
following newline — a newline that is either
+ // the in-array separator for the NEXT element of an
enclosing array or a meaningful boundary
+ // for an outer scope. This prevents adjacent
newline-separated inner arrays like
+ // `[[1,2]\n[3,4]]` from being silently re-merged via
HOCON array-concatenation above us.
+ // The loop terminates on the cached `]` via the
while-condition above.
t.skipWhitespaceAndComments();
if (t.peek().type() == HoconTokenizer.TokenType.COMMA
|| t.peek().type() == HoconTokenizer.TokenType.NEWLINE)
t.read();
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconTokenizer.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconTokenizer.java
index 75d1f0964b..7026cf9359 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconTokenizer.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/hocon/HoconTokenizer.java
@@ -255,9 +255,21 @@ public class HoconTokenizer {
/**
* Skips whitespace and comments.
*
+ * <p>
+ * No-op when a token has already been peeked (i.e. {@link #peek()} or
{@link #peekNoSkip()}
+ * cached a token). Once a token is cached, the underlying reader is
positioned <em>after</em>
+ * that token, so skipping whitespace here would consume characters
that semantically come after
+ * the cached token rather than before it. The leading whitespace
before the cached token was
+ * already consumed when the token was produced, so this method has
nothing left to do until the
+ * cached token is read. This invariant keeps callers from accidentally
reading past a peeked
+ * structural token (such as a closing {@code ]} or {@code }}) and
swallowing a separator that
+ * belongs to an enclosing scope.
+ *
* @throws IOException If a read error occurs.
*/
public void skipWhitespaceAndComments() throws IOException {
+ if (peeked != null)
+ return;
while (true) {
var c = readChar();
if (c < 0)
diff --git
a/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconParser_Test.java
b/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconParser_Test.java
index 4d17e9c35e..c94c152263 100644
--- a/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconParser_Test.java
+++ b/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconParser_Test.java
@@ -259,4 +259,67 @@ class HoconParser_Test {
assertEquals(1, ((Number) obj.get("a")).intValue());
assertEquals(2, ((Number) obj.get("b")).intValue());
}
+
+ // Regression for the tokenizer peek/skip invariant (work item 135 /
FINISHED-57 OQ #13).
+ // Newline-separated inner arrays inside an outer array must stay
NESTED — the newline is an
+ // element separator, NOT a value-concatenation join. If
skipWhitespaceAndComments wrongly read
+ // past the closing ']' cached by parseValueOrConcat's peekNoSkip and
ate the following newline,
+ // the two inner arrays would silently flatten via HOCON
array-concatenation.
+ @Test
+ void b27_nestedArraysNewlineSeparatedStayNested() throws Exception {
+ var hocon = "outer = [\n [1, 2]\n [3, 4]\n]";
+ var m = (Map<String, Object>) HoconParser.DEFAULT.parse(hocon,
Map.class, String.class, Object.class);
+ var outer = (List<?>) m.get("outer");
+ assertNotNull(outer);
+ assertEquals(2, outer.size());
+ var first = (List<?>) outer.get(0);
+ var second = (List<?>) outer.get(1);
+ assertEquals(2, first.size());
+ assertEquals(2, second.size());
+ assertEquals(1, ((Number) first.get(0)).intValue());
+ assertEquals(2, ((Number) first.get(1)).intValue());
+ assertEquals(3, ((Number) second.get(0)).intValue());
+ assertEquals(4, ((Number) second.get(1)).intValue());
+ }
+
+ // Companion to b25: space-separated inner arrays on the SAME line
still concatenate per the
+ // HOCON array-concatenation rule. Confirms the structural fix did not
break that feature.
+ @Test
+ void b28_sameLineInnerArraysStillConcatenate() throws Exception {
+ var hocon = "outer = [\n [1, 2] [3, 4]\n]";
+ var m = (Map<String, Object>) HoconParser.DEFAULT.parse(hocon,
Map.class, String.class, Object.class);
+ var outer = (List<?>) m.get("outer");
+ assertNotNull(outer);
+ assertEquals(1, outer.size());
+ var inner = (List<?>) outer.get(0);
+ assertEquals(4, inner.size());
+ assertEquals(1, ((Number) inner.get(0)).intValue());
+ assertEquals(4, ((Number) inner.get(3)).intValue());
+ }
+
+ // Regression for the parseObject side of the invariant: when a nested
object's closing '}' is
+ // cached as a peeked token, the newline that separates the next
sibling key must survive so the
+ // sibling is parsed at the correct scope rather than being swallowed.
+ @Test
+ void b29_nestedObjectClosePreservesSiblingNewline() throws Exception {
+ var hocon = "a {\n b = 1\n}\nc = 2";
+ var m = (Map<String, Object>) HoconParser.DEFAULT.parse(hocon,
Map.class, String.class, Object.class);
+ var a = (Map<String, Object>) m.get("a");
+ assertNotNull(a);
+ assertEquals(1, ((Number) a.get("b")).intValue());
+ assertEquals(2, ((Number) m.get("c")).intValue());
+ }
+
+ // Newline-separated objects inside an array must remain distinct
elements rather than merging
+ // via HOCON object-concatenation (which only applies to same-line
adjacency).
+ @Test
+ void b30_arrayOfObjectsNewlineSeparatedStayDistinct() throws Exception {
+ var hocon = "items = [\n { id = 1 }\n { id = 2 }\n]";
+ var m = (Map<String, Object>) HoconParser.DEFAULT.parse(hocon,
Map.class, String.class, Object.class);
+ var items = (List<?>) m.get("items");
+ assertNotNull(items);
+ assertEquals(2, items.size());
+ assertEquals(1, ((Number) ((Map<String, Object>)
items.get(0)).get("id")).intValue());
+ assertEquals(2, ((Number) ((Map<String, Object>)
items.get(1)).get("id")).intValue());
+ }
}
diff --git
a/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconTokenizer_Test.java
b/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconTokenizer_Test.java
index 9db659da43..f7cbd86320 100644
---
a/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconTokenizer_Test.java
+++
b/juneau-utest/src/test/java/org/apache/juneau/hocon/HoconTokenizer_Test.java
@@ -178,4 +178,32 @@ class HoconTokenizer_Test extends TestBase {
assertEquals(PLUS_EQUALS, t.read().type());
assertEquals(EOF, t.read().type());
}
+
+ // Structural invariant (work item 135): skipWhitespaceAndComments is a
no-op while a token is
+ // cached. peekNoSkip() reads a structural token and leaves the
underlying reader positioned
+ // AFTER it. A subsequent skipWhitespaceAndComments() must NOT advance
past the cached token and
+ // swallow the whitespace/newline that follows it — that whitespace
belongs to the enclosing
+ // scope (e.g. an in-array element separator).
+ @Test
+ void g15_skipIsNoopWhileTokenCached() throws Exception {
+ var t = tokenizer("]\nx");
+ // peekNoSkip caches RBRACKET; the reader is now positioned at
the '\n' that follows ']'.
+ assertEquals(RBRACKET, t.peekNoSkip().type());
+ // Must be a no-op: the trailing '\n' must survive.
+ t.skipWhitespaceAndComments();
+ // The cached RBRACKET is still returned first...
+ assertEquals(RBRACKET, t.read().type());
+ // ...and the '\n' that followed ']' was preserved, not
swallowed.
+ assertEquals(NEWLINE, t.peekNoSkip().type());
+ }
+
+ // Companion to g15: the no-op guard must not regress the normal
(nothing-cached) skip behavior.
+ @Test
+ void g16_skipStillWorksWhenNothingCached() throws Exception {
+ var t = tokenizer(" # comment\n z");
+ t.skipWhitespaceAndComments();
+ var tok = t.read();
+ assertEquals(UNQUOTED_STRING, tok.type());
+ assertEquals("z", tok.stringValue());
+ }
}
diff --git a/juneau-utest/test-run-history.tsv
b/juneau-utest/test-run-history.tsv
index e1cfd1da41..4818b939d6 100644
--- a/juneau-utest/test-run-history.tsv
+++ b/juneau-utest/test-run-history.tsv
@@ -51,3 +51,4 @@ timestamp git_sha branch tests_run failures
errors skipped surefire_sec wall_sec
2026-05-28T20:22:45Z 5e454b3a7988 master 125891 0 0 21
141
2026-05-28T20:45:00Z 007293082747 master 125891 0 0 21
146
2026-05-28T21:29:51Z b0dc55154f95 master 125891 0 0 21
143
+2026-05-29T10:45:54Z 3dc947b24d41 master 125897 0 0 21
151
