[
https://issues.apache.org/jira/browse/TIKA-4943?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18125610#comment-18125610
]
ASF GitHub Bot commented on TIKA-4943:
--------------------------------------
Copilot commented on code in PR #3277:
URL: https://github.com/apache/tika/pull/3277#discussion_r4232259656
##########
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java:
##########
@@ -236,6 +243,7 @@ private enum ParseWarningKind {
public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata,
XHTMLContentHandler xhtml, ParseContext parseContext)
throws SAXException, TikaException, IOException {
+ depthSkippedObjects.clear();
Review Comment:
walkTree()` clears `depthSkippedObjects`, but the newly added caches
(`numberListInfoCache` and `listStyleComparisonCache`) are never cleared per
parse/walk. If a single `MSOneStorePackage` instance is reused (or if very
large numbers of unique list formats/indents appear), these caches can retain
substantial memory (including large `byte[]` keys). Consider clearing these
caches at the same lifecycle points as `depthSkippedObjects` (e.g., at the
start of `walkTree()` and `walkCell()`), or otherwise bounding them.
##########
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java:
##########
@@ -622,25 +627,471 @@ private void walkObject(RevisionStoreObject object,
}
int objectType = jcidIndex(object);
recordEntityGuid(object, objectType);
- // The title structure of a page (StructureElementChildNodes) appears
above the page
- // body on screen, but is declared after the body child nodes. Emit it
first so the
- // text comes out in visual order.
- for (PropertyAction action : actions) {
- if (action.oneNotePropertyEnum ==
OneNotePropertyEnum.StructureElementChildNodes) {
- processAction(action, objectsById, visited, authorRole,
options, metadata, xhtml,
- depth, resourceInfo);
+ ListStyle listStyle = listStyleForObject(object, actions, objectsById);
+ startObjectStructure(objectType, listStyle, xhtml);
+ try {
+ // Emit the title structure before page body nodes to preserve
visual order.
+ List<PropertyAction> structureActions = new ArrayList<>();
+ for (PropertyAction action : actions) {
+ if (action.oneNotePropertyEnum ==
OneNotePropertyEnum.StructureElementChildNodes) {
+ structureActions.add(action);
+ }
}
+ processStructureChildActions(structureActions, objectsById,
visited, authorRole,
+ options, metadata, xhtml, depth, resourceInfo);
+ List<PropertyAction> remainingActions = new ArrayList<>();
+ for (PropertyAction action : actions) {
+ if (hasPrimaryPicture &&
+ action.oneNotePropertyEnum ==
OneNotePropertyEnum.WebPictureContainer14) {
+ continue;
+ }
+ if (action.oneNotePropertyEnum !=
OneNotePropertyEnum.StructureElementChildNodes) {
+ remainingActions.add(action);
+ }
+ }
+ processStructureChildActions(remainingActions, objectsById,
visited, authorRole,
+ options, metadata, xhtml, depth, resourceInfo);
+ } finally {
+ endObjectStructure(objectType, listStyle, xhtml);
}
- for (PropertyAction action : actions) {
- if (hasPrimaryPicture &&
- action.oneNotePropertyEnum ==
OneNotePropertyEnum.WebPictureContainer14) {
+ }
+
+ private void startObjectStructure(int objectType, ListStyle listStyle,
+ XHTMLContentHandler xhtml) throws
SAXException {
+ switch (objectType) {
+ case OneNoteStructureJcid.TABLE_NODE:
+ xhtml.startElement("table");
+ break;
+ case OneNoteStructureJcid.TABLE_ROW_NODE:
+ xhtml.startElement("tr");
+ break;
+ case OneNoteStructureJcid.TABLE_CELL_NODE:
+ xhtml.startElement("td");
+ break;
+ case OneNoteStructureJcid.OUTLINE_ELEMENT_NODE:
+ if (listStyle != null) {
+ if (listStyle.restartValue == null) {
+ xhtml.startElement("li");
+ } else {
+ xhtml.startElement("li", "value",
listStyle.restartValue.toString());
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void endObjectStructure(int objectType, ListStyle listStyle,
+ XHTMLContentHandler xhtml) throws
SAXException {
+ switch (objectType) {
+ case OneNoteStructureJcid.TABLE_NODE:
+ xhtml.endElement("table");
+ break;
+ case OneNoteStructureJcid.TABLE_ROW_NODE:
+ xhtml.endElement("tr");
+ break;
+ case OneNoteStructureJcid.TABLE_CELL_NODE:
+ xhtml.endElement("td");
+ break;
+ case OneNoteStructureJcid.OUTLINE_ELEMENT_NODE:
+ if (listStyle != null) {
+ xhtml.endElement("li");
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void walkObjectGroupRoots(List<RevisionStoreObjectGroup>
objectGroups,
+ Map<ExGuid, RevisionStoreObject>
objectsById,
+ Set<ExGuid> visited,
OneNoteTreeWalkerOptions options,
+ Metadata metadata, XHTMLContentHandler
xhtml)
+ throws SAXException, TikaException, IOException {
+ Set<ExGuid> structureDescendantIds =
collectStructureDescendantIds(objectsById);
+ List<RevisionStoreObject> deferredObjects = new ArrayList<>();
+ for (RevisionStoreObjectGroup objectGroup : objectGroups) {
+ if (objectGroup == null || objectGroup.objects == null) {
continue;
}
- if (action.oneNotePropertyEnum !=
OneNotePropertyEnum.StructureElementChildNodes) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ if (object != null && object.objectID != null &&
+ structureDescendantIds.contains(object.objectID)) {
+ deferredObjects.add(object);
+ continue;
+ }
+ walkRootObject(object, objectsById, visited, options,
metadata, xhtml);
+ }
+ }
+ // Cyclic owners may have no independent root; start tables before
list containers.
+ for (RevisionStoreObject object : deferredObjects) {
+ if (!visited.contains(object.objectID) &&
+ jcidIndex(object) == OneNoteStructureJcid.TABLE_NODE) {
+ walkRootObject(object, objectsById, visited, options,
metadata, xhtml);
+ }
+ }
+ for (RevisionStoreObject object : deferredObjects) {
+ if (!visited.contains(object.objectID) && isListContainer(object,
objectsById)) {
+ walkRootObject(object, objectsById, visited, options,
metadata, xhtml);
+ }
+ }
+ // Recover only depth-skipped objects, not intentionally suppressed
renditions.
+ while (!depthSkippedObjects.isEmpty()) {
+ Map.Entry<RevisionStoreObject, Map<AuthorRole,
EmbeddedResourceInfo>> skipped =
+ depthSkippedObjects.entrySet().iterator().next();
+ depthSkippedObjects.remove(skipped.getKey());
+ for (Map.Entry<AuthorRole, EmbeddedResourceInfo> reference :
skipped.getValue().entrySet()) {
+ walkRootObject(skipped.getKey(), objectsById, visited,
reference.getKey(), options,
+ metadata, xhtml, reference.getValue());
+ }
+ }
+ }
+
+ private boolean isListContainer(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject>
objectsById) {
+ int type = jcidIndex(object);
+ if (type == OneNoteStructureJcid.OUTLINE_NODE ||
+ type == OneNoteStructureJcid.OUTLINE_ELEMENT_NODE) {
+ return true;
+ }
+ if (object == null || object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ return false;
+ }
+ for (PropertyAction action : collectObjectActions(object)) {
+ if (action.isChildReference &&
+ jcidIndex(objectsById.get(action.childReference)) ==
+ OneNoteStructureJcid.OUTLINE_ELEMENT_NODE) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Set<ExGuid> collectStructureDescendantIds(
+ Map<ExGuid, RevisionStoreObject> objectsById) {
+ Set<ExGuid> descendants = new HashSet<>();
+ Set<ExGuid> traversed = new HashSet<>();
+ List<RevisionStoreObject> pending = new ArrayList<>();
+ for (RevisionStoreObject object : objectsById.values()) {
+ if (jcidIndex(object) == OneNoteStructureJcid.TABLE_NODE ||
+ isListContainer(object, objectsById)) {
+ traversed.add(object.objectID);
+ pending.add(object);
+ }
+ }
+ for (int index = 0; index < pending.size(); index++) {
+ RevisionStoreObject object = pending.get(index);
+ if (object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ continue;
+ }
+ for (PropertyAction action : collectObjectActions(object)) {
+ if (!action.isChildReference || action.childReference == null)
{
+ continue;
+ }
+ ExGuid childId = action.childReference;
+ descendants.add(childId);
+ RevisionStoreObject child = objectsById.get(childId);
+ if (child != null && traversed.add(childId)) {
+ pending.add(child);
+ }
+ }
+ }
+ return descendants;
+ }
+
+ private void walkRootObject(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById,
Set<ExGuid> visited,
+ OneNoteTreeWalkerOptions options, Metadata
metadata,
+ XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ walkRootObject(object, objectsById, visited, AuthorRole.NONE, options,
metadata, xhtml, null);
+ }
+
+ private void walkRootObject(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById,
Set<ExGuid> visited,
+ AuthorRole authorRole,
OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml,
EmbeddedResourceInfo resourceInfo)
+ throws SAXException, TikaException, IOException {
+ if (!canWalkObject(object, visited, 0)) {
+ walkObject(object, objectsById, visited, authorRole, options,
metadata, xhtml, 0, resourceInfo);
+ return;
+ }
Review Comment:
walkRootObject` can be invoked with `object == null` from
`walkObjectGroupRoots` (it iterates raw `objectGroup.objects` and does not skip
nulls). In that case `canWalkObject` returns false and this code calls
`walkObject(null, ...)`, which will NPE if `walkObject` dereferences `object`
(the shown implementation does). Fix by short-circuiting nulls (e.g., early
return in `walkRootObject` when `object == null`, and/or skip nulls in
`walkObjectGroupRoots` before calling `walkRootObject`).
##########
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java:
##########
@@ -79,6 +81,699 @@
public class MSOneStorePackageTest {
+ @Test
+ public void testTableAndListObjectsRetainXhtmlStructure() throws Exception
{
+ ExGuid tableId = id(1100);
+ ExGuid rowId = id(1101);
+ ExGuid cellId = id(1102);
+ ExGuid listId = id(1103);
+ RevisionStoreObject tableCell = object(cellId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("table text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreObject row = object(rowId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(cellId),
Collections.emptyList());
+ RevisionStoreObject table = object(tableId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(rowId),
Collections.emptyList());
+ ExGuid bulletListId = id(1104);
+ RevisionStoreObject bulletList = numberListNode(bulletListId,
+ new byte[] {0x01, 0x00, 0x22, 0x20}, null);
+ RevisionStoreObject listItem = listItem(listId, bulletListId, "list
text");
+ setJcid(table, OneNoteStructureJcid.TABLE_NODE);
+ setJcid(row, OneNoteStructureJcid.TABLE_ROW_NODE);
+ setJcid(tableCell, OneNoteStructureJcid.TABLE_CELL_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(table, row, tableCell, listItem,
bulletList));
+ cell.rootDeclares.add(rootDeclare(tableId));
+ cell.rootDeclares.add(rootDeclare(listId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertTrue(xml.contains("<table>"));
+ assertTrue(xml.contains("<tr>"));
+ assertTrue(xml.contains("<td>"));
+ assertTrue(xml.contains("table text"));
+ assertTrue(xml.contains("<ul>"));
+ assertTrue(xml.contains("<li>"));
+ assertTrue(xml.contains("list text"));
+ }
+
+ @Test
+ public void testNumberedListItemsShareOrderedListMarkup() throws Exception
{
+ ExGuid outlineId = id(1150);
+ ExGuid firstItemId = id(1151);
+ ExGuid secondItemId = id(1152);
+ ExGuid firstNumberListId = id(1153);
+ ExGuid secondNumberListId = id(1154);
+ byte[] numberFormat = new byte[] {0x02, 0x00, (byte) 0xfd, (byte)
0xff, 0x01, 0x00};
+
+ RevisionStoreObject firstNumberList =
numberListNode(firstNumberListId, numberFormat, null);
+ RevisionStoreObject secondNumberList =
numberListNode(secondNumberListId, numberFormat, 5);
+ RevisionStoreObject firstItem = listItem(firstItemId,
firstNumberListId,
+ "first numbered item");
+ RevisionStoreObject secondItem = listItem(secondItemId,
secondNumberListId,
+ "second numbered item");
+ RevisionStoreObject outline = object(outlineId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(2))), Arrays.asList(firstItemId,
secondItemId),
+ Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, firstItem, secondItem,
firstNumberList,
+ secondNumberList));
+ cell.rootDeclares.add(rootDeclare(outlineId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol type=\"I\">"), xml);
+ assertEquals(1, countOccurrences(xml, "<li value=\"5\">"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"));
+ assertTrue(xml.contains("first numbered item"));
+ assertTrue(xml.contains("second numbered item"));
+ assertFalse(xml.contains("<ul>"));
+ }
+
+ @Test
+ public void testListFallbackPreservesGroupingRegardlessOfRootOrder()
throws Exception {
+ for (boolean useCellFallback : new boolean[] {false, true}) {
+ for (boolean parentFirst : new boolean[] {false, true}) {
+ for (boolean typedParent : new boolean[] {false, true}) {
+ RevisionStoreObject numberList = numberListNode(id(1703),
+ numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1701),
numberList.objectID, "first item");
+ RevisionStoreObject second = listItem(id(1702),
numberList.objectID, "second item");
+ RevisionStoreObject outline = object(id(1700),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001C20,
+ arrayNumber(2))),
Arrays.asList(first.objectID, second.objectID),
+ Collections.emptyList());
+ if (typedParent) {
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ }
+ RevisionStoreObjectGroup objects = parentFirst ?
+ group(outline, first, second, numberList) :
+ group(first, second, numberList, outline);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ if (useCellFallback) {
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(objects);
+ pkg.cells.add(cell);
+ } else {
+ pkg.OtherFileNodeList.add(objects);
+ }
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*first
item.*second item.*</ol>.*"), xml);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testVisitedListReferencesDoNotCreateOrSplitContainers() throws
Exception {
+ RevisionStoreObject roman = numberListNode(id(1724),
numberListFormat(1), null);
+ RevisionStoreObject decimal = numberListNode(id(1725),
numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1721), roman.objectID,
"visited item");
+ RevisionStoreObject second = listItem(id(1722), decimal.objectID,
"second item");
+ RevisionStoreObject third = listItem(id(1723), decimal.objectID,
"third item");
+ RevisionStoreObject outline = object(id(1720),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(5))), Arrays.asList(first.objectID,
second.objectID,
+ first.objectID, second.objectID, third.objectID),
Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, first, second, third, roman,
decimal));
+ cell.rootDeclares.add(rootDeclare(first.objectID));
+ cell.rootDeclares.add(rootDeclare(outline.objectID));
+ cell.rootDeclares.add(rootDeclare(second.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(2, countOccurrences(xml, "<ol"), xml);
+ assertEquals(3, countOccurrences(xml, "<li"), xml);
+ assertEquals(1, countOccurrences(xml, "visited item"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*second item.*third
item.*</ol>.*"), xml);
+ }
+
+ @Test
+ public void testRepeatedLargeListStylesReuseExactComparisons() throws
Exception {
+ byte[] format = new byte[1024 * 1024];
+ byte[] equalFormat = format.clone();
+ byte[] anotherEqualFormat = format.clone();
+ byte[] differentFormat = format.clone();
+ differentFormat[differentFormat.length - 1] = 1;
+ byte[] indent = new byte[16];
+ byte[] equalIndent = indent.clone();
+ RevisionStoreObject[] nodes = {
+ numberListNode(id(1750), format, null),
+ numberListNode(id(1751), equalFormat, null),
+ numberListNode(id(1752), anotherEqualFormat, null),
+ numberListNode(id(1753), differentFormat, null)
+ };
+ List<RevisionStoreObject> objects = new
ArrayList<>(Arrays.asList(nodes));
+ List<ExGuid> itemIds = new ArrayList<>();
+ for (int i = 0; i <= 2000; i++) {
+ ExGuid itemId = id(1800 + i);
+ RevisionStoreObject node = nodes[i == 2000 ? 3 : i % 3];
+ objects.add(listItem(itemId, node.objectID, "large item " + i,
+ i == 0 ? indent : equalIndent));
+ itemIds.add(itemId);
+ }
Review Comment:
This unit test allocates multiple 1MB arrays and builds 2001 list items; in
constrained CI environments this can noticeably increase test runtime and
memory pressure. You can likely keep the intent (verifying cache reuse
behavior) with smaller arrays (still large enough to discourage accidental
interning) and/or fewer items, since the assertions are about cache cardinality
rather than scale.
##########
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java:
##########
@@ -79,6 +81,699 @@
public class MSOneStorePackageTest {
+ @Test
+ public void testTableAndListObjectsRetainXhtmlStructure() throws Exception
{
+ ExGuid tableId = id(1100);
+ ExGuid rowId = id(1101);
+ ExGuid cellId = id(1102);
+ ExGuid listId = id(1103);
+ RevisionStoreObject tableCell = object(cellId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("table text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreObject row = object(rowId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(cellId),
Collections.emptyList());
+ RevisionStoreObject table = object(tableId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(rowId),
Collections.emptyList());
+ ExGuid bulletListId = id(1104);
+ RevisionStoreObject bulletList = numberListNode(bulletListId,
+ new byte[] {0x01, 0x00, 0x22, 0x20}, null);
+ RevisionStoreObject listItem = listItem(listId, bulletListId, "list
text");
+ setJcid(table, OneNoteStructureJcid.TABLE_NODE);
+ setJcid(row, OneNoteStructureJcid.TABLE_ROW_NODE);
+ setJcid(tableCell, OneNoteStructureJcid.TABLE_CELL_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(table, row, tableCell, listItem,
bulletList));
+ cell.rootDeclares.add(rootDeclare(tableId));
+ cell.rootDeclares.add(rootDeclare(listId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertTrue(xml.contains("<table>"));
+ assertTrue(xml.contains("<tr>"));
+ assertTrue(xml.contains("<td>"));
+ assertTrue(xml.contains("table text"));
+ assertTrue(xml.contains("<ul>"));
+ assertTrue(xml.contains("<li>"));
+ assertTrue(xml.contains("list text"));
+ }
+
+ @Test
+ public void testNumberedListItemsShareOrderedListMarkup() throws Exception
{
+ ExGuid outlineId = id(1150);
+ ExGuid firstItemId = id(1151);
+ ExGuid secondItemId = id(1152);
+ ExGuid firstNumberListId = id(1153);
+ ExGuid secondNumberListId = id(1154);
+ byte[] numberFormat = new byte[] {0x02, 0x00, (byte) 0xfd, (byte)
0xff, 0x01, 0x00};
+
+ RevisionStoreObject firstNumberList =
numberListNode(firstNumberListId, numberFormat, null);
+ RevisionStoreObject secondNumberList =
numberListNode(secondNumberListId, numberFormat, 5);
+ RevisionStoreObject firstItem = listItem(firstItemId,
firstNumberListId,
+ "first numbered item");
+ RevisionStoreObject secondItem = listItem(secondItemId,
secondNumberListId,
+ "second numbered item");
+ RevisionStoreObject outline = object(outlineId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(2))), Arrays.asList(firstItemId,
secondItemId),
+ Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, firstItem, secondItem,
firstNumberList,
+ secondNumberList));
+ cell.rootDeclares.add(rootDeclare(outlineId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol type=\"I\">"), xml);
+ assertEquals(1, countOccurrences(xml, "<li value=\"5\">"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"));
+ assertTrue(xml.contains("first numbered item"));
+ assertTrue(xml.contains("second numbered item"));
+ assertFalse(xml.contains("<ul>"));
+ }
+
+ @Test
+ public void testListFallbackPreservesGroupingRegardlessOfRootOrder()
throws Exception {
+ for (boolean useCellFallback : new boolean[] {false, true}) {
+ for (boolean parentFirst : new boolean[] {false, true}) {
+ for (boolean typedParent : new boolean[] {false, true}) {
+ RevisionStoreObject numberList = numberListNode(id(1703),
+ numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1701),
numberList.objectID, "first item");
+ RevisionStoreObject second = listItem(id(1702),
numberList.objectID, "second item");
+ RevisionStoreObject outline = object(id(1700),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001C20,
+ arrayNumber(2))),
Arrays.asList(first.objectID, second.objectID),
+ Collections.emptyList());
+ if (typedParent) {
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ }
+ RevisionStoreObjectGroup objects = parentFirst ?
+ group(outline, first, second, numberList) :
+ group(first, second, numberList, outline);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ if (useCellFallback) {
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(objects);
+ pkg.cells.add(cell);
+ } else {
+ pkg.OtherFileNodeList.add(objects);
+ }
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*first
item.*second item.*</ol>.*"), xml);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testVisitedListReferencesDoNotCreateOrSplitContainers() throws
Exception {
+ RevisionStoreObject roman = numberListNode(id(1724),
numberListFormat(1), null);
+ RevisionStoreObject decimal = numberListNode(id(1725),
numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1721), roman.objectID,
"visited item");
+ RevisionStoreObject second = listItem(id(1722), decimal.objectID,
"second item");
+ RevisionStoreObject third = listItem(id(1723), decimal.objectID,
"third item");
+ RevisionStoreObject outline = object(id(1720),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(5))), Arrays.asList(first.objectID,
second.objectID,
+ first.objectID, second.objectID, third.objectID),
Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, first, second, third, roman,
decimal));
+ cell.rootDeclares.add(rootDeclare(first.objectID));
+ cell.rootDeclares.add(rootDeclare(outline.objectID));
+ cell.rootDeclares.add(rootDeclare(second.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(2, countOccurrences(xml, "<ol"), xml);
+ assertEquals(3, countOccurrences(xml, "<li"), xml);
+ assertEquals(1, countOccurrences(xml, "visited item"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*second item.*third
item.*</ol>.*"), xml);
+ }
+
+ @Test
+ public void testRepeatedLargeListStylesReuseExactComparisons() throws
Exception {
+ byte[] format = new byte[1024 * 1024];
+ byte[] equalFormat = format.clone();
+ byte[] anotherEqualFormat = format.clone();
+ byte[] differentFormat = format.clone();
+ differentFormat[differentFormat.length - 1] = 1;
+ byte[] indent = new byte[16];
+ byte[] equalIndent = indent.clone();
+ RevisionStoreObject[] nodes = {
+ numberListNode(id(1750), format, null),
+ numberListNode(id(1751), equalFormat, null),
+ numberListNode(id(1752), anotherEqualFormat, null),
+ numberListNode(id(1753), differentFormat, null)
+ };
+ List<RevisionStoreObject> objects = new
ArrayList<>(Arrays.asList(nodes));
+ List<ExGuid> itemIds = new ArrayList<>();
+ for (int i = 0; i <= 2000; i++) {
+ ExGuid itemId = id(1800 + i);
+ RevisionStoreObject node = nodes[i == 2000 ? 3 : i % 3];
+ objects.add(listItem(itemId, node.objectID, "large item " + i,
+ i == 0 ? indent : equalIndent));
+ itemIds.add(itemId);
+ }
+ RevisionStoreObject outline = object(id(1740),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(itemIds.size()))), itemIds,
Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ objects.add(outline);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(objects.toArray(new
RevisionStoreObject[0])));
+ cell.rootDeclares.add(rootDeclare(outline.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(2, countOccurrences(xml, "<ul"), xml);
+ assertEquals(2001, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.contains("large item 2000"), xml);
+ // Cache size, rather than wall-clock time, verifies repeated pair
reuse.
+ java.lang.reflect.Field cacheField = MSOneStorePackage.class
+ .getDeclaredField("listStyleComparisonCache");
+ cacheField.setAccessible(true);
+ Map<?, ?> cache = (Map<?, ?>) cacheField.get(pkg);
+ assertEquals(2, cache.size());
+ Map<?, ?> formatComparisons = (Map<?, ?>) cache.get(format);
+ assertEquals(3, formatComparisons.size());
+ assertEquals(Boolean.TRUE, formatComparisons.get(equalFormat));
+ assertEquals(Boolean.TRUE, formatComparisons.get(anotherEqualFormat));
+ assertEquals(Boolean.FALSE, formatComparisons.get(differentFormat));
+ assertEquals(Collections.singletonMap(equalIndent, Boolean.TRUE),
cache.get(indent));
Review Comment:
This test reaches into a private field by name, which tightly couples it to
an implementation detail (renaming the field or changing the cache strategy
will break the test even if behavior is still correct). Consider asserting
cache behavior indirectly (e.g., via observable output structure) or exposing a
package-private accessor/testing hook that returns cache stats without relying
on reflection and field names.
##########
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java:
##########
@@ -79,6 +81,699 @@
public class MSOneStorePackageTest {
+ @Test
+ public void testTableAndListObjectsRetainXhtmlStructure() throws Exception
{
+ ExGuid tableId = id(1100);
+ ExGuid rowId = id(1101);
+ ExGuid cellId = id(1102);
+ ExGuid listId = id(1103);
+ RevisionStoreObject tableCell = object(cellId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("table text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreObject row = object(rowId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(cellId),
Collections.emptyList());
+ RevisionStoreObject table = object(tableId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))), Collections.singletonList(rowId),
Collections.emptyList());
+ ExGuid bulletListId = id(1104);
+ RevisionStoreObject bulletList = numberListNode(bulletListId,
+ new byte[] {0x01, 0x00, 0x22, 0x20}, null);
+ RevisionStoreObject listItem = listItem(listId, bulletListId, "list
text");
+ setJcid(table, OneNoteStructureJcid.TABLE_NODE);
+ setJcid(row, OneNoteStructureJcid.TABLE_ROW_NODE);
+ setJcid(tableCell, OneNoteStructureJcid.TABLE_CELL_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(table, row, tableCell, listItem,
bulletList));
+ cell.rootDeclares.add(rootDeclare(tableId));
+ cell.rootDeclares.add(rootDeclare(listId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertTrue(xml.contains("<table>"));
+ assertTrue(xml.contains("<tr>"));
+ assertTrue(xml.contains("<td>"));
+ assertTrue(xml.contains("table text"));
+ assertTrue(xml.contains("<ul>"));
+ assertTrue(xml.contains("<li>"));
+ assertTrue(xml.contains("list text"));
+ }
+
+ @Test
+ public void testNumberedListItemsShareOrderedListMarkup() throws Exception
{
+ ExGuid outlineId = id(1150);
+ ExGuid firstItemId = id(1151);
+ ExGuid secondItemId = id(1152);
+ ExGuid firstNumberListId = id(1153);
+ ExGuid secondNumberListId = id(1154);
+ byte[] numberFormat = new byte[] {0x02, 0x00, (byte) 0xfd, (byte)
0xff, 0x01, 0x00};
+
+ RevisionStoreObject firstNumberList =
numberListNode(firstNumberListId, numberFormat, null);
+ RevisionStoreObject secondNumberList =
numberListNode(secondNumberListId, numberFormat, 5);
+ RevisionStoreObject firstItem = listItem(firstItemId,
firstNumberListId,
+ "first numbered item");
+ RevisionStoreObject secondItem = listItem(secondItemId,
secondNumberListId,
+ "second numbered item");
+ RevisionStoreObject outline = object(outlineId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(2))), Arrays.asList(firstItemId,
secondItemId),
+ Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, firstItem, secondItem,
firstNumberList,
+ secondNumberList));
+ cell.rootDeclares.add(rootDeclare(outlineId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol type=\"I\">"), xml);
+ assertEquals(1, countOccurrences(xml, "<li value=\"5\">"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"));
+ assertTrue(xml.contains("first numbered item"));
+ assertTrue(xml.contains("second numbered item"));
+ assertFalse(xml.contains("<ul>"));
+ }
+
+ @Test
+ public void testListFallbackPreservesGroupingRegardlessOfRootOrder()
throws Exception {
+ for (boolean useCellFallback : new boolean[] {false, true}) {
+ for (boolean parentFirst : new boolean[] {false, true}) {
+ for (boolean typedParent : new boolean[] {false, true}) {
+ RevisionStoreObject numberList = numberListNode(id(1703),
+ numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1701),
numberList.objectID, "first item");
+ RevisionStoreObject second = listItem(id(1702),
numberList.objectID, "second item");
+ RevisionStoreObject outline = object(id(1700),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001C20,
+ arrayNumber(2))),
Arrays.asList(first.objectID, second.objectID),
+ Collections.emptyList());
+ if (typedParent) {
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ }
+ RevisionStoreObjectGroup objects = parentFirst ?
+ group(outline, first, second, numberList) :
+ group(first, second, numberList, outline);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ if (useCellFallback) {
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(objects);
+ pkg.cells.add(cell);
+ } else {
+ pkg.OtherFileNodeList.add(objects);
+ }
+ String xml = walkXml(pkg);
+ assertEquals(1, countOccurrences(xml, "<ol"), xml);
+ assertEquals(2, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*first
item.*second item.*</ol>.*"), xml);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testVisitedListReferencesDoNotCreateOrSplitContainers() throws
Exception {
+ RevisionStoreObject roman = numberListNode(id(1724),
numberListFormat(1), null);
+ RevisionStoreObject decimal = numberListNode(id(1725),
numberListFormat(0), null);
+ RevisionStoreObject first = listItem(id(1721), roman.objectID,
"visited item");
+ RevisionStoreObject second = listItem(id(1722), decimal.objectID,
"second item");
+ RevisionStoreObject third = listItem(id(1723), decimal.objectID,
"third item");
+ RevisionStoreObject outline = object(id(1720),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(5))), Arrays.asList(first.objectID,
second.objectID,
+ first.objectID, second.objectID, third.objectID),
Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(outline, first, second, third, roman,
decimal));
+ cell.rootDeclares.add(rootDeclare(first.objectID));
+ cell.rootDeclares.add(rootDeclare(outline.objectID));
+ cell.rootDeclares.add(rootDeclare(second.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(2, countOccurrences(xml, "<ol"), xml);
+ assertEquals(3, countOccurrences(xml, "<li"), xml);
+ assertEquals(1, countOccurrences(xml, "visited item"), xml);
+ assertTrue(xml.matches("(?s).*<ol type=\"1\">.*second item.*third
item.*</ol>.*"), xml);
+ }
+
+ @Test
+ public void testRepeatedLargeListStylesReuseExactComparisons() throws
Exception {
+ byte[] format = new byte[1024 * 1024];
+ byte[] equalFormat = format.clone();
+ byte[] anotherEqualFormat = format.clone();
+ byte[] differentFormat = format.clone();
+ differentFormat[differentFormat.length - 1] = 1;
+ byte[] indent = new byte[16];
+ byte[] equalIndent = indent.clone();
+ RevisionStoreObject[] nodes = {
+ numberListNode(id(1750), format, null),
+ numberListNode(id(1751), equalFormat, null),
+ numberListNode(id(1752), anotherEqualFormat, null),
+ numberListNode(id(1753), differentFormat, null)
+ };
+ List<RevisionStoreObject> objects = new
ArrayList<>(Arrays.asList(nodes));
+ List<ExGuid> itemIds = new ArrayList<>();
+ for (int i = 0; i <= 2000; i++) {
+ ExGuid itemId = id(1800 + i);
+ RevisionStoreObject node = nodes[i == 2000 ? 3 : i % 3];
+ objects.add(listItem(itemId, node.objectID, "large item " + i,
+ i == 0 ? indent : equalIndent));
+ itemIds.add(itemId);
+ }
+ RevisionStoreObject outline = object(id(1740),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(itemIds.size()))), itemIds,
Collections.emptyList());
+ setJcid(outline, OneNoteStructureJcid.OUTLINE_NODE);
+ objects.add(outline);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(objects.toArray(new
RevisionStoreObject[0])));
+ cell.rootDeclares.add(rootDeclare(outline.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertEquals(2, countOccurrences(xml, "<ul"), xml);
+ assertEquals(2001, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.contains("large item 2000"), xml);
+ // Cache size, rather than wall-clock time, verifies repeated pair
reuse.
+ java.lang.reflect.Field cacheField = MSOneStorePackage.class
+ .getDeclaredField("listStyleComparisonCache");
+ cacheField.setAccessible(true);
+ Map<?, ?> cache = (Map<?, ?>) cacheField.get(pkg);
+ assertEquals(2, cache.size());
+ Map<?, ?> formatComparisons = (Map<?, ?>) cache.get(format);
+ assertEquals(3, formatComparisons.size());
+ assertEquals(Boolean.TRUE, formatComparisons.get(equalFormat));
+ assertEquals(Boolean.TRUE, formatComparisons.get(anotherEqualFormat));
+ assertEquals(Boolean.FALSE, formatComparisons.get(differentFormat));
+ assertEquals(Collections.singletonMap(equalIndent, Boolean.TRUE),
cache.get(indent));
+ }
+
+ @Test
+ public void testValidListMetadataWinsInEitherReferenceOrder() throws
Exception {
+ for (boolean validFirst : new boolean[] {false, true}) {
+ RevisionStoreObject valid = numberListNode(id(1731),
numberListFormat(1), 5);
+ RevisionStoreObject malformed = numberListNode(id(1732),
+ new byte[] {3, 0, (byte) 0xfd, (byte) 0xff, 1, 0}, 7);
+ RevisionStoreObject item = object(id(1730),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001C26,
+ arrayNumber(2)),
+ new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("multi-node item"))),
+ validFirst ? Arrays.asList(valid.objectID,
malformed.objectID) :
+ Arrays.asList(malformed.objectID, valid.objectID),
Collections.emptyList());
+ setJcid(item, OneNoteStructureJcid.OUTLINE_ELEMENT_NODE);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(item, valid, malformed));
+ cell.rootDeclares.add(rootDeclare(item.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+ assertTrue(xml.contains("<ol type=\"I\">"), xml);
+ assertTrue(xml.contains("<li value=\"5\">"), xml);
+ assertTrue(xml.contains("multi-node item"), xml);
+ assertFalse(xml.contains("<ul"), xml);
+ }
+ }
+
+ @Test
+ public void testNumberListLengthPrefixIsNotNumberingMarker() throws
Exception {
+ ExGuid listId = id(1160);
+ ExGuid numberListId = id(1161);
+ byte[] numberFormat = new byte[(0xfffd + 1) * 2];
+ numberFormat[0] = (byte) 0xfd;
+ numberFormat[1] = (byte) 0xff;
+ for (int i = 2; i < numberFormat.length; i += 2) {
+ numberFormat[i] = 0x2e;
+ }
+ RevisionStoreObject numberList = numberListNode(numberListId,
numberFormat, null);
+ RevisionStoreObject listItem = listItem(listId, numberListId, "bullet
item");
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(listItem, numberList));
+ cell.rootDeclares.add(rootDeclare(listId));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ String xml = walkXml(pkg);
+
+ assertTrue(xml.contains("<ul>"), xml);
+ assertFalse(xml.contains("<ol"), xml);
+ }
+
+ @Test
+ public void testNumberListFormatsMapHtmlTypesAndRejectMalformedValues()
throws Exception {
+ String[] types = {"1", "I", "i", "A", "a"};
+ for (int code = 0; code < types.length; code++) {
+ String xml = singleListXml(1200 + code * 10,
numberListFormat(code), null);
+ assertTrue(xml.contains("<ol type=\"" + types[code] + "\">"), xml);
+ }
+
+ String unknown = singleListXml(1260, numberListFormat(5), null);
+ assertTrue(unknown.contains("<ol>"), unknown);
+ assertFalse(unknown.contains("<ol type="), unknown);
+
+ byte[][] malformedFormats = {null, {0}, {1, 0, 0}, {3, 0, (byte) 0xfd,
(byte) 0xff,
+ 1, 0}, {1, 0, (byte) 0xfd, (byte) 0xff}, {1, 0, 0x2e, 0}};
+ for (int i = 0; i < malformedFormats.length; i++) {
+ String xml = singleListXml(1270 + i * 10, malformedFormats[i],
null);
+ assertTrue(xml.contains("<ul>"), xml);
+ assertFalse(xml.contains("<ol"), xml);
+ }
+ }
+
+ @Test
+ public void testMalformedListFormatsDoNotShareUnorderedList() throws
Exception {
+ ExGuid outlineId = id(1320);
+ ExGuid firstItemId = id(1321);
+ ExGuid secondItemId = id(1322);
+ byte[] firstMalformedFormat = {3, 0, (byte) 0xfd, (byte) 0xff, 1, 0};
+ byte[] secondMalformedFormat = {4, 0, (byte) 0xfd, (byte) 0xff, 1, 0};
+ RevisionStoreObject firstNumberList = numberListNode(id(1323),
firstMalformedFormat,
+ null);
+ RevisionStoreObject secondNumberList = numberListNode(id(1324),
secondMalformedFormat,
+ null);
+ List<ExGuid> itemIds = Arrays.asList(firstItemId, secondItemId);
+ List<RevisionStoreObject> items = Arrays.asList(
+ listItem(firstItemId, firstNumberList.objectID, "first
malformed item"),
+ listItem(secondItemId, secondNumberList.objectID, "second
malformed item"));
+
+ String xml = walkListItems(outlineId, itemIds, items,
+ Arrays.asList(firstNumberList, secondNumberList), "");
+
+ assertEquals(2, countOccurrences(xml, "<ul>"), xml);
+ assertEquals(2, countOccurrences(xml, "<li>"), xml);
+ assertFalse(xml.contains("<ol"), xml);
+ }
+
+ @Test
+ public void testListStylesSplitOnFormatAndIndentAndShareWhenEqual() throws
Exception {
+ int seed = 1350;
+ ExGuid outlineId = id(seed);
+ RevisionStoreObject upperRoman = numberListNode(id(seed + 30),
numberListFormat(1), null);
+ RevisionStoreObject lowerRoman = numberListNode(id(seed + 31),
numberListFormat(2), null);
+ byte[] lowerRomanWithSuffix = {3, 0, (byte) 0xfd, (byte) 0xff, 2, 0,
0x78, 0};
+ RevisionStoreObject lowerRomanSuffix = numberListNode(id(seed + 32),
+ lowerRomanWithSuffix, null);
+ RevisionStoreObject bullet = numberListNode(id(seed + 33),
+ new byte[] {1, 0, 0x22, 0x20}, null);
+ RevisionStoreObject unknown = numberListNode(id(seed + 34),
numberListFormat(5), null);
+ RevisionStoreObject decimal = numberListNode(id(seed + 35),
numberListFormat(0), null);
+
+ List<RevisionStoreObject> items = new ArrayList<>();
+ List<ExGuid> itemIds = new ArrayList<>();
+ RevisionStoreObject[] listNodes = {upperRoman, lowerRoman,
lowerRomanSuffix,
+ lowerRomanSuffix, lowerRomanSuffix, bullet, bullet, unknown,
decimal, unknown,
+ unknown};
+ for (int i = 0; i < listNodes.length; i++) {
+ ExGuid itemId = id(seed + 1 + i);
+ byte[] indent = i == 3 ? new byte[] {1, 0, 0, 0} : null;
+ RevisionStoreObject item = i == 4 ?
+ listItemWithWrongIndent(itemId, listNodes[i].objectID,
"item " + i) :
+ listItem(itemId, listNodes[i].objectID, "item " + i,
indent);
+ items.add(item);
+ itemIds.add(itemId);
+ }
+ ExGuid plainOutlineId = id(seed + 20);
+ RevisionStoreObject plainOutline = object(plainOutlineId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("plain outline text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(plainOutline, OneNoteStructureJcid.OUTLINE_ELEMENT_NODE);
+ items.add(plainOutline);
+ itemIds.add(plainOutlineId);
+
+ List<RevisionStoreObject> nodes = Arrays.asList(upperRoman, lowerRoman,
+ lowerRomanSuffix, bullet, unknown, decimal);
+ String xml = walkListItems(outlineId, itemIds, items, nodes, "outline
caption");
+
+ assertEquals(8, countOccurrences(xml, "<ol"), xml);
+ assertEquals(1, countOccurrences(xml, "<ul>"), xml);
+ assertEquals(11, countOccurrences(xml, "<li"), xml);
+ assertTrue(xml.contains("plain outline text"), xml);
+ assertTrue(xml.contains("outline caption"), xml);
+ }
+
+ @Test
+ public void testInvalidNumberListNodesAndRestartDataFallBackSafely()
throws Exception {
+ int seed = 1400;
+ ExGuid outlineId = id(seed);
+ ExGuid missingNodeId = id(seed + 50);
+ RevisionStoreObject noProperties = numberListNode(id(seed + 30), null,
null);
+ noProperties.propertySet = null;
+ RevisionStoreObject noObjectSpace = numberListNode(id(seed + 31),
numberListFormat(0), null);
+ noObjectSpace.propertySet.objectSpaceObjectPropSet = null;
+ RevisionStoreObject wrongJcid = numberListNode(id(seed + 32),
numberListFormat(0), null);
+ setJcid(wrongJcid, OneNoteStructureJcid.OUTLINE_ELEMENT_NODE);
+ RevisionStoreObject missingFormat = numberListNode(id(seed + 33),
null, null);
+ RevisionStoreObject wrongFormatType =
numberListNodeWithProperties(id(seed + 34),
+ new PropertySpec(PropertyType.FourBytesOfData, 0x1C001C1A,
fourBytes(1)));
+ RevisionStoreObject wrongRestartType =
numberListNodeWithProperties(id(seed + 35),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
0x1C001C1A,
+ bytes(numberListFormat(0))),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
0x14001CB7,
+ bytes(new byte[] {1, 0, 0, 0})));
+ RevisionStoreObject nullRestartData =
numberListNodeWithProperties(id(seed + 36),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
0x1C001C1A,
+ bytes(numberListFormat(0))),
+ new PropertySpec(PropertyType.FourBytesOfData, 0x14001CB7,
+ fourBytes((byte[]) null)));
+ RevisionStoreObject shortRestartData =
numberListNodeWithProperties(id(seed + 37),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
0x1C001C1A,
+ bytes(numberListFormat(0))),
+ new PropertySpec(PropertyType.FourBytesOfData, 0x14001CB7,
+ fourBytes(new byte[] {1, 0, 0})));
+
+ RevisionStoreObject[] listNodes = {null, null, noProperties,
noObjectSpace, wrongJcid,
+ missingFormat, missingFormat, wrongFormatType,
wrongRestartType, nullRestartData,
+ shortRestartData};
+ List<RevisionStoreObject> items = new ArrayList<>();
+ List<ExGuid> itemIds = new ArrayList<>();
+ for (int i = 0; i < listNodes.length; i++) {
+ ExGuid itemId = id(seed + 1 + i);
+ ExGuid numberListId = listNodes[i] == null ?
+ (i == 0 ? missingNodeId : null) : listNodes[i].objectID;
+ items.add(listItem(itemId, numberListId, "invalid item " + i));
+ itemIds.add(itemId);
+ }
+ ExGuid plainItemId = id(seed + 20);
+ RevisionStoreObject plainItem = object(plainItemId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("unstyled outline item"))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(plainItem, OneNoteStructureJcid.OUTLINE_ELEMENT_NODE);
+ items.add(plainItem);
+ itemIds.add(plainItemId);
+ itemIds.add(id(seed + 21));
+
+ List<RevisionStoreObject> nodes = new ArrayList<>();
+ for (RevisionStoreObject node : listNodes) {
+ if (node != null && !nodes.contains(node)) {
+ nodes.add(node);
+ }
+ }
+ String xml = walkListItems(outlineId, itemIds, items, nodes, "outline
text");
+
+ assertEquals(1, countOccurrences(xml, "<ul>"), xml);
+ assertEquals(1, countOccurrences(xml, "<ol type=\"1\">"), xml);
+ assertEquals(0, countOccurrences(xml, "<li value="), xml);
+ assertTrue(xml.contains("unstyled outline item"), xml);
+ for (int i = 0; i < listNodes.length; i++) {
+ assertTrue(xml.contains("<p>invalid item " + i + "</p>"), xml);
+ }
+ }
+
+ @Test
+ public void testFallbackRecoversDepthSkippedTableDescendants() throws
Exception {
+ RevisionStoreObject leaf = object(id(4003),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("recovered leaf"))),
Collections.emptyList(), Collections.emptyList());
+ RevisionStoreObject tableCell = object(id(4002),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))),
Collections.singletonList(leaf.objectID), Collections.emptyList());
+ RevisionStoreObject author = object(id(4005),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D75, utf16Text("Depth Author"))),
Collections.emptyList(), Collections.emptyList());
+ RevisionStoreObject picture = object(id(4006), propertySet(),
+ Collections.emptyList(), Collections.emptyList());
+ picture.fileDataObject = fileData("depth image");
+ RevisionStoreObject row = object(id(4001),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1)),
+ new PropertySpec(PropertyType.ObjectID, 0x20001D79,
new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x20001D78,
new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x20001C3F,
new NoData()),
+ new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001DD7, utf16Text("depth-image.png"))),
+ Arrays.asList(tableCell.objectID, author.objectID,
author.objectID, picture.objectID),
+ Collections.emptyList());
+ RevisionStoreObject table = object(id(4000),
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(1))),
Collections.singletonList(row.objectID), Collections.emptyList());
+ RevisionStoreObject shallow = object(id(4004),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("shallow text"))),
Collections.emptyList(), Collections.emptyList());
+ setJcid(table, OneNoteStructureJcid.TABLE_NODE);
+ setJcid(row, OneNoteStructureJcid.TABLE_ROW_NODE);
+ setJcid(tableCell, OneNoteStructureJcid.TABLE_CELL_NODE);
+ List<RevisionStoreObjectGroup> groups =
Collections.singletonList(group(shallow, table,
+ row, tableCell, leaf, author, picture));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ Map<ExGuid, RevisionStoreObject> objectsById =
pkg.indexObjectsById(groups);
+ Set<ExGuid> visited = new HashSet<>();
+ Metadata metadata = new Metadata();
+ ToXMLContentHandler xml = new ToXMLContentHandler();
+ ParseContext context = new ParseContext();
+ List<String> recoveredResourceNames = new ArrayList<>();
+ context.set(EmbeddedDocumentExtractor.class, new
EmbeddedDocumentExtractor() {
+ @Override
+ public boolean shouldParseEmbedded(Metadata embeddedMetadata,
ParseContext parseContext) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(org.apache.tika.io.TikaInputStream
stream,
+ org.xml.sax.ContentHandler handler,
Metadata embeddedMetadata,
+ ParseContext parseContext, boolean
outputHtml) {
+
recoveredResourceNames.add(embeddedMetadata.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ }
+ });
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(xml, metadata,
context);
+ xhtml.startDocument();
+ pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context);
+ Method walkObject =
MSOneStorePackage.class.getDeclaredMethod("walkObject",
+ RevisionStoreObject.class, Map.class, Set.class,
+ Class.forName(MSOneStorePackage.class.getName() +
"$AuthorRole"),
+ OneNoteTreeWalkerOptions.class, Metadata.class,
XHTMLContentHandler.class, int.class);
+ walkObject.setAccessible(true);
+ // Begin near the cap to exercise the real cutoff without a thousand
stack frames.
+ walkObject.invoke(pkg, table, objectsById, visited, null,
+ new OneNoteTreeWalkerOptions(), metadata, xhtml, 998);
Review Comment:
This test depends on private method names/signatures and even the binary
name of a nested type (`$AuthorRole`). That makes the test brittle to internal
refactors (e.g., parameter changes, renames, moving the enum). A more
maintainable approach would be to introduce a package-private testing hook
(method or helper) that exercises the depth-limit/fallback path without
reflection, or to validate the behavior through the public `walkTree(...)` API
by constructing inputs that naturally exceed the depth limit.
> Emit onenote lists and tables as html lists and tables
> ------------------------------------------------------
>
> Key: TIKA-4943
> URL: https://issues.apache.org/jira/browse/TIKA-4943
> Project: Tika
> Issue Type: Improvement
> Components: parser
> Reporter: Henry Lindeman
> Priority: Minor
>
> currently table cells and list items are emitted in p tags
> instead use <tr>/<td> and <ul>/<li>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
