This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch 3x-port-sax-ooxml in repository https://gitbox.apache.org/repos/asf/tika.git
commit 478049cfa904b914a05b2df999d3621869b2581f Author: tallison <[email protected]> AuthorDate: Wed Jul 8 11:11:11 2026 -0400 First steps in porting 4.x sax ooxml -> 3.x - checkpoint --- .../microsoft/ooxml/OOXMLPartContentCollector.java | 227 +++++++++++++++++++++ .../ooxml/SXSLFPowerPointExtractorDecorator.java | 13 +- .../ooxml/SXWPFWordExtractorDecorator.java | 142 ++++++++++++- .../parser/microsoft/ooxml/SXSLFExtractorTest.java | 5 + .../parser/microsoft/ooxml/SXWPFExtractorTest.java | 5 + 5 files changed, 378 insertions(+), 14 deletions(-) diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPartContentCollector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPartContentCollector.java new file mode 100644 index 0000000000..6cece158e8 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPartContentCollector.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Generic SAX handler that collects raw XML content by ID from OOXML part files. + * Works with any part that contains wrapper elements with {@code w:id} attributes + * containing body content (paragraphs, tables, formatting, etc.). + * <p> + * Used for: + * <ul> + * <li>footnotes.xml — wrapper element "footnote"</li> + * <li>endnotes.xml — wrapper element "endnote"</li> + * <li>comments.xml — wrapper element "comment"</li> + * </ul> + * <p> + * IDs "0" and "-1" are skipped (these are separator/continuation elements in + * footnotes/endnotes). + */ +class OOXMLPartContentCollector extends DefaultHandler { + + private static final String W_NS = + "http://schemas.openxmlformats.org/wordprocessingml/2006/main";; + + private final Set<String> wrapperElementNames; + private final Set<String> skipIds; + private final Map<String, byte[]> contentMap = new HashMap<>(); + private final Map<String, String> namespaceMappings = new HashMap<>(); + + private String currentId = null; + private ByteArrayOutputStream buffer = null; + private int depth = 0; + + /** + * @param wrapperElementNames local names of wrapper elements to collect + * (e.g., "footnote", "endnote", "comment") + */ + OOXMLPartContentCollector(Set<String> wrapperElementNames) { + this(wrapperElementNames, Set.of("0", "-1")); + } + + /** + * @param wrapperElementNames local names of wrapper elements to collect + * @param skipIds IDs to skip (e.g., "0", "-1" for footnote + * separator/continuation elements) + */ + OOXMLPartContentCollector(Set<String> wrapperElementNames, Set<String> skipIds) { + this.wrapperElementNames = wrapperElementNames; + this.skipIds = skipIds; + } + + @Override + public void startPrefixMapping(String prefix, String uri) { + namespaceMappings.put(prefix, uri); + } + + Map<String, byte[]> getContentMap() { + return contentMap; + } + + @Override + public void startElement(String uri, String localName, String qName, + Attributes atts) throws SAXException { + if (currentId != null) { + depth++; + appendStartTag(localName, qName, atts); + return; + } + + if (wrapperElementNames.contains(localName)) { + String id = atts.getValue(W_NS, "id"); + if (id != null && !skipIds.contains(id)) { + currentId = id; + buffer = new ByteArrayOutputStream(); + // Don't write wrapper open tag yet — inline xmlns declarations + // (e.g., xmlns:a on nested elements) haven't been captured via + // startPrefixMapping. Defer to endElement when all are known. + depth = 0; + } + } + } + + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + if (currentId == null) { + return; + } + + if (depth == 0) { + // Build the wrapper now — all startPrefixMapping calls from nested + // elements have been captured, so inline xmlns declarations are included. + byte[] wrapperOpen = buildWrapperOpenTag().getBytes(StandardCharsets.UTF_8); + byte[] content = buffer.toByteArray(); + ByteArrayOutputStream combined = + new ByteArrayOutputStream(wrapperOpen.length + content.length + 16); + combined.write(wrapperOpen, 0, wrapperOpen.length); + combined.write(content, 0, content.length); + writeString(combined, "</w:body>"); + contentMap.put(currentId, combined.toByteArray()); + currentId = null; + buffer = null; + return; + } + + depth--; + if (qName != null && !qName.isEmpty()) { + writeString("</" + qName + ">"); + } else { + writeString("</" + localName + ">"); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if (currentId != null) { + writeString(escape(new String(ch, start, length))); + } + } + + private String buildWrapperOpenTag() { + StringBuilder sb = new StringBuilder("<w:body"); + // include all namespace declarations from the source document + for (Map.Entry<String, String> entry : namespaceMappings.entrySet()) { + String prefix = entry.getKey(); + String nsUri = entry.getValue(); + if (prefix == null || prefix.isEmpty()) { + sb.append(" xmlns=\"").append(escape(nsUri)).append("\""); + } else { + sb.append(" xmlns:").append(prefix).append("=\"") + .append(escape(nsUri)).append("\""); + } + } + // ensure w namespace is present + if (!namespaceMappings.containsKey("w")) { + sb.append(" xmlns:w=\"").append(W_NS).append("\""); + } + sb.append(">"); + return sb.toString(); + } + + private void appendStartTag(String localName, String qName, Attributes atts) { + String tagName = (qName != null && !qName.isEmpty()) ? qName : localName; + StringBuilder sb = new StringBuilder(); + sb.append('<').append(tagName); + for (int i = 0; i < atts.getLength(); i++) { + String attName = atts.getQName(i); + if (attName == null || attName.isEmpty()) { + attName = atts.getLocalName(i); + } + sb.append(' ').append(attName).append("=\""); + sb.append(escape(atts.getValue(i))); + sb.append('"'); + } + sb.append('>'); + writeString(sb.toString()); + } + + private void writeString(String s) { + writeString(buffer, s); + } + + private static void writeString(ByteArrayOutputStream target, String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + target.write(bytes, 0, bytes.length); + } + + static String escape(String s) { + if (s == null) { + return ""; + } + StringBuilder sb = null; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + String replacement = null; + switch (c) { + case '&': + replacement = "&"; + break; + case '<': + replacement = "<"; + break; + case '>': + replacement = ">"; + break; + case '"': + replacement = """; + break; + default: + if (sb != null) { + sb.append(c); + } + continue; + } + if (sb == null) { + sb = new StringBuilder(s.length() + 16); + sb.append(s, 0, i); + } + sb.append(replacement); + } + return sb != null ? sb.toString() : s; + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java index c036f086f0..ee5d7050c0 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java @@ -41,6 +41,7 @@ import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.microsoft.ooxml.xslf.XSLFEventBasedPowerPointExtractor; @@ -177,11 +178,17 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor { // Map<String, String> hyperlinks = loadHyperlinkRelationships(packagePart); xhtml.startElement("div", "class", "slide-content"); + //pass metadata so the body handler can emit HAS_* signals for the slide, and keep a + //reference so we can flag hidden slides after the parse + OOXMLWordAndPowerPointTextHandler wordAndPPTHandler = + new OOXMLWordAndPowerPointTextHandler( + new OOXMLTikaBodyPartHandler(xhtml, metadata), linkedRelationships); try (InputStream stream = slidePart.getInputStream()) { XMLReaderUtils.parseSAX(CloseShieldInputStream.wrap(stream), - new EmbeddedContentHandler(new OOXMLWordAndPowerPointTextHandler( - new OOXMLTikaBodyPartHandler(xhtml), linkedRelationships)), context); - + new EmbeddedContentHandler(wordAndPPTHandler), context); + if (wordAndPPTHandler.isHiddenSlide()) { + metadata.set(Office.HAS_HIDDEN_SLIDES, true); + } } catch (TikaException | IOException e) { metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, ExceptionUtils.getStackTrace(e)); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java index ed8759c9af..38426a1279 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java @@ -16,11 +16,14 @@ */ package org.apache.tika.parser.microsoft.ooxml; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.ZipException; import org.apache.commons.io.input.CloseShieldInputStream; @@ -40,6 +43,7 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; @@ -256,6 +260,10 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor { ExceptionUtils.getStackTrace(e)); } + //pre-collect footnotes, endnotes, and comments so they can be inlined at their + //reference points in the main document (matching the DOM parser's behavior) + OOXMLInlineBodyPartMap inlinePartMap = collectInlineParts(documentPart); + if (config.isIncludeHeadersAndFooters()) { //headers try { @@ -265,7 +273,8 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor { for (int i = 0; i < headersPRC.size(); i++) { PackagePart header = documentPart.getRelatedPart(headersPRC.getRelationship(i)); - handlePart(header, styles, listManager, xhtml); + handlePart(header, styles, listManager, xhtml, + OOXMLInlineBodyPartMap.EMPTY); } } } catch (InvalidFormatException | ZipException e) { @@ -274,18 +283,18 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor { } } - //main document + //main document -- footnotes/endnotes/comments are inlined via inlinePartMap + OOXMLTikaBodyPartHandler mainHandler = null; try { - handlePart(documentPart, styles, listManager, xhtml); + mainHandler = handlePart(documentPart, styles, listManager, xhtml, inlinePartMap); } catch (ZipException e) { metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, ExceptionUtils.getStackTrace(e)); } - //for now, just dump other components at end + //dump remaining components at end; footnotes/endnotes/comments are now inlined + //above, so they are no longer dumped here for (String rel : new String[]{AbstractOOXMLExtractor.RELATION_DIAGRAM_DATA, - XSSFRelation.CHART.getRelation(), XWPFRelation.FOOTNOTE.getRelation(), - XWPFRelation.COMMENT.getRelation(), XWPFRelation.FOOTER.getRelation(), - XWPFRelation.ENDNOTE.getRelation(),}) { + XSSFRelation.CHART.getRelation(), XWPFRelation.FOOTER.getRelation(),}) { //skip footers if we shouldn't extract them if (!config.isIncludeHeadersAndFooters() && rel.equals(XWPFRelation.FOOTER.getRelation())) { @@ -297,7 +306,8 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor { for (int i = 0; i < prc.size(); i++) { PackagePart packagePart = documentPart.getRelatedPart(prc.getRelationship(i)); - handlePart(packagePart, styles, listManager, xhtml); + handlePart(packagePart, styles, listManager, xhtml, + OOXMLInlineBodyPartMap.EMPTY); } } } catch (InvalidFormatException | ZipException e) { @@ -305,26 +315,136 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor { ExceptionUtils.getStackTrace(e)); } } + + //dump any comments that were NOT inlined via commentReference + if (mainHandler != null) { + handleUnreferencedComments(xhtml, inlinePartMap, mainHandler.getEmittedCommentIds()); + } } - private void handlePart(PackagePart packagePart, XWPFStylesShim styles, - XWPFListManager listManager, XHTMLContentHandler xhtml) + private OOXMLTikaBodyPartHandler handlePart(PackagePart packagePart, XWPFStylesShim styles, + XWPFListManager listManager, XHTMLContentHandler xhtml, + OOXMLInlineBodyPartMap inlinePartMap) throws IOException, SAXException { Map<String, String> linkedRelationships = loadLinkedRelationships(packagePart, true, metadata); + OOXMLTikaBodyPartHandler bodyHandler = + new OOXMLTikaBodyPartHandler(xhtml, styles, listManager, config, metadata); + bodyHandler.setInlineBodyPartMap(inlinePartMap, context); try (InputStream stream = packagePart.getInputStream()) { XMLReaderUtils.parseSAX(CloseShieldInputStream.wrap(stream), new EmbeddedContentHandler(new OOXMLWordAndPowerPointTextHandler( - new OOXMLTikaBodyPartHandler(xhtml, styles, listManager, config, metadata), + bodyHandler, linkedRelationships, config.isIncludeShapeBasedContent(), config.isConcatenatePhoneticRuns(), config.isPreferAlternateContentChoice())), context); + } catch (SAXException e) { + WriteLimitReachedException.throwIfWriteLimitReached(e); + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + //the partial parse may have left <p>/<td>/<tr>/<table> or formatting tags open; + //close them so subsequent parts and the outer </body></html> land balanced + bodyHandler.closeAnyPending(); } catch (TikaException | IOException e) { metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, ExceptionUtils.getStackTrace(e)); + bodyHandler.closeAnyPending(); + } + return bodyHandler; + } + + /** + * Dumps comments that were not inlined at a {@code commentReference} in the main document + * (matching the DOM parser, which appends orphaned comments at the end). + */ + private void handleUnreferencedComments(XHTMLContentHandler xhtml, + OOXMLInlineBodyPartMap inlinePartMap, Set<String> emittedCommentIds) { + if (!inlinePartMap.hasComments()) { + return; + } + Map<String, String> linkedRelationships = inlinePartMap.getLinkedRelationships(); + for (Map.Entry<String, byte[]> entry : inlinePartMap.getCommentEntries()) { + if (emittedCommentIds.contains(entry.getKey())) { + continue; + } + try { + xhtml.startElement("div", "class", "comment"); + XMLReaderUtils.parseSAX(new ByteArrayInputStream(entry.getValue()), + new EmbeddedContentHandler(new OOXMLWordAndPowerPointTextHandler( + new OOXMLTikaBodyPartHandler(xhtml), linkedRelationships)), + context); + xhtml.endElement("div"); + } catch (TikaException | IOException | SAXException e) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + } + } + } + + /** + * Pre-parses the footnote, endnote, and comment parts into raw XML fragments keyed by their + * {@code w:id}, so {@link OOXMLTikaBodyPartHandler} can inline each at its reference point. + */ + private OOXMLInlineBodyPartMap collectInlineParts(PackagePart documentPart) { + Map<String, String> allRelationships = new java.util.HashMap<>(); + Map<String, byte[]> footnoteMap = collectPartContent(documentPart, + XWPFRelation.FOOTNOTE.getRelation(), Set.of("footnote"), allRelationships); + String endnoteRel = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/endnotes";; + Map<String, byte[]> endnoteMap = collectPartContent(documentPart, + endnoteRel, Set.of("endnote"), allRelationships); + String commentsRel = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments";; + Map<String, byte[]> commentMap = collectPartContent(documentPart, + commentsRel, Set.of("comment"), allRelationships, Collections.emptySet()); + return new OOXMLInlineBodyPartMap(footnoteMap, endnoteMap, commentMap, allRelationships); + } + + private Map<String, byte[]> collectPartContent(PackagePart documentPart, + String relationshipType, Set<String> wrapperElements, + Map<String, String> allRelationships) { + //footnotes/endnotes reserve ids 0 and -1 for separator/continuation-separator + return collectPartContent(documentPart, relationshipType, wrapperElements, + allRelationships, Set.of("0", "-1")); + } + + private Map<String, byte[]> collectPartContent(PackagePart documentPart, + String relationshipType, Set<String> wrapperElements, + Map<String, String> allRelationships, Set<String> skipIds) { + try { + PackageRelationshipCollection prc = + documentPart.getRelationshipsByType(relationshipType); + if (prc == null || prc.size() == 0) { + return Collections.emptyMap(); + } + OOXMLPartContentCollector collector = + new OOXMLPartContentCollector(wrapperElements, skipIds); + for (int i = 0; i < prc.size(); i++) { + PackagePart part = safeGetRelatedPart(documentPart, prc.getRelationship(i)); + if (part == null) { + continue; + } + //collect the part's linked relationships (for picture/hyperlink resolution) + allRelationships.putAll(loadLinkedRelationships(part, true, metadata)); + try (InputStream stream = part.getInputStream()) { + XMLReaderUtils.parseSAX(stream, collector, context); + } + } + return collector.getContentMap(); + } catch (InvalidFormatException | IOException | TikaException | SAXException e) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + return Collections.emptyMap(); } + } + private PackagePart safeGetRelatedPart(PackagePart parent, PackageRelationship rel) { + try { + return parent.getRelatedPart(rel); + } catch (Exception e) { + return null; + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java index b76b2567b9..a33139ad38 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java @@ -604,4 +604,9 @@ public class SXSLFExtractorTest extends TikaTest { metadataList.get(2).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH)); } + @Test + public void testPPTXDiagramData() throws Exception { + assertContains("President", getXML("testPPT_diagramData.pptx", parseContext).xml); + } + } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java index 80db8b71d7..65a73faf3a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java @@ -211,12 +211,17 @@ public class SXWPFExtractorTest extends TikaTest { assertEquals("application/vnd.openxmlformats-officedocument.wordprocessingml.document", xmlResult.metadata.get(Metadata.CONTENT_TYPE)); assertTrue(xmlResult.xml.contains("snoska")); + //footnote content is inlined as a div, emitted after the paragraph closes + //(not nested inside the <p>, which would be malformed) + assertContains("<div class=\"footnote\">", xmlResult.xml); + assertNotContained("<p><div class=\"footnote\">", xmlResult.xml); } @Test public void testEndnoteWithTable() throws Exception { XMLResult xmlResult = getXML("testWORD_endnote_table.docx", parseContext); assertContains("Cat Property Act", xmlResult.xml); + assertContains("<div class=\"endnote\">", xmlResult.xml); } /**
