This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch 3x-port-sax-ooxml in repository https://gitbox.apache.org/repos/asf/tika.git
commit c11e0377784e3eaaece27e0d9634635dbd3570d1 Author: tallison <[email protected]> AuthorDate: Wed Jul 8 11:36:36 2026 -0400 First steps in porting 4.x sax ooxml -> 3.x - checkpoint --- .../microsoft/ooxml/OOXMLExtractorFactory.java | 17 + .../microsoft/ooxml/TikaSheetContentsHandler.java | 36 ++ .../microsoft/ooxml/TikaSheetXMLHandler.java | 398 ++++++++++++++++++++ .../microsoft/ooxml/TikaXSSFBCommentsTable.java | 137 +++++++ .../ooxml/TikaXSSFBSharedStringsTable.java | 159 ++++++++ .../microsoft/ooxml/VSDXExtractorDecorator.java | 177 +++++++++ .../ooxml/XSSFBExcelExtractorDecorator.java | 106 +++--- .../parser/microsoft/ooxml/XSSFCommentsShim.java | 187 +++++++++ .../ooxml/XSSFExcelExtractorDecorator.java | 418 ++++++++++++++------- .../microsoft/ooxml/XSSFSharedStringsShim.java | 156 ++++++++ .../parser/microsoft/ooxml/XSSFStylesShim.java | 146 +++++++ .../parser/microsoft/ooxml/VSDXParserTest.java | 41 ++ 12 files changed, 1800 insertions(+), 178 deletions(-) diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java index 7e1e3c0fc6..4ddccbe158 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.util.Locale; +import java.util.Set; import org.apache.poi.extractor.ExtractorFactory; import org.apache.poi.ooxml.POIXMLDocument; @@ -182,6 +183,10 @@ public class OOXMLExtractorFactory { XSLFEventBasedPowerPointExtractor.class.getCanonicalName()); } else if (poiExtractor instanceof XPSTextExtractor) { extractor = new XPSExtractorDecorator(context, poiExtractor); + } else if (isVisioType(type)) { + //SAX-based .vsdx extractor; the XDGF-based poiExtractor is used only for its + //OPCPackage (VSDXExtractorDecorator reads the package directly) + extractor = new VSDXExtractorDecorator(context, poiExtractor); } else if (document == null) { throw new TikaException( "Expecting UserModel based POI OOXML extractor with a document, but none" + @@ -295,5 +300,17 @@ public class OOXMLExtractorFactory { return null; } + private static final Set<String> VISIO_SUBTYPES = Set.of( + "vnd.ms-visio.drawing", + "vnd.ms-visio.drawing.macroenabled.12", + "vnd.ms-visio.stencil", + "vnd.ms-visio.stencil.macroenabled.12", + "vnd.ms-visio.template", + "vnd.ms-visio.template.macroenabled.12" + ); + + private static boolean isVisioType(MediaType type) { + return type != null && VISIO_SUBTYPES.contains(type.getSubtype()); + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetContentsHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetContentsHandler.java new file mode 100644 index 0000000000..44173ec322 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetContentsHandler.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +/** + * Sheet contents handler that uses {@link XSSFCommentsShim.CommentData} + * instead of POI's XMLBeans-dependent {@code XSSFComment}. + */ +interface TikaSheetContentsHandler { + + void startRow(int rowNum); + + void endRow(int rowNum); + + void cell(String cellRef, String formattedValue, XSSFCommentsShim.CommentData comment); + + default void headerFooter(String text, boolean isHeader, String tagName) { + } + + default void endSheet() { + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetXMLHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetXMLHandler.java new file mode 100644 index 0000000000..e95506be5d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaSheetXMLHandler.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Queue; + +import org.apache.poi.ss.usermodel.BuiltinFormats; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.util.CellAddress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Sheet XML handler for XLSX event-based parsing that uses {@link XSSFStylesShim} + * and {@link XSSFCommentsShim} instead of POI's XMLBeans-dependent + * {@code StylesTable} and {@code CommentsTable}. + * <p> + * Adapted from Apache POI's {@code XSSFSheetXMLHandler} (Apache 2.0 license). + */ +class TikaSheetXMLHandler extends DefaultHandler { + + private static final Logger LOG = LoggerFactory.getLogger(TikaSheetXMLHandler.class); + + private static final String NS_SPREADSHEETML = + "http://schemas.openxmlformats.org/spreadsheetml/2006/main";; + + enum XssfDataType { + BOOLEAN, + ERROR, + FORMULA, + INLINE_STRING, + SST_STRING, + NUMBER, + } + + private final XSSFStylesShim stylesShim; + private final XSSFCommentsShim commentsShim; + private final XSSFSharedStringsShim sharedStringsShim; + private final TikaSheetContentsHandler output; + private final DataFormatter formatter; + private final boolean formulasNotResults; + + private boolean vIsOpen; + private boolean fIsOpen; + private boolean isIsOpen; + private boolean hfIsOpen; + + private XssfDataType nextDataType; + private short formatIndex; + private String formatString; + + private int rowNum; + private int nextRowNum; + private String cellRef; + + private final StringBuilder value = new StringBuilder(64); + private final StringBuilder formula = new StringBuilder(64); + private final StringBuilder headerFooter = new StringBuilder(64); + + private Queue<CellAddress> commentCellRefs; + + TikaSheetXMLHandler(XSSFStylesShim stylesShim, + XSSFCommentsShim commentsShim, + XSSFSharedStringsShim sharedStringsShim, + TikaSheetContentsHandler sheetContentsHandler, + DataFormatter dataFormatter, + boolean formulasNotResults) { + this.stylesShim = stylesShim; + this.commentsShim = commentsShim; + this.sharedStringsShim = sharedStringsShim; + this.output = sheetContentsHandler; + this.formatter = dataFormatter; + this.formulasNotResults = formulasNotResults; + this.nextDataType = XssfDataType.NUMBER; + initComments(commentsShim); + } + + TikaSheetXMLHandler(XSSFStylesShim stylesShim, + XSSFSharedStringsShim sharedStringsShim, + TikaSheetContentsHandler sheetContentsHandler, + DataFormatter dataFormatter, + boolean formulasNotResults) { + this(stylesShim, null, sharedStringsShim, sheetContentsHandler, dataFormatter, + formulasNotResults); + } + + private void initComments(XSSFCommentsShim commentsShim) { + if (commentsShim != null) { + commentCellRefs = new LinkedList<>(); + for (Iterator<CellAddress> iter = commentsShim.getCellAddresses(); + iter.hasNext(); ) { + commentCellRefs.add(iter.next()); + } + } + } + + private boolean isTextTag(String name) { + if ("v".equals(name)) { + return true; + } + if ("inlineStr".equals(name)) { + return true; + } + return "t".equals(name) && isIsOpen; + } + + @Override + public void startElement(String uri, String localName, String qName, + Attributes attributes) throws SAXException { + if (uri != null && !uri.equals(NS_SPREADSHEETML)) { + return; + } + + if (isTextTag(localName)) { + vIsOpen = true; + if (!isIsOpen) { + value.setLength(0); + } + } else if ("is".equals(localName)) { + isIsOpen = true; + } else if ("f".equals(localName)) { + formula.setLength(0); + if (this.nextDataType == XssfDataType.NUMBER) { + this.nextDataType = XssfDataType.FORMULA; + } + String type = attributes.getValue("t"); + if (type != null && type.equals("shared")) { + String ref = attributes.getValue("ref"); + if (ref != null) { + fIsOpen = true; + } + // shared-formula reference without a `ref` attribute is not yet supported + } else { + fIsOpen = true; + } + } else if ("oddHeader".equals(localName) || "evenHeader".equals(localName) || + "firstHeader".equals(localName) || "firstFooter".equals(localName) || + "oddFooter".equals(localName) || "evenFooter".equals(localName)) { + hfIsOpen = true; + headerFooter.setLength(0); + } else if ("row".equals(localName)) { + String rowNumStr = attributes.getValue("r"); + if (rowNumStr != null) { + rowNum = Integer.parseInt(rowNumStr.trim()) - 1; + } else { + rowNum = nextRowNum; + } + output.startRow(rowNum); + } else if ("c".equals(localName)) { + // Cell element — resolve style to format index/string + this.formula.setLength(0); + this.nextDataType = XssfDataType.NUMBER; + this.formatIndex = -1; + this.formatString = null; + cellRef = attributes.getValue("r"); + String cellType = attributes.getValue("t"); + String cellStyleStr = attributes.getValue("s"); + + if ("b".equals(cellType)) { + nextDataType = XssfDataType.BOOLEAN; + } else if ("e".equals(cellType)) { + nextDataType = XssfDataType.ERROR; + } else if ("inlineStr".equals(cellType)) { + nextDataType = XssfDataType.INLINE_STRING; + } else if ("s".equals(cellType)) { + nextDataType = XssfDataType.SST_STRING; + } else if ("str".equals(cellType)) { + nextDataType = XssfDataType.FORMULA; + } else { + // Number — resolve format via our styles shim + if (stylesShim != null) { + int styleIndex; + if (cellStyleStr != null) { + styleIndex = Integer.parseInt(cellStyleStr.trim()); + } else if (stylesShim.getNumCellStyles() > 0) { + styleIndex = 0; + } else { + styleIndex = -1; + } + if (styleIndex >= 0) { + this.formatIndex = stylesShim.getFormatIndex(styleIndex); + this.formatString = stylesShim.getFormatString(styleIndex); + if (this.formatString == null) { + this.formatString = + BuiltinFormats.getBuiltinFormat(this.formatIndex); + } + } + } + } + } + } + + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + if (uri != null && !uri.equals(NS_SPREADSHEETML)) { + return; + } + + if (isTextTag(localName)) { + vIsOpen = false; + if (!isIsOpen) { + outputCell(); + value.setLength(0); + } + } else if ("f".equals(localName)) { + fIsOpen = false; + } else if ("is".equals(localName)) { + isIsOpen = false; + outputCell(); + value.setLength(0); + } else if ("row".equals(localName)) { + checkForEmptyCellComments(EmptyCellCommentsCheckType.END_OF_ROW); + output.endRow(rowNum); + nextRowNum = rowNum + 1; + } else if ("sheetData".equals(localName)) { + checkForEmptyCellComments(EmptyCellCommentsCheckType.END_OF_SHEET_DATA); + output.endSheet(); + } else if ("oddHeader".equals(localName) || "evenHeader".equals(localName) || + "firstHeader".equals(localName)) { + hfIsOpen = false; + output.headerFooter(headerFooter.toString(), true, localName); + } else if ("oddFooter".equals(localName) || "evenFooter".equals(localName) || + "firstFooter".equals(localName)) { + hfIsOpen = false; + output.headerFooter(headerFooter.toString(), false, localName); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if (vIsOpen) { + value.append(ch, start, length); + } + if (fIsOpen) { + formula.append(ch, start, length); + } + if (hfIsOpen) { + headerFooter.append(ch, start, length); + } + } + + private void outputCell() { + String thisStr = null; + + if (formulasNotResults && formula.length() > 0) { + thisStr = formula.toString(); + } else { + switch (nextDataType) { + case BOOLEAN: + char first = value.charAt(0); + thisStr = first == '0' ? "FALSE" : "TRUE"; + break; + case ERROR: + thisStr = "ERROR:" + value; + break; + case FORMULA: + if (formulasNotResults) { + thisStr = formula.toString(); + } else { + String fv = value.toString(); + if (this.formatString != null) { + try { + double d = Double.parseDouble(fv.trim()); + thisStr = formatter.formatRawCellContents( + d, this.formatIndex, this.formatString); + } catch (Exception e) { + thisStr = fv; + } + } else { + thisStr = fv; + } + } + break; + case INLINE_STRING: + thisStr = value.toString(); + break; + case SST_STRING: + String sstIndex = value.toString().trim(); + if (!sstIndex.isEmpty() && sharedStringsShim != null) { + try { + int idx = Integer.parseInt(sstIndex); + thisStr = sharedStringsShim.getItemAt(idx); + } catch (NumberFormatException ex) { + LOG.error("Failed to parse SST index '{}'", sstIndex, ex); + } + } + break; + case NUMBER: + String n = value.toString(); + if (this.formatString != null && !n.isEmpty()) { + try { + thisStr = formatter.formatRawCellContents( + Double.parseDouble(n.trim()), + this.formatIndex, this.formatString); + } catch (Exception e) { + thisStr = n; + } + } else { + thisStr = n; + } + break; + default: + thisStr = "(TODO: Unexpected type: " + nextDataType + ")"; + break; + } + } + + checkForEmptyCellComments(EmptyCellCommentsCheckType.CELL); + XSSFCommentsShim.CommentData comment = commentsShim != null ? + commentsShim.findCellComment(new CellAddress(cellRef)) : null; + output.cell(cellRef, thisStr, comment); + } + + private void checkForEmptyCellComments(EmptyCellCommentsCheckType type) { + if (commentCellRefs != null && !commentCellRefs.isEmpty()) { + if (type == EmptyCellCommentsCheckType.END_OF_SHEET_DATA) { + while (!commentCellRefs.isEmpty()) { + outputEmptyCellComment(commentCellRefs.remove()); + } + return; + } + + if (this.cellRef == null) { + if (type == EmptyCellCommentsCheckType.END_OF_ROW) { + while (!commentCellRefs.isEmpty()) { + if (commentCellRefs.peek().getRow() == rowNum) { + outputEmptyCellComment(commentCellRefs.remove()); + } else { + return; + } + } + return; + } else { + throw new IllegalStateException( + "Cell ref should be null only if there are only empty " + + "cells in the row; rowNum: " + rowNum); + } + } + + CellAddress nextCommentCellRef; + do { + CellAddress cellAddr = new CellAddress(this.cellRef); + CellAddress peekCellRef = commentCellRefs.peek(); + if (type == EmptyCellCommentsCheckType.CELL && + cellAddr.equals(peekCellRef)) { + commentCellRefs.remove(); + return; + } else { + int comparison = peekCellRef.compareTo(cellAddr); + if (comparison > 0 && + type == EmptyCellCommentsCheckType.END_OF_ROW && + peekCellRef.getRow() <= rowNum) { + nextCommentCellRef = commentCellRefs.remove(); + outputEmptyCellComment(nextCommentCellRef); + } else if (comparison < 0 && + type == EmptyCellCommentsCheckType.CELL && + peekCellRef.getRow() <= rowNum) { + nextCommentCellRef = commentCellRefs.remove(); + outputEmptyCellComment(nextCommentCellRef); + } else { + nextCommentCellRef = null; + } + } + } while (nextCommentCellRef != null && !commentCellRefs.isEmpty()); + } + } + + private void outputEmptyCellComment(CellAddress cellRef) { + XSSFCommentsShim.CommentData comment = commentsShim.findCellComment(cellRef); + output.cell(cellRef.formatAsString(), null, comment); + } + + private enum EmptyCellCommentsCheckType { + CELL, + END_OF_ROW, + END_OF_SHEET_DATA + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBCommentsTable.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBCommentsTable.java new file mode 100644 index 0000000000..79a968a0ba --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBCommentsTable.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.poi.ss.util.CellAddress; +import org.apache.poi.util.LittleEndian; +import org.apache.poi.util.LittleEndianConsts; +import org.apache.poi.xssf.binary.XSSFBParser; +import org.apache.poi.xssf.binary.XSSFBRecordType; +import org.apache.poi.xssf.binary.XSSFBUtils; +import org.xml.sax.SAXException; + +import org.apache.tika.sax.XHTMLContentHandler; + +/** + * Replacement for POI's {@code XSSFBCommentsTable} that does not depend on + * {@code XSSFBComment}/{@code XSSFBRichTextString}/{@code XSSFRichTextString} + * (which pull in poi-ooxml-lite / xmlbeans via {@code CTRst}). + * <p> + * Stores comments as plain author + text strings. + */ +class TikaXSSFBCommentsTable extends XSSFBParser { + + private final Map<CellAddress, CommentEntry> comments = new TreeMap<>(); + private final List<String> authors = new ArrayList<>(); + + private int authorId = -1; + private int cellRow = -1; + private int cellCol = -1; + private String commentText; + private final StringBuilder buffer = new StringBuilder(); + + TikaXSSFBCommentsTable(InputStream is) throws IOException { + super(is); + parse(); + } + + @Override + public void handleRecord(int id, byte[] data) { + XSSFBRecordType recordType = XSSFBRecordType.lookup(id); + switch (recordType) { + case BrtBeginComment: + authorId = (int) LittleEndian.getUInt(data, 0); + // cell range: firstRow at offset 4, firstCol at offset 12 + cellRow = (int) LittleEndian.getUInt(data, LittleEndianConsts.INT_SIZE); + cellCol = (int) LittleEndian.getUInt(data, + LittleEndianConsts.INT_SIZE + 2 * LittleEndianConsts.INT_SIZE); + break; + case BrtCommentText: + buffer.setLength(0); + XSSFBUtils.readXLWideString(data, 1, buffer); + commentText = buffer.toString(); + break; + case BrtEndComment: + CellAddress addr = new CellAddress(cellRow, cellCol); + String author = (authorId >= 0 && authorId < authors.size()) + ? authors.get(authorId) : ""; + comments.put(addr, new CommentEntry(author, commentText)); + authorId = -1; + cellRow = -1; + cellCol = -1; + commentText = null; + break; + case BrtCommentAuthor: + buffer.setLength(0); + XSSFBUtils.readXLWideString(data, 0, buffer); + authors.add(buffer.toString()); + break; + default: + break; + } + } + + CommentEntry get(CellAddress cellAddress) { + return cellAddress == null ? null : comments.get(cellAddress); + } + + boolean hasComments() { + return !comments.isEmpty(); + } + + /** + * Emits all comments as cell content. Called after sheet processing + * since we bypass POI's built-in comment handling. + */ + void emitAllComments(XHTMLContentHandler xhtml) throws SAXException { + for (Map.Entry<CellAddress, CommentEntry> entry : comments.entrySet()) { + CommentEntry comment = entry.getValue(); + xhtml.startElement("p", "class", "cell-comment"); + String author = comment.getAuthor(); + if (author != null && !author.isEmpty()) { + xhtml.characters(author + ": "); + } + xhtml.characters(comment.getText()); + xhtml.endElement("p"); + } + } + + static class CommentEntry { + private final String author; + private final String text; + + CommentEntry(String author, String text) { + this.author = author; + this.text = text; + } + + String getAuthor() { + return author; + } + + String getText() { + return text; + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBSharedStringsTable.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBSharedStringsTable.java new file mode 100644 index 0000000000..6e1ec853d5 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/TikaXSSFBSharedStringsTable.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackagePart; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.RichTextString; +import org.apache.poi.util.LittleEndian; +import org.apache.poi.xssf.binary.XSSFBParser; +import org.apache.poi.xssf.binary.XSSFBRecordType; +import org.apache.poi.xssf.binary.XSSFBUtils; +import org.apache.poi.xssf.model.SharedStrings; + +/** + * Replacement for POI's {@code XSSFBSharedStringsTable} that does not depend on + * {@code XSSFRichTextString} (which pulls in poi-ooxml-lite / xmlbeans via {@code CTRst}). + * <p> + * The binary parsing logic is identical to POI's implementation; only + * {@link #getItemAt(int)} is changed to return a lightweight {@link RichTextString} + * wrapper instead of {@code XSSFRichTextString}. + */ +class TikaXSSFBSharedStringsTable implements SharedStrings { + + private static final String SHARED_STRINGS_BINARY_CT = + "application/vnd.ms-excel.sharedStrings"; + + private int count; + private int uniqueCount; + private final List<String> strings = new ArrayList<>(); + + TikaXSSFBSharedStringsTable(OPCPackage pkg) throws IOException { + ArrayList<PackagePart> parts = + pkg.getPartsByContentType(SHARED_STRINGS_BINARY_CT); + if (!parts.isEmpty()) { + PackagePart sstPart = parts.get(0); + try (InputStream stream = sstPart.getInputStream()) { + readFrom(stream); + } + } + } + + private void readFrom(InputStream inputStream) throws IOException { + new SSTBinaryReader(inputStream).parse(); + } + + @Override + public RichTextString getItemAt(int idx) { + return new PlainRichTextString(strings.get(idx)); + } + + @Override + public int getCount() { + return count; + } + + @Override + public int getUniqueCount() { + return uniqueCount; + } + + private class SSTBinaryReader extends XSSFBParser { + + SSTBinaryReader(InputStream is) { + super(is); + } + + @Override + public void handleRecord(int recordType, byte[] data) { + XSSFBRecordType type = XSSFBRecordType.lookup(recordType); + switch (type) { + case BrtSstItem: + // Inline XSSFBRichStr.build() logic — that class is package-private in POI + StringBuilder sb = new StringBuilder(); + XSSFBUtils.readXLWideString(data, 1, sb); + strings.add(sb.toString()); + break; + case BrtBeginSst: + count = (int) LittleEndian.getUInt(data, 0); + uniqueCount = (int) LittleEndian.getUInt(data, 4); + break; + default: + break; + } + } + } + + /** + * Minimal {@link RichTextString} that just wraps a plain string, + * avoiding the xmlbeans dependency in {@code XSSFRichTextString}. + */ + private static class PlainRichTextString implements RichTextString { + + private final String text; + + PlainRichTextString(String text) { + this.text = text; + } + + @Override + public String getString() { + return text; + } + + @Override + public int length() { + return text == null ? 0 : text.length(); + } + + @Override + public int numFormattingRuns() { + return 0; + } + + @Override + public int getIndexOfFormattingRun(int index) { + return 0; + } + + @Override + public void applyFont(int startIndex, int endIndex, short fontIndex) { + } + + @Override + public void applyFont(int startIndex, int endIndex, Font font) { + } + + @Override + public void applyFont(Font font) { + } + + @Override + public void clearFormatting() { + } + + @Override + public void applyFont(short fontIndex) { + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java new file mode 100644 index 0000000000..197ccf6eda --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.poi.ooxml.extractor.POIXMLTextExtractor; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackagePart; +import org.apache.poi.openxml4j.opc.PackageRelationship; +import org.apache.poi.openxml4j.opc.PackageRelationshipCollection; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.XMLReaderUtils; + +/** + * SAX-based extractor for Visio OOXML (.vsdx) files. + * Extracts text from {@code <Text>} elements inside shapes on each page. + */ +public class VSDXExtractorDecorator extends AbstractOOXMLExtractor { + + private static final String VISIO_DOCUMENT_REL = + "http://schemas.microsoft.com/visio/2010/relationships/document";; + private static final String VISIO_PAGES_REL = + "http://schemas.microsoft.com/visio/2010/relationships/pages";; + private static final String VISIO_PAGE_REL = + "http://schemas.microsoft.com/visio/2010/relationships/page";; + + private final ParseContext context; + + public VSDXExtractorDecorator(ParseContext context, POIXMLTextExtractor extractor) { + //keep the 3x extractor-based ctor (factory + AbstractOOXMLExtractor unchanged); + //the body reads from the base opcPackage, derived from the extractor + super(context, extractor); + this.context = context; + } + + @Override + protected void buildXHTML(XHTMLContentHandler xhtml) + throws SAXException, IOException { + try { + List<PackagePart> pageParts = getPageParts(); + for (PackagePart pagePart : pageParts) { + xhtml.startElement("div", "class", "page"); + try (InputStream is = pagePart.getInputStream()) { + XMLReaderUtils.parseSAX(is, new VisioPageHandler(xhtml), context); + } catch (TikaException e) { + throw new SAXException(e); + } + xhtml.endElement("div"); + } + } catch (InvalidFormatException e) { + throw new SAXException("Error reading VSDX pages", e); + } + } + + private List<PackagePart> getPageParts() throws InvalidFormatException { + // Root -> visio/document.xml + PackagePart documentPart = getRelatedPart(opcPackage, VISIO_DOCUMENT_REL); + if (documentPart == null) { + return Collections.emptyList(); + } + + // document.xml -> pages/pages.xml + PackagePart pagesPart = getRelatedPart(documentPart, VISIO_PAGES_REL); + if (pagesPart == null) { + return Collections.emptyList(); + } + + // pages.xml -> page1.xml, page2.xml, ... + List<PackagePart> pageParts = new ArrayList<>(); + PackageRelationshipCollection pageRels = + pagesPart.getRelationshipsByType(VISIO_PAGE_REL); + for (PackageRelationship rel : pageRels) { + PackagePart pagePart = pagesPart.getRelatedPart(rel); + if (pagePart != null) { + pageParts.add(pagePart); + } + } + return pageParts; + } + + private PackagePart getRelatedPart(OPCPackage pkg, String relType) + throws InvalidFormatException { + PackageRelationshipCollection rels = pkg.getRelationshipsByType(relType); + if (rels.isEmpty()) { + return null; + } + return pkg.getPart(rels.getRelationship(0)); + } + + private PackagePart getRelatedPart(PackagePart part, String relType) + throws InvalidFormatException { + PackageRelationshipCollection rels = part.getRelationshipsByType(relType); + if (rels.isEmpty()) { + return null; + } + return part.getRelatedPart(rels.getRelationship(0)); + } + + @Override + protected List<PackagePart> getMainDocumentParts() { + return Collections.emptyList(); + } + + /** + * SAX handler for Visio page XML. Extracts text from {@code <Text>} + * elements inside {@code <Shape>} elements. + */ + private static class VisioPageHandler extends DefaultHandler { + + private static final String VISIO_NS = + "http://schemas.microsoft.com/office/visio/2012/main";; + + private final XHTMLContentHandler xhtml; + private boolean inText; + private final StringBuilder textBuffer = new StringBuilder(); + + VisioPageHandler(XHTMLContentHandler xhtml) { + this.xhtml = xhtml; + } + + @Override + public void startElement(String uri, String localName, String qName, + Attributes atts) { + if ("Text".equals(localName) && VISIO_NS.equals(uri)) { + inText = true; + textBuffer.setLength(0); + } + } + + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + if ("Text".equals(localName) && VISIO_NS.equals(uri)) { + inText = false; + String text = textBuffer.toString().trim(); + if (!text.isEmpty()) { + xhtml.startElement("p"); + xhtml.characters(text); + xhtml.endElement("p"); + } + } + } + + @Override + public void characters(char[] ch, int start, int length) { + if (inText) { + textBuffer.append(ch, start, length); + } + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java index 77000b9a9a..f9e87301d8 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java @@ -18,27 +18,27 @@ package org.apache.tika.parser.microsoft.ooxml; import java.io.IOException; import java.io.InputStream; -import java.util.List; import java.util.Locale; import org.apache.poi.ooxml.extractor.POIXMLTextExtractor; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackagePart; -import org.apache.poi.xssf.binary.XSSFBCommentsTable; -import org.apache.poi.xssf.binary.XSSFBSharedStringsTable; +import org.apache.poi.openxml4j.opc.PackagePartName; +import org.apache.poi.openxml4j.opc.PackageRelationship; +import org.apache.poi.openxml4j.opc.PackageRelationshipCollection; +import org.apache.poi.openxml4j.opc.PackagingURIHelper; import org.apache.poi.xssf.binary.XSSFBSheetHandler; import org.apache.poi.xssf.binary.XSSFBStylesTable; import org.apache.poi.xssf.eventusermodel.XSSFBReader; -import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler.SheetContentsHandler; -import org.apache.poi.xssf.extractor.XSSFBEventBasedExcelExtractor; -import org.apache.poi.xssf.usermodel.XSSFShape; import org.apache.xmlbeans.XmlException; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.XHTMLContentHandler; @@ -50,17 +50,9 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator { super(context, extractor, locale); } - @Override - protected void configureExtractor(POIXMLTextExtractor extractor, Locale locale) { - //need to override this because setFormulasNotResults is not yet available - //for xlsb - //((XSSFBEventBasedExcelExtractor)extractor).setFormulasNotResults(false); - ((XSSFBEventBasedExcelExtractor) extractor).setLocale(locale); - } - @Override public void getXHTML(ContentHandler handler, Metadata metadata, ParseContext context) - throws SAXException, XmlException, IOException, TikaException { + throws SAXException, IOException, TikaException, XmlException { this.metadata = metadata; this.parseContext = context; @@ -69,15 +61,12 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator { super.getXHTML(handler, metadata, context); } - /** - * @see org.apache.poi.xssf.extractor.XSSFBEventBasedExcelExtractor#getText() - */ @Override protected void buildXHTML(XHTMLContentHandler xhtml) - throws SAXException, XmlException, IOException { - OPCPackage container = extractor.getPackage(); + throws SAXException, IOException { + OPCPackage container = opcPackage; - XSSFBSharedStringsTable strings; + TikaXSSFBSharedStringsTable strings; XSSFBReader.SheetIterator iter; XSSFBReader xssfReader; XSSFBStylesTable styles; @@ -89,9 +78,9 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator { } styles = xssfReader.getXSSFBStylesTable(); iter = (XSSFBReader.SheetIterator) xssfReader.getSheetsData(); - strings = new XSSFBSharedStringsTable(container); + strings = new TikaXSSFBSharedStringsTable(container); } catch (OpenXML4JException e) { - throw new XmlException(e); + throw new IOException(e); } while (iter.hasNext()) { @@ -101,44 +90,71 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator { sheetParts.add(sheetPart); SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config, xhtml); - XSSFBCommentsTable comments = iter.getXSSFBSheetComments(); - // Start, and output the sheet name + // Parse comments with our own binary parser that avoids xmlbeans + TikaXSSFBCommentsTable tikaComments = parseBinaryComments(sheetPart); + if (tikaComments != null && tikaComments.hasComments()) { + metadata.set(Office.HAS_COMMENTS, true); + } + xhtml.startElement("div"); xhtml.element("h1", iter.getSheetName()); - // Extract the main sheet contents xhtml.startElement("table"); xhtml.startElement("tbody"); - processSheet(sheetExtractor, comments, styles, strings, stream); + // Pass null for POI's comments table to avoid xmlbeans dependency. + // Comments are emitted separately after sheet processing. + XSSFBSheetHandler xssfbSheetHandler = + new XSSFBSheetHandler(stream, styles, null, strings, + sheetExtractor, formatter, false); + xssfbSheetHandler.parse(); xhtml.endElement("tbody"); xhtml.endElement("table"); - // Output any headers and footers - // (Need to process the sheet to get them, so we can't - // do the headers before the contents) + // Emit comments after the table (since we bypass POI's inline + // comment handling to avoid xmlbeans dependency) + if (tikaComments != null) { + tikaComments.emitAllComments(xhtml); + } + for (String header : sheetExtractor.headers) { extractHeaderFooter(header, xhtml); } for (String footer : sheetExtractor.footers) { extractHeaderFooter(footer, xhtml); } - List<XSSFShape> shapes = iter.getShapes(); - - processShapes(shapes, xhtml); - - //for now dump sheet hyperlinks at bottom of page - //consider a double-pass of the inputstream to reunite hyperlinks with cells/textboxes - //step 1: extract hyperlink info from bottom of page - //step 2: process as we do now, but with cached hyperlink relationship info + processDrawings(sheetPart, xhtml); extractHyperLinks(sheetPart, xhtml); - // All done with this sheet xhtml.endElement("div"); } } + private static final String RELATION_COMMENTS = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments";; + + private TikaXSSFBCommentsTable parseBinaryComments(PackagePart sheetPart) { + try { + PackageRelationshipCollection rels = + sheetPart.getRelationshipsByType(RELATION_COMMENTS); + if (rels.isEmpty()) { + return null; + } + PackageRelationship rel = rels.getRelationship(0); + PackagePartName partName = + PackagingURIHelper.createPartName(rel.getTargetURI()); + PackagePart commentsPart = rel.getPackage().getPart(partName); + if (commentsPart == null) { + return null; + } + try (InputStream is = commentsPart.getInputStream()) { + return new TikaXSSFBCommentsTable(is); + } + } catch (InvalidFormatException | IOException e) { + return null; + } + } @Override protected void extractHeaderFooter(String hf, XHTMLContentHandler xhtml) throws SAXException { @@ -146,16 +162,4 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator { xhtml.element("p", hf); } } - - - private void processSheet(SheetContentsHandler sheetContentsExtractor, - XSSFBCommentsTable comments, XSSFBStylesTable styles, - XSSFBSharedStringsTable strings, InputStream sheetInputStream) - throws IOException, SAXException { - - XSSFBSheetHandler xssfbSheetHandler = - new XSSFBSheetHandler(sheetInputStream, styles, comments, strings, - sheetContentsExtractor, formatter, false); - xssfbSheetHandler.parse(); - } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFCommentsShim.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFCommentsShim.java new file mode 100644 index 0000000000..f3293a0d3c --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFCommentsShim.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.poi.ss.util.CellAddress; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.utils.XMLReaderUtils; + +/** + * SAX-based shim that parses {@code xl/commentsN.xml} without XMLBeans. + * Replaces POI's {@code CommentsTable} (which depends on poi-ooxml-lite) + * for Tika's text extraction needs. + * + * <p>Only extracts what Tika needs: cell reference → (author, text) mapping.</p> + */ +class XSSFCommentsShim { + + private final Map<CellAddress, CommentData> commentsByCell; + + /** + * Simple holder for comment data needed by Tika. + */ + static class CommentData { + private final String author; + private final String text; + + CommentData(String author, String text) { + this.author = author; + this.text = text; + } + + public String getAuthor() { + return author; + } + + public String getText() { + return text; + } + } + + /** + * Parse a comments XML stream. + * + * @param is the {@code xl/commentsN.xml} stream (may be null) + * @param parseContext parse context for SAX parser configuration + */ + XSSFCommentsShim(InputStream is, ParseContext parseContext) + throws IOException, TikaException, SAXException { + commentsByCell = new LinkedHashMap<>(); + if (is != null) { + CommentsHandler handler = new CommentsHandler(); + XMLReaderUtils.parseSAX(is, handler, parseContext); + } + } + + /** + * @return the number of comments parsed + */ + int getNumberOfComments() { + return commentsByCell.size(); + } + + /** + * Find comment data for a given cell address. + * + * @return CommentData or null if no comment at that cell + */ + CommentData findCellComment(CellAddress cellAddress) { + return commentsByCell.get(cellAddress); + } + + /** + * @return iterator over all cell addresses that have comments, in document order + */ + Iterator<CellAddress> getCellAddresses() { + return commentsByCell.keySet().iterator(); + } + + /** + * SAX handler for comments XML. Structure: + * <pre> + * <comments> + * <authors> + * <author>Name</author> + * </authors> + * <commentList> + * <comment ref="A1" authorId="0"> + * <text> + * <r><t>Comment text</t></r> + * or plain <t>Comment text</t> + * </text> + * </comment> + * </commentList> + * </comments> + * </pre> + */ + private class CommentsHandler extends DefaultHandler { + + private final List<String> authors = new ArrayList<>(); + private final StringBuilder textBuffer = new StringBuilder(); + + private boolean inAuthor; + private boolean inT; + private boolean inText; + + private String currentRef; + private int currentAuthorId; + private final StringBuilder commentText = new StringBuilder(); + + @Override + public void startElement(String uri, String localName, String qName, + Attributes atts) { + if ("author".equals(localName)) { + inAuthor = true; + textBuffer.setLength(0); + } else if ("comment".equals(localName)) { + currentRef = atts.getValue("ref"); + String authorIdStr = atts.getValue("authorId"); + currentAuthorId = authorIdStr != null ? Integer.parseInt(authorIdStr) : -1; + commentText.setLength(0); + } else if ("text".equals(localName)) { + inText = true; + } else if ("t".equals(localName) && inText) { + inT = true; + textBuffer.setLength(0); + } + } + + @Override + public void endElement(String uri, String localName, String qName) { + if ("author".equals(localName)) { + inAuthor = false; + authors.add(textBuffer.toString()); + } else if ("t".equals(localName) && inT) { + inT = false; + if (commentText.length() > 0) { + commentText.append(' '); + } + commentText.append(textBuffer); + } else if ("text".equals(localName)) { + inText = false; + } else if ("comment".equals(localName)) { + if (currentRef != null) { + String author = (currentAuthorId >= 0 && currentAuthorId < authors.size()) + ? authors.get(currentAuthorId) : ""; + commentsByCell.put(new CellAddress(currentRef), + new CommentData(author, commentText.toString())); + } + currentRef = null; + } + } + + @Override + public void characters(char[] ch, int start, int length) { + if (inAuthor || inT) { + textBuffer.append(ch, start, length); + } + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java index fe63fe156d..194d206a0e 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java @@ -21,11 +21,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import org.apache.poi.hssf.extractor.ExcelExtractor; import org.apache.poi.ooxml.extractor.POIXMLTextExtractor; @@ -42,24 +40,11 @@ import org.apache.poi.openxml4j.opc.TargetMode; import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.HeaderFooter; import org.apache.poi.ss.util.CellReference; -import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable; import org.apache.poi.xssf.eventusermodel.XSSFReader; -import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler; import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler.SheetContentsHandler; -import org.apache.poi.xssf.extractor.XSSFEventBasedExcelExtractor; -import org.apache.poi.xssf.model.Comments; -import org.apache.poi.xssf.model.StylesTable; import org.apache.poi.xssf.usermodel.XSSFComment; -import org.apache.poi.xssf.usermodel.XSSFDrawing; -import org.apache.poi.xssf.usermodel.XSSFRelation; -import org.apache.poi.xssf.usermodel.XSSFShape; -import org.apache.poi.xssf.usermodel.XSSFSimpleShape; import org.apache.poi.xssf.usermodel.helpers.HeaderFooterHelper; import org.apache.xmlbeans.XmlException; -import org.openxmlformats.schemas.drawingml.x2006.main.CTHyperlink; -import org.openxmlformats.schemas.drawingml.x2006.main.CTNonVisualDrawingProps; -import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTShape; -import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTShapeNonVisual; import org.xml.sax.Attributes; import org.xml.sax.ContentHandler; import org.xml.sax.Locator; @@ -68,6 +53,7 @@ import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.exception.RuntimeSAXException; import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; @@ -75,6 +61,7 @@ import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.microsoft.OfficeParserConfig; import org.apache.tika.parser.microsoft.TikaExcelDataFormatter; import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.ExceptionUtils; import org.apache.tika.utils.StringUtils; import org.apache.tika.utils.XMLReaderUtils; @@ -92,6 +79,20 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { // Power Query stores data in customData parts private static final String POWER_QUERY_CONTENT_TYPE = "application/vnd.ms-excel.customDataProperties+xml"; + private static final String RELATION_DRAWING = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/drawing";; + private static final String RELATION_CHART = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/chart";; + private static final String RELATION_HYPERLINK = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink";; + private static final String NS_DRAWING_ML = + "http://schemas.openxmlformats.org/drawingml/2006/main";; + private static final String NS_RELATIONSHIPS = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships";; + private static final String RELATION_VML_DRAWING = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/vmlDrawing";; + private static final String RELATION_COMMENTS = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments";; /** * Allows access to headers/footers from raw xml strings @@ -105,11 +106,11 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { public XSSFExcelExtractorDecorator(ParseContext context, POIXMLTextExtractor extractor, Locale locale) { + //keep the 3x extractor-based ctor so the factory and AbstractOOXMLExtractor are + //unchanged; the base derives opcPackage from the extractor (used by the body below) super(context, extractor); this.parseContext = context; - this.extractor = (XSSFEventBasedExcelExtractor) extractor; - configureExtractor(this.extractor, locale); if (locale == null) { formatter = new TikaExcelDataFormatter(); @@ -123,19 +124,14 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { } } - protected void configureExtractor(POIXMLTextExtractor extractor, Locale locale) { - ((XSSFEventBasedExcelExtractor) extractor) - .setIncludeTextBoxes(config.isIncludeShapeBasedContent()); - ((XSSFEventBasedExcelExtractor) extractor).setFormulasNotResults(false); - ((XSSFEventBasedExcelExtractor) extractor).setLocale(locale); - //given that we load our own shared strings table, setting: - //((XSSFEventBasedExcelExtractor)extractor).setConcatenatePhoneticRuns(); - //does no good here. + @Override + public MetadataExtractor getMetadataExtractor() { + return new SAXBasedMetadataExtractor(opcPackage, parseContext); } @Override public void getXHTML(ContentHandler handler, Metadata metadata, ParseContext context) - throws SAXException, XmlException, IOException, TikaException { + throws SAXException, IOException, TikaException, XmlException { this.metadata = metadata; this.parseContext = context; @@ -149,34 +145,66 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { */ @Override protected void buildXHTML(XHTMLContentHandler xhtml) - throws SAXException, XmlException, IOException { - OPCPackage container = extractor.getPackage(); + throws SAXException, IOException { + OPCPackage container = opcPackage; - ReadOnlySharedStringsTable strings; + XSSFSharedStringsShim stringsShim = null; XSSFReader.SheetIterator iter; XSSFReader xssfReader; - StylesTable styles; + XSSFStylesShim stylesShim = null; try { xssfReader = new XSSFReader(container); - styles = xssfReader.getStylesTable(); - iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); - strings = new ReadOnlySharedStringsTable(container, config.isConcatenatePhoneticRuns()); - } catch (OpenXML4JException e) { - throw new XmlException(e); + } catch (OpenXML4JException | RuntimeException e) { + throw new IOException(e); } - - while (iter.hasNext()) { + // Styles and shared strings are optional — if either part is missing or + // unreadable, log to metadata and continue with degraded extraction. + try { + stylesShim = new XSSFStylesShim(xssfReader.getStylesData(), parseContext); + } catch (Exception e) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + } + try { + stringsShim = new XSSFSharedStringsShim(xssfReader.getSharedStringsData(), + config.isConcatenatePhoneticRuns(), parseContext); + } catch (Exception e) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + } + while (true) { + try { + if (!iter.hasNext()) { + break; + } + } catch (RuntimeException e) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + break; + } SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config, xhtml); PackagePart sheetPart = null; - try (InputStream stream = iter.next()) { + InputStream nextStream; + try { + nextStream = iter.next(); + } catch (RuntimeException e) { + // POI can throw POIXMLException for missing sheet parts (e.g., + // truncated workbook references a sheet that isn't in the zip). + // Break rather than continue — POI's iterator state may not have + // advanced, which would cause an infinite loop. + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + break; + } + try (InputStream stream = nextStream) { sheetPart = iter.getSheetPart(); addDrawingHyperLinks(sheetPart); sheetParts.add(sheetPart); - Comments comments = iter.getSheetComments(); - if (comments != null && comments.getNumberOfComments() > 0) { + XSSFCommentsShim commentsShim = parseSheetComments(sheetPart); + if (commentsShim != null && commentsShim.getNumberOfComments() > 0) { metadata.set(Office.HAS_COMMENTS, true); } @@ -188,7 +216,26 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { xhtml.startElement("table"); xhtml.startElement("tbody"); - processSheet(sheetExtractor, comments, styles, strings, stream); + try { + processSheet(sheetExtractor, commentsShim, stylesShim, stringsShim, stream); + } catch (SAXException e) { + // Truncated/malformed sheet XML — keep prior sheets and + // record the failure as a warning. + WriteLimitReachedException.throwIfWriteLimitReached(e); + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + // Balance any <tr>/<td> left open by the partial parse so + // the </tbody></table></div> emitted below land in the + // right place. + sheetExtractor.closeAnyPending(); + } catch (IOException e) { + // Truncated stream — same risk: partial <tr>/<td> still + // open. Close them so the surrounding </tbody></table> + // stays balanced, record the failure, and keep going. + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.getStackTrace(e)); + sheetExtractor.closeAnyPending(); + } try { getThreadedComments(container, sheetPart, xhtml); } catch (InvalidFormatException | TikaException | IOException e) { @@ -210,8 +257,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { // Do text held in shapes, if required if (config.isIncludeShapeBasedContent()) { - List<XSSFShape> shapes = iter.getShapes(); - processShapes(shapes, xhtml); + processDrawings(sheetPart, xhtml); } //for now dump sheet hyperlinks at bottom of page @@ -670,7 +716,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { protected void addDrawingHyperLinks(PackagePart sheetPart) { try { for (PackageRelationship rel : sheetPart - .getRelationshipsByType(XSSFRelation.DRAWINGS.getRelation())) { + .getRelationshipsByType(RELATION_DRAWING)) { if (rel.getTargetMode() == TargetMode.INTERNAL) { PackagePartName relName = PackagingURIHelper.createPartName(rel.getTargetURI()); PackagePart part = rel.getPackage().getPart(relName); @@ -679,7 +725,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { continue; } for (PackageRelationship drawRel : part - .getRelationshipsByType(XSSFRelation.SHEET_HYPERLINKS.getRelation())) { + .getRelationshipsByType(RELATION_HYPERLINK)) { drawingHyperlinks.put(drawRel.getId(), drawRel.getTargetURI().toString()); } } @@ -696,8 +742,13 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { protected void extractHyperLinks(PackagePart sheetPart, XHTMLContentHandler xhtml) throws SAXException { try { + boolean first = true; for (PackageRelationship rel : sheetPart - .getRelationshipsByType(XSSFRelation.SHEET_HYPERLINKS.getRelation())) { + .getRelationshipsByType(RELATION_HYPERLINK)) { + if (!first) { + xhtml.characters(" "); + } + first = false; xhtml.startElement("a", "href", rel.getTargetURI().toString()); xhtml.characters(rel.getTargetURI().toString()); xhtml.endElement("a"); @@ -714,101 +765,125 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { } } - protected void processShapes(List<XSSFShape> shapes, XHTMLContentHandler xhtml) + protected void processDrawings(PackagePart sheetPart, XHTMLContentHandler xhtml) throws SAXException { - if (shapes == null) { - return; - } - //We don't currently have an obvious way to get drawings - //directly from sheetIter. Therefore, we grab the shapes and process those. - //To get the diagrams and charts, we need to get the parent drawing for each - //shape, and we need to make sure that we only process each parent shape once! - //SEE TIKA-2703 TODO: add unit test - Set<String> seenParentDrawings = new HashSet<>(); - for (XSSFShape shape : shapes) { - if (shape instanceof XSSFSimpleShape) { - String sText = ((XSSFSimpleShape) shape).getText(); - if (sText != null && sText.length() > 0) { - xhtml.element("p", sText); + try { + for (PackageRelationship rel : sheetPart + .getRelationshipsByType(RELATION_DRAWING)) { + if (rel.getTargetMode() != TargetMode.INTERNAL) { + continue; } - extractHyperLinksFromShape(((XSSFSimpleShape) shape).getCTShape(), xhtml); - } - - XSSFDrawing parentDrawing = shape.getDrawing(); - if (parentDrawing != null) { - if (!seenParentDrawings - .contains(parentDrawing.getPackagePart().getPartName().toString())) { - //dump diagram data - handleGeneralTextContainingPart(AbstractOOXMLExtractor.RELATION_DIAGRAM_DATA, - "diagram-data", parentDrawing.getPackagePart(), metadata, - new OOXMLWordAndPowerPointTextHandler( - new OOXMLTikaBodyPartHandler(xhtml), - new HashMap<>()//empty - )); - //dump chart data - handleGeneralTextContainingPart(XSSFRelation.CHART.getRelation(), "chart", - parentDrawing.getPackagePart(), metadata, - new OOXMLWordAndPowerPointTextHandler( - new OOXMLTikaBodyPartHandler(xhtml), - new HashMap<>()//empty - )); + PackagePartName relName = + PackagingURIHelper.createPartName(rel.getTargetURI()); + PackagePart drawingPart = rel.getPackage().getPart(relName); + if (drawingPart == null) { + continue; + } + // SAX-parse drawing XML for shape text and hyperlinks + try (InputStream is = drawingPart.getInputStream()) { + XMLReaderUtils.parseSAX(is, + new DrawingShapeHandler(xhtml, drawingHyperlinks), + parseContext); + } catch (IOException | TikaException e) { + //swallow } - seenParentDrawings.add(parentDrawing.getPackagePart().getPartName().toString()); + // Process diagram and chart data through drawing part relationships + handleGeneralTextContainingPart( + AbstractOOXMLExtractor.RELATION_DIAGRAM_DATA, + "diagram-data", drawingPart, metadata, + new OOXMLWordAndPowerPointTextHandler( + new OOXMLTikaBodyPartHandler(xhtml), + new HashMap<>())); + handleGeneralTextContainingPart(RELATION_CHART, "chart", + drawingPart, metadata, + new OOXMLWordAndPowerPointTextHandler( + new OOXMLTikaBodyPartHandler(xhtml), + new HashMap<>())); } + } catch (InvalidFormatException e) { + //swallow } } - private void extractHyperLinksFromShape(CTShape ctShape, XHTMLContentHandler xhtml) - throws SAXException { - - if (ctShape == null) { - return; - } + /** + * SAX handler for drawing XML that extracts shape text and hyperlinks + * without requiring XMLBeans or the POI usermodel (XSSFShape, etc.). + */ + private static class DrawingShapeHandler extends DefaultHandler { - CTShapeNonVisual nvSpPR = ctShape.getNvSpPr(); - if (nvSpPR == null) { - return; - } + private final XHTMLContentHandler xhtml; + private final Map<String, String> hyperlinks; - CTNonVisualDrawingProps cNvPr = nvSpPR.getCNvPr(); - if (cNvPr == null) { - return; - } + private boolean inTxBody; + private boolean inT; + private final StringBuilder textBuffer = new StringBuilder(); + private final StringBuilder shapeText = new StringBuilder(); - CTHyperlink ctHyperlink = cNvPr.getHlinkClick(); - if (ctHyperlink == null) { - return; + DrawingShapeHandler(XHTMLContentHandler xhtml, Map<String, String> hyperlinks) { + this.xhtml = xhtml; + this.hyperlinks = hyperlinks; } - String url = drawingHyperlinks.get(ctHyperlink.getId()); - if (url != null) { - xhtml.startElement("a", "href", url); - xhtml.characters(url); - xhtml.endElement("a"); + @Override + public void startElement(String uri, String localName, String qName, + Attributes atts) throws SAXException { + if ("txBody".equals(localName)) { + inTxBody = true; + shapeText.setLength(0); + } else if ("t".equals(localName) && inTxBody) { + inT = true; + textBuffer.setLength(0); + } else if ("hlinkClick".equals(localName) || "hlinkHover".equals(localName)) { + String rId = atts.getValue(NS_RELATIONSHIPS, "id"); + if (rId == null) { + // try non-namespace-aware fallback + rId = atts.getValue("r:id"); + } + if (rId != null) { + String url = hyperlinks.get(rId); + if (url != null) { + xhtml.startElement("a", "href", url); + xhtml.characters(url); + xhtml.endElement("a"); + } + } + } } - CTHyperlink ctHoverHyperlink = cNvPr.getHlinkHover(); - if (ctHoverHyperlink == null) { - return; + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + if ("t".equals(localName) && inT) { + inT = false; + shapeText.append(textBuffer); + } else if ("p".equals(localName) && inTxBody && + shapeText.length() > 0) { + shapeText.append('\n'); + } else if ("txBody".equals(localName)) { + inTxBody = false; + String text = shapeText.toString().trim(); + if (!text.isEmpty()) { + xhtml.element("p", text); + } + } } - url = drawingHyperlinks.get(ctHoverHyperlink.getId()); - if (url != null) { - xhtml.startElement("a", "href", url); - xhtml.characters(url); - xhtml.endElement("a"); + @Override + public void characters(char[] ch, int start, int length) { + if (inT) { + textBuffer.append(ch, start, length); + } } - } - public void processSheet(SheetContentsHandler sheetContentsHandler, Comments comments, - StylesTable styles, ReadOnlySharedStringsTable strings, + public void processSheet(TikaSheetContentsHandler sheetContentsHandler, + XSSFCommentsShim commentsShim, + XSSFStylesShim stylesShim, XSSFSharedStringsShim stringsShim, InputStream sheetInputStream) throws IOException, SAXException { try { - XSSFSheetInterestingPartsCapturer handler = new XSSFSheetInterestingPartsCapturer( - new XSSFSheetXMLHandler(styles, comments, strings, sheetContentsHandler, - formatter, false)); + new TikaSheetXMLHandler(stylesShim, commentsShim, stringsShim, + sheetContentsHandler, formatter, false)); XMLReaderUtils.parseSAX(sheetInputStream, handler, parseContext); sheetInputStream.close(); @@ -826,6 +901,33 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { } } + /** + * Parse the comments XML for a sheet part via SAX, avoiding XMLBeans. + */ + private XSSFCommentsShim parseSheetComments(PackagePart sheetPart) { + try { + PackageRelationshipCollection rels = + sheetPart.getRelationshipsByType(RELATION_COMMENTS); + if (rels.isEmpty()) { + return null; + } + PackageRelationship rel = rels.getRelationship(0); + PackagePartName partName = + PackagingURIHelper.createPartName(rel.getTargetURI()); + PackagePart commentsPart = rel.getPackage().getPart(partName); + if (commentsPart == null) { + return null; + } + try (InputStream is = commentsPart.getInputStream()) { + return new XSSFCommentsShim(is, parseContext); + } + } catch (InvalidFormatException | IOException | TikaException | SAXException e) { + //swallow — comments are not critical + return null; + } + } + + /** * In Excel files, sheets have things embedded in them, * and sheet drawings which have the images @@ -833,26 +935,33 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { @Override protected List<PackagePart> getMainDocumentParts() throws TikaException { List<PackagePart> parts = new ArrayList<>(); + // The sheet order in sheetParts mirrors the workbook's sheet + // ordering (populated in buildXHTML), so the index here is the + // 1-based sheet number. + int sheetNumber = 0; for (PackagePart part : sheetParts) { + sheetNumber++; // Add the sheet parts.add(part); // If it has drawings, return those too try { for (PackageRelationship rel : part - .getRelationshipsByType(XSSFRelation.DRAWINGS.getRelation())) { + .getRelationshipsByType(RELATION_DRAWING)) { if (rel.getTargetMode() == TargetMode.INTERNAL) { PackagePartName relName = PackagingURIHelper.createPartName(rel.getTargetURI()); - parts.add(rel.getPackage().getPart(relName)); + PackagePart drawingPart = rel.getPackage().getPart(relName); + parts.add(drawingPart); } } for (PackageRelationship rel : part - .getRelationshipsByType(XSSFRelation.VML_DRAWINGS.getRelation())) { + .getRelationshipsByType(RELATION_VML_DRAWING)) { if (rel.getTargetMode() == TargetMode.INTERNAL) { PackagePartName relName = PackagingURIHelper.createPartName(rel.getTargetURI()); - parts.add(rel.getPackage().getPart(relName)); + PackagePart vmlPart = rel.getPackage().getPart(relName); + parts.add(vmlPart); } } } catch (InvalidFormatException e) { @@ -862,7 +971,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { //add main document so that macros can be extracted //by AbstractOOXMLExtractor - parts.addAll(extractor.getPackage() + parts.addAll(opcPackage .getPartsByRelationshipType(PackageRelationshipTypes.CORE_DOCUMENT)); return parts; @@ -871,7 +980,8 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { /** * Turns formatted sheet events into HTML */ - protected static class SheetTextAsHTML implements SheetContentsHandler { + protected static class SheetTextAsHTML + implements TikaSheetContentsHandler, SheetContentsHandler { private final boolean includeHeadersFooters; private final boolean includeMissingRows; protected List<String> headers; @@ -879,6 +989,12 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { private XHTMLContentHandler xhtml; private int lastSeenRow = -1; private int lastSeenCol = -1; + // Track open <tr>/<td> so the outer catch can emit balanced closes + // when processSheet throws part-way through a row (e.g., a malformed + // sheet XML). Without this, the outer code would emit </tbody></table> + // while <tr> (or <td>) was still on the stack, producing malformed XHTML. + private boolean rowOpen; + private boolean cellOpen; protected SheetTextAsHTML(OfficeParserConfig config, XHTMLContentHandler xhtml) { this.includeHeadersFooters = config.isIncludeHeadersAndFooters(); @@ -894,14 +1010,19 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { if (includeMissingRows && rowNum > (lastSeenRow + 1)) { for (int rn = lastSeenRow + 1; rn < rowNum; rn++) { xhtml.startElement("tr"); + rowOpen = true; xhtml.startElement("td"); + cellOpen = true; xhtml.endElement("td"); + cellOpen = false; xhtml.endElement("tr"); + rowOpen = false; } } // Start the new row xhtml.startElement("tr"); + rowOpen = true; lastSeenCol = -1; } catch (SAXException e) { //swallow @@ -913,24 +1034,45 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { public void endRow(int rowNum) { try { xhtml.endElement("tr"); + rowOpen = false; } catch (SAXException e) { throw new RuntimeSAXException(e); } } - public void cell(String cellRef, String formattedValue, XSSFComment comment) { + /** + * Closes any pending {@code <tr>} or {@code <td>} that was opened + * before a {@link SAXException} interrupted sheet processing. Safe to + * call when nothing is open. + */ + void closeAnyPending() throws SAXException { + if (cellOpen) { + xhtml.endElement("td"); + cellOpen = false; + } + if (rowOpen) { + xhtml.endElement("tr"); + rowOpen = false; + } + } + + public void cell(String cellRef, String formattedValue, + XSSFCommentsShim.CommentData comment) { try { // Handle any missing cells int colNum = (cellRef == null) ? lastSeenCol + 1 : (new CellReference(cellRef)).getCol(); for (int cn = lastSeenCol + 1; cn < colNum; cn++) { xhtml.startElement("td"); + cellOpen = true; xhtml.endElement("td"); + cellOpen = false; } lastSeenCol = colNum; // Start this cell xhtml.startElement("td"); + cellOpen = true; // Main cell contents if (formattedValue != null) { @@ -943,15 +1085,31 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { xhtml.endElement("br"); xhtml.characters(comment.getAuthor()); xhtml.characters(": "); - xhtml.characters(comment.getString().getString()); + xhtml.characters(comment.getText()); } xhtml.endElement("td"); + cellOpen = false; } catch (SAXException e) { throw new RuntimeSAXException(e); } } + /** + * Bridge for POI's {@link SheetContentsHandler} interface, used by the + * XLSB (binary) path via {@link org.apache.poi.xssf.binary.XSSFBSheetHandler}. + */ + public void cell(String cellRef, String formattedValue, XSSFComment comment) { + XSSFCommentsShim.CommentData commentData = null; + if (comment != null) { + String text = comment.getString() != null ? + comment.getString().getString() : ""; + commentData = new XSSFCommentsShim.CommentData( + comment.getAuthor(), text); + } + cell(cellRef, formattedValue, commentData); + } + public void headerFooter(String text, boolean isHeader, String tagName) { if (!includeHeadersFooters) { return; @@ -962,6 +1120,11 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { footers.add(text); } } + + @Override + public void endSheet() { + // no-op — satisfies both TikaSheetContentsHandler and SheetContentsHandler + } } protected static class HeaderFooterFromString implements HeaderFooter { @@ -1143,4 +1306,5 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor { } } } + } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java new file mode 100644 index 0000000000..8556d0fbb3 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.utils.XMLReaderUtils; + +/** + * SAX-based shim that replaces POI's {@code ReadOnlySharedStringsTable} + * for XLSX event-based parsing. + * <p> + * Parses {@code xl/sharedStrings.xml} and stores each shared string entry + * as a plain {@code String}, avoiding the XMLBeans dependency that + * {@code XSSFRichTextString} requires. Rich text runs within a single + * {@code <si>} are concatenated into a single string. + */ +class XSSFSharedStringsShim { + + private final List<String> strings; + private final boolean includePhoneticRuns; + + XSSFSharedStringsShim(InputStream sharedStringsData, + boolean includePhoneticRuns, + ParseContext parseContext) + throws IOException, SAXException, TikaException { + this.includePhoneticRuns = includePhoneticRuns; + SharedStringsHandler handler = new SharedStringsHandler(); + if (sharedStringsData != null) { + try { + XMLReaderUtils.parseSAX(sharedStringsData, handler, parseContext); + } finally { + sharedStringsData.close(); + } + } + this.strings = handler.strings; + } + + String getItemAt(int idx) { + return strings.get(idx); + } + + int getCount() { + return strings.size(); + } + + private class SharedStringsHandler extends DefaultHandler { + + private static final String NS = + "http://schemas.openxmlformats.org/spreadsheetml/2006/main";; + + final List<String> strings = new ArrayList<>(); + private StringBuilder characters; + private boolean tIsOpen; + private boolean inRPh; + + @Override + public void startElement(String uri, String localName, String qName, + Attributes attributes) { + if (uri != null && !NS.equals(uri)) { + return; + } + switch (localName) { + case "sst": + String uniqueCount = attributes.getValue("uniqueCount"); + if (uniqueCount != null) { + try { + int hint = (int) Long.parseLong(uniqueCount); + // guard against corrupt files with absurd counts + ((ArrayList<String>) strings).ensureCapacity( + Math.min(hint, 100_000)); + } catch (NumberFormatException e) { + // ignore + } + } + characters = new StringBuilder(64); + break; + case "si": + if (characters != null) { + characters.setLength(0); + } + break; + case "t": + tIsOpen = true; + break; + case "rPh": + inRPh = true; + if (includePhoneticRuns && characters != null && + characters.length() > 0) { + characters.append(" "); + } + break; + default: + break; + } + } + + @Override + public void endElement(String uri, String localName, String qName) { + if (uri != null && !NS.equals(uri)) { + return; + } + switch (localName) { + case "si": + if (characters != null) { + strings.add(characters.toString()); + } + break; + case "t": + tIsOpen = false; + break; + case "rPh": + inRPh = false; + break; + default: + break; + } + } + + @Override + public void characters(char[] ch, int start, int length) { + if (tIsOpen && characters != null) { + if (inRPh) { + if (includePhoneticRuns) { + characters.append(ch, start, length); + } + } else { + characters.append(ch, start, length); + } + } + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFStylesShim.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFStylesShim.java new file mode 100644 index 0000000000..ca99c7512e --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFStylesShim.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.poi.ss.usermodel.BuiltinFormats; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.utils.XMLReaderUtils; + +/** + * SAX-based shim that replaces POI's {@code StylesTable} for XLSX event-based parsing. + * <p> + * Parses {@code xl/styles.xml} and extracts only the information needed for text + * extraction: the number format resolution chain (cellXfs index to format string). + * This avoids the XMLBeans dependency that {@code StylesTable} requires. + */ +class XSSFStylesShim { + + private final Map<Short, String> numberFormats = new HashMap<>(); + private final List<Short> cellXfFormatIds = new ArrayList<>(); + + XSSFStylesShim(InputStream stylesData, ParseContext parseContext) + throws IOException, SAXException, TikaException { + if (stylesData != null) { + try { + XMLReaderUtils.parseSAX(stylesData, new StylesHandler(), parseContext); + } finally { + stylesData.close(); + } + } + } + + int getNumCellStyles() { + return cellXfFormatIds.size(); + } + + short getFormatIndex(int styleIndex) { + if (styleIndex < 0 || styleIndex >= cellXfFormatIds.size()) { + return -1; + } + return cellXfFormatIds.get(styleIndex); + } + + String getFormatString(int styleIndex) { + short fmtId = getFormatIndex(styleIndex); + if (fmtId == -1) { + return null; + } + String fmt = numberFormats.get(fmtId); + if (fmt == null) { + fmt = BuiltinFormats.getBuiltinFormat(fmtId); + } + return fmt; + } + + private class StylesHandler extends DefaultHandler { + + private static final String NS = + "http://schemas.openxmlformats.org/spreadsheetml/2006/main";; + + private boolean inCellXfs; + private boolean inNumFmts; + + @Override + public void startElement(String uri, String localName, String qName, + Attributes attributes) { + if (!NS.equals(uri)) { + return; + } + switch (localName) { + case "numFmts": + inNumFmts = true; + break; + case "numFmt": + if (inNumFmts) { + String idStr = attributes.getValue("numFmtId"); + String code = attributes.getValue("formatCode"); + if (idStr != null && code != null) { + try { + numberFormats.put(Short.parseShort(idStr), code); + } catch (NumberFormatException e) { + // skip malformed + } + } + } + break; + case "cellXfs": + inCellXfs = true; + break; + case "xf": + if (inCellXfs) { + String numFmtIdStr = attributes.getValue("numFmtId"); + short numFmtId = 0; + if (numFmtIdStr != null) { + try { + numFmtId = Short.parseShort(numFmtIdStr); + } catch (NumberFormatException e) { + // default to 0 (General) + } + } + cellXfFormatIds.add(numFmtId); + } + break; + default: + break; + } + } + + @Override + public void endElement(String uri, String localName, String qName) { + if (!NS.equals(uri)) { + return; + } + if ("numFmts".equals(localName)) { + inNumFmts = false; + } else if ("cellXfs".equals(localName)) { + inCellXfs = false; + } + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/VSDXParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/VSDXParserTest.java new file mode 100644 index 0000000000..c4a832bd96 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/VSDXParserTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +public class VSDXParserTest extends TikaTest { + + @Test + public void testBasicTextExtraction() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testVISIO.vsdx"); + String content = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT); + assertEquals("application/vnd.ms-visio.drawing", + metadataList.get(0).get(Metadata.CONTENT_TYPE)); + assertContains("test", content); + assertContains("This is a test.", content); + assertContains("Nothing fancy.", content); + } +}
