This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4759-markdown-writer in repository https://gitbox.apache.org/repos/asf/tika.git
commit 0238d761b133d2130c2e1a680fad1b8045141de5 Author: tallison <[email protected]> AuthorDate: Wed Jun 24 06:27:44 2026 -0400 TIKA-4759 - use a markdown writer --- .../src/main/appended-resources/META-INF/LICENSE | 29 + .../test/java/org/apache/tika/bundle/BundleIT.java | 9 +- tika-bundles/tika-bundle-standard/test-bundles.xml | 3 + tika-core/pom.xml | 17 + .../apache/tika/sax/ToMarkdownContentHandler.java | 694 +++++++++++---------- .../tika/sax/ToMarkdownContentHandlerTest.java | 132 +++- tika-parent/pom.xml | 1 + tika-parsers/tika-parsers-ml/tika-vlm/pom.xml | 4 - 8 files changed, 538 insertions(+), 351 deletions(-) diff --git a/tika-app/src/main/appended-resources/META-INF/LICENSE b/tika-app/src/main/appended-resources/META-INF/LICENSE index 43dae22216..ac394dcdaa 100644 --- a/tika-app/src/main/appended-resources/META-INF/LICENSE +++ b/tika-app/src/main/appended-resources/META-INF/LICENSE @@ -900,3 +900,32 @@ Sqlite (included in the "provided" org.xerial's sqlite-jdbc) Sqlite is in the Public Domain. For details see: https://www.sqlite.org/copyright.html +commonmark-java libraries (commonmark, commonmark-ext-gfm-tables, and +commonmark-ext-gfm-strikethrough) + + The BSD 2-Clause License + + Copyright (c) 2015, Atlassian Pty Ltd + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java index 1a2d4fdd0d..76123b9b24 100644 --- a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java +++ b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java @@ -85,12 +85,19 @@ public class BundleIT { // Install all bundles first, then start. // tika-core requires osgi.serviceloader capabilities that are // provided by tika-bundle-standard, so both must be installed - // before either can resolve. + // before either can resolve. tika-core also imports the org.commonmark + // packages (Markdown serialization), so those bundles must be present too. Bundle commonsIo = install("commons-io.jar"); + Bundle commonmark = install("commonmark.jar"); + Bundle commonmarkTables = install("commonmark-ext-gfm-tables.jar"); + Bundle commonmarkStrikethrough = install("commonmark-ext-gfm-strikethrough.jar"); Bundle tikaCore = install("tika-core.jar"); Bundle tikaBundle = install("tika-bundle-standard.jar"); commonsIo.start(); + commonmark.start(); + commonmarkTables.start(); + commonmarkStrikethrough.start(); tikaCore.start(); tikaBundle.start(); } diff --git a/tika-bundles/tika-bundle-standard/test-bundles.xml b/tika-bundles/tika-bundle-standard/test-bundles.xml index 4da4310920..821846ec9c 100644 --- a/tika-bundles/tika-bundle-standard/test-bundles.xml +++ b/tika-bundles/tika-bundle-standard/test-bundles.xml @@ -30,6 +30,9 @@ <include>org.apache.tika:tika-core</include> <include>org.apache.tika:tika-bundle-standard</include> <include>commons-io:commons-io</include> + <include>org.commonmark:commonmark</include> + <include>org.commonmark:commonmark-ext-gfm-tables</include> + <include>org.commonmark:commonmark-ext-gfm-strikethrough</include> </includes> </dependencySet> <dependencySet> diff --git a/tika-core/pom.xml b/tika-core/pom.xml index dcd0b780f2..668cba8a25 100644 --- a/tika-core/pom.xml +++ b/tika-core/pom.xml @@ -44,6 +44,23 @@ <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> + <!-- Markdown serialization (ToMarkdownContentHandler). BSD 2-Clause, ASF Category A, + zero transitive dependencies. --> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark</artifactId> + <version>${commonmark.version}</version> + </dependency> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark-ext-gfm-tables</artifactId> + <version>${commonmark.version}</version> + </dependency> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark-ext-gfm-strikethrough</artifactId> + <version>${commonmark.version}</version> + </dependency> <!-- Optional OSGi dependencies, used only when running within OSGi --> <dependency> diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java index c92ae55549..4228955394 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java +++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java @@ -23,69 +23,105 @@ import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.util.ArrayDeque; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Deque; import java.util.List; import java.util.Locale; +import org.commonmark.Extension; +import org.commonmark.ext.gfm.strikethrough.Strikethrough; +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension; +import org.commonmark.ext.gfm.tables.TableBlock; +import org.commonmark.ext.gfm.tables.TableBody; +import org.commonmark.ext.gfm.tables.TableCell; +import org.commonmark.ext.gfm.tables.TableHead; +import org.commonmark.ext.gfm.tables.TableRow; +import org.commonmark.ext.gfm.tables.TablesExtension; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Document; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.Image; +import org.commonmark.node.Link; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.renderer.markdown.MarkdownRenderer; import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; /** * SAX event handler that writes content as Markdown. - * Supports headings, paragraphs, bold, italic, links, images, lists (ordered - * and unordered, including nested), tables (GFM pipe tables), code blocks, - * inline code, blockquotes, horizontal rules, and definition lists. + * Supports headings, paragraphs, bold, italic, strikethrough, links, images, + * lists (ordered and unordered, including nested), tables (GFM pipe tables), + * code blocks, inline code, blockquotes, horizontal rules, and definition + * lists. + * <p> + * The handler builds a <a href="https://github.com/commonmark/commonmark-java";> + * commonmark-java</a> document model from the SAX event stream and renders it + * with commonmark's {@code MarkdownRenderer}. Document text is added to the + * model as raw literals, so escaping of Markdown metacharacters — including + * characters that would otherwise break out of a link, image, or table cell — + * is performed in one place by the renderer rather than at each emit site. + * <p> + * The handler tolerates malformed input (unbalanced or misnested tags): block + * elements are always attached at a structurally valid point so a well-formed + * document model is rendered regardless of the event stream. * <p> * Content within <script> and <style> tags is ignored. - * </p> * * @since Apache Tika 3.2 */ public class ToMarkdownContentHandler extends DefaultHandler { - private static final String STYLE = "STYLE"; - private static final String SCRIPT = "SCRIPT"; + private static final List<Extension> EXTENSIONS = Arrays.asList( + TablesExtension.create(), StrikethroughExtension.create()); private final Writer writer; + private final MarkdownRenderer renderer = + MarkdownRenderer.builder().extensions(EXTENSIONS).build(); - private final Deque<String> elementStack = new ArrayDeque<>(); - private final Deque<ListState> listStack = new ArrayDeque<>(); - - // Link buffering - private StringBuilder linkText; - private String linkHref; + private final Document document = new Document(); + private final Deque<Node> stack = new ArrayDeque<>(); - // Table buffering (only the outermost table is rendered; nested tables are ignored) - private int tableDepth = 0; - private List<List<String>> tableRows; - private List<String> currentRow; - private StringBuilder currentCell; + // An implicit Paragraph opened to hold inline content that appeared in a + // block container (e.g. text directly under <body>/<li>/<blockquote>). + private Paragraph implicitParagraph; - // Blockquote - private int blockquoteDepth = 0; + // Buffer for inline <code> literal (commonmark Code holds a literal, not children). + private Code inlineCode; + private StringBuilder inlineCodeText; - // Code - private boolean inPreBlock = false; - private boolean inInlineCode = false; + // Buffer for <pre> fenced code block literal. + private FencedCodeBlock codeBlock; + private StringBuilder codeBlockText; - // Script/style suppression - private int scriptDepth = 0; - private int styleDepth = 0; + // Table state (outermost table only). + private int tableDepth; + private TableBlock tableBlock; + private TableBody tableBody; + private int rowIndex; + private boolean inHeaderRow; - // Spacing - private boolean needsBlockSeparator = false; - private boolean atLineStart = true; + // <script>/<style> content is dropped. + private int suppressDepth; - // Track if we've written any content at all - private boolean hasContent = false; - - // Track if meaningful (non-whitespace) content was written since last block separator - private boolean hasContentSinceLastSeparator = false; + // True once endDocument has rendered the full document to the writer. + private boolean finished; public ToMarkdownContentHandler(Writer writer) { this.writer = writer; + this.stack.push(document); } public ToMarkdownContentHandler(OutputStream stream, String encoding) @@ -102,136 +138,132 @@ public class ToMarkdownContentHandler extends DefaultHandler { throws SAXException { String name = localName(localName, qName); - // Track script/style depth - if (name.equals("script")) { - scriptDepth++; - elementStack.push(name); - return; - } - if (name.equals("style")) { - styleDepth++; - elementStack.push(name); + if (name.equals("script") || name.equals("style")) { + suppressDepth++; return; } - - if (scriptDepth > 0 || styleDepth > 0) { - elementStack.push(name); + if (suppressDepth > 0) { return; } - elementStack.push(name); - switch (name) { + // --- block-level --- case "h1": case "h2": case "h3": case "h4": case "h5": case "h6": - emitBlockSeparator(); - int level = name.charAt(1) - '0'; - write(repeatChar('#', level) + " "); + Heading heading = new Heading(); + heading.setLevel(name.charAt(1) - '0'); + openBlock(heading); break; case "p": - emitBlockSeparator(); + openBlock(new Paragraph()); break; - case "b": - case "strong": - write("**"); - break; - case "i": - case "em": - write("*"); - break; - case "a": - linkHref = atts.getValue("href"); - linkText = new StringBuilder(); - break; - case "img": - String alt = atts.getValue("alt"); - String src = atts.getValue("src"); - write("![" + (alt != null ? alt : "") + "](" + (src != null ? src : "") + ")"); + case "blockquote": + openBlock(new BlockQuote()); break; case "ul": + openBlock(newBulletList()); + break; case "ol": - if (!listStack.isEmpty()) { - // nested list — no extra block separator - } else { - emitBlockSeparator(); - } - listStack.push(new ListState(name.equals("ol"), listStack.size())); + OrderedList ol = new OrderedList(); + ol.setMarkerStartNumber(1); + ol.setMarkerDelimiter("."); + ol.setTight(true); + openBlock(ol); break; case "li": - if (!listStack.isEmpty()) { - ListState state = listStack.peek(); - String indent = repeatChar(' ', state.depth * 4); - if (state.ordered) { - state.counter++; - write(indent + state.counter + ". "); - } else { - write(indent + "- "); - } - } - break; - case "blockquote": - emitBlockSeparator(); - blockquoteDepth++; + openListItem(); break; case "pre": - emitBlockSeparator(); - inPreBlock = true; - write("```\n"); - break; - case "code": - if (!inPreBlock) { - inInlineCode = true; - write("`"); - } - break; - case "br": - write("\n"); - atLineStart = true; + popToBlockHost(); + codeBlock = new FencedCodeBlock(); + codeBlock.setFenceCharacter("`"); + codeBlock.setOpeningFenceLength(3); + codeBlock.setLiteral(""); + stack.peek().appendChild(codeBlock); + codeBlockText = new StringBuilder(); break; case "hr": - emitBlockSeparator(); - write("---"); - needsBlockSeparator = true; - hasContent = true; + ThematicBreak hr = new ThematicBreak(); + hr.setLiteral("---"); + addBlockLeaf(hr); + break; + case "div": + // Block boundary: close any open paragraph so adjacent divs separate. + popToBlockHost(); break; case "table": - tableDepth++; - if (tableDepth == 1) { - emitBlockSeparator(); - tableRows = new ArrayList<>(); - } + startTable(); break; case "tr": - if (tableDepth == 1 && tableRows != null) { - currentRow = new ArrayList<>(); - } + startRow(); break; case "th": - if (tableDepth == 1 && currentRow != null) { - currentCell = new StringBuilder(); - } + startCell(true); break; case "td": - if (tableDepth == 1 && currentRow != null) { - currentCell = new StringBuilder(); - } + startCell(false); break; case "dt": - emitBlockSeparator(); - write("**"); + Paragraph term = new Paragraph(); + openBlock(term); + StrongEmphasis bold = new StrongEmphasis(); + term.appendChild(bold); + stack.push(bold); break; case "dd": - write("\n: "); + Paragraph def = new Paragraph(); + openBlock(def); + def.appendChild(new Text(": ")); break; - case "div": - emitBlockSeparator(); + + // --- inline-level --- + case "b": + case "strong": + openInline(new StrongEmphasis()); + break; + case "i": + case "em": + openInline(new Emphasis()); break; + case "s": + case "strike": + case "del": + openInline(new Strikethrough("~~")); + break; + case "a": + String href = atts.getValue("href"); + openInline(new Link(href != null ? href : "", null)); + break; + case "code": + if (codeBlockText == null) { + ensureInlineContainer(); + inlineCode = new Code(""); + stack.peek().appendChild(inlineCode); + inlineCodeText = new StringBuilder(); + } + break; + case "br": + ensureInlineContainer(); + stack.peek().appendChild(new HardLineBreak()); + break; + case "img": + ensureInlineContainer(); + Image img = new Image(value(atts, "src"), null); + String alt = atts.getValue("alt"); + if (alt != null && !alt.isEmpty()) { + img.appendChild(new Text(alt)); + } + stack.peek().appendChild(img); + break; + default: - // Ignore structural elements like html, head, body, title, meta + // html, head, body, title, meta, div, span, dl, thead, tbody... + // structurally transparent; their inline children flow to the + // nearest real container. break; } } @@ -240,21 +272,11 @@ public class ToMarkdownContentHandler extends DefaultHandler { public void endElement(String uri, String localName, String qName) throws SAXException { String name = localName(localName, qName); - if (!elementStack.isEmpty()) { - elementStack.pop(); - } - - // Track script/style depth - if (name.equals("script")) { - scriptDepth--; + if (name.equals("script") || name.equals("style")) { + suppressDepth = Math.max(0, suppressDepth - 1); return; } - if (name.equals("style")) { - styleDepth--; - return; - } - - if (scriptDepth > 0 || styleDepth > 0) { + if (suppressDepth > 0) { return; } @@ -265,99 +287,61 @@ public class ToMarkdownContentHandler extends DefaultHandler { case "h4": case "h5": case "h6": - needsBlockSeparator = true; - hasContent = true; - break; case "p": - needsBlockSeparator = true; - hasContent = true; - break; + case "blockquote": + case "ul": + case "ol": + case "li": case "b": case "strong": - write("**"); - break; case "i": case "em": - write("*"); - break; + case "s": + case "strike": + case "del": case "a": - if (linkText != null) { - String text = linkText.toString(); - String href = linkHref != null ? linkHref : ""; - write("[" + text + "](" + href + ")"); - linkText = null; - linkHref = null; - } - break; - case "ul": - case "ol": - if (!listStack.isEmpty()) { - listStack.pop(); - } - if (listStack.isEmpty()) { - needsBlockSeparator = true; - hasContent = true; - } + closeImplicitParagraph(); + pop(); break; - case "li": - write("\n"); - atLineStart = true; + case "dt": + pop(); // StrongEmphasis + pop(); // Paragraph break; - case "blockquote": - blockquoteDepth--; - needsBlockSeparator = true; - hasContent = true; + case "dd": + pop(); break; case "pre": - if (!endsWithNewline()) { - write("\n"); + if (codeBlock != null) { + codeBlock.setLiteral(withTrailingNewline(codeBlockText.toString())); + codeBlock = null; + codeBlockText = null; } - write("```"); - inPreBlock = false; - needsBlockSeparator = true; - hasContent = true; break; case "code": - if (!inPreBlock) { - inInlineCode = false; - write("`"); + if (inlineCode != null) { + inlineCode.setLiteral(inlineCodeText.toString()); + inlineCode = null; + inlineCodeText = null; } break; + case "div": + closeImplicitParagraph(); + break; case "table": - if (tableDepth == 1) { - emitTable(); - tableRows = null; - currentRow = null; - currentCell = null; - needsBlockSeparator = true; - hasContent = true; - } - tableDepth = Math.max(0, tableDepth - 1); + endTable(); break; case "tr": - if (tableDepth == 1 && tableRows != null && currentRow != null) { - tableRows.add(currentRow); - currentRow = null; + if (tableDepth == 1) { + pop(); + rowIndex++; } break; case "th": case "td": - if (tableDepth == 1 && currentRow != null && currentCell != null) { - currentRow.add(currentCell.toString().trim()); - currentCell = null; + if (tableDepth == 1) { + pop(); } break; - case "dt": - write("**"); - break; - case "dd": - needsBlockSeparator = true; - hasContent = true; - break; - case "div": - needsBlockSeparator = true; - hasContent = true; - break; default: break; } @@ -365,58 +349,34 @@ public class ToMarkdownContentHandler extends DefaultHandler { @Override public void characters(char[] ch, int start, int length) throws SAXException { - if (scriptDepth > 0 || styleDepth > 0) { + if (suppressDepth > 0) { return; } - - // Buffer into link text - if (linkText != null) { - linkText.append(ch, start, length); + if (codeBlockText != null) { + codeBlockText.append(ch, start, length); return; } - - // Buffer into table cell - if (currentCell != null) { - currentCell.append(ch, start, length); + if (inlineCodeText != null) { + inlineCodeText.append(ch, start, length); return; } String text = new String(ch, start, length); - - // In pre blocks, write raw (no escaping) - if (inPreBlock) { - write(text); - return; - } - - // In inline code, write raw (no escaping) - if (inInlineCode) { - write(text); - return; - } - - // Skip whitespace-only text at line start; preserve inline spaces if (text.trim().isEmpty()) { - if (!atLineStart) { - write(" "); + // Inter-element whitespace: drop between blocks, collapse within inline runs. + if (!isBlockContainer(stack.peek())) { + stack.peek().appendChild(new Text(" ")); } return; } - // Escape markdown special characters in normal text - text = escapeMarkdown(text); - - // Add blockquote prefix if needed at line start - if (blockquoteDepth > 0 && atLineStart && !text.isEmpty()) { - write(repeatChar('>', blockquoteDepth) + " "); - atLineStart = false; - } - - if (!text.isEmpty()) { - write(text); - hasContent = true; - hasContentSinceLastSeparator = true; + if (isStructuralOnly(stack.peek())) { + // Stray text inside a list/table wrapper — not a valid inline position; drop. + return; } + ensureInlineContainer(); + // Newlines inside prose would otherwise survive as literal line breaks. + stack.peek().appendChild(new Text(collapseLineBreaks(text))); } @Override @@ -427,110 +387,193 @@ public class ToMarkdownContentHandler extends DefaultHandler { @Override public void endDocument() throws SAXException { try { + renderer.render(document, writer); writer.flush(); + finished = true; } catch (IOException e) { - throw new SAXException("Error flushing character output", e); + throw new SAXException("Error writing markdown", e); } } @Override public String toString() { - return writer.toString(); + if (finished) { + return writer.toString(); + } + // Parsing did not complete (e.g. the parser threw mid-document). Render the + // content accumulated so far so partial content is not lost on failure — the + // streaming writer this replaced exposed partial content the same way. + return renderer.render(document); } - private void write(String s) throws SAXException { - try { - writer.write(s); - if (!s.isEmpty()) { - atLineStart = s.charAt(s.length() - 1) == '\n'; - if (!s.trim().isEmpty()) { - hasContentSinceLastSeparator = true; - } + // --- tree construction helpers --- + + /** + * Append a block-level node at a structurally valid point and descend into + * it. commonmark requires a block's parent to be a block, so any inline (or + * other non-hosting) nodes left open are closed first. + */ + private void openBlock(Node block) { + popToBlockHost(); + stack.peek().appendChild(block); + stack.push(block); + } + + /** Append a block-level leaf (hr, code block, table) without descending. */ + private void addBlockLeaf(Node block) { + popToBlockHost(); + stack.peek().appendChild(block); + } + + /** + * A list item is only valid inside a list. If the current point isn't a + * list (a stray {@code <li>} or misnested markup), recover by opening an + * implicit bullet list so the model stays renderable. + */ + private void openListItem() { + if (!(stack.peek() instanceof ListBlock)) { + popToBlockHost(); + BulletList implicit = newBulletList(); + stack.peek().appendChild(implicit); + stack.push(implicit); + } + ListItem li = new ListItem(); + stack.peek().appendChild(li); + stack.push(li); + } + + private void openInline(Node node) { + ensureInlineContainer(); + stack.peek().appendChild(node); + stack.push(node); + } + + private void pop() { + if (stack.size() > 1) { + Node popped = stack.pop(); + if (popped == implicitParagraph) { + implicitParagraph = null; } - } catch (IOException e) { - throw new SAXException("Error writing: " + s, e); } } - private void emitBlockSeparator() throws SAXException { - if (needsBlockSeparator && hasContent && hasContentSinceLastSeparator) { - write("\n\n"); - needsBlockSeparator = false; - atLineStart = true; - hasContentSinceLastSeparator = false; - } else { - needsBlockSeparator = false; + /** Pop open nodes until the top can host block children (Document/blockquote/list item). */ + private void popToBlockHost() { + while (stack.size() > 1 && !canHostBlocks(stack.peek())) { + Node popped = stack.pop(); + if (popped == implicitParagraph) { + implicitParagraph = null; + } } } - private void emitTable() throws SAXException { - if (tableRows == null || tableRows.isEmpty()) { - return; + /** + * If the current insertion point is a block container that cannot hold + * inline content directly, open an implicit {@link Paragraph} to hold it. + */ + private void ensureInlineContainer() { + Node top = stack.peek(); + if (canHostBlocks(top)) { + Paragraph p = new Paragraph(); + top.appendChild(p); + stack.push(p); + implicitParagraph = p; } + } - // Determine column count - int cols = 0; - for (List<String> row : tableRows) { - cols = Math.max(cols, row.size()); + private void closeImplicitParagraph() { + if (implicitParagraph != null && stack.peek() == implicitParagraph) { + stack.pop(); + implicitParagraph = null; } + } - // Emit rows - for (int r = 0; r < tableRows.size(); r++) { - List<String> row = tableRows.get(r); - StringBuilder sb = new StringBuilder("|"); - for (int c = 0; c < cols; c++) { - String cell = c < row.size() ? row.get(c) : ""; - sb.append(" ").append(cell).append(" |"); - } - write(sb.toString()); - write("\n"); - - // Insert separator after first row - if (r == 0) { - StringBuilder sep = new StringBuilder("|"); - for (int c = 0; c < cols; c++) { - sep.append(" --- |"); - } - write(sep.toString()); - write("\n"); - } + private void startTable() { + tableDepth++; + if (tableDepth == 1) { + popToBlockHost(); + tableBlock = new TableBlock(); + tableBody = null; + rowIndex = 0; + stack.peek().appendChild(tableBlock); } } - private boolean endsWithNewline() { - String s = writer.toString(); - return !s.isEmpty() && s.charAt(s.length() - 1) == '\n'; + private void endTable() { + if (tableDepth == 1) { + tableBlock = null; + tableBody = null; + } + tableDepth = Math.max(0, tableDepth - 1); } - private static String escapeMarkdown(String text) { - StringBuilder sb = new StringBuilder(text.length()); - for (int i = 0; i < text.length(); i++) { - char c = text.charAt(i); - switch (c) { - case '\\': - case '`': - case '*': - case '_': - case '[': - case ']': - case '#': - case '|': - sb.append('\\').append(c); - break; - default: - sb.append(c); - break; + private void startRow() { + if (tableDepth != 1) { + return; + } + TableRow row = new TableRow(); + if (rowIndex == 0) { + TableHead head = new TableHead(); + tableBlock.appendChild(head); + head.appendChild(row); + inHeaderRow = true; + } else { + if (tableBody == null) { + tableBody = new TableBody(); + tableBlock.appendChild(tableBody); } + tableBody.appendChild(row); + inHeaderRow = false; } - return sb.toString(); + stack.push(row); } - private static String repeatChar(char c, int count) { - StringBuilder sb = new StringBuilder(count); - for (int i = 0; i < count; i++) { - sb.append(c); + private void startCell(boolean header) { + if (tableDepth != 1 || !(stack.peek() instanceof TableRow)) { + return; } - return sb.toString(); + TableCell cell = new TableCell(); + cell.setHeader(header || inHeaderRow); + stack.peek().appendChild(cell); + stack.push(cell); + } + + private static BulletList newBulletList() { + BulletList list = new BulletList(); + list.setMarker("-"); + list.setTight(true); + return list; + } + + private static boolean canHostBlocks(Node n) { + return n instanceof Document || n instanceof BlockQuote || n instanceof ListItem; + } + + private static boolean isBlockContainer(Node n) { + return n instanceof Document || n instanceof BlockQuote || n instanceof ListItem + || n instanceof ListBlock || n instanceof TableBlock || n instanceof TableHead + || n instanceof TableBody || n instanceof TableRow; + } + + private static boolean isStructuralOnly(Node n) { + return n instanceof ListBlock || n instanceof TableBlock || n instanceof TableHead + || n instanceof TableBody || n instanceof TableRow; + } + + private static String collapseLineBreaks(String s) { + return s.replace('\r', ' ').replace('\n', ' '); + } + + private static String withTrailingNewline(String s) { + if (s.isEmpty() || s.charAt(s.length() - 1) == '\n') { + return s; + } + return s + "\n"; + } + + private static String value(Attributes atts, String name) { + String v = atts.getValue(name); + return v != null ? v : ""; } private static String localName(String localName, String qName) { @@ -538,23 +581,10 @@ public class ToMarkdownContentHandler extends DefaultHandler { return localName.toLowerCase(Locale.ROOT); } if (qName != null) { - // Strip namespace prefix int colon = qName.indexOf(':'); String name = colon >= 0 ? qName.substring(colon + 1) : qName; return name.toLowerCase(Locale.ROOT); } return ""; } - - private static class ListState { - final boolean ordered; - final int depth; - int counter; - - ListState(boolean ordered, int depth) { - this.ordered = ordered; - this.depth = depth; - this.counter = 0; - } - } } diff --git a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java index 298d11c70f..992fe6ac4d 100644 --- a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java +++ b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java @@ -314,8 +314,8 @@ public class ToMarkdownContentHandlerTest { String result = handler.toString(); assertTrue(result.contains("- Fruit")); - assertTrue(result.contains(" - Apple")); - assertTrue(result.contains(" - Banana")); + assertTrue(result.contains(" - Apple")); + assertTrue(result.contains(" - Banana")); assertTrue(result.contains("- Vegetable")); } @@ -351,9 +351,9 @@ public class ToMarkdownContentHandlerTest { handler.endDocument(); String result = handler.toString(); - assertTrue(result.contains("| Name | Age |")); - assertTrue(result.contains("| --- | --- |")); - assertTrue(result.contains("| Alice | 30 |")); + assertTrue(result.contains("|Name|Age|")); + assertTrue(result.contains("|---|---|")); + assertTrue(result.contains("|Alice|30|")); } @Test @@ -445,7 +445,8 @@ public class ToMarkdownContentHandlerTest { handler.endDocument(); - assertTrue(handler.toString().contains("Line one\nLine two")); + // GFM hard line break: two trailing spaces before the newline + assertTrue(handler.toString().contains("Line one \nLine two")); } @Test @@ -544,7 +545,9 @@ public class ToMarkdownContentHandlerTest { assertTrue(result.contains("\\_")); assertTrue(result.contains("\\[")); assertTrue(result.contains("\\]")); - assertTrue(result.contains("\\#")); + // '#' only needs escaping at line start, so it is left bare mid-line + assertTrue(result.contains("#")); + assertFalse(result.contains("\\#")); assertTrue(result.contains("\\|")); assertTrue(result.contains("\\\\")); assertTrue(result.contains("\\`")); @@ -751,9 +754,9 @@ public class ToMarkdownContentHandlerTest { handler.endDocument(); String result = handler.toString(); - assertTrue(result.contains("| A | B |")); - assertTrue(result.contains("| --- | --- |")); - assertTrue(result.contains("| C | D |")); + assertTrue(result.contains("|A|B|")); + assertTrue(result.contains("|---|---|")); + assertTrue(result.contains("|C|D|")); } @Test @@ -799,12 +802,12 @@ public class ToMarkdownContentHandlerTest { String result = handler.toString(); // Outer table should be rendered - assertTrue(result.contains("| Outer1 | Outer2 |")); - assertTrue(result.contains("| --- | --- |")); + assertTrue(result.contains("|Outer1|Outer2|")); + assertTrue(result.contains("|---|---|")); // Inner cell text gets folded into the outer cell ("B" + "Inner" = "BInner") - assertTrue(result.contains("| A | BInner |")); + assertTrue(result.contains("|A|BInner|")); // Inner table structure should not appear as a separate table - assertFalse(result.contains("| Inner |")); + assertFalse(result.contains("|Inner|")); } private static final String[] ALL_ELEMENTS = { @@ -1006,4 +1009,105 @@ public class ToMarkdownContentHandlerTest { handler.endDocument(); }); } + + // --- untrusted content cannot break out of inline/table structure --- + + @Test + public void testLinkTextCannotBreakOut() throws Exception { + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(); + handler.startDocument(); + startElement(handler, "p"); + startElement(handler, "a", "href", "https://good.example";); + chars(handler, "x](https://evil.example)"); + endElement(handler, "a"); + endElement(handler, "p"); + handler.endDocument(); + + String result = handler.toString(); + // the ] in the link text is escaped, so it cannot close the link early + assertTrue(result.contains("x\\]"), result); + assertFalse(result.contains("x](https://evil.example)"), result); + assertTrue(result.contains("](https://good.example)"), result); + } + + @Test + public void testLinkDestinationWithSpaceAndParenWrapped() throws Exception { + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(); + handler.startDocument(); + startElement(handler, "p"); + startElement(handler, "a", "href", "https://evil.example) text"); + chars(handler, "click"); + endElement(handler, "a"); + endElement(handler, "p"); + handler.endDocument(); + + String result = handler.toString(); + // a destination with spaces/parens is wrapped in <> so it cannot terminate early + assertTrue(result.contains("](<https://evil.example) text>)"), result); + } + + @Test + public void testTableCellPipeAndNewlineContained() throws Exception { + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(); + handler.startDocument(); + startElement(handler, "table"); + startElement(handler, "tr"); + startElement(handler, "th"); + chars(handler, "h1"); + endElement(handler, "th"); + startElement(handler, "th"); + chars(handler, "h2"); + endElement(handler, "th"); + endElement(handler, "tr"); + startElement(handler, "tr"); + startElement(handler, "td"); + chars(handler, "a|b\nc"); + endElement(handler, "td"); + startElement(handler, "td"); + chars(handler, "d"); + endElement(handler, "td"); + endElement(handler, "tr"); + endElement(handler, "table"); + handler.endDocument(); + + String result = handler.toString(); + // pipe escaped, newline folded: cell stays one cell, row stays two columns + assertTrue(result.contains("|a\\|b c|d|"), result); + } + + @Test + public void testImageAltAndSrcCannotBreakOut() throws Exception { + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(); + handler.startDocument(); + startElement(handler, "p"); + AttributesImpl atts = new AttributesImpl(); + atts.addAttribute("", "alt", "alt", "CDATA", "a]b"); + atts.addAttribute("", "src", "src", "CDATA", "https://evil.example) x"); + startElement(handler, "img", atts); + endElement(handler, "img"); + endElement(handler, "p"); + handler.endDocument(); + + String result = handler.toString(); + assertTrue(result.contains("a\\]b"), result); + assertFalse(result.contains("](https://evil.example) x)"), result); + } + + @Test + public void testBoldInsideLinkIsNested() throws Exception { + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(); + handler.startDocument(); + startElement(handler, "p"); + startElement(handler, "a", "href", "https://example.com";); + startElement(handler, "b"); + chars(handler, "bold link"); + endElement(handler, "b"); + endElement(handler, "a"); + endElement(handler, "p"); + handler.endDocument(); + + String result = handler.toString(); + // inline markup nests correctly inside the link text + assertTrue(result.contains("[**bold link**](https://example.com)"), result); + } } diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml index c175ef3d5a..0fcc273ab1 100644 --- a/tika-parent/pom.xml +++ b/tika-parent/pom.xml @@ -329,6 +329,7 @@ <bouncycastle.version>1.84</bouncycastle.version> <!-- NOTE: sync brotli version with commons-compress--> <brotli.version>0.1.2</brotli.version> + <commonmark.version>0.29.0</commonmark.version> <commons.cli.version>1.11.0</commons.cli.version> <commons.codec.version>1.22.0</commons.codec.version> <commons.collections4.version>4.5.0</commons.collections4.version> diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml index eeddec6075..446a68f97d 100644 --- a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml +++ b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml @@ -29,10 +29,6 @@ <artifactId>tika-vlm</artifactId> <name>Apache Tika VLM module</name> - <properties> - <commonmark.version>0.29.0</commonmark.version> - </properties> - <dependencies> <dependency> <groupId>${project.groupId}</groupId>
