This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new aca20dc144 TIKA-4770: Add a Markdown parser with structured, lossless
XHTML output (#2922)
aca20dc144 is described below
commit aca20dc144a398420bc2193ad97303c81fe2fff1
Author: Kristian Rickert <[email protected]>
AuthorDate: Thu Jul 2 14:26:35 2026 -0400
TIKA-4770: Add a Markdown parser with structured, lossless XHTML output
(#2922)
---
tika-bom/pom.xml | 5 +
.../tika/config/TikaEncodingDetectorTest.java | 6 +-
.../tika-parsers-standard-modules/pom.xml | 1 +
.../pom.xml | 55 +--
.../apache/tika/parser/datauri}/DataURIScheme.java | 2 +-
.../datauri}/DataURISchemeParseException.java | 2 +-
.../tika/parser/datauri}/DataURISchemeUtil.java | 5 +-
.../parser/datauri}/DataURISchemeParserTest.java | 2 +-
.../tika-parser-html-module/pom.xml | 5 +-
.../org/apache/tika/parser/html/HtmlHandler.java | 3 +
.../tika-parser-text-module/pom.xml | 17 +-
.../tika/parser/markdown/MarkdownParser.java | 503 +++++++++++++++++++++
.../tika/parser/markdown/MarkdownParserTest.java | 137 ++++++
.../test/resources/test-documents/testMARKDOWN.md | 25 +
.../test-documents/testMARKDOWN_dataURI.md | 7 +
15 files changed, 713 insertions(+), 62 deletions(-)
diff --git a/tika-bom/pom.xml b/tika-bom/pom.xml
index 3f8a473509..e208ff61b7 100644
--- a/tika-bom/pom.xml
+++ b/tika-bom/pom.xml
@@ -255,6 +255,11 @@
<version>${revision}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parser-datauri-commons</artifactId>
+ <version>${revision}</version>
+ </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parser-digest-commons</artifactId>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index c75d427890..f01cc0804d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -147,7 +147,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(7, parsers.size());
+ assertEquals(8, parsers.size());
EncodingDetector encodingDetector =
((AbstractEncodingDetectorParser)
parsers.get(0)).getEncodingDetector();
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
@@ -177,7 +177,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(8, parsers.size());
+ assertEquals(9, parsers.size());
for (Parser encodingDetectingParser : parsers) {
EncodingDetector encodingDetector =
@@ -203,7 +203,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(7, parsers.size());
+ assertEquals(8, parsers.size());
for (Parser childParser : parsers) {
EncodingDetector encodingDetector =
((AbstractEncodingDetectorParser)
childParser).getEncodingDetector();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
index 1021e59c82..c9e5946ccd 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
@@ -61,6 +61,7 @@
</dependencies>
<modules>
<module>tika-parser-jdbc-commons</module>
+ <module>tika-parser-datauri-commons</module>
<module>tika-parser-digest-commons</module>
<module>tika-parser-mail-commons</module>
<module>tika-parser-xmp-commons</module>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/pom.xml
similarity index 52%
copy from
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
copy to
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/pom.xml
index 0e18b8ce22..312fe15a59 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/pom.xml
@@ -26,40 +26,17 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>tika-parser-text-module</artifactId>
- <name>Apache Tika text parser module</name>
+ <artifactId>tika-parser-datauri-commons</artifactId>
+ <name>Apache Tika data URI commons</name>
<dependencies>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-encoding-detector-mojibuster</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- </dependency>
- <!-- full default chain for tests: html base detector + junk arbiter -->
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-encoding-detector-html</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-ml-junkdetect</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>test</scope>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</dependency>
</dependencies>
@@ -71,31 +48,11 @@
<configuration>
<archive>
<manifestEntries>
-
<Automatic-Module-Name>org.apache.tika.parser.txt</Automatic-Module-Name>
+
<Automatic-Module-Name>org.apache.tika.parser.datauri.commons</Automatic-Module-Name>
</manifestEntries>
</archive>
</configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <inputExcludes>
- <inputExclude>src/test/resources/test-documents/**</inputExclude>
- </inputExcludes>
- </configuration>
</plugin>
</plugins>
</build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
+</project>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURIScheme.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURIScheme.java
similarity index 98%
rename from
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURIScheme.java
rename to
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURIScheme.java
index ee589786e0..ae616f64a6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURIScheme.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURIScheme.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.html;
+package org.apache.tika.parser.datauri;
import java.io.InputStream;
import java.util.Arrays;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeParseException.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeParseException.java
similarity index 96%
rename from
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeParseException.java
rename to
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeParseException.java
index d8c2d8fd03..08cfedb73d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeParseException.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeParseException.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.html;
+package org.apache.tika.parser.datauri;
import org.apache.tika.exception.TikaException;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeUtil.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeUtil.java
similarity index 97%
rename from
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeUtil.java
rename to
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeUtil.java
index 68471d3fb1..ffae401873 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeUtil.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.html;
+package org.apache.tika.parser.datauri;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
@@ -96,7 +96,6 @@ public class DataURISchemeUtil {
}
list.add(dataURIScheme);
}
- return (list == null) ? Collections.EMPTY_LIST : list;
+ return (list == null) ? Collections.emptyList() : list;
}
-
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/DataURISchemeParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/test/java/org/apache/tika/parser/datauri/DataURISchemeParserTest.java
similarity index 98%
rename from
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/DataURISchemeParserTest.java
rename to
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/test/java/org/apache/tika/parser/datauri/DataURISchemeParserTest.java
index 5f3f763d07..16f4a59e0b 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/DataURISchemeParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/test/java/org/apache/tika/parser/datauri/DataURISchemeParserTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.html;
+package org.apache.tika.parser.datauri;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
index b579cb8d50..482701c049 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
@@ -45,8 +45,9 @@
</dependency>
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parser-datauri-commons</artifactId>
+ <version>${project.version}</version>
</dependency>
<!-- test scope -->
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
index 94a4593a95..d68f250979 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
@@ -45,6 +45,9 @@ import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.datauri.DataURIScheme;
+import org.apache.tika.parser.datauri.DataURISchemeParseException;
+import org.apache.tika.parser.datauri.DataURISchemeUtil;
import org.apache.tika.sax.TextContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.StringUtils;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
index 0e18b8ce22..f61c1cc74e 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
@@ -36,13 +36,26 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parser-datauri-commons</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-tables</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-strikethrough</artifactId>
+ </dependency>
<!-- full default chain for tests: html base detector + junk arbiter -->
<dependency>
<groupId>org.apache.tika</groupId>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/markdown/MarkdownParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/markdown/MarkdownParser.java
new file mode 100644
index 0000000000..1c3266a7b4
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/markdown/MarkdownParser.java
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.markdown;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.commonmark.Extension;
+import org.commonmark.ext.gfm.strikethrough.Strikethrough;
+import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension;
+import org.commonmark.ext.gfm.tables.TableBlock;
+import org.commonmark.ext.gfm.tables.TableBody;
+import org.commonmark.ext.gfm.tables.TableCell;
+import org.commonmark.ext.gfm.tables.TableHead;
+import org.commonmark.ext.gfm.tables.TableRow;
+import org.commonmark.ext.gfm.tables.TablesExtension;
+import org.commonmark.node.AbstractVisitor;
+import org.commonmark.node.BlockQuote;
+import org.commonmark.node.BulletList;
+import org.commonmark.node.Code;
+import org.commonmark.node.CustomBlock;
+import org.commonmark.node.CustomNode;
+import org.commonmark.node.Emphasis;
+import org.commonmark.node.FencedCodeBlock;
+import org.commonmark.node.HardLineBreak;
+import org.commonmark.node.Heading;
+import org.commonmark.node.HtmlBlock;
+import org.commonmark.node.HtmlInline;
+import org.commonmark.node.Image;
+import org.commonmark.node.IndentedCodeBlock;
+import org.commonmark.node.Link;
+import org.commonmark.node.LinkReferenceDefinition;
+import org.commonmark.node.ListBlock;
+import org.commonmark.node.ListItem;
+import org.commonmark.node.Node;
+import org.commonmark.node.OrderedList;
+import org.commonmark.node.Paragraph;
+import org.commonmark.node.SoftLineBreak;
+import org.commonmark.node.StrongEmphasis;
+import org.commonmark.node.Text;
+import org.commonmark.node.ThematicBreak;
+import org.commonmark.parser.Parser;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
+
+import org.apache.tika.config.TikaComponent;
+import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.exception.RuntimeSAXException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractEncodingDetectorParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.datauri.DataURIScheme;
+import org.apache.tika.parser.datauri.DataURISchemeParseException;
+import org.apache.tika.parser.datauri.DataURISchemeUtil;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+/**
+ * Parser for Markdown ({@code text/markdown}). Uses commonmark-java (with the
GFM
+ * tables and strikethrough extensions) to emit structured XHTML. Raw HTML in
the
+ * source is emitted as escaped text; data: URIs in image/link destinations
and in
+ * raw HTML are extracted as embedded documents.
+ */
+@TikaComponent
+public class MarkdownParser extends AbstractEncodingDetectorParser {
+
+ private static final long serialVersionUID = 7690358159471186692L;
+
+ private static final MediaType MARKDOWN = MediaType.text("markdown");
+
+ private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MARKDOWN);
+
+ private static final List<Extension> EXTENSIONS =
+ List.of(TablesExtension.create(), StrikethroughExtension.create());
+
+ //immutable and thread-safe
+ private static final Parser COMMONMARK =
Parser.builder().extensions(EXTENSIONS).build();
+
+ public MarkdownParser() {
+ super();
+ }
+
+ public MarkdownParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
+ ParseContext context) throws IOException, SAXException,
TikaException {
+ Node document;
+ try (AutoDetectReader reader = new AutoDetectReader(tis, metadata,
+ getEncodingDetector(context))) {
+ Charset charset = reader.getCharset();
+ metadata.set(Metadata.CONTENT_TYPE, new MediaType(MARKDOWN,
charset).toString());
+ metadata.set(Metadata.CONTENT_ENCODING, charset.name());
+ document = COMMONMARK.parseReader(reader);
+ }
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata,
context);
+ xhtml.startDocument();
+ try {
+ document.accept(new XhtmlMarkdownVisitor(xhtml, metadata,
context));
+ } catch (RuntimeSAXException e) {
+ throw (SAXException) e.getCause();
+ }
+ xhtml.endDocument();
+ }
+
+ /**
+ * Walks the commonmark AST and emits XHTML SAX events.
+ */
+ private static final class XhtmlMarkdownVisitor extends AbstractVisitor {
+
+ private final XHTMLContentHandler xhtml;
+ private final Metadata metadata;
+ private final ParseContext context;
+ private final DataURISchemeUtil dataURISchemeUtil = new
DataURISchemeUtil();
+
+ XhtmlMarkdownVisitor(XHTMLContentHandler xhtml, Metadata metadata,
ParseContext context) {
+ this.xhtml = xhtml;
+ this.metadata = metadata;
+ this.context = context;
+ }
+
+ @Override
+ public void visit(Heading heading) {
+ String tag = "h" + Math.max(1, Math.min(6, heading.getLevel()));
+ start(tag);
+ visitChildren(heading);
+ end(tag);
+ }
+
+ @Override
+ public void visit(Paragraph paragraph) {
+ //tight list items render their paragraph content inline, as in
markdown
+ if (isTightListItemParagraph(paragraph)) {
+ visitChildren(paragraph);
+ return;
+ }
+ start("p");
+ visitChildren(paragraph);
+ end("p");
+ }
+
+ @Override
+ public void visit(BlockQuote blockQuote) {
+ start("blockquote");
+ visitChildren(blockQuote);
+ end("blockquote");
+ }
+
+ @Override
+ public void visit(BulletList bulletList) {
+ start("ul");
+ visitChildren(bulletList);
+ end("ul");
+ }
+
+ @Override
+ public void visit(OrderedList orderedList) {
+ AttributesImpl atts = new AttributesImpl();
+ Integer startNumber = orderedList.getMarkerStartNumber();
+ if (startNumber != null && startNumber != 1) {
+ addAttribute(atts, "start", String.valueOf(startNumber));
+ }
+ start("ol", atts);
+ visitChildren(orderedList);
+ end("ol");
+ }
+
+ @Override
+ public void visit(ListItem listItem) {
+ start("li");
+ visitChildren(listItem);
+ end("li");
+ }
+
+ @Override
+ public void visit(ThematicBreak thematicBreak) {
+ start("hr");
+ end("hr");
+ }
+
+ @Override
+ public void visit(FencedCodeBlock codeBlock) {
+ emitCodeBlock(codeBlock.getInfo(), codeBlock.getLiteral());
+ }
+
+ @Override
+ public void visit(IndentedCodeBlock codeBlock) {
+ emitCodeBlock(null, codeBlock.getLiteral());
+ }
+
+ @Override
+ public void visit(HtmlBlock htmlBlock) {
+ //raw html: keep the text escaped so nothing is injected
+ start("p");
+ characters(htmlBlock.getLiteral());
+ end("p");
+ extractDataURIs(htmlBlock.getLiteral());
+ }
+
+ @Override
+ public void visit(Text text) {
+ characters(text.getLiteral());
+ }
+
+ @Override
+ public void visit(Emphasis emphasis) {
+ start("em");
+ visitChildren(emphasis);
+ end("em");
+ }
+
+ @Override
+ public void visit(StrongEmphasis strongEmphasis) {
+ start("strong");
+ visitChildren(strongEmphasis);
+ end("strong");
+ }
+
+ @Override
+ public void visit(Code code) {
+ start("code");
+ characters(code.getLiteral());
+ end("code");
+ }
+
+ @Override
+ public void visit(Link link) {
+ AttributesImpl atts = new AttributesImpl();
+ addAttribute(atts, "href", link.getDestination());
+ if (link.getTitle() != null) {
+ addAttribute(atts, "title", link.getTitle());
+ }
+ start("a", atts);
+ visitChildren(link);
+ end("a");
+ maybeEmbedDataURI(link.getDestination());
+ }
+
+ @Override
+ public void visit(Image image) {
+ AttributesImpl atts = new AttributesImpl();
+ addAttribute(atts, "src", image.getDestination());
+ String alt = collectText(image);
+ if (!alt.isEmpty()) {
+ addAttribute(atts, "alt", alt);
+ }
+ if (image.getTitle() != null) {
+ addAttribute(atts, "title", image.getTitle());
+ }
+ start("img", atts);
+ end("img");
+ maybeEmbedDataURI(image.getDestination());
+ }
+
+ @Override
+ public void visit(HardLineBreak hardLineBreak) {
+ start("br");
+ end("br");
+ }
+
+ @Override
+ public void visit(SoftLineBreak softLineBreak) {
+ characters("\n");
+ }
+
+ @Override
+ public void visit(HtmlInline htmlInline) {
+ characters(htmlInline.getLiteral());
+ extractDataURIs(htmlInline.getLiteral());
+ }
+
+ @Override
+ public void visit(LinkReferenceDefinition linkReferenceDefinition) {
+ //no rendered output
+ }
+
+ @Override
+ public void visit(CustomBlock customBlock) {
+ if (customBlock instanceof TableBlock) {
+ start("table");
+ visitChildren(customBlock);
+ end("table");
+ } else {
+ visitChildren(customBlock);
+ }
+ }
+
+ @Override
+ public void visit(CustomNode customNode) {
+ if (customNode instanceof Strikethrough) {
+ start("del");
+ visitChildren(customNode);
+ end("del");
+ } else if (customNode instanceof TableHead) {
+ start("thead");
+ visitChildren(customNode);
+ end("thead");
+ } else if (customNode instanceof TableBody) {
+ start("tbody");
+ visitChildren(customNode);
+ end("tbody");
+ } else if (customNode instanceof TableRow) {
+ start("tr");
+ visitChildren(customNode);
+ end("tr");
+ } else if (customNode instanceof TableCell) {
+ TableCell cell = (TableCell) customNode;
+ String tag = cell.isHeader() ? "th" : "td";
+ AttributesImpl atts = new AttributesImpl();
+ String align = alignment(cell.getAlignment());
+ if (align != null) {
+ addAttribute(atts, "align", align);
+ }
+ start(tag, atts);
+ visitChildren(customNode);
+ end(tag);
+ } else {
+ visitChildren(customNode);
+ }
+ }
+
+ private void emitCodeBlock(String info, String literal) {
+ AttributesImpl atts = new AttributesImpl();
+ String language = language(info);
+ if (language != null) {
+ addAttribute(atts, "class", "language-" + language);
+ }
+ //keep any fence info beyond the language token
+ if (info != null && !info.trim().isEmpty() &&
!info.trim().equals(language)) {
+ addAttribute(atts, "data-info", info.trim());
+ }
+ start("pre");
+ start("code", atts);
+ characters(literal);
+ end("code");
+ end("pre");
+ }
+
+ private void maybeEmbedDataURI(String destination) {
+ if (destination == null || !destination.startsWith("data:")) {
+ return;
+ }
+ try {
+ embed(dataURISchemeUtil.parse(destination));
+ } catch (DataURISchemeParseException e) {
+ //swallow
+ }
+ }
+
+ private void extractDataURIs(String literal) {
+ if (literal == null) {
+ return;
+ }
+ for (DataURIScheme dataURIScheme :
dataURISchemeUtil.extract(literal)) {
+ embed(dataURIScheme);
+ }
+ }
+
+ private void embed(DataURIScheme dataURIScheme) {
+ Metadata m = Metadata.newInstance(context);
+ m.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
+ TikaCoreProperties.EmbeddedResourceType.INLINE.toString());
+ if (dataURIScheme.getMediaType() != null) {
+ m.set(Metadata.CONTENT_TYPE,
dataURIScheme.getMediaType().toString());
+ }
+ EmbeddedDocumentExtractor extractor =
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
+ if (extractor.shouldParseEmbedded(m)) {
+ try (TikaInputStream tis =
TikaInputStream.get(dataURIScheme.getInputStream())) {
+ extractor.parseEmbedded(tis, xhtml, m, context, true);
+ } catch (IOException e) {
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e,
metadata);
+ } catch (SAXException e) {
+ throw new RuntimeSAXException(e);
+ }
+ }
+ }
+
+ private void start(String name) {
+ try {
+ xhtml.startElement(name);
+ } catch (SAXException e) {
+ throw new RuntimeSAXException(e);
+ }
+ }
+
+ private void start(String name, AttributesImpl atts) {
+ try {
+ xhtml.startElement(name, atts);
+ } catch (SAXException e) {
+ throw new RuntimeSAXException(e);
+ }
+ }
+
+ private void end(String name) {
+ try {
+ xhtml.endElement(name);
+ } catch (SAXException e) {
+ throw new RuntimeSAXException(e);
+ }
+ }
+
+ private void characters(String text) {
+ try {
+ xhtml.characters(text);
+ } catch (SAXException e) {
+ throw new RuntimeSAXException(e);
+ }
+ }
+
+ private static boolean isTightListItemParagraph(Paragraph paragraph) {
+ Node parent = paragraph.getParent();
+ if (parent instanceof ListItem) {
+ Node list = parent.getParent();
+ return list instanceof ListBlock && ((ListBlock)
list).isTight();
+ }
+ return false;
+ }
+
+ private static String language(String info) {
+ if (info == null) {
+ return null;
+ }
+ String trimmed = info.trim();
+ if (trimmed.isEmpty()) {
+ return null;
+ }
+ int space = trimmed.indexOf(' ');
+ return space < 0 ? trimmed : trimmed.substring(0, space);
+ }
+
+ private static String alignment(TableCell.Alignment alignment) {
+ if (alignment == null) {
+ return null;
+ }
+ switch (alignment) {
+ case LEFT:
+ return "left";
+ case CENTER:
+ return "center";
+ case RIGHT:
+ return "right";
+ default:
+ return null;
+ }
+ }
+
+ /** Flattens inline content to plain text (for img alt), losing no
literals. */
+ private static String collectText(Node node) {
+ StringBuilder sb = new StringBuilder();
+ for (Node child = node.getFirstChild(); child != null; child =
child.getNext()) {
+ if (child instanceof Text) {
+ sb.append(((Text) child).getLiteral());
+ } else if (child instanceof Code) {
+ sb.append(((Code) child).getLiteral());
+ } else if (child instanceof HtmlInline) {
+ sb.append(((HtmlInline) child).getLiteral());
+ } else if (child instanceof SoftLineBreak || child instanceof
HardLineBreak) {
+ sb.append(' ');
+ } else {
+ sb.append(collectText(child));
+ }
+ }
+ return sb.toString();
+ }
+
+ private static void addAttribute(AttributesImpl atts, String name,
String value) {
+ atts.addAttribute("", name, name, "CDATA", value == null ? "" :
value);
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/markdown/MarkdownParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/markdown/MarkdownParserTest.java
new file mode 100644
index 0000000000..88bcb14448
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/markdown/MarkdownParserTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.markdown;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.ToMarkdownContentHandler;
+
+public class MarkdownParserTest extends TikaTest {
+
+ /** Parses markdown from a string via AutoDetectParser (glob detection on
a dummy name). */
+ private XMLResult parseString(String markdown) throws Exception {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.md");
+ try (TikaInputStream tis =
+
TikaInputStream.get(markdown.getBytes(StandardCharsets.UTF_8))) {
+ return getXML(tis, AUTO_DETECT_PARSER, metadata);
+ }
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ XMLResult r = getXML("testMARKDOWN.md", AUTO_DETECT_PARSER);
+ String xhtml = r.xml;
+
+ assertContains("<h1>Title</h1>", xhtml);
+ assertContains("<strong>bold</strong>", xhtml);
+ assertContains("<em>italic</em>", xhtml);
+ assertContains("<del>struck</del>", xhtml);
+ assertContains("<code>inline</code>", xhtml);
+ assertContains("<li>one</li>", xhtml);
+ assertContains("<ol start=\"3\">", xhtml);
+ assertContains("<blockquote>", xhtml);
+ assertContains("href=\"https://example.com\"";, xhtml);
+
+ //full fence info survives alongside the language class
+ assertContains("class=\"language-java\"", xhtml);
+ assertContains("data-info=\"java title="Example"\"", xhtml);
+
+ //gfm table with alignment sections
+ assertContains("<thead>", xhtml);
+ assertContains("<th align=\"left\">L</th>", xhtml);
+ assertContains("<th align=\"center\">C</th>", xhtml);
+ assertContains("<td align=\"right\">c</td>", xhtml);
+
+ //alt text keeps code spans
+ assertContains("alt=\"see config.json for details\"", xhtml);
+ assertContains("title=\"Tika\"", xhtml);
+
+ //detected charset lands in the metadata
+
assertTrue(r.metadata.get(Metadata.CONTENT_TYPE).startsWith("text/markdown;
charset="),
+ r.metadata.get(Metadata.CONTENT_TYPE));
+ assertContains("Café naïve résumé", xhtml);
+ }
+
+ @Test
+ public void testRawHtmlIsEscaped() throws Exception {
+ String xhtml =
+ parseString("before\n\n<script>alert(1)</script>\n\nand inline
<b>bold</b>\n").xml;
+
+ assertFalse(xhtml.contains("<script>"), xhtml);
+ assertContains("<script>alert(1)</script>", xhtml);
+ assertFalse(xhtml.contains("<b>"), xhtml);
+ assertContains("<b>bold</b>", xhtml);
+ }
+
+ @Test
+ public void testOrderedListStartsAtOneNeedsNoAttribute() throws Exception {
+ assertFalse(parseString("1. a\n2. b\n").xml.contains("start="));
+ }
+
+ @Test
+ public void testLineBreaks() throws Exception {
+ assertContains("<br", parseString("hard \nbreak\n").xml);
+ }
+
+ @Test
+ public void testDataURIsBecomeEmbeddedDocuments() throws Exception {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY,
"testMARKDOWN_dataURI.md");
+ List<Metadata> metadataList =
getRecursiveMetadata("testMARKDOWN_dataURI.md", metadata);
+
+ //container + image destination + data uri scraped from the raw html
block
+ assertEquals(3, metadataList.size());
+ assertEquals("image/gif",
metadataList.get(1).get(Metadata.CONTENT_TYPE));
+ assertEquals("image/gif",
metadataList.get(2).get(Metadata.CONTENT_TYPE));
+ assertEquals(TikaCoreProperties.EmbeddedResourceType.INLINE.toString(),
+
metadataList.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ }
+
+ @Test
+ public void testRoundTripsBackToMarkdown() throws Exception {
+ String markdown = "# Title\n\nSome **bold** and *italic* and
~~struck~~ `inline`.\n";
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.md");
+ StringWriter writer = new StringWriter();
+ try (TikaInputStream tis =
+
TikaInputStream.get(markdown.getBytes(StandardCharsets.UTF_8))) {
+ AUTO_DETECT_PARSER.parse(tis, new
ToMarkdownContentHandler(writer), metadata,
+ new ParseContext());
+ }
+ String md = writer.toString();
+
+ assertContains("# Title", md);
+ assertContains("**bold**", md);
+ assertContains("*italic*", md);
+ assertContains("~~struck~~", md);
+ assertContains("`inline`", md);
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN.md
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN.md
new file mode 100644
index 0000000000..313bad8c16
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN.md
@@ -0,0 +1,25 @@
+# Title
+
+Some **bold** and *italic* and ~~struck~~ text with `inline` code.
+
+- one
+- two
+
+3. third
+4. fourth
+
+> a quote
+
+```java title="Example"
+System.out.println("hi");
+```
+
+[link](https://example.com)
+
+| L | C | R |
+|:--|:-:|--:|
+| a | b | c |
+
+
+
+Café naïve résumé
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN_dataURI.md
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN_dataURI.md
new file mode 100644
index 0000000000..c8f7e9967e
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/testMARKDOWN_dataURI.md
@@ -0,0 +1,7 @@
+# Embedded resources
+
+
+
+<script>
+var img =
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7";
+</script>
