This is an automated email from the ASF dual-hosted git repository. nddipiazza pushed a commit to branch TIKA-4735-content-only-batch-fix in repository https://gitbox.apache.org/repos/asf/tika.git
commit 45ffe1e755125b89f05159723f8682dbf5826f23 Author: Nicholas DiPiazza <[email protected]> AuthorDate: Tue Jun 30 09:03:16 2026 -0500 TIKA-4735: Fix --content-only batch output and remove -h/--help short option conflict - ParseHandler now writes the resolved ParseMode back into ParseContext so EmitHandler (which reads ParseContext directly with no defaultParseMode fallback) correctly invokes content-only emit when parseMode is set only as a PipesConfig default and not explicitly in the per-request context. - Removed the -h short option from --help in TikaAsyncCLI to eliminate the conflict with the documented handler shorthand; --help still works as a long option. - Added reproducer tests: AsyncCliParserTest: testTIKA4735_ContentOnlyHandlerMarkdown, testTIKA4735_ContentOnlyDefaultHandler TikaConfigAsyncWriterTest: testTIKA4735_ContentOnlyMarkdownConfig, testTIKA4735_ContentOnlyDefaultMarkdownConfig AsyncProcessorTest: testContentOnlyFromConfigDefault (integration test that sends a request without ParseMode in the context and asserts the output file is raw content, not a JSON metadata wrapper) Co-authored-by: Copilot <[email protected]> --- .../org/apache/tika/async/cli/TikaAsyncCLI.java | 2 +- .../apache/tika/async/cli/AsyncCliParserTest.java | 30 +++++++++++++ .../tika/async/cli/TikaConfigAsyncWriterTest.java | 51 ++++++++++++++++++++++ .../tika/pipes/core/server/ParseHandler.java | 3 ++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java index d9b96d359b..cd06371b20 100644 --- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java +++ b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java @@ -69,7 +69,7 @@ public class TikaAsyncCLI { options.addOption("o", "outputDir", true, "output directory"); options.addOption("n", "numClients", true, "number of forked clients"); options.addOption(null, "Xmx", true, "heap for the forked clients, e.g. --Xmx 1g"); - options.addOption("h", "help", false, "this help message"); + options.addOption(null, "help", false, "this help message"); options.addOption("T", "timeoutMs", true, "timeout for each parse in milliseconds"); options.addOption(null, "handler", true, "handler type: t=text, h=html, x=xml, m=markdown, b=body, i=ignore (default: m)"); options.addOption("p", "pluginsDir", true, "plugins directory"); diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java index 975506989e..439b17a44c 100644 --- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java +++ b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java @@ -239,4 +239,34 @@ public class AsyncCliParserTest { // Clean up Files.deleteIfExists(result); } + + // TIKA-4735: --content-only --handler m must correctly parse into SimpleAsyncConfig + @Test + public void testTIKA4735_ContentOnlyHandlerMarkdown(@TempDir Path tmp) throws Exception { + Path inputDir = tmp.resolve("input"); + Files.createDirectories(inputDir); + + SimpleAsyncConfig config = TikaAsyncCLI.parseCommandLine( + new String[]{"--inputDir", inputDir.toString(), "--outputDir", "output", + "--content-only", "--handler", "m"}); + + assertTrue(config.isContentOnly(), "TIKA-4735: contentOnly should be true"); + assertTrue(config.isConcatenate(), "TIKA-4735: --content-only implies --concatenate"); + assertEquals(BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, config.getHandlerType(), + "TIKA-4735: handler type should be MARKDOWN"); + } + + // TIKA-4735: --content-only without --handler should still set contentOnly=true with default MARKDOWN handler + @Test + public void testTIKA4735_ContentOnlyDefaultHandler(@TempDir Path tmp) throws Exception { + Path inputDir = tmp.resolve("input"); + Files.createDirectories(inputDir); + + SimpleAsyncConfig config = TikaAsyncCLI.parseCommandLine( + new String[]{"--inputDir", inputDir.toString(), "--outputDir", "output", "--content-only"}); + + assertTrue(config.isContentOnly(), "TIKA-4735: contentOnly should be true"); + assertEquals(BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, config.getHandlerType(), + "TIKA-4735: default handler type should be MARKDOWN"); + } } diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaConfigAsyncWriterTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaConfigAsyncWriterTest.java index c9b0f534bd..39f9728cf4 100644 --- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaConfigAsyncWriterTest.java +++ b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaConfigAsyncWriterTest.java @@ -22,10 +22,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.apache.tika.config.loader.TikaJsonConfig; +import org.apache.tika.pipes.api.ParseMode; import org.apache.tika.pipes.core.PipesConfig; import org.apache.tika.sax.BasicContentHandlerFactory; @@ -49,4 +52,52 @@ public class TikaConfigAsyncWriterTest { assertEquals("-Xmx1g", pipesConfig.getForkedJvmArgs().get(0)); } + // TIKA-4735: --content-only --handler m must produce parseMode=CONTENT_ONLY and fileExtension=md + @Test + public void testTIKA4735_ContentOnlyMarkdownConfig(@TempDir Path dir) throws Exception { + SimpleAsyncConfig simpleAsyncConfig = new SimpleAsyncConfig("input", "output", 1, + 30000L, "-Xmx1g", null, null, + BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, SimpleAsyncConfig.ExtractBytesMode.NONE, null, + true, true, null, null, false); + + PluginsWriter pluginsWriter = new PluginsWriter(simpleAsyncConfig, null); + Path tmp = Files.createTempFile(dir, "plugins-content-only-", ".json"); + pluginsWriter.write(tmp); + + TikaJsonConfig tikaJsonConfig = TikaJsonConfig.load(tmp); + PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig); + assertEquals(ParseMode.CONTENT_ONLY, pipesConfig.getParseMode(), + "TIKA-4735: parseMode must be CONTENT_ONLY when --content-only is set"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(tmp.toFile()); + JsonNode fileExt = root.at("/emitters/fse/file-system-emitter/fileExtension"); + assertEquals("md", fileExt.asText(), + "TIKA-4735: file extension must be 'md' when --handler m and --content-only"); + } + + // TIKA-4735: --content-only without a handler should produce parseMode=CONTENT_ONLY and fileExtension=md (MARKDOWN default) + @Test + public void testTIKA4735_ContentOnlyDefaultMarkdownConfig(@TempDir Path dir) throws Exception { + SimpleAsyncConfig simpleAsyncConfig = new SimpleAsyncConfig("input", "output", 1, + 30000L, "-Xmx1g", null, null, + BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, SimpleAsyncConfig.ExtractBytesMode.NONE, null, + true, true, null, null, false); + + PluginsWriter pluginsWriter = new PluginsWriter(simpleAsyncConfig, null); + Path tmp = Files.createTempFile(dir, "plugins-content-only-default-", ".json"); + pluginsWriter.write(tmp); + + TikaJsonConfig tikaJsonConfig = TikaJsonConfig.load(tmp); + PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig); + assertEquals(ParseMode.CONTENT_ONLY, pipesConfig.getParseMode(), + "TIKA-4735: parseMode must be CONTENT_ONLY when --content-only is set"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(tmp.toFile()); + JsonNode fileExt = root.at("/emitters/fse/file-system-emitter/fileExtension"); + assertEquals("md", fileExt.asText(), + "TIKA-4735: file extension must be 'md' for MARKDOWN handler with --content-only"); + } + } diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ParseHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ParseHandler.java index cd02d99767..c52008d0d0 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ParseHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ParseHandler.java @@ -87,6 +87,9 @@ class ParseHandler { List<Metadata> metadataList; //this adds the EmbeddedDocumentByteStore to the parsecontext ParseMode parseMode = getParseMode(parseContext); + // Ensure the resolved mode is visible to EmitHandler, which checks parseContext directly + // and does not have access to the defaultParseMode fallback. + parseContext.set(ParseMode.class, parseMode); ContentHandlerFactory contentHandlerFactory = getContentHandlerFactory(parseContext); if (parseMode == ParseMode.NO_PARSE) { metadataList = detectOnly(fetchEmitTuple, stream, metadata, parseContext);
