janhoy commented on code in PR #3670: URL: https://github.com/apache/solr/pull/3670#discussion_r2425441971
########## solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java: ########## @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.extraction; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Utils; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +public class TikaServerParser { + private final SAXParser saxParser; + + public TikaServerParser() { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setNamespaceAware(true); + try { + factory.setFeature("http://xml.org/sax/features/external-general-entities";, false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities";, false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd";, false); + } catch (Throwable ignore) { + // Some parsers may not support all features; ignore + } + try { + saxParser = factory.newSAXParser(); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + + /** + * Parses response in XML format from Tika Server /tika endpoint. The result is that the metadata + * object is populated and the content handler is called with extracted text. + */ + public void parseXml(InputStream inputStream, ContentHandler handler, ExtractionMetadata metadata) + throws IOException, SAXException { + DefaultHandler xmlHandler = new TikaXmlResponseSaxContentHandler(handler, metadata); + try (Reader reader = + new XmlSanitizingReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + saxParser.parse(new InputSource(reader), xmlHandler); + } + } + + /** + * Parses response in JSON format from Tika Server /rmeta endpoint. The result is that the + * metadata object is populated, and the content handler is called with extracted text. + * + * @param jsonStream - JSON stream to parse + * @param handler - SAX content handler to call with extracted text + * @param md - metadata object to populate + */ + @SuppressWarnings({"rawtypes", "PatternVariableCanBeUsed"}) + void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMetadata md) + throws IOException, SAXException { + Object parsed = Utils.fromJSON(jsonStream); Review Comment: In case user uses `tikaserver.recursive` option, this code path will be triggered, as the stream is a huge JSON object. Do we want to provide som OOM prevention guardrails here? What if a document misbehaves and produces several Gb of content? Then we'd risk OOM in Solr process. We also grab the content part of the JSON into String `xhtml` further down, doubling heap usage. I imagine some way of aborting the parsing after at a configurable number of bytes when reading stream from Tika. But that would probably have to be implemented in `Utils.fromJSON`? Thoughts? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
