dschmidt commented on code in PR #3044: URL: https://github.com/apache/tika/pull/3044#discussion_r3871463552
########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java: ########## @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.geogebra; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import org.apache.tika.annotation.TikaComponent; +import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.BoundedInputStream; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PageAnchoring; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.XMLReaderUtils; +import org.apache.tika.zip.utils.ZipFileHelper; + +/** + * Parser for the zip-based GeoGebra formats: worksheets (*.ggb), Notes/Slides + * (*.ggs) and tools (*.ggt). + * <p> + * The construction metadata (title, author, date) and the application + * name/version are read from {@code geogebra.xml} (or, for a tool, from + * {@code geogebra_macro.xml}), and the user-visible text (text objects, inline + * text, captions, tool names and help) is emitted as XHTML paragraphs. For + * Notes/Slides, each {@code _slideN/geogebra.xml} becomes a + * {@code <div class="slide">}, in the order given by {@code structure.json}. + * <p> + * The representative rendering of the document, {@code geogebra_thumbnail.png} + * at the root of a worksheet or tool, or the first available slide thumbnail + * of a Notes/Slides file, is emitted as an embedded document marked with + * {@link TikaCoreProperties.EmbeddedResourceType#THUMBNAIL}, so that clients + * (e.g. the unpacker's sidecar metadata) can pick it as the preview image. + * Thumbnails of the remaining slides are renderings of content that is already + * extracted, so they are skipped. The document script + * {@code geogebra_javascript.js} is emitted as a + * {@link TikaCoreProperties.EmbeddedResourceType#MACRO}, and any other + * embedded file (e.g. inserted pictures) as an embedded document. + * <p> + * A part that cannot be read (an unsupported zip entry, malformed XML) is + * recorded in the metadata and skipped; the remaining parts are still parsed. + */ +@TikaComponent(name = "geogebra-parser") +public class GeoGebraParser implements Parser { + + /** + * Serial version UID + */ + private static final long serialVersionUID = 2114923339149498692L; + + public static final String GEOGEBRA_PREFIX = "geogebra:"; + + /** + * The GeoGebra application flavor the file was written with, + * e.g. "classic", "notes", "graphing". + */ + public static final Property APP_NAME = + Property.internalText(GEOGEBRA_PREFIX + "app-name"); + + /** + * The GeoGebra application version the file was written with. + */ + public static final Property APP_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "app-version"); + + /** + * The GeoGebra XML format version. + */ + public static final Property FORMAT_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "format-version"); + + /** + * The unique id GeoGebra assigns to the document. + */ + public static final Property ID = Property.internalText(GEOGEBRA_PREFIX + "id"); + + /** + * The free-form date string of the construction. This is user-entered + * text, not necessarily a parseable date. + */ + public static final Property DATE = Property.internalText(GEOGEBRA_PREFIX + "date"); + + /** + * The tool names of the macros in a tool file (or in a worksheet with + * embedded macros). The name is the {@code toolName} attribute of the + * macro element. + */ + public static final Property TOOL_NAME = + Property.internalTextBag(GEOGEBRA_PREFIX + "toolName"); + + private static final Set<MediaType> SUPPORTED_TYPES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(MediaType.application("vnd.geogebra.file"), + MediaType.application("vnd.geogebra.slides"), + MediaType.application("vnd.geogebra.tool")))); + + private static final String GEOGEBRA_XML = "geogebra.xml"; + private static final String MACRO_XML = "geogebra_macro.xml"; + private static final String STRUCTURE_JSON = "structure.json"; + private static final String THUMBNAIL_PNG = "geogebra_thumbnail.png"; + private static final String JAVASCRIPT_JS = "geogebra_javascript.js"; + + /** + * Housekeeping entries at the root or in a slide directory that carry no + * user content of their own. The XML files are parsed for text and the + * thumbnails handled separately. + */ + private static final Set<String> HOUSEKEEPING_NAMES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(GEOGEBRA_XML, MACRO_XML, THUMBNAIL_PNG, + "geogebra_defaults2d.xml", "geogebra_defaults3d.xml"))); + + private static final String SLIDE_DIR_PREFIX = "_slide"; + + private static final Pattern SLIDE_XML_PATTERN = + Pattern.compile("^(" + SLIDE_DIR_PREFIX + "\\d+)/" + Pattern.quote(GEOGEBRA_XML) + "$"); + + /** + * structure.json only lists chapters, pages and element ids; a real one is + * a few kilobytes. + */ + private static final long MAX_STRUCTURE_JSON_LENGTH = 1024 * 1024; + + static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Override + public Set<MediaType> getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, + ParseContext context) throws IOException, SAXException, TikaException { + EmbeddedDocumentExtractor embeddedDocumentExtractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); + + ZipFile zipFile; + Object container = tis.getOpenContainer(); + if (container instanceof ZipFile) { + zipFile = (ZipFile) container; + } else { + zipFile = ZipFileHelper.open(tis, null); + tis.setOpenContainer(zipFile); + } + + XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); + xhtml.startDocument(); + List<String> slideIds = getSlideIds(zipFile); + ZipArchiveEntry rootXml = zipFile.getEntry(GEOGEBRA_XML); + ZipArchiveEntry macroXml = zipFile.getEntry(MACRO_XML); + //document metadata comes from the first XML parsed: a worksheet's + //geogebra.xml, a tool's geogebra_macro.xml, or the first slide + boolean documentMetadataPending = true; + if (rootXml != null) { + documentMetadataPending = false; + parseGeoGebraXml(zipFile, rootXml, xhtml, metadata, true, context); + } + if (macroXml != null) { + //a worksheet with macros carries both XMLs; the macro one only + //contributes the tool names then, not the document metadata + parseGeoGebraXml(zipFile, macroXml, xhtml, metadata, documentMetadataPending, context); + documentMetadataPending = false; + } + Map<String, Integer> pageNumbers = new HashMap<>(); + if (!slideIds.isEmpty()) { + metadata.set(PagedText.N_PAGES, slideIds.size()); + int page = 1; + for (String slideId : slideIds) { + pageNumbers.put(slideId, page++); + xhtml.startElement("div", "class", "slide"); + try { + ZipArchiveEntry slideXml = zipFile.getEntry(slideId + "/" + GEOGEBRA_XML); + parseGeoGebraXml(zipFile, slideXml, xhtml, metadata, documentMetadataPending, + context); + documentMetadataPending = false; + } finally { + xhtml.endElement("div"); + } + } + } + handleThumbnail(zipFile, slideIds, xhtml, metadata, context, embeddedDocumentExtractor); + handleOtherEntries(zipFile, pageNumbers, xhtml, metadata, context, + embeddedDocumentExtractor); + xhtml.endDocument(); Review Comment: tis.setOpenContainer registers the ZipFile as a closeable resource of the TikaInputStream, so it is closed with the stream; this is the same lifecycle ZipParser and OpenDocumentParser use. ########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-zip-commons/src/test/java/org/apache/tika/detect/zip/GeoGebraDetectionTest.java: ########## @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect.zip; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +/** + * Test case for detecting the zip-based GeoGebra formats by their contents. + * The streams are parsed without a resource name, so detection must rely on + * the zip entry names, not the *.ggb/*.ggs/*.ggt globs. + */ +public class GeoGebraDetectionTest extends TikaTest { + + private List<Metadata> getRecursiveMetadataWithoutName(String fileName) throws Exception { + try (TikaInputStream tis = TikaInputStream.get( + getClass().getResourceAsStream("/test-documents/" + fileName), new Metadata())) { + return getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(), + new ParseContext(), true); Review Comment: Fixed: the test now asserts the resource exists with a clear message. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
