dschmidt commented on code in PR #3044: URL: https://github.com/apache/tika/pull/3044#discussion_r3870821441
########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java: ########## @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.geogebra; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import org.apache.tika.annotation.TikaComponent; +import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.BoundedInputStream; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PageAnchoring; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.XMLReaderUtils; +import org.apache.tika.zip.utils.ZipFileHelper; + +/** + * Parser for the zip-based GeoGebra formats: worksheets (*.ggb), Notes/Slides + * (*.ggs) and tools (*.ggt). + * <p> + * The construction metadata (title, author, date) and the application + * name/version are read from {@code geogebra.xml} (or, for a tool, from + * {@code geogebra_macro.xml}), and the user-visible text (text objects, inline + * text, captions, tool names and help) is emitted as XHTML paragraphs. For + * Notes/Slides, each {@code _slideN/geogebra.xml} becomes a + * {@code <div class="slide">}, in the order given by {@code structure.json}. + * <p> + * The representative rendering of the document, {@code geogebra_thumbnail.png} + * at the root of a worksheet or tool, or the first available slide thumbnail + * of a Notes/Slides file, is emitted as an embedded document marked with + * {@link TikaCoreProperties.EmbeddedResourceType#THUMBNAIL}, so that clients + * (e.g. the unpacker's sidecar metadata) can pick it as the preview image. + * Thumbnails of the remaining slides are renderings of content that is already + * extracted, so they are skipped. The document script + * {@code geogebra_javascript.js} is emitted as a + * {@link TikaCoreProperties.EmbeddedResourceType#MACRO}, and any other + * embedded file (e.g. inserted pictures) as an embedded document. + * <p> + * A part that cannot be read (an unsupported zip entry, malformed XML) is + * recorded in the metadata and skipped; the remaining parts are still parsed. + */ +@TikaComponent(name = "geogebra-parser") +public class GeoGebraParser implements Parser { + + /** + * Serial version UID + */ + private static final long serialVersionUID = 2114923339149498692L; + + public static final String GEOGEBRA_PREFIX = "geogebra:"; + + /** + * The GeoGebra application flavor the file was written with, + * e.g. "classic", "notes", "graphing". + */ + public static final Property APP_NAME = + Property.internalText(GEOGEBRA_PREFIX + "app-name"); + + /** + * The GeoGebra application version the file was written with. + */ + public static final Property APP_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "app-version"); + + /** + * The GeoGebra XML format version. + */ + public static final Property FORMAT_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "format-version"); + + /** + * The unique id GeoGebra assigns to the document. + */ + public static final Property ID = Property.internalText(GEOGEBRA_PREFIX + "id"); + + /** + * The free-form date string of the construction. This is user-entered + * text, not necessarily a parseable date. + */ + public static final Property DATE = Property.internalText(GEOGEBRA_PREFIX + "date"); + + /** + * The tool names of the macros in a tool file (or in a worksheet with + * embedded macros). The name is the {@code toolName} attribute of the + * macro element. + */ + public static final Property TOOL_NAME = + Property.internalTextBag(GEOGEBRA_PREFIX + "toolName"); + + private static final Set<MediaType> SUPPORTED_TYPES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(MediaType.application("vnd.geogebra.file"), + MediaType.application("vnd.geogebra.slides"), + MediaType.application("vnd.geogebra.tool")))); + + private static final String GEOGEBRA_XML = "geogebra.xml"; + private static final String MACRO_XML = "geogebra_macro.xml"; + private static final String STRUCTURE_JSON = "structure.json"; + private static final String THUMBNAIL_PNG = "geogebra_thumbnail.png"; + private static final String JAVASCRIPT_JS = "geogebra_javascript.js"; + + /** + * Housekeeping entries at the root or in a slide directory that carry no + * user content of their own. The XML files are parsed for text and the + * thumbnails handled separately. + */ + private static final Set<String> HOUSEKEEPING_NAMES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(GEOGEBRA_XML, MACRO_XML, THUMBNAIL_PNG, + "geogebra_defaults2d.xml", "geogebra_defaults3d.xml"))); + + private static final String SLIDE_DIR_PREFIX = "_slide"; + + private static final Pattern SLIDE_XML_PATTERN = + Pattern.compile("^(" + SLIDE_DIR_PREFIX + "\\d+)/" + Pattern.quote(GEOGEBRA_XML) + "$"); + + /** + * structure.json only lists chapters, pages and element ids; a real one is + * a few kilobytes. + */ + private static final long MAX_STRUCTURE_JSON_LENGTH = 1024 * 1024; + + static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Override + public Set<MediaType> getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, + ParseContext context) throws IOException, SAXException, TikaException { + EmbeddedDocumentExtractor embeddedDocumentExtractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); + + ZipFile zipFile; + Object container = tis.getOpenContainer(); + if (container instanceof ZipFile) { + zipFile = (ZipFile) container; + } else { + zipFile = ZipFileHelper.open(tis, null); + tis.setOpenContainer(zipFile); + } + + XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); + xhtml.startDocument(); + List<String> slideIds = getSlideIds(zipFile); + ZipArchiveEntry rootXml = zipFile.getEntry(GEOGEBRA_XML); + ZipArchiveEntry macroXml = zipFile.getEntry(MACRO_XML); + //document metadata comes from the first XML parsed: a worksheet's + //geogebra.xml, a tool's geogebra_macro.xml, or the first slide + boolean documentMetadataPending = true; + if (rootXml != null) { + documentMetadataPending = false; + parseGeoGebraXml(zipFile, rootXml, xhtml, metadata, true, context); + } + if (macroXml != null) { + //a worksheet with macros carries both XMLs; the macro one only + //contributes the tool names then, not the document metadata + parseGeoGebraXml(zipFile, macroXml, xhtml, metadata, documentMetadataPending, context); + documentMetadataPending = false; + } + Map<String, Integer> pageNumbers = new HashMap<>(); + if (!slideIds.isEmpty()) { + metadata.set(PagedText.N_PAGES, slideIds.size()); + int page = 1; + for (String slideId : slideIds) { + pageNumbers.put(slideId, page++); + xhtml.startElement("div", "class", "slide"); + try { + ZipArchiveEntry slideXml = zipFile.getEntry(slideId + "/" + GEOGEBRA_XML); + parseGeoGebraXml(zipFile, slideXml, xhtml, metadata, documentMetadataPending, + context); + documentMetadataPending = false; + } finally { + xhtml.endElement("div"); + } + } + } + handleThumbnail(zipFile, slideIds, xhtml, metadata, context, embeddedDocumentExtractor); + handleOtherEntries(zipFile, pageNumbers, xhtml, metadata, context, + embeddedDocumentExtractor); + xhtml.endDocument(); + } + + /** + * Returns the ordered slide directory names of a Notes/Slides file, or an + * empty list if there are no slides. The slides are the + * {@code _slideN/geogebra.xml} entries; {@code structure.json} only + * supplies their order, slides it does not list (or all of them, if it is + * missing or unreadable) follow in numeric order. + */ + private List<String> getSlideIds(ZipFile zipFile) { + List<String> numericallySorted = new ArrayList<>(); + Enumeration<ZipArchiveEntry> entries = zipFile.getEntries(); + while (entries.hasMoreElements()) { + Matcher m = SLIDE_XML_PATTERN.matcher(entries.nextElement().getName()); + if (m.matches()) { + numericallySorted.add(m.group(1)); + } + } + if (numericallySorted.isEmpty()) { + return Collections.emptyList(); + } + numericallySorted.sort(GeoGebraParser::compareSlideIds); + + Set<String> ordered = new LinkedHashSet<>(); + ZipArchiveEntry structure = zipFile.getEntry(STRUCTURE_JSON); + if (structure != null && zipFile.canReadEntryData(structure)) { + Set<String> knownSlideIds = new HashSet<>(numericallySorted); + try (InputStream is = new BoundedInputStream(MAX_STRUCTURE_JSON_LENGTH, + zipFile.getInputStream(structure))) { + JsonNode root = OBJECT_MAPPER.readTree(is); + for (JsonNode chapter : root.path("chapters")) { + for (JsonNode page : chapter.path("pages")) { + for (JsonNode element : page.path("elements")) { + String id = element.path("id").asText(""); + if (knownSlideIds.contains(id)) { + ordered.add(id); + } + } + } + } + } catch (IOException e) { + //fall through to the numeric order + } + } + ordered.addAll(numericallySorted); + return new ArrayList<>(ordered); + } + + /** + * Compares the digit suffixes of two slide ids numerically without + * parsing them (a crafted id may carry more digits than a long holds): + * leading zeros aside, a shorter digit string is the smaller number and + * equal lengths compare lexicographically. + */ + private static int compareSlideIds(String a, String b) { + String da = stripLeadingZeros(a.substring(SLIDE_DIR_PREFIX.length())); + String db = stripLeadingZeros(b.substring(SLIDE_DIR_PREFIX.length())); + if (da.length() != db.length()) { + return Integer.compare(da.length(), db.length()); + } + int byValue = da.compareTo(db); + return byValue != 0 ? byValue : a.compareTo(b); + } + + private static String stripLeadingZeros(String digits) { + int i = 0; + while (i < digits.length() - 1 && digits.charAt(i) == '0') { + i++; + } + return digits.substring(i); + } + + /** + * Parses one GeoGebra XML for its text and, if {@code documentMetadata} + * is set, the document metadata. A part that cannot be read or is not + * well-formed is recorded in the metadata and skipped. + */ + private void parseGeoGebraXml(ZipFile zipFile, ZipArchiveEntry entry, + XHTMLContentHandler xhtml, Metadata metadata, + boolean documentMetadata, ParseContext context) + throws SAXException { + if (entry == null) { + return; + } + if (!zipFile.canReadEntryData(entry)) { + EmbeddedDocumentUtil.recordEmbeddedStreamException( + new IOException("Unsupported zip entry: " + entry.getName()), metadata); + return; + } + try (InputStream is = zipFile.getInputStream(entry)) { + XMLReaderUtils.parseSAX(is, new EmbeddedContentHandler( + new GeoGebraXMLHandler(xhtml, metadata, documentMetadata)), context); + } catch (SAXException e) { + if (WriteLimitReachedException.isWriteLimitReached(e)) { + throw e; + } + EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata); + } catch (IOException | TikaException e) { + EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata); + } + } + + /** + * Emits the representative thumbnail: the root one of a worksheet or + * tool, or the first slide thumbnail (in slide order) of a Notes/Slides + * file. + */ + private void handleThumbnail(ZipFile zipFile, List<String> slideIds, XHTMLContentHandler xhtml, + Metadata metadata, ParseContext context, + EmbeddedDocumentExtractor embeddedDocumentExtractor) + throws IOException, SAXException { + ZipArchiveEntry entry = zipFile.getEntry(THUMBNAIL_PNG); + for (int i = 0; entry == null && i < slideIds.size(); i++) { + entry = zipFile.getEntry(slideIds.get(i) + "/" + THUMBNAIL_PNG); + } + if (entry != null) { + handleEmbedded(zipFile, entry, TikaCoreProperties.EmbeddedResourceType.THUMBNAIL, + null, xhtml, metadata, context, embeddedDocumentExtractor); + } + } + + /** + * Emits everything that is not GeoGebra housekeeping: the document script + * as a macro, and inserted pictures and other files as embedded documents. + * Housekeeping is matched at the root and in the slide directories only, + * so a file of the same name elsewhere is still emitted. + */ + private void handleOtherEntries(ZipFile zipFile, Map<String, Integer> pageNumbers, + XHTMLContentHandler xhtml, Metadata metadata, + ParseContext context, + EmbeddedDocumentExtractor embeddedDocumentExtractor) + throws IOException, SAXException { + Enumeration<ZipArchiveEntry> entries = zipFile.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + String name = entry.getName(); + String dir = ""; + String basename = name; + int slash = name.indexOf('/'); + if (slash >= 0) { + dir = name.substring(0, slash); + basename = name.substring(slash + 1); + } + boolean knownDir = dir.isEmpty() || pageNumbers.containsKey(dir); + if (knownDir && (HOUSEKEEPING_NAMES.contains(basename) + || (dir.isEmpty() && STRUCTURE_JSON.equals(basename)))) { + continue; + } + TikaCoreProperties.EmbeddedResourceType type = null; + if (knownDir && JAVASCRIPT_JS.equals(basename)) { + type = TikaCoreProperties.EmbeddedResourceType.MACRO; + } + handleEmbedded(zipFile, entry, type, pageNumbers.get(dir), xhtml, metadata, context, + embeddedDocumentExtractor); + } + } + + /** + * Emits one zip entry as an embedded document. Without a given resource + * type, pictures are marked {@link TikaCoreProperties.EmbeddedResourceType#INLINE} + * and other files {@link TikaCoreProperties.EmbeddedResourceType#ATTACHMENT}. + * An entry in a slide directory is tagged with the slide's page number. + */ + private void handleEmbedded(ZipFile zipFile, ZipArchiveEntry entry, + TikaCoreProperties.EmbeddedResourceType type, Integer page, + XHTMLContentHandler xhtml, Metadata parentMetadata, + ParseContext context, + EmbeddedDocumentExtractor embeddedDocumentExtractor) + throws IOException, SAXException { + if (!zipFile.canReadEntryData(entry)) { + EmbeddedDocumentUtil.recordEmbeddedStreamException( + new IOException("Unsupported zip entry: " + entry.getName()), parentMetadata); + return; + } + Metadata embeddedMetadata = Metadata.newInstance(context); + embeddedMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, entry.getName()); + embeddedMetadata.set(TikaCoreProperties.INTERNAL_PATH, entry.getName()); + if (page != null) { + PageAnchoring.applyPageMetadata(embeddedMetadata, Collections.singleton(page)); + } + try (TikaInputStream tisZip = TikaInputStream.get(zipFile.getInputStream(entry))) { + if (type == null) { + MediaType mediaType = EmbeddedDocumentUtil.getDetector(context) + .detect(tisZip, embeddedMetadata, context); + if (mediaType != null) { + embeddedMetadata.set(HttpHeaders.CONTENT_TYPE, mediaType.toString()); + } + type = mediaType != null && "image".equals(mediaType.getType()) + ? TikaCoreProperties.EmbeddedResourceType.INLINE + : TikaCoreProperties.EmbeddedResourceType.ATTACHMENT; + } + embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, type.toString()); + if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata, context)) { + embeddedDocumentExtractor.parseEmbedded(tisZip, new EmbeddedContentHandler(xhtml), + embeddedMetadata, context, false); Review Comment: Fixed: the stream is spooled and reset around detection, the way OpenDocumentParser does it. ########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java: ########## @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.geogebra; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import org.apache.tika.annotation.TikaComponent; +import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.BoundedInputStream; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PageAnchoring; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.XMLReaderUtils; +import org.apache.tika.zip.utils.ZipFileHelper; + +/** + * Parser for the zip-based GeoGebra formats: worksheets (*.ggb), Notes/Slides + * (*.ggs) and tools (*.ggt). + * <p> + * The construction metadata (title, author, date) and the application + * name/version are read from {@code geogebra.xml} (or, for a tool, from + * {@code geogebra_macro.xml}), and the user-visible text (text objects, inline + * text, captions, tool names and help) is emitted as XHTML paragraphs. For + * Notes/Slides, each {@code _slideN/geogebra.xml} becomes a + * {@code <div class="slide">}, in the order given by {@code structure.json}. + * <p> + * The representative rendering of the document, {@code geogebra_thumbnail.png} + * at the root of a worksheet or tool, or the first available slide thumbnail + * of a Notes/Slides file, is emitted as an embedded document marked with + * {@link TikaCoreProperties.EmbeddedResourceType#THUMBNAIL}, so that clients + * (e.g. the unpacker's sidecar metadata) can pick it as the preview image. + * Thumbnails of the remaining slides are renderings of content that is already + * extracted, so they are skipped. The document script + * {@code geogebra_javascript.js} is emitted as a + * {@link TikaCoreProperties.EmbeddedResourceType#MACRO}, and any other + * embedded file (e.g. inserted pictures) as an embedded document. + * <p> + * A part that cannot be read (an unsupported zip entry, malformed XML) is + * recorded in the metadata and skipped; the remaining parts are still parsed. + */ +@TikaComponent(name = "geogebra-parser") +public class GeoGebraParser implements Parser { + + /** + * Serial version UID + */ + private static final long serialVersionUID = 2114923339149498692L; + + public static final String GEOGEBRA_PREFIX = "geogebra:"; + + /** + * The GeoGebra application flavor the file was written with, + * e.g. "classic", "notes", "graphing". + */ + public static final Property APP_NAME = + Property.internalText(GEOGEBRA_PREFIX + "app-name"); + + /** + * The GeoGebra application version the file was written with. + */ + public static final Property APP_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "app-version"); + + /** + * The GeoGebra XML format version. + */ + public static final Property FORMAT_VERSION = + Property.internalText(GEOGEBRA_PREFIX + "format-version"); + + /** + * The unique id GeoGebra assigns to the document. + */ + public static final Property ID = Property.internalText(GEOGEBRA_PREFIX + "id"); + + /** + * The free-form date string of the construction. This is user-entered + * text, not necessarily a parseable date. + */ + public static final Property DATE = Property.internalText(GEOGEBRA_PREFIX + "date"); + + /** + * The tool names of the macros in a tool file (or in a worksheet with + * embedded macros). The name is the {@code toolName} attribute of the + * macro element. + */ + public static final Property TOOL_NAME = + Property.internalTextBag(GEOGEBRA_PREFIX + "toolName"); Review Comment: Intentional: toolName is the verbatim attribute name from the GeoGebra XML, and per the casing rule in metadata-changes-4x.adoc only Tika-coined names are kebab-cased (app-name, app-version, format-version are), verbatim names keep their spelling. ########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraXMLHandler.java: ########## @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.geogebra; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.StringUtils; + +/** + * SAX handler for {@code geogebra.xml} and {@code geogebra_macro.xml}. + * <p> + * Extracts the document metadata from the {@code <geogebra>} root and its + * {@code <construction>} child (when asked to), and emits the user-visible + * text as XHTML paragraphs: the string literals of text object + * {@code <expression>}s, the text runs of {@code <content>} elements (inline + * text, tables, mind maps), element {@code <caption>}s and macro names and + * help texts. + */ +class GeoGebraXMLHandler extends DefaultHandler { + + /** + * A GeoGebra string literal. GeoGebra writes strings between plain + * double quotes without any escaping, so a literal never contains one. + */ + private static final Pattern STRING_LITERAL = Pattern.compile("\"([^\"]*)\""); + + private final XHTMLContentHandler xhtml; + private final Metadata metadata; + private final boolean documentMetadata; + private int depth = 0; + + /** + * @param xhtml the handler paragraphs are written to + * @param metadata the metadata tool names are added to + * @param documentMetadata whether to also fill the document metadata from + * the root and construction elements + */ + GeoGebraXMLHandler(XHTMLContentHandler xhtml, Metadata metadata, boolean documentMetadata) { + this.xhtml = xhtml; + this.metadata = metadata; + this.documentMetadata = documentMetadata; + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) + throws SAXException { + if (depth == 0 && "geogebra".equals(localName)) { + if (documentMetadata) { + setIfNotBlank(GeoGebraParser.APP_NAME, attributes.getValue("app")); + setIfNotBlank(GeoGebraParser.APP_VERSION, attributes.getValue("version")); + setIfNotBlank(GeoGebraParser.FORMAT_VERSION, attributes.getValue("format")); + setIfNotBlank(GeoGebraParser.ID, attributes.getValue("id")); + } + } else if (depth == 1 && "construction".equals(localName)) { + //only the document's own construction; a macro's construction is + //nested one level deeper inside its <macro> element + if (documentMetadata) { + setIfNotBlank(TikaCoreProperties.TITLE, attributes.getValue("title")); + setIfNotBlank(TikaCoreProperties.CREATOR, attributes.getValue("author")); + setIfNotBlank(GeoGebraParser.DATE, attributes.getValue("date")); + } + } else if ("expression".equals(localName)) { + handleExpression(attributes.getValue("exp")); + } else if ("content".equals(localName)) { + handleContent(attributes.getValue("val")); + } else if ("caption".equals(localName)) { + paragraph(attributes.getValue("val")); + } else if ("macro".equals(localName)) { + String toolName = attributes.getValue("toolName"); + if (StringUtils.isBlank(toolName)) { + toolName = attributes.getValue("cmdName"); + } + if (!StringUtils.isBlank(toolName)) { + metadata.add(GeoGebraParser.TOOL_NAME, toolName.trim()); + } + paragraph(toolName); + paragraph(attributes.getValue("toolHelp")); + } + depth++; + } + + @Override + public void endElement(String uri, String localName, String qName) { + depth--; + } + + /** + * Emits the string literals of an expression. A text object's expression + * is either a single literal like {@code "some text"} or, for a dynamic + * text, literals combined with values like {@code "Area = " + a}; the + * literals are the user's text, everything else is geometry and skipped. + */ + private void handleExpression(String exp) throws SAXException { + if (exp == null || exp.indexOf('"') < 0) { + return; + } + StringBuilder sb = new StringBuilder(); + Matcher m = STRING_LITERAL.matcher(exp); + while (m.find()) { + sb.append(m.group(1)); + } + paragraph(sb.toString()); + } + + /** + * Emits the text runs of a rich-text {@code content} value, a JSON array + * of text runs like {@code [{"text":"Hello\n"}]}. All {@code text} fields + * are collected recursively (tables and mind maps nest them), joined, and + * emitted one paragraph per line. + */ + private void handleContent(String val) throws SAXException { + if (val == null) { + return; + } + String trimmed = val.trim(); + if (trimmed.isEmpty() || (trimmed.charAt(0) != '[' && trimmed.charAt(0) != '{')) { + //not a JSON document; a plain string carries no text runs + return; + } + JsonNode root; + try { + root = GeoGebraParser.OBJECT_MAPPER.readTree(trimmed); + } catch (IOException e) { + return; + } + StringBuilder sb = new StringBuilder(); + for (String text : root.findValuesAsText("text")) { + sb.append(text); + } + for (String line : sb.toString().split("\n")) { Review Comment: Fixed: text runs are split on CRLF and CR as well, with a test. ########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/geogebra/GeoGebraParserTest.java: ########## @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.geogebra; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.TikaPagedText; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; + +public class GeoGebraParserTest extends TikaTest { + + private static final String XML_HEAD = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"; + + private static final String PNG = "\u0089PNG"; + + @Test + public void testGGB() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testGeoGebra.ggb"); + Metadata metadata = metadataList.get(0); + assertEquals("application/vnd.geogebra.file", metadata.get(HttpHeaders.CONTENT_TYPE)); + assertEquals("Pythagorean theorem", metadata.get(TikaCoreProperties.TITLE)); + assertEquals("Ada Lovelace", metadata.get(TikaCoreProperties.CREATOR)); + assertEquals("15 January 2026", metadata.get(GeoGebraParser.DATE)); + assertEquals("classic", metadata.get(GeoGebraParser.APP_NAME)); + assertEquals("5.0.815.0", metadata.get(GeoGebraParser.APP_VERSION)); + assertEquals("5.0", metadata.get(GeoGebraParser.FORMAT_VERSION)); + assertEquals("0c34397e-e3e1-4d1c-9cb6-fe6e54b1e88f", metadata.get(GeoGebraParser.ID)); + + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("In a right triangle a² + b² = c²", content); + assertContains("Theorem statement", content); + assertContains("Drag the vertices to explore.", content); + + //the embedded macro is parsed alongside geogebra.xml + assertEquals("Midpoint tool", metadata.get(GeoGebraParser.TOOL_NAME)); + assertContains("Select two points to construct their midpoint", content); + + assertEquals(3, metadataList.size()); + Metadata thumbnail = byName(metadataList, "geogebra_thumbnail.png"); + assertEquals("image/png", thumbnail.get(HttpHeaders.CONTENT_TYPE)); + assertEquals(TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString(), + thumbnail.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + //the document script is user code + Metadata script = byName(metadataList, "geogebra_javascript.js"); + assertEquals(TikaCoreProperties.EmbeddedResourceType.MACRO.toString(), + script.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + + @Test + public void testGGS() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testGeoGebraSlides.ggs"); + Metadata metadata = metadataList.get(0); + assertEquals("application/vnd.geogebra.slides", metadata.get(HttpHeaders.CONTENT_TYPE)); + assertEquals("notes", metadata.get(GeoGebraParser.APP_NAME)); + assertEquals(2, (int) metadata.getInt(PagedText.N_PAGES)); + + //structure.json orders _slide1 before _slide0 + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("First slide text", content); + assertContains("Second slide text", content); + assertTrue(content.indexOf("First slide text") < content.indexOf("Second slide text"), + "slide order should follow structure.json"); + assertContains("<div class=\"slide\">", content); + + //only the first slide's thumbnail is emitted, marked THUMBNAIL + assertEquals(5, metadataList.size()); + Metadata thumbnail = byName(metadataList, "_slide1/geogebra_thumbnail.png"); + assertEquals("image/png", thumbnail.get(HttpHeaders.CONTENT_TYPE)); + assertEquals(TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString(), + thumbnail.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertNull(byName(metadataList, "_slide0/geogebra_thumbnail.png")); + + //the inserted picture is emitted under its full zip entry name, as an + //inline image anchored to its slide (the second page) + Metadata picture = byName(metadataList, "_slide0/8c6976e5b541/photo.png"); + assertEquals("_slide0/8c6976e5b541/photo.png", + picture.get(TikaCoreProperties.INTERNAL_PATH)); + assertEquals("image/png", picture.get(HttpHeaders.CONTENT_TYPE)); + assertEquals(TikaCoreProperties.EmbeddedResourceType.INLINE.toString(), + picture.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertEquals("2", picture.get(TikaPagedText.PAGE_NUMBERS)); + + Metadata script = byName(metadataList, "_slide0/geogebra_javascript.js"); + assertEquals(TikaCoreProperties.EmbeddedResourceType.MACRO.toString(), + script.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + + @Test + public void testGGT() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testGeoGebraTool.ggt"); + Metadata metadata = metadataList.get(0); + assertEquals("application/vnd.geogebra.tool", metadata.get(HttpHeaders.CONTENT_TYPE)); + assertEquals("Midpoint tool", metadata.get(GeoGebraParser.TOOL_NAME)); + assertEquals("classic", metadata.get(GeoGebraParser.APP_NAME)); + + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("Midpoint tool", content); + assertContains("Select two points to construct their midpoint", content); + + //no thumbnail in this tool file; the macro's own construction carries + //no document metadata + assertEquals(1, metadataList.size()); + assertNull(metadata.get(TikaCoreProperties.TITLE)); + } + + @Test + public void testMacroDoesNotOverrideWorksheetMetadata() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("geogebra.xml", geogebra("classic", "5.0.1.0", "doc-id", + "<construction title=\"Worksheet\" author=\"Ada\" date=\"today\">" + + "<expression label=\"t\" exp=\""Body"\"/></construction>")); + entries.put("geogebra_macro.xml", geogebra("other", "9.9.9.9", "macro-id", + "<macro cmdName=\"Mid\" toolName=\"Midpoint\" toolHelp=\"Two points\">" + + "<construction title=\"Macro\" author=\"Bob\" date=\"never\"/></macro>")); + Metadata metadata = parse(entries).get(0); + assertEquals("Worksheet", metadata.get(TikaCoreProperties.TITLE)); + assertEquals("Ada", metadata.get(TikaCoreProperties.CREATOR)); + assertEquals("today", metadata.get(GeoGebraParser.DATE)); + assertEquals("classic", metadata.get(GeoGebraParser.APP_NAME)); + assertEquals("5.0.1.0", metadata.get(GeoGebraParser.APP_VERSION)); + assertEquals("doc-id", metadata.get(GeoGebraParser.ID)); + assertEquals("Midpoint", metadata.get(GeoGebraParser.TOOL_NAME)); + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("Body", content); + assertContains("Two points", content); + } + + @Test + public void testTextExpressions() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("geogebra.xml", geogebra("classic", "5.0.1.0", "id", "<construction>" + + "<expression label=\"t1\" exp=\""plain text"\"/>" + //a dynamic text: literals combined with a value + + "<expression label=\"t2\" exp=\""Area = " + a\"/>" + + "<expression label=\"t3\" exp=\""a"+"b"\"/>" + //geometry, not text + + "<expression label=\"f\" exp=\"x^2 + 1\"/>" + + "<expression label=\"empty\" exp=\"""\"/>" + + "</construction>")); + String content = parse(entries).get(0).get(TikaCoreProperties.TIKA_CONTENT); + assertContains("<p>plain text</p>", content); + assertContains("<p>Area =</p>", content); + assertContains("<p>ab</p>", content); + assertNotContained("x^2", content); + } + + @Test + public void testSlidesWithoutStructureJson() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("_slide10/geogebra.xml", slide("Tenth")); + entries.put("_slide007/geogebra.xml", slide("Seventh")); + entries.put("_slide2/geogebra.xml", slide("Second")); + Metadata metadata = parse(entries, new GeoGebraParser()).get(0); + assertEquals(3, (int) metadata.getInt(PagedText.N_PAGES)); + //numeric order, leading zeros aside + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertTrue(content.indexOf("Second") < content.indexOf("Seventh"), content); + assertTrue(content.indexOf("Seventh") < content.indexOf("Tenth"), content); + } + + @Test + public void testStructureJsonSuppliesOrderOnly() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + //a structure.json that is not JSON at all, and one slide it would not list + entries.put("structure.json", "not json"); + entries.put("_slide0/geogebra.xml", slide("Zero")); + entries.put("_slide1/geogebra.xml", slide("One")); + Metadata metadata = parse(entries).get(0); + assertEquals(2, (int) metadata.getInt(PagedText.N_PAGES)); + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("Zero", content); + assertContains("One", content); + } + + @Test + public void testMalformedSlideDoesNotAbortTheRest() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("structure.json", "{\"chapters\":[{\"pages\":[{\"elements\":" + + "[{\"id\":\"_slide0\"},{\"id\":\"_slide1\"}]}]}]}"); + entries.put("_slide0/geogebra.xml", XML_HEAD + "<geogebra><construction>" + + "<expression label=\"t\" exp=\""Broken"\"/><unclosed>"); + entries.put("_slide1/geogebra.xml", slide("Fine")); + entries.put("_slide1/geogebra_thumbnail.png", PNG); + List<Metadata> metadataList = parse(entries); + Metadata metadata = metadataList.get(0); + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("Fine", content); + //the slide div was closed and the failure recorded + assertContains("</div>", content); + assertNotNull(metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM)); + //the thumbnail that follows is still emitted + assertNotNull(byName(metadataList, "_slide1/geogebra_thumbnail.png")); + } + + @Test + public void testThumbnailFallsBackToNextSlide() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("_slide0/geogebra.xml", slide("Zero")); + entries.put("_slide1/geogebra.xml", slide("One")); + entries.put("_slide1/geogebra_thumbnail.png", PNG); + entries.put("_slide2/geogebra.xml", slide("Two")); + entries.put("_slide2/geogebra_thumbnail.png", PNG); + List<Metadata> metadataList = parse(entries, new GeoGebraParser()); + assertEquals(2, metadataList.size()); + Metadata thumbnail = byName(metadataList, "_slide1/geogebra_thumbnail.png"); + assertEquals(TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString(), + thumbnail.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + + @Test + public void testRootWorksheetAlongsideSlides() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("geogebra.xml", geogebra("notes", "5.2.0.0", "root-id", + "<construction title=\"Root\"><expression label=\"t\" " + + "exp=\""Root text"\"/></construction>")); + entries.put("_slide0/geogebra.xml", slide("Slide text")); + Metadata metadata = parse(entries).get(0); + assertEquals("Root", metadata.get(TikaCoreProperties.TITLE)); + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + assertContains("Root text", content); + assertContains("Slide text", content); + } + + @Test + public void testHousekeepingNamesOnlyMatchAtKnownPlaces() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("geogebra.xml", geogebra("classic", "5.0.1.0", "id", "<construction/>")); + //a script hidden in a subdirectory is not the document script + entries.put("dir/geogebra_javascript.js", "alert(1)"); + entries.put("dir/geogebra.xml", "<geogebra/>"); + List<Metadata> metadataList = parse(entries); + assertEquals(3, metadataList.size()); + Metadata script = byName(metadataList, "dir/geogebra_javascript.js"); + assertEquals(TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString(), + script.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertNotNull(byName(metadataList, "dir/geogebra.xml")); + } + + /** + * A crafted slide id can carry more digits than an int holds; sorting the + * slide ids must not throw a NumberFormatException out of parse(). + */ + @Test + public void testSlideNumberLargerThanIntParses() throws Exception { + Map<String, String> entries = new LinkedHashMap<>(); + entries.put("structure.json", "{\"chapters\":[{\"pages\":[{\"elements\":" + + "[{\"id\":\"_slide0\"},{\"id\":\"_slide99999999999\"}]}]}]}"); + //two slides so that sorting actually compares the ids + entries.put("_slide0/geogebra.xml", "<geogebra format=\"5.0\"></geogebra>"); + entries.put("_slide99999999999/geogebra.xml", "<geogebra format=\"5.0\"></geogebra>"); + assertEquals("application/vnd.geogebra.slides", + parse(entries).get(0).get(HttpHeaders.CONTENT_TYPE)); + } + + private static String geogebra(String app, String version, String id, String body) { + return XML_HEAD + "<geogebra format=\"5.0\" version=\"" + version + "\" app=\"" + app + + "\" id=\"" + id + "\">" + body + "</geogebra>"; + } + + private static String slide(String text) { + return geogebra("notes", "5.2.0.0", "slide-" + text, "<construction>" + + "<element type=\"inlinetext\" label=\"a\"><content val=\"[{"text":"" + + text + "\\n"}]\"/></element></construction>"); + } + + private List<Metadata> parse(Map<String, String> entries) throws Exception { + return parse(entries, null); + } + + /** + * Parses an in-memory zip, through detection or, for a container that + * detection would not attribute to GeoGebra (a slides file without + * structure.json), with the parser directly. + */ + private List<Metadata> parse(Map<String, String> entries, Parser parser) throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(bos)) { + for (Map.Entry<String, String> e : entries.entrySet()) { + zos.putNextEntry(new ZipEntry(e.getKey())); + zos.write(e.getValue().getBytes(StandardCharsets.ISO_8859_1)); Review Comment: Fixed: the helper writes text entries as UTF-8 and takes binary entries as raw bytes. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
