This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 3ae604cb51 TIKA-4769 - improve metadata robustness (#2923)
3ae604cb51 is described below
commit 3ae604cb5179b0fbed71dd5ec2b6b4479f45a9ca
Author: Tim Allison <[email protected]>
AuthorDate: Wed Jul 8 08:24:50 2026 -0400
TIKA-4769 - improve metadata robustness (#2923)
---
.../apache/tika/digest/InputStreamDigester.java | 12 +-
.../java/org/apache/tika/metadata/Metadata.java | 78 ++++++++++++-
.../metadata/filter/CompositeMetadataFilter.java | 2 +-
.../tika/metadata/filter/MetadataFilter.java | 26 ++++-
.../tika/metadata/filter/MetadataFilterBase.java | 3 +-
.../apache/tika/metadata/filter/NoOpFilter.java | 2 +-
.../filter/RemoveByMimeMetadataFilter.java | 2 +-
.../org/apache/tika/parser/CompositeParser.java | 2 +-
.../parser/multiple/AbstractMultipleParser.java | 10 +-
.../java/org/apache/tika/utils/ParserUtils.java | 7 +-
.../metadata/MetadataInternalKeyGuardTest.java | 125 +++++++++++++++++++++
.../geoinfo/GeographicInformationParser.java | 7 +-
.../org/apache/tika/parser/grib/GribParser.java | 27 +++--
.../java/org/apache/tika/parser/hdf/HDFParser.java | 2 +-
.../apache/tika/parser/netcdf/NetCDFParser.java | 29 +++--
.../org/apache/tika/parser/hdf/HDFParserTest.java | 2 +-
.../tika/parser/netcdf/NetCDFParserTest.java | 2 +-
.../tika/inference/AbstractEmbeddingFilter.java | 2 +-
.../tika/inference/JinaEmbeddingFilterTest.java | 6 +-
.../tika/inference/OpenAIEmbeddingFilterTest.java | 12 +-
.../apache/tika/parser/geo/topic/GeoParser.java | 5 +
.../tika/parser/journal/GrobidRESTParser.java | 4 +
.../tika/parser/iwork/NumbersContentHandler.java | 32 +++---
.../tika/parser/iwork/PagesContentHandler.java | 2 +-
.../org/apache/tika/parser/audio/AudioParser.java | 1 -
.../java/org/apache/tika/parser/mp3/Mp3Parser.java | 1 -
.../org/apache/tika/parser/ogg/OggAudioParser.java | 4 +-
.../org/apache/tika/parser/video/FLVParser.java | 6 +-
.../apache/tika/parser/audio/AudioParserTest.java | 7 +-
.../org/apache/tika/parser/mp3/Mp3ParserTest.java | 18 +--
.../apache/tika/parser/video/FLVParserTest.java | 10 +-
.../org/apache/tika/parser/dwg/DWGReadParser.java | 4 +-
.../java/org/apache/tika/parser/mat/MatParser.java | 3 +-
.../org/apache/tika/parser/mat/MatParserTest.java | 3 +-
.../apache/tika/parser/font/TrueTypeParser.java | 2 +-
.../apache/tika/parser/font/FontParsersTest.java | 2 +-
.../rtf/jflex/RTFEmbeddedHandlerTest.java | 2 +-
.../apache/tika/parser/indesign/IDMLParser.java | 3 +-
.../apache/tika/parser/mif/MIFContentHandler.java | 3 +-
.../tika/parser/indesign/IDMLParserTest.java | 5 +-
.../org/apache/tika/parser/mif/MIFParserTest.java | 5 +-
.../tika/parser/pdf/PDMetadataExtractor.java | 25 +++--
.../serialization/FetchEmitTupleDeserializer.java | 4 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 17 ++-
.../filter/AttachmentCountingListFilter.java | 2 +-
.../serialization/serdes/MetadataDeserializer.java | 4 +-
.../filter/AttachmentCountingListFilter.java | 2 +-
.../CustomClassSerializationTest.java | 2 +-
.../tika/serialization/JsonMetadataTest.java | 22 ++++
49 files changed, 422 insertions(+), 136 deletions(-)
diff --git
a/tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java
b/tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java
index 2d1180435b..6371d23c9e 100644
--- a/tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java
+++ b/tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java
@@ -20,9 +20,12 @@ import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.Provider;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.utils.StringUtils;
@@ -32,8 +35,10 @@ import org.apache.tika.utils.StringUtils;
*/
public class InputStreamDigester implements Digester {
+ private static final Map<String, Property> METADATA_KEY_PROPERTIES = new
ConcurrentHashMap<>();
+
private final String algorithm;
- private final String metadataKey;
+ private final Property metadataProperty;
private final Encoder encoder;
/**
@@ -45,7 +50,8 @@ public class InputStreamDigester implements Digester {
*/
public InputStreamDigester(String algorithm, String metadataKey, Encoder
encoder) {
this.algorithm = algorithm;
- this.metadataKey = metadataKey;
+ this.metadataProperty =
+ METADATA_KEY_PROPERTIES.computeIfAbsent(metadataKey,
Property::internalText);
this.encoder = encoder;
}
@@ -107,7 +113,7 @@ public class InputStreamDigester implements Digester {
}
setContentLength(total, metadata);
- metadata.set(metadataKey, encoder.encode(messageDigest.digest()));
+ metadata.set(metadataProperty, encoder.encode(messageDigest.digest()));
tis.rewind();
}
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
index b0ede3526c..54d9c9c5a9 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
@@ -34,6 +34,9 @@ import java.util.Objects;
import java.util.Properties;
import java.util.TimeZone;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.metadata.Property.PropertyType;
import org.apache.tika.metadata.writefilter.MetadataWriteLimiter;
import org.apache.tika.metadata.writefilter.MetadataWriteLimiterFactory;
@@ -47,6 +50,7 @@ public class Metadata
implements CreativeCommons, Geographic, HttpHeaders, Message,
ClimateForcast, TIFF,
TikaMimeKeys, Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(Metadata.class);
private static final MetadataWriteLimiter ACCEPT_ALL = new
MetadataWriteLimiter() {
@Override
@@ -96,6 +100,7 @@ public class Metadata
private MetadataWriteLimiter writeLimiter = ACCEPT_ALL;
+ private transient boolean trusted;
/**
* Constructs a new, empty metadata.
*/
@@ -304,9 +309,66 @@ public class Metadata
* @param value the metadata value.
*/
public void add(final String name, final String value) {
+ if (blockReservedKeyWrite(name)) {
+ return;
+ }
+ addUnchecked(name, value);
+ }
+
+ /** Trusted add, bypassing the reserved-key guard. */
+ private void addUnchecked(final String name, final String value) {
writeLimiter.add(name, value, metadata);
}
+ /**
+ * Mark this Metadata as a trusted transformation target (e.g. a metadata
filter), letting
+ * String writes reach reserved {@code X-TIKA:} keys. Reset when the
transformation is done.
+ */
+ public void setTrusted(boolean trusted) {
+ this.trusted = trusted;
+ }
+
+ public boolean isTrusted() {
+ return trusted;
+ }
+
+ /** Drop String writes to reserved {@code X-TIKA:} keys unless trusted;
use their Property. */
+ private boolean blockReservedKeyWrite(String name) {
+ if (!trusted && name != null &&
name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
+ LOG.debug("Dropping String write to reserved metadata key '{}';
use its Property.", name);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Trusted write for clone/merge/deserialize; reserved keys go via their
Property.
+ *
+ * @param append add rather than set
+ */
+ public void reconstruct(String name, String value, boolean append) {
+ if (name != null &&
name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
+ Property property = Property.get(name);
+ if (property != null) {
+ if (append) {
+ add(property, value);
+ } else {
+ set(property, value);
+ }
+ } else if (append) {
+ addUnchecked(name, value);
+ } else {
+ setUnchecked(name, value);
+ }
+ return;
+ }
+ if (append) {
+ add(name, value);
+ } else {
+ set(name, value);
+ }
+ }
+
/**
* Add a metadata name/value mapping. Add the specified value to the list
of
* values associated to the specified metadata name.
@@ -320,7 +382,7 @@ public class Metadata
set(name, newValues);
} else {
for (String val : newValues) {
- add(name, val);
+ addUnchecked(name, val);
}
}
}
@@ -351,7 +413,7 @@ public class Metadata
set(property, value);
} else {
if (property.isMultiValuePermitted()) {
- add(property.getName(), value);
+ addUnchecked(property.getName(), value);
} else {
throw new PropertyTypeException(
property.getName() + " : " +
property.getPropertyType());
@@ -384,6 +446,14 @@ public class Metadata
* @param value the metadata value, or <code>null</code>
*/
public void set(String name, String value) {
+ if (blockReservedKeyWrite(name)) {
+ return;
+ }
+ setUnchecked(name, value);
+ }
+
+ /** Trusted set, bypassing the reserved-key guard. */
+ private void setUnchecked(String name, String value) {
writeLimiter.set(name, value, metadata);
}
@@ -393,7 +463,7 @@ public class Metadata
if (values != null) {
metadata.remove(name);
for (String v : values) {
- add(name, v);
+ addUnchecked(name, v);
}
} else {
metadata.remove(name);
@@ -419,7 +489,7 @@ public class Metadata
}
}
} else {
- set(property.getName(), value);
+ setUnchecked(property.getName(), value);
}
}
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java
b/tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java
index b5c8e2a647..e2a51f0cd4 100644
---
a/tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java
+++
b/tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java
@@ -46,7 +46,7 @@ public class CompositeMetadataFilter extends MetadataFilter {
}
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
for (MetadataFilter filter : filters) {
filter.filter(metadataList, parseContext);
}
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java
b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java
index 05b0a9bbb3..7b66a3f815 100644
---
a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java
+++
b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java
@@ -19,7 +19,9 @@ package org.apache.tika.metadata.filter;
import java.io.Closeable;
import java.io.IOException;
import java.io.Serializable;
+import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -37,7 +39,29 @@ public abstract class MetadataFilter implements
Serializable, Closeable {
* @param parseContext per-request context (e.g. skip flags, runtime
config)
* @throws TikaException if filtering fails
*/
- public abstract void filter(List<Metadata> metadataList, ParseContext
parseContext)
+ public final void filter(List<Metadata> metadataList, ParseContext
parseContext)
+ throws TikaException {
+ Map<Metadata, Boolean> previous = new IdentityHashMap<>();
+ for (Metadata m : metadataList) {
+ previous.put(m, m.isTrusted());
+ m.setTrusted(true);
+ }
+ try {
+ doFilter(metadataList, parseContext);
+ } finally {
+ previous.forEach(Metadata::setTrusted);
+ }
+ }
+
+ /**
+ * Applies the filter in place. Reserved-key String writes are permitted
here: filters run
+ * on already-parsed, trusted metadata.
+ *
+ * @param metadataList the list to filter (must be mutable)
+ * @param parseContext per-request context
+ * @throws TikaException if filtering fails
+ */
+ protected abstract void doFilter(List<Metadata> metadataList, ParseContext
parseContext)
throws TikaException;
/**
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java
b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java
index 0b66d7018c..2e8dfb5b6a 100644
---
a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java
+++
b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java
@@ -30,7 +30,8 @@ import org.apache.tika.parser.ParseContext;
public abstract class MetadataFilterBase extends MetadataFilter {
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext)
+ throws TikaException {
for (Metadata m : metadataList) {
filter(m);
}
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java
b/tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java
index 92ca54ad6c..7fafc2a67a 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java
@@ -34,7 +34,7 @@ public class NoOpFilter extends MetadataFilter {
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
// no-op
}
}
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java
b/tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java
index 5fb930b7c8..c2b73957e9 100644
---
a/tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java
+++
b/tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java
@@ -74,7 +74,7 @@ public class RemoveByMimeMetadataFilter extends
MetadataFilter {
}
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
metadataList.removeIf(this::shouldRemove);
}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
index 4832da01c4..5e744518ba 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
@@ -345,7 +345,7 @@ public class CompositeParser implements Parser {
for (Metadata m : record.getMetadataList()) {
for (String n : m.names()) {
for (String v : m.getValues(n)) {
- metadata.add(n, v);
+ metadata.reconstruct(n, v, true);
}
}
}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java
b/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java
index 17eff0515e..4e10fdb398 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java
@@ -125,7 +125,7 @@ public abstract class AbstractMultipleParser implements
Parser {
if (newVals == null || newVals.length == 0) {
// Metadata only in previous run, keep old values
for (String val : oldVals) {
- newMetadata.add(n, val);
+ newMetadata.reconstruct(n, val, true);
}
} else if (Arrays.deepEquals(oldVals, newVals)) {
// Metadata is the same, nothing to do
@@ -136,7 +136,7 @@ public abstract class AbstractMultipleParser implements
Parser {
// Use the earlier value(s) in place of this/these
one/s
newMetadata.remove(n);
for (String val : oldVals) {
- newMetadata.add(n, val);
+ newMetadata.reconstruct(n, val, true);
}
continue;
case LAST_WINS:
@@ -147,11 +147,11 @@ public abstract class AbstractMultipleParser implements
Parser {
List<String> vals = new
ArrayList<>(Arrays.asList(oldVals));
newMetadata.remove(n);
for (String oldVal : oldVals) {
- newMetadata.add(n, oldVal);
+ newMetadata.reconstruct(n, oldVal, true);
}
for (String newVal : newVals) {
if (!vals.contains(newVal)) {
- newMetadata.add(n, newVal);
+ newMetadata.reconstruct(n, newVal, true);
vals.add(newVal);
}
}
@@ -318,7 +318,7 @@ public abstract class AbstractMultipleParser implements
Parser {
for (String n : metadata.names()) {
originalMetadata.remove(n);
for (String val : metadata.getValues(n)) {
- originalMetadata.add(n, val);
+ originalMetadata.reconstruct(n, val, true);
}
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
index 552e418859..ecd5287894 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
@@ -43,11 +43,10 @@ public class ParserUtils {
for (String n : m.names()) {
if (!m.isMultiValued(n)) {
- clone.set(n, m.get(n));
+ clone.reconstruct(n, m.get(n), false);
} else {
- String[] vals = m.getValues(n);
- for (String val : vals) {
- clone.add(n, val);
+ for (String val : m.getValues(n)) {
+ clone.reconstruct(n, val, true);
}
}
}
diff --git
a/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java
b/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java
new file mode 100644
index 0000000000..d6bafa5f6e
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.junit.jupiter.api.Test;
+
+/** Reserved {@code X-TIKA:} keys can't be overwritten by String writes, only
via Property. */
+public class MetadataInternalKeyGuardTest {
+
+ @Test
+ public void testStringWriteToInternalKeyIsDropped() {
+ Metadata metadata = new Metadata();
+ // hostile scrape
+ metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "injected");
+ assertNull(metadata.get(TikaCoreProperties.TIKA_CONTENT),
+ "String write to an internal key must be dropped");
+ assertNull(metadata.get(TikaCoreProperties.TIKA_CONTENT.getName()));
+ }
+
+ @Test
+ public void testStringAddToInternalMultiValueKeyIsDropped() {
+ Metadata metadata = new Metadata();
+ metadata.add(TikaCoreProperties.TIKA_PARSED_BY.getName(),
"org.evil.FakeParser");
+ assertArrayEquals(new String[0],
metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY),
+ "String add to an internal key must be dropped");
+ }
+
+ @Test
+ public void testStringWriteCannotOverwriteTrustedInternalValue() {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "trusted");
+ // String-path attempt must not clobber
+ metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "injected");
+ assertEquals("trusted", metadata.get(TikaCoreProperties.TIKA_CONTENT));
+ }
+
+ @Test
+ public void testPropertyWriteToInternalKeyStillWorks() {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "legit");
+ assertEquals("legit", metadata.get(TikaCoreProperties.TIKA_CONTENT));
+
+ metadata.add(TikaCoreProperties.TIKA_PARSED_BY, "p1");
+ metadata.add(TikaCoreProperties.TIKA_PARSED_BY, "p2");
+ assertArrayEquals(new String[] {"p1", "p2"},
+ metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY));
+ }
+
+ @Test
+ public void testNonReservedStringKeysAreUnaffected() {
+ Metadata metadata = new Metadata();
+ // unregistered key passes through
+ metadata.set("my:customKey", "value");
+ assertEquals("value", metadata.get("my:customKey"));
+
+ // a registered non-reserved property (dc:description) written by name
still works
+ metadata.set(TikaCoreProperties.DESCRIPTION.getName(), "hello");
+ assertEquals("hello", metadata.get(TikaCoreProperties.DESCRIPTION));
+ }
+
+ @Test
+ public void testNullNameDoesNotThrow() {
+ Metadata metadata = new Metadata();
+ // no NPE on a null name
+ metadata.set((String) null, "x");
+ }
+
+ @Test
+ public void testReconstructPreservesRegisteredReservedKey() {
+ Metadata metadata = new Metadata();
+ metadata.reconstruct(TikaCoreProperties.TIKA_CONTENT.getName(), "the
content", false);
+ assertEquals("the content",
metadata.get(TikaCoreProperties.TIKA_CONTENT));
+
+ metadata.reconstruct(TikaCoreProperties.TIKA_PARSED_BY.getName(),
"p1", true);
+ metadata.reconstruct(TikaCoreProperties.TIKA_PARSED_BY.getName(),
"p2", true);
+ assertArrayEquals(new String[] {"p1", "p2"},
+ metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY));
+ }
+
+ @Test
+ public void testReconstructPreservesUnregisteredReservedKey() {
+ Metadata metadata = new Metadata();
+ String unregistered = TikaCoreProperties.TIKA_META_PREFIX +
"noSuchRegisteredProperty";
+ assertNull(Property.get(unregistered), "precondition: key must be
unregistered");
+
+ metadata.set(unregistered, "dropped");
+ assertNull(metadata.get(unregistered));
+
+ metadata.reconstruct(unregistered, "kept", false);
+ assertEquals("kept", metadata.get(unregistered));
+
+ metadata.reconstruct(unregistered, "kept2", true);
+ assertArrayEquals(new String[] {"kept", "kept2"},
metadata.getValues(unregistered));
+ }
+
+ @Test
+ public void testTrustedModeAllowsReservedStringWrites() {
+ Metadata metadata = new Metadata();
+ metadata.setTrusted(true);
+ metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "trusted");
+ assertEquals("trusted", metadata.get(TikaCoreProperties.TIKA_CONTENT));
+
+ metadata.setTrusted(false);
+ metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "blocked");
+ assertEquals("trusted", metadata.get(TikaCoreProperties.TIKA_CONTENT));
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
index b385955d8e..ebd964b8f0 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
@@ -64,6 +64,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -234,7 +235,7 @@ public class GeographicInformationParser implements Parser {
for (Identification i : identifications) {
DefaultDataIdentification defaultDataIdentification =
(DefaultDataIdentification) i;
if (i.getCitation() != null && i.getCitation().getTitle() != null)
{
- metadata.add("IdentificationInfoCitationTitle ",
+ metadata.set(TikaCoreProperties.TITLE,
i.getCitation().getTitle().toString());
}
@@ -271,7 +272,7 @@ public class GeographicInformationParser implements Parser {
}
}
if (i.getAbstract() != null) {
- metadata.add("IdentificationInfoAbstract ",
i.getAbstract().toString());
+ metadata.set(TikaCoreProperties.DESCRIPTION,
i.getAbstract().toString());
}
for (Progress p : i.getStatus()) {
metadata.add("IdentificationInfoStatus ", p.name());
@@ -286,7 +287,7 @@ public class GeographicInformationParser implements Parser {
Map<Locale, Charset> localeCharsetMap =
defaultDataIdentification.getLocalesAndCharsets();
for (Locale l : localeCharsetMap.keySet()) {
- metadata.add("IdentificationInfoLanguage-->",
l.getDisplayLanguage(Locale.ENGLISH));
+ metadata.set(TikaCoreProperties.LANGUAGE,
l.getDisplayLanguage(Locale.ENGLISH));
}
CodeListSet<TopicCategory> categoryList =
(CodeListSet<TopicCategory>)
defaultDataIdentification.getTopicCategories();
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
index 94f8d77911..1f98243e06 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
@@ -35,8 +35,8 @@ import ucar.nc2.dataset.NetcdfDataset;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.ClimateForcast;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
@@ -74,12 +74,11 @@ public class GribParser implements Parser {
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
- Property property = resolveMetadataKey(attr.getFullName());
if (attr.getDataType().isString()) {
- metadata.add(property, attr.getStringValue());
+ addGlobalAttribute(metadata, attr.getFullName(),
attr.getStringValue());
} else if (attr.getDataType().isNumeric()) {
- int value = attr.getNumericValue().intValue();
- metadata.add(property, String.valueOf(value));
+ addGlobalAttribute(metadata, attr.getFullName(),
+
String.valueOf(attr.getNumericValue().intValue()));
}
}
@@ -122,11 +121,21 @@ public class GribParser implements Parser {
}
}
- private Property resolveMetadataKey(String localName) {
- if ("title".equals(localName)) {
- return TikaCoreProperties.TITLE;
+ private static final Set<String> CF_GLOBAL_ATTRIBUTES = Set.of(
+ ClimateForcast.PROGRAM_ID, ClimateForcast.COMMAND_LINE,
ClimateForcast.HISTORY,
+ ClimateForcast.TABLE_ID, ClimateForcast.INSTITUTION,
ClimateForcast.SOURCE,
+ ClimateForcast.CONTACT, ClimateForcast.PROJECT_ID,
ClimateForcast.CONVENTIONS,
+ ClimateForcast.REFERENCES, ClimateForcast.ACKNOWLEDGEMENT,
ClimateForcast.REALIZATION,
+ ClimateForcast.EXPERIMENT_ID, ClimateForcast.COMMENT,
ClimateForcast.MODEL_NAME_ENGLISH);
+
+ private static void addGlobalAttribute(Metadata metadata, String name,
String value) {
+ if ("title".equals(name)) {
+ metadata.add(TikaCoreProperties.TITLE, value);
+ } else if (CF_GLOBAL_ATTRIBUTES.contains(name)) {
+ metadata.add(name, value);
+ } else {
+ metadata.add("grib:" + name, value);
}
- return Property.internalText(localName);
}
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
index d3f7c36d85..673f5485fc 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
@@ -105,7 +105,7 @@ public class HDFParser implements Parser {
}
// get file type
- met.set("File-Type-Description", ncFile.getFileTypeDescription());
+ met.set("hdf:File-Type-Description", ncFile.getFileTypeDescription());
// unravel its string attrs
for (Attribute attribute : group.getAttributes()) {
if (attribute.isString()) {
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
index 364a3a5d83..f40f31215c 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
@@ -33,8 +33,8 @@ import ucar.nc2.Variable;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.ClimateForcast;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
@@ -81,15 +81,14 @@ public class NetCDFParser implements Parser {
ParseContext context) throws IOException, SAXException,
TikaException {
try (NetcdfFile ncFile =
NetcdfFile.open(tis.getFile().getAbsolutePath())) {
- metadata.set("File-Type-Description",
ncFile.getFileTypeDescription());
+ metadata.set("netcdf:File-Type-Description",
ncFile.getFileTypeDescription());
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
- Property property = resolveMetadataKey(attr.getFullName());
if (attr.getDataType().isString()) {
- metadata.add(property, attr.getStringValue());
+ addGlobalAttribute(metadata, attr.getFullName(),
attr.getStringValue());
} else if (attr.getDataType().isNumeric()) {
- int value = attr.getNumericValue().intValue();
- metadata.add(property, String.valueOf(value));
+ addGlobalAttribute(metadata, attr.getFullName(),
+ String.valueOf(attr.getNumericValue().intValue()));
}
}
@@ -130,10 +129,20 @@ public class NetCDFParser implements Parser {
}
}
- private Property resolveMetadataKey(String localName) {
- if ("title".equals(localName)) {
- return TikaCoreProperties.TITLE;
+ private static final Set<String> CF_GLOBAL_ATTRIBUTES = Set.of(
+ ClimateForcast.PROGRAM_ID, ClimateForcast.COMMAND_LINE,
ClimateForcast.HISTORY,
+ ClimateForcast.TABLE_ID, ClimateForcast.INSTITUTION,
ClimateForcast.SOURCE,
+ ClimateForcast.CONTACT, ClimateForcast.PROJECT_ID,
ClimateForcast.CONVENTIONS,
+ ClimateForcast.REFERENCES, ClimateForcast.ACKNOWLEDGEMENT,
ClimateForcast.REALIZATION,
+ ClimateForcast.EXPERIMENT_ID, ClimateForcast.COMMENT,
ClimateForcast.MODEL_NAME_ENGLISH);
+
+ private static void addGlobalAttribute(Metadata metadata, String name,
String value) {
+ if ("title".equals(name)) {
+ metadata.add(TikaCoreProperties.TITLE, value);
+ } else if (CF_GLOBAL_ATTRIBUTES.contains(name)) {
+ metadata.add(name, value);
+ } else {
+ metadata.add("netcdf:" + name, value);
}
- return Property.internalText(localName);
}
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
index 5f5f2c16c8..69f61863df 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
@@ -79,6 +79,6 @@ public class HDFParserTest extends TikaTest {
assertNotNull(metadata);
assertEquals("Direct read of HDF4 file through CDM library",
metadata.get("_History"));
assertEquals("Ascending", metadata.get("Pass"));
- assertEquals("Hierarchical Data Format, version 4",
metadata.get("File-Type-Description"));
+ assertEquals("Hierarchical Data Format, version 4",
metadata.get("hdf:File-Type-Description"));
}
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetCDFParserTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetCDFParserTest.java
index 2e04296c80..735bbd5a38 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetCDFParserTest.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetCDFParserTest.java
@@ -52,7 +52,7 @@ public class NetCDFParserTest {
assertEquals(metadata.get(Metadata.REALIZATION), "1");
assertEquals(metadata.get(Metadata.EXPERIMENT_ID),
"720 ppm stabilization experiment (SRESA1B)");
- assertEquals(metadata.get("File-Type-Description"), "NetCDF-3/CDM");
+ assertEquals(metadata.get("netcdf:File-Type-Description"),
"NetCDF-3/CDM");
String content = handler.toString();
assertContains("long_name = \"Surface area\"", content);
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
index a89573c29b..4dffd47929 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
@@ -86,7 +86,7 @@ public abstract class AbstractEmbeddingFilter extends
MetadataFilter {
throws IOException, TikaException;
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
InferenceConfig requestConfig =
parseContext.get(InferenceConfig.class);
if (requestConfig != null && requestConfig.isSkipEmbedding()) {
return;
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/JinaEmbeddingFilterTest.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/JinaEmbeddingFilterTest.java
index 803eb48523..ae25f3915a 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/JinaEmbeddingFilterTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/JinaEmbeddingFilterTest.java
@@ -66,7 +66,7 @@ public class JinaEmbeddingFilterTest {
buildEmbeddingResponse(1, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "Some document
text.");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "Some document text.");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
@@ -85,7 +85,7 @@ public class JinaEmbeddingFilterTest {
buildEmbeddingResponse(1, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "What is
Tika?");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "What is Tika?");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
@@ -119,7 +119,7 @@ public class JinaEmbeddingFilterTest {
+ "# Background\n\nSecond section text.";
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), content);
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, content);
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
index 3c58a21b71..cda29abe2d 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
@@ -72,7 +72,7 @@ public class OpenAIEmbeddingFilterTest {
buildEmbeddingResponse(2, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), content);
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, content);
List<Metadata> metadataList = new ArrayList<>();
metadataList.add(metadata);
@@ -107,7 +107,7 @@ public class OpenAIEmbeddingFilterTest {
buildEmbeddingResponse(1, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "Some text.");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "Some text.");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
@@ -131,7 +131,7 @@ public class OpenAIEmbeddingFilterTest {
@Test
void testBlankContent() throws Exception {
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), " ");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, " ");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
@@ -145,7 +145,7 @@ public class OpenAIEmbeddingFilterTest {
server.enqueue(new TikaTestHttpServer.MockResponse(500,
"{\"error\":\"boom\"}"));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "Some text.");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "Some text.");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
@@ -183,7 +183,7 @@ public class OpenAIEmbeddingFilterTest {
buildEmbeddingResponse(1, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "Single chunk
of text.");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "Single chunk of text.");
List<Metadata> list = new ArrayList<>();
list.add(metadata);
filter.filter(list);
@@ -205,7 +205,7 @@ public class OpenAIEmbeddingFilterTest {
buildEmbeddingResponse(1, 3)));
Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "Some text.");
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "Some text.");
// Pre-populate with an image chunk
Chunk imgChunk = new Chunk(null,
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/geo/topic/GeoParser.java
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/geo/topic/GeoParser.java
index 45556c927a..73efd34d49 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/geo/topic/GeoParser.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/geo/topic/GeoParser.java
@@ -44,6 +44,11 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.geo.topic.gazetteer.GeoGazetteerClient;
import org.apache.tika.parser.geo.topic.gazetteer.Location;
+/**
+ * Locations here are inferred from the text (NER + gazetteer), so they are
stored under
+ * {@code Geographic_*} keys rather than the standard {@code Geographic}
coordinate properties, which
+ * denote coordinates the file asserts about itself.
+ */
@TikaComponent
public class GeoParser implements Parser {
private static final long serialVersionUID = -2241391757440215491L;
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java
index 26461a8282..f6c761ac40 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java
@@ -37,6 +37,10 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
+/**
+ * GROBID header fields are namespaced under {@code grobid:header_*} rather
than mapped to standard
+ * properties (e.g. {@code dc:title}): they are model inferences, not what the
file asserts about itself.
+ */
@TikaComponent
public class GrobidRESTParser {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java
index bda113df79..bb5ffa52a0 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java
@@ -25,7 +25,6 @@ import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Office;
-import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -40,7 +39,7 @@ class NumbersContentHandler extends DefaultHandler {
private boolean parseText = false;
private boolean inMetadata = false;
- private Property metadataKey;
+ private String metadataLocalName;
private String metadataPropertyQName;
private boolean inTable = false;
@@ -69,7 +68,7 @@ class NumbersContentHandler extends DefaultHandler {
numberOfSheets++;
xhtml.startElement("div");
String sheetName = attributes.getValue("ls:workspace-name");
- metadata.add("sheetNames", sheetName);
+ metadata.add("numbers:sheetNames", sheetName);
}
if ("sf:text".equals(qName)) {
@@ -86,13 +85,13 @@ class NumbersContentHandler extends DefaultHandler {
return;
}
- if (inMetadata && metadataKey == null) {
- metadataKey = resolveMetadataKey(localName);
+ if (inMetadata && metadataLocalName == null) {
+ metadataLocalName = localName;
metadataPropertyQName = qName;
}
- if (inMetadata && metadataKey != null && "sf:string".equals(qName)) {
- metadata.add(metadataKey, attributes.getValue("sfa:string"));
+ if (inMetadata && metadataLocalName != null &&
"sf:string".equals(qName)) {
+ addMetadata(metadata, metadataLocalName,
attributes.getValue("sfa:string"));
}
if (!inSheet) {
@@ -198,7 +197,7 @@ class NumbersContentHandler extends DefaultHandler {
if (inMetadata && qName.equals(metadataPropertyQName)) {
metadataPropertyQName = null;
- metadataKey = null;
+ metadataLocalName = null;
}
if (!inSheet) {
@@ -219,16 +218,15 @@ class NumbersContentHandler extends DefaultHandler {
}
}
- private Property resolveMetadataKey(String localName) {
+ private static void addMetadata(Metadata metadata, String localName,
String value) {
if ("authors".equals(localName)) {
- return TikaCoreProperties.CREATOR;
+ metadata.add(TikaCoreProperties.CREATOR, value);
+ } else if ("title".equals(localName)) {
+ metadata.add(TikaCoreProperties.TITLE, value);
+ } else if ("comment".equals(localName)) {
+ metadata.add(TikaCoreProperties.COMMENTS, value);
+ } else {
+ metadata.add("numbers:" + localName, value);
}
- if ("title".equals(localName)) {
- return TikaCoreProperties.TITLE;
- }
- if ("comment".equals(localName)) {
- return TikaCoreProperties.COMMENTS;
- }
- return Property.internalText(localName);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/PagesContentHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/PagesContentHandler.java
index 2298983d1f..dd726ee27a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/PagesContentHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/PagesContentHandler.java
@@ -293,7 +293,7 @@ class PagesContentHandler extends DefaultHandler {
* @return a resolved key that is common in other document types
*/
private Object resolveMetaDataKey(String metaDataLocalName) {
- Object metaDataKey = metaDataLocalName;
+ Object metaDataKey = "pages:" + metaDataLocalName;
if ("sf:authors".equals(metaDataQName)) {
metaDataKey = TikaCoreProperties.CREATOR;
} else if ("sf:title".equals(metaDataQName)) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/AudioParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/AudioParser.java
index da1a0e9a39..543c54fe73 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/AudioParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/AudioParser.java
@@ -92,7 +92,6 @@ public class AudioParser implements Parser {
}
float rate = audioFormat.getSampleRate();
if (rate != AudioSystem.NOT_SPECIFIED) {
- metadata.set("samplerate", String.valueOf(rate));
metadata.set(XMPDM.AUDIO_SAMPLE_RATE, Integer.toString((int)
rate));
}
int bits = audioFormat.getSampleSizeInBits();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
index 7f23209b0f..0fedb73b35 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
@@ -158,7 +158,6 @@ public class Mp3Parser implements Parser {
}
if (audioAndTags.audio != null) {
- metadata.set("samplerate",
String.valueOf(audioAndTags.audio.getSampleRate()));
metadata.set("channels",
String.valueOf(audioAndTags.audio.getChannels()));
metadata.set("version", audioAndTags.audio.getVersion());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
index 563bf40bed..31af6758d4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
@@ -78,7 +78,7 @@ public abstract class OggAudioParser extends AbstractParser {
metadata.set(XMPDM.GENRE, comments.getGenre());
metadata.set(XMPDM.RELEASE_DATE, comments.getDate());
metadata.add(XMP.CREATOR_TOOL, comments.getVendor());
- metadata.add("vendor", comments.getVendor());
+ metadata.add("vorbis:vendor", comments.getVendor());
for (String comment : comments.getComments("comment")) {
metadata.add(XMPDM.LOG_COMMENT.getName(), comment);
@@ -94,7 +94,7 @@ public abstract class OggAudioParser extends AbstractParser {
for (String key : comments.getAllComments().keySet()) {
if (!done.contains(key)) {
for (String value : comments.getAllComments().get(key)) {
- metadata.add(key, value);
+ metadata.add("vorbis:" + key, value);
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java
index fdbf7c06fb..a3c36ebeea 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java
@@ -195,8 +195,8 @@ public class FLVParser implements Parser {
}
metadata.set(Metadata.CONTENT_TYPE, "video/x-flv");
- metadata.set("hasVideo", Boolean.toString((typeFlags & MASK_VIDEO) !=
0));
- metadata.set("hasAudio", Boolean.toString((typeFlags & MASK_AUDIO) !=
0));
+ metadata.set("flv:hasVideo", Boolean.toString((typeFlags & MASK_VIDEO)
!= 0));
+ metadata.set("flv:hasAudio", Boolean.toString((typeFlags & MASK_AUDIO)
!= 0));
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata,
context);
xhtml.startDocument();
@@ -245,7 +245,7 @@ public class FLVParser implements Parser {
if (entry.getValue() == null) {
continue;
}
- metadata.set(entry.getKey(),
entry.getValue().toString());
+ metadata.set("flv:" + entry.getKey(),
entry.getValue().toString());
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/audio/AudioParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/audio/AudioParserTest.java
index 12f753356d..d6e40c1dfd 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/audio/AudioParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/audio/AudioParserTest.java
@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.Tika;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.XMPDM;
public class AudioParserTest {
@@ -33,7 +34,7 @@ public class AudioParserTest {
new
Tika().parseToString(AudioParserTest.class.getResourceAsStream(path), metadata);
assertEquals("audio/vnd.wave", metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("44100.0", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("2", metadata.get("channels"));
assertEquals("16", metadata.get("bits"));
assertEquals("PCM_SIGNED", metadata.get("encoding"));
@@ -49,7 +50,7 @@ public class AudioParserTest {
new
Tika().parseToString(AudioParserTest.class.getResourceAsStream(path), metadata);
assertEquals("audio/x-aiff", metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("44100.0", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("2", metadata.get("channels"));
assertEquals("16", metadata.get("bits"));
assertEquals("PCM_SIGNED", metadata.get("encoding"));
@@ -65,7 +66,7 @@ public class AudioParserTest {
new
Tika().parseToString(AudioParserTest.class.getResourceAsStream(path), metadata);
assertEquals("audio/basic", metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("44100.0", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("2", metadata.get("channels"));
assertEquals("16", metadata.get("bits"));
assertEquals("PCM_SIGNED", metadata.get("encoding"));
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp3/Mp3ParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp3/Mp3ParserTest.java
index 40a9870408..c5d098ffc7 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp3/Mp3ParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp3/Mp3ParserTest.java
@@ -65,7 +65,7 @@ public class Mp3ParserTest extends TikaTest {
assertContains("Rock", content);
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
checkDuration(metadata, 2);
}
@@ -96,7 +96,7 @@ public class Mp3ParserTest extends TikaTest {
// Check un-typed audio properties
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
// Check XMPDM-typed audio properties
@@ -151,7 +151,7 @@ public class Mp3ParserTest extends TikaTest {
assertContains("Rock", content);
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
checkDuration(metadata, 2);
}
@@ -178,7 +178,7 @@ public class Mp3ParserTest extends TikaTest {
assertContains(", disc 1", content);
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
checkDuration(metadata, 2);
@@ -214,7 +214,7 @@ public class Mp3ParserTest extends TikaTest {
metadata.get(XMPDM.LOG_COMMENT));
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
checkDuration(metadata, 2);
}
@@ -239,7 +239,7 @@ public class Mp3ParserTest extends TikaTest {
metadata.get(XMPDM.LOG_COMMENT));
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("1", metadata.get("channels"));
checkDuration(metadata, 2);
}
@@ -269,7 +269,7 @@ public class Mp3ParserTest extends TikaTest {
assertContains("Rock", content);
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("2", metadata.get("channels"));
checkDuration(metadata, 1);
}
@@ -321,7 +321,7 @@ public class Mp3ParserTest extends TikaTest {
assertContains("Plus loin vers l'ouest", content);
assertEquals("MPEG 3 Layer III Version 1", metadata.get("version"));
- assertEquals("44100", metadata.get("samplerate"));
+ assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("2", metadata.get("channels"));
}
@@ -349,7 +349,7 @@ public class Mp3ParserTest extends TikaTest {
// File lacks any audio frames, so we can't know these
assertEquals(null, metadata.get("version"));
- assertEquals(null, metadata.get("samplerate"));
+ assertEquals(null, metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals(null, metadata.get("channels"));
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/video/FLVParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/video/FLVParserTest.java
index 2259f8a3d0..bd37dc3671 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/video/FLVParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/video/FLVParserTest.java
@@ -35,11 +35,11 @@ public class FLVParserTest {
assertEquals("", content);
assertEquals("video/x-flv", metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("true", metadata.get("hasVideo"));
- assertEquals("false", metadata.get("stereo"));
- assertEquals("true", metadata.get("hasAudio"));
- assertEquals("120.0", metadata.get("height"));
- assertEquals("16.0", metadata.get("audiosamplesize"));
+ assertEquals("true", metadata.get("flv:hasVideo"));
+ assertEquals("false", metadata.get("flv:stereo"));
+ assertEquals("true", metadata.get("flv:hasAudio"));
+ assertEquals("120.0", metadata.get("flv:height"));
+ assertEquals("16.0", metadata.get("flv:audiosamplesize"));
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java
index 682005ccee..f774723486 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java
@@ -294,7 +294,7 @@ public class DWGReadParser extends AbstractDWGParser {
if (nextToken.isStructStart()) {
jsonParser.skipChildren();
} else if (nextToken.isScalarValue()) {
- metadata.set(nextFieldName, jsonParser.getText());
+ metadata.set("dwg:" + nextFieldName, jsonParser.getText());
}
}
}
@@ -347,7 +347,7 @@ public class DWGReadParser extends AbstractDWGParser {
} else if ("HYPERLINKBASE".equals(nextFieldName)) {
metadata.set(TikaCoreProperties.RELATION, textVal);
} else if (!Strings.CI.startsWith(nextFieldName,
"unknown")) {
- metadata.set(nextFieldName, textVal);
+ metadata.set("dwg:" + nextFieldName, textVal);
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/main/java/org/apache/tika/parser/mat/MatParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/main/java/org/apache/tika/parser/mat/MatParser.java
index d203b4e7b7..4eac3bf833 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/main/java/org/apache/tika/parser/mat/MatParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/main/java/org/apache/tika/parser/mat/MatParser.java
@@ -36,6 +36,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -79,7 +80,7 @@ public class MatParser implements Parser {
if (parts[2].contains("Created")) {
int lastIndex1 = parts[2].lastIndexOf("Created on:");
String dateCreated = parts[2].substring(lastIndex1 + "Created
on:".length()).trim();
- metadata.set("createdOn", dateCreated);
+ metadata.set(TikaCoreProperties.CREATED, dateCreated);
}
if (parts[1].contains("Platform")) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
index a93e347988..1d640fff2b 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-code-module/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
@@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.ToXMLContentHandler;
@@ -42,7 +43,7 @@ public class MatParserTest extends TikaTest {
assertEquals("PCWIN64", metadata.get("platform"));
assertEquals("MATLAB 5.0 MAT-file", metadata.get("fileType"));
assertEquals("IM", metadata.get("endian"));
- assertEquals("Thu Feb 21 15:52:49 2013", metadata.get("createdOn"));
+ assertEquals("Thu Feb 21 15:52:49 2013",
metadata.get(TikaCoreProperties.CREATED));
assertContains("<li>[1x909 double array]</li>", content);
assertContains("<p>c1:[1x1 struct array]</p>", content);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
index 6e4b67f92f..1be6a091cd 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
@@ -103,7 +103,7 @@ public class TrueTypeParser implements Parser {
metadata.set(AdobeFontMetricParser.MET_PS_NAME,
nr.getString());
}
if (nr.getNameId() == NameRecord.NAME_COPYRIGHT) {
- metadata.set("Copyright", nr.getString());
+ metadata.set(TikaCoreProperties.RIGHTS, nr.getString());
}
if (nr.getNameId() == NameRecord.NAME_TRADEMARK) {
metadata.set("Trademark", nr.getString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/test/java/org/apache/tika/parser/font/FontParsersTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/test/java/org/apache/tika/parser/font/FontParsersTest.java
index ef99442b11..9cdb45be26 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/test/java/org/apache/tika/parser/font/FontParsersTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/test/java/org/apache/tika/parser/font/FontParsersTest.java
@@ -95,7 +95,7 @@ public class FontParsersTest extends TikaTest {
assertEquals("Bold", metadata.get(MET_FONT_SUB_FAMILY_NAME));
assertEquals("OpenSans-Bold", metadata.get(MET_PS_NAME));
- assertEquals("Digitized", metadata.get("Copyright").substring(0, 9));
+ assertEquals("Digitized",
metadata.get(TikaCoreProperties.RIGHTS).substring(0, 9));
assertEquals("Open Sans", metadata.get("Trademark").substring(0, 9));
// Not extracted
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/jflex/RTFEmbeddedHandlerTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/jflex/RTFEmbeddedHandlerTest.java
index 32b8ae58f9..c119ada099 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/jflex/RTFEmbeddedHandlerTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/jflex/RTFEmbeddedHandlerTest.java
@@ -59,7 +59,7 @@ public class RTFEmbeddedHandlerTest {
Metadata copy = new Metadata();
for (String name : metadata.names()) {
for (String val : metadata.getValues(name)) {
- copy.add(name, val);
+ copy.reconstruct(name, val, true);
}
}
extracted.add(copy);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/indesign/IDMLParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/indesign/IDMLParser.java
index 1b8f7a5577..10261e9054 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/indesign/IDMLParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/indesign/IDMLParser.java
@@ -35,6 +35,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -115,7 +116,7 @@ public class IDMLParser implements Parser {
metadata.set("SpreadPageCount", Integer.toString(pageCount));
metadata.set("MasterSpreadPageCount",
Integer.toString(masterSpreadCount));
- metadata.set("TotalPageCount", Integer.toString(pageCount +
masterSpreadCount));
+ metadata.set(Office.PAGE_COUNT, Integer.toString(pageCount +
masterSpreadCount));
xhtml.endDocument();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/mif/MIFContentHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/mif/MIFContentHandler.java
index e76d15a9ee..a41ae1147c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/mif/MIFContentHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/mif/MIFContentHandler.java
@@ -23,6 +23,7 @@ import org.xml.sax.helpers.AttributesImpl;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
import org.apache.tika.sax.XHTMLContentHandler;
/**
@@ -113,7 +114,7 @@ public class MIFContentHandler extends DefaultHandler {
@Override
public void endDocument() {
- metadata.set("PageCount", String.valueOf(bodyPageCount));
+ metadata.set(Office.PAGE_COUNT, String.valueOf(bodyPageCount));
metadata.set("MasterPageCount", String.valueOf(masterPageCount));
metadata.set("ReferencePageCount", String.valueOf(referencePageCount));
metadata.set("TotalPageCount",
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/indesign/IDMLParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/indesign/IDMLParserTest.java
index 505a1e6f05..99293522c5 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/indesign/IDMLParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/indesign/IDMLParserTest.java
@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
import org.apache.tika.metadata.XMP;
import org.apache.tika.parser.Parser;
@@ -40,7 +41,7 @@ public class IDMLParserTest extends TikaTest {
public void testParserToText() throws Exception {
Metadata metadata = new Metadata();
String content = getText("testIndesign.idml", parser, metadata);
- assertEquals("3", metadata.get("TotalPageCount"));
+ assertEquals("3", metadata.get(Office.PAGE_COUNT));
assertEquals("2", metadata.get("MasterSpreadPageCount"));
assertEquals("1", metadata.get("SpreadPageCount"));
assertEquals("application/vnd.adobe.indesign-idml-package",
metadata.get(Metadata.CONTENT_TYPE));
@@ -59,7 +60,7 @@ public class IDMLParserTest extends TikaTest {
// failure below could be because of missing javax.xml.bind if xmpbox
2.* is used
// check the swallowed throwable in XMPMetadataExtractor.parse()
assertEquals("Adobe InDesign CC 14.0 (Windows)",
metadata.get(XMP.CREATOR_TOOL));
- assertEquals("3", metadata.get("TotalPageCount"));
+ assertEquals("3", metadata.get(Office.PAGE_COUNT));
assertContains("<p>Lorem ipsum dolor sit amet, consectetur adipiscing
elit", xml);
assertContains("<meta name=\"xmp:CreatorTool\" content=\"Adobe
InDesign CC 14.0 (Windows)\" />", xml);
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/mif/MIFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/mif/MIFParserTest.java
index ed98728b02..066625bada 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/mif/MIFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/mif/MIFParserTest.java
@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
import org.apache.tika.parser.Parser;
/**
@@ -38,7 +39,7 @@ public class MIFParserTest extends TikaTest {
public void testParserToText() throws Exception {
Metadata metadata = new Metadata();
String content = getText("testFramemakerMif.mif", parser, metadata);
- assertEquals("1", metadata.get("PageCount"));
+ assertEquals("1", metadata.get(Office.PAGE_COUNT));
assertEquals("application/x-mif", metadata.get(Metadata.CONTENT_TYPE));
assertContains("Lorem ipsum dolor sit amet, consectetur adipiscing
elit", content);
}
@@ -47,7 +48,7 @@ public class MIFParserTest extends TikaTest {
public void testParserToXML() throws Exception {
Metadata metadata = new Metadata();
String xml = getXML("testFramemakerMif.mif", parser, metadata).xml;
- assertEquals("1", metadata.get("PageCount"));
+ assertEquals("1", metadata.get(Office.PAGE_COUNT));
assertContains("<p>Lorem ipsum dolor sit amet, consectetur adipiscing
elit", xml);
assertContains("<meta name=\"Content-Type\"
content=\"application/x-mif\" />", xml);
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
index 8ffb16a628..ffe5474899 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
@@ -55,6 +55,7 @@ import org.apache.tika.parser.pdf.xmpschemas.XMPSchemaPDFVT;
import org.apache.tika.parser.pdf.xmpschemas.XMPSchemaPDFX;
import org.apache.tika.parser.pdf.xmpschemas.XMPSchemaPDFXId;
import org.apache.tika.parser.xmp.JempboxExtractor;
+import org.apache.tika.utils.ExceptionUtils;
import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
@@ -95,8 +96,8 @@ public class PDMetadataExtractor {
try {
schema = (XMPSchemaIllustrator)
xmp.getSchemaByClass(XMPSchemaIllustrator.class);
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
if (schema == null) {
@@ -145,8 +146,8 @@ public class PDMetadataExtractor {
try {
schema = (XMPSchemaPDFVT)
xmp.getSchemaByClass(XMPSchemaPDFVT.class);
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
if (schema == null) {
@@ -178,8 +179,8 @@ public class PDMetadataExtractor {
}
}
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
try {
XMPSchemaPDFX XMPSchemaPDFX = (XMPSchemaPDFX)
xmp.getSchemaByClass(XMPSchemaPDFX.class);
@@ -194,8 +195,8 @@ public class PDMetadataExtractor {
}
}
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
}
@@ -207,8 +208,8 @@ public class PDMetadataExtractor {
try {
schema = (XMPSchemaPDFUA)
xmp.getSchemaByClass(XMPSchemaPDFUA.class);
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
if (schema == null) {
@@ -232,8 +233,8 @@ public class PDMetadataExtractor {
try {
schema = (XMPSchemaPDFAId)
xmp.getSchemaByClass(XMPSchemaPDFAId.class);
} catch (IOException e) {
- metadata.set(TikaCoreProperties.TIKA_META_PREFIX +
"pdf:metadata-xmp-parse-failed",
- "" + e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
}
if (schema == null) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
index f7ab29295b..338da15e9d 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
@@ -96,10 +96,10 @@ public class FetchEmitTupleDeserializer extends
JsonDeserializer<FetchEmitTuple>
String k = e.getKey();
if (vals.isArray()) {
for (JsonNode arrVal : vals) {
- metadata.add(k, arrVal.textValue());
+ metadata.reconstruct(k, arrVal.textValue(), true);
}
} else {
- metadata.set(k, vals.asText());
+ metadata.reconstruct(k, vals.asText(), false);
}
}
return metadata;
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
index 94074e048a..4245b9e631 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
@@ -231,12 +231,19 @@ class EmitHandler {
}
private void injectUserMetadata(Metadata userMetadata, List<Metadata>
metadataList) {
- for (String n : userMetadata.names()) {
- //overwrite whatever was there
- metadataList.get(0).set(n, null);
- for (String val : userMetadata.getValues(n)) {
- metadataList.get(0).add(n, val);
+ Metadata target = metadataList.get(0);
+ boolean prev = target.isTrusted();
+ target.setTrusted(true);
+ try {
+ for (String n : userMetadata.names()) {
+ //overwrite whatever was there
+ target.set(n, null);
+ for (String val : userMetadata.getValues(n)) {
+ target.add(n, val);
+ }
}
+ } finally {
+ target.setTrusted(prev);
}
}
diff --git
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
index fe72ceef11..4706dbe9d0 100644
---
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
+++
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
@@ -29,7 +29,7 @@ public class AttachmentCountingListFilter extends
MetadataFilter {
private Integer count = 0;
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
if (metadataList == null || metadataList.isEmpty()) {
return;
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
index f02e12f30e..248783f97f 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
@@ -50,14 +50,14 @@ public class MetadataDeserializer extends
JsonDeserializer<Metadata> {
}
if (token.isScalarValue()) {
- metadata.set(field, jsonParser.getText());
+ metadata.reconstruct(field, jsonParser.getText(), false);
} else if (jsonParser.isExpectedStartArrayToken()) {
token = jsonParser.nextToken();
while (token != null) {
if (token == JsonToken.END_ARRAY) {
return true;
} else if (token.isScalarValue()) {
- metadata.add(field, jsonParser.getText());
+ metadata.reconstruct(field, jsonParser.getText(), true);
} else {
break;
}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
index 1d1e6083f5..5cde289d8e 100644
---
a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
+++
b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java
@@ -28,7 +28,7 @@ public class AttachmentCountingListFilter extends
MetadataFilter {
private Integer count = 0;
@Override
- public void filter(List<Metadata> metadataList, ParseContext parseContext)
throws TikaException {
+ protected void doFilter(List<Metadata> metadataList, ParseContext
parseContext) throws TikaException {
if (metadataList == null || metadataList.isEmpty()) {
return;
}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java
index 426493b1b3..e660cc2acc 100644
---
a/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java
@@ -64,7 +64,7 @@ public class CustomClassSerializationTest {
}
@Override
- public void filter(java.util.List<Metadata> metadataList, ParseContext
parseContext) {
+ protected void doFilter(java.util.List<Metadata> metadataList,
ParseContext parseContext) {
for (Metadata metadata : metadataList) {
for (String name : metadata.names()) {
String[] values = metadata.getValues(name);
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/JsonMetadataTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/JsonMetadataTest.java
index e62289ba5d..d2fa94b6b0 100644
---
a/tika-serialization/src/test/java/org/apache/tika/serialization/JsonMetadataTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/JsonMetadataTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.tika.serialization;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -130,4 +131,25 @@ public class JsonMetadataTest {
Metadata deserialized = JsonMetadata.fromJson(new
StringReader(writer.toString()));
assertEquals(m, deserialized);
}
+
+ @Test
+ public void testInternalKeysRoundTrip() throws Exception {
+ //reserved keys must survive serialize->deserialize (reconstructed via
Property).
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.TIKA_CONTENT, "the content");
+ m.add(TikaCoreProperties.TIKA_PARSED_BY,
"org.apache.tika.parser.DefaultParser");
+ m.add(TikaCoreProperties.TIKA_PARSED_BY,
"org.apache.tika.parser.pdf.PDFParser");
+ m.add("k1", "v1");
+
+ StringWriter writer = new StringWriter();
+ JsonMetadata.toJson(m, writer);
+ Metadata deserialized = JsonMetadata.fromJson(new
StringReader(writer.toString()));
+
+ assertEquals("the content",
deserialized.get(TikaCoreProperties.TIKA_CONTENT));
+ assertArrayEquals(
+ new String[] {"org.apache.tika.parser.DefaultParser",
+ "org.apache.tika.parser.pdf.PDFParser"},
+ deserialized.getValues(TikaCoreProperties.TIKA_PARSED_BY));
+ assertEquals(m, deserialized);
+ }
}
