This is an automated email from the ASF dual-hosted git repository.
krickert pushed a change to branch OPENNLP-1850_Whitespace-UTF-Normalizae
in repository https://gitbox.apache.org/repos/asf/opennlp.git
from be2ad3dd1 OPENNLP-1850 Unicode-aware text normalization and UAX #29
word tokenizer
add 2d980fba9 OPENNLP-1850 Move normalizer engine out of opennlp-api;
unify token analysis on Term
add fbfc4c925 OPENNLP-1850 Single-source the shared normalization rungs on
Dimension
add 5ab7f8735 OPENNLP-1850 Document bundled Unicode data file licensing
No new revisions were added by this update.
Summary of changes:
LICENSE | 48 ++++++++++
NOTICE | 41 +++++----
.../tools/util/normalizer/AnalyzedToken.java | 34 -------
.../tools/util/normalizer/TextAnalyzer.java | 93 -------------------
.../tools/util/normalizer/TextAnalyzerTest.java | 102 ---------------------
opennlp-core/opennlp-ml/opennlp-dl/pom.xml | 12 +--
.../opennlp/tools/util/normalizer/CharClass.java | 0
.../tools/util/normalizer/CodePointSet.java | 0
.../opennlp/tools/util/normalizer/Dimension.java | 51 ++++++++---
.../tools/util/normalizer/TermAnalyzer.java | 21 ++---
.../tools/util/normalizer/TextNormalizer.java | 12 +--
.../opennlp/tools/util/normalizer/UnicodeDash.java | 0
.../tools/util/normalizer/UnicodeWhitespace.java | 0
.../tools/util/normalizer/CharClassTest.java | 0
.../tools/util/normalizer/CodePointSetTest.java | 0
.../tools/util/normalizer/DimensionTest.java | 45 +++++++++
.../tools/util/normalizer/UnicodeDashTest.java | 0
.../util/normalizer/UnicodeWhitespaceTest.java | 0
opennlp-docs/src/docbkx/normalizer.xml | 31 ++-----
rat-excludes | 6 ++
src/license/NOTICE.template | 30 ++++++
21 files changed, 213 insertions(+), 313 deletions(-)
delete mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/AnalyzedToken.java
delete mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/TextAnalyzer.java
delete mode 100644
opennlp-api/src/test/java/opennlp/tools/util/normalizer/TextAnalyzerTest.java
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/util/normalizer/CharClass.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
(100%)
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/DimensionTest.java
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/util/normalizer/UnicodeDashTest.java
(100%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/util/normalizer/UnicodeWhitespaceTest.java
(100%)
