This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch OPENNLP-1220
in repository https://gitbox.apache.org/repos/asf/opennlp.git
from a56b281c Checkstyle
add f0741075 OPENNLP-1220 - Address review feedback for BPE tokenization
No new revisions were added by this update.
Summary of changes:
.../main/java/opennlp/tools/tokenize/BPEModel.java | 7 +-
.../java/opennlp/tools/tokenize/BPETokenizer.java | 33 ++-
.../tools/tokenize/BPETokenizerFactory.java | 41 ++--
.../tools/tokenize/BPETokenizerTrainer.java | 67 +++++-
.../tools/tokenize/AbstractBPEModelTest.java | 158 +++++++++++++
...java => AbstractBPETokenizerRealisticTest.java} | 250 +++++++++++----------
.../opennlp/tools/tokenize/BPEModelDeTest.java} | 34 +--
.../opennlp/tools/tokenize/BPEModelEnTest.java} | 31 ++-
.../opennlp/tools/tokenize/BPEModelFrTest.java} | 33 ++-
.../java/opennlp/tools/tokenize/BPEModelTest.java | 159 -------------
.../tools/tokenize/BPETokenizerFactoryTest.java | 16 +-
.../tokenize/BPETokenizerRealisticDeTest.java | 119 ++++++++++
.../tokenize/BPETokenizerRealisticEnTest.java | 124 ++++++++++
.../tokenize/BPETokenizerRealisticEsTest.java | 119 ++++++++++
.../tokenize/BPETokenizerRealisticFrTest.java | 119 ++++++++++
.../tokenize/BPETokenizerRealisticItTest.java | 119 ++++++++++
.../opennlp/tools/tokenize/BPETokenizerTest.java | 8 +-
.../tools/tokenize/BPETokenizerTrainerTest.java | 4 +-
18 files changed, 1070 insertions(+), 371 deletions(-)
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AbstractBPEModelTest.java
rename
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/{BPETokenizerRealisticTest.java
=> AbstractBPETokenizerRealisticTest.java} (51%)
copy
opennlp-core/opennlp-runtime/src/{main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
=> test/java/opennlp/tools/tokenize/BPEModelDeTest.java} (53%)
copy
opennlp-core/opennlp-runtime/src/{main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
=> test/java/opennlp/tools/tokenize/BPEModelEnTest.java} (56%)
copy
opennlp-core/opennlp-runtime/src/{main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
=> test/java/opennlp/tools/tokenize/BPEModelFrTest.java} (54%)
delete mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPEModelTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPETokenizerRealisticDeTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPETokenizerRealisticEnTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPETokenizerRealisticEsTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPETokenizerRealisticFrTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BPETokenizerRealisticItTest.java
