That was it. Reversing the order fixed it. Thanks for wading through the lack of whitespace. ThanksKyle
> From: luc...@mikemccandless.com > Date: Tue, 15 Oct 2013 09:24:17 -0400 > Subject: Re: Payloads disabled in 4.5? > To: java-user@lucene.apache.org > > Something catastrophic went wrong with the whitespace in your email ... > > But I think the problem is in your incrementToken impl: you cannot set > the payload, and then call input.incrementToken, because > input.incrementToken will clear all attributes (including payload). > Try reversing the order of those two? > > Mike McCandless > > http://blog.mikemccandless.com > > > On Tue, Oct 15, 2013 at 8:13 AM, Kyle Judson <kvjud...@hotmail.com> wrote: > > > > > > > > > > > > > > > > > > Hi All, > > > > > > I'm trying to create an index with payloads using Lucene 4.5 but the > > payloads don't seem to be getting into the index. > > > > > > I looked into the code and it looks like storePayloads argument to > > addOrUpdateInternal when called from addOrUpdate in FieldInfos is set to > > false. Is this my problem? If so should I be using a different indexer? > > > > > > The indexing and reading coded is below. > > ThanksKyle > > The indexing code:import java.io.File;import java.io.IOException;import > > java.nio.ByteBuffer; > > import org.apache.commons.logging.Log;import > > org.apache.commons.logging.LogFactory;import > > org.apache.lucene.analysis.Analyzer;import > > org.apache.lucene.analysis.AnalyzerWrapper;import > > org.apache.lucene.analysis.TokenFilter;import > > org.apache.lucene.analysis.TokenStream;import > > org.apache.lucene.analysis.standard.StandardAnalyzer;import > > org.apache.lucene.analysis.tokenattributes.PayloadAttribute;import > > org.apache.lucene.document.Document;import > > org.apache.lucene.document.Field;import > > org.apache.lucene.document.FieldType;import > > org.apache.lucene.document.TextField;import > > org.apache.lucene.index.IndexWriter;import > > org.apache.lucene.index.IndexWriterConfig;import > > org.apache.lucene.index.IndexWriterConfig.OpenMode;import > > org.apache.lucene.index.IndexableField;import > > org.apache.lucene.store.Directory;import > > org.apache.lucene.store.FSDirectory;import > > org.apache.lucene.util.BytesRef;import org.apache.lucene.util.Version; > > > > public class TestIndex2 { private static Log LOG = > > LogFactory.getLog(TestIndex2.class); MetricAnalyzer analyzer; > > Directory directory; IndexWriterConfig configIndex; IndexWriter > > iwriter; public TestIndex2(File fileOut) throws IOException { > > analyzer = new MetricAnalyzer(Analyzer.PER_FIELD_REUSE_STRATEGY); > > directory = FSDirectory.open(fileOut); > > configIndex = new IndexWriterConfig(Version.LUCENE_45, analyzer); > > configIndex.setOpenMode(OpenMode.CREATE); iwriter = new > > IndexWriter(directory, configIndex); } // end TestIndex2 > > public void close() { if (iwriter != null) { > > try { iwriter.close(); > > } catch (IOException e) { LOG.error("Unable > > to close IndexWriter", e); } } > > if (directory != null) { try { > > directory.close(); } catch (IOException e) { > > LOG.error("Unable to close Directory", e); > > } } } // end close BytesRef > > encodeMetrics(long[] metrics) { ByteBuffer buffer = > > ByteBuffer.allocate(metrics.length * 8); for (long metric : > > metrics) { buffer.putLong(metric); } > > return new BytesRef(buffer.array()); } // end encodeMetrics > > public void index() throws IOException { FieldType type = > > new FieldType(TextField.TYPE_NOT_STORED); > > type.setStoreTermVectorPayloads(true); > > type.setStoreTermVectorPositions(true); > > type.setStoreTermVectors(true); Document doc = new > > Document(); > > long[] metrics = { 1, 2, 3 }; > > analyzer.setPayload(encodeMetrics(metrics)); > > IndexableField indexField = new Field("test", "one", type); > > doc.add(indexField); indexField = new Field("test", "two", > > type); doc.add(indexField); > > indexField = new Field("test_query", "one", type); > > doc.add(indexField); indexField = new Field("test_query", > > "two", type); doc.add(indexField); > > iwriter.addDocument(doc); } // end index public class > > MetricTokenFilter extends TokenFilter { private > > MetricAnalyzer analyzer; private PayloadAttribute > > payloadAttr; > > protected MetricTokenFilter(MetricAnalyzer analyzer, > > TokenStream input) { super(input); > > this.analyzer = analyzer; this.payloadAttr = > > addAttribute(PayloadAttribute.class); } // end > > MetricTokenFilter @Override > > public boolean incrementToken() throws IOException { > > payloadAttr.setPayload(analyzer.getPayload()); return > > input.incrementToken(); } // end incrementToken } > > // end class MetricTokenFilter public class MetricAnalyzer > > extends AnalyzerWrapper { BytesRef payload; > > protected MetricAnalyzer(Analyzer.ReuseStrategy > > reuseStrategy) { super(reuseStrategy); } > > // end MetricAnalyzer protected void > > setPayload(BytesRef payload) { this.payload = payload; } protected > > BytesRef getPayload() { return payload; } > > @Override protected Analyzer > > getWrappedAnalyzer(String fieldName) { return new > > StandardAnalyzer(Version.LUCENE_45); } // end getWrappedAnalyzer > > @Override protected > > TokenStreamComponents wrapComponents(String fieldName, > > TokenStreamComponents components) { if > > (fieldName.endsWith("_query")) { return > > components; } else { > > MetricTokenFilter filter = new MetricTokenFilter(this, > > components.getTokenStream()); return new > > TokenStreamComponents(components.getTokenizer(), filter); > > } } // end wrapComponents > > } // end class MetricAnalyzer public static void > > main(String[] args) throws IOException { File fileOut = new > > File("test_index"); TestIndex2 iwriter = new TestIndex2(fileOut); > > try { iwriter.index(); } finally > > { iwriter.close(); } } // end main > > } // end class TestIndex2 > > > > The reading code:import java.io.File;import java.io.IOException;import > > java.nio.ByteBuffer;import java.util.List; > > import org.apache.lucene.index.AtomicReaderContext;import > > org.apache.lucene.index.DirectoryReader;import > > org.apache.lucene.index.DocsAndPositionsEnum;import > > org.apache.lucene.index.IndexReader;import > > org.apache.lucene.index.Terms;import > > org.apache.lucene.index.TermsEnum;import > > org.apache.lucene.search.DocIdSetIterator;import > > org.apache.lucene.store.Directory;import > > org.apache.lucene.store.FSDirectory;import > > org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef; > > > > public class TestQuery2 { > > public static long[] decodeMetrics(BytesRef metrics, int count, > > long[] reuse) { if (reuse == null) { reuse = new > > long[count]; } ByteBuffer > > buffer = ByteBuffer.wrap(metrics.bytes); for (int x=0; x<count; > > x++) { reuse[x] = buffer.getLong(); } > > return reuse; } // end decodeMetrics public static void > > report (IndexReader ireader, String field) throws IOException { > > int count = 3; List<AtomicReaderContext> leaves = > > ireader.leaves(); for (AtomicReaderContext leaf : leaves) { > > Terms t = leaf.reader().terms(field); if > > (t == null) { System.out.println("Invalid > > field: '"+field+"'"); return; > > } TermsEnum terms = t.iterator(null); > > BytesRef term; long[] metrics = null; > > while ((term = terms.next()) != null) { int > > docCount = terms.docFreq(); Bits bits = new > > Bits.MatchAllBits(docCount); > > System.out.print ("\t" + > > term.utf8ToString() + "\t"); > > boolean first = true; > > DocsAndPositionsEnum docPositions = terms.docsAndPositions(bits, null, > > DocsAndPositionsEnum.FLAG_PAYLOADS); if > > (docPositions == null) { > > System.out.println("No doc positions"); } else { > > while (docPositions.nextDoc() != > > DocIdSetIterator.NO_MORE_DOCS) { > > for (int pos=0; pos<docPositions.freq(); pos++) { > > docPositions.nextPosition(); > > BytesRef > > payload = docPositions.getPayload(); > > if (payload != null) { > > metrics = TestQuery2.decodeMetrics(payload, count, metrics); > > for (int x=0; > > x<count; x++) { > > if (!first) { > > System.out.print(","); > > } else { > > first = false; > > } > > System.out.print(metrics[x]); > > } > > } > > } } > > } System.out.println(); } > > } } // end report public static void > > main(String[] args) throws IOException { File fileOut = > > new File("test_index"); Directory directory = > > FSDirectory.open(fileOut); > > DirectoryReader ireader = DirectoryReader.open(directory); > > try { TestQuery2.report(ireader, "test"); > > } finally { directory.close(); } > > } // end main } // end class TestQuery2 > > > > --------------------------------------------------------------------- > To unsubscribe, e-mail: java-user-unsubscr...@lucene.apache.org > For additional commands, e-mail: java-user-h...@lucene.apache.org >