I have been experimenting with payloads and BoostingTermQuery,
which I think
are excellent additions to Lucene core. Currently,
BoostingTermQuery extends
SpanQuery. I would suggest changing this class to extend TermQuery and
refactor the current version to something like 'BoostingSpanQuery'.
The reason is rooted in performance. In my testing, I compared query
throughput using TermQuery against 2 versions of BoostingTermQuery
- the
current one that extends SpanQuery and one that extends TermQuery
(which
I've included, below). Here are the results (qps = queries per
second):
TermQuery: 200 qps
BoostingTermQuery (extends SpanQuery): 97 qps
BoostingTermQuery (extends TermQuery): 130 qps
Here is a version of BoostingTermQuery that extends TermQuery. I
had to
modify TermQuery and TermScorer to make them public. A code review
would be
in order, and I would appreciate your comments on this suggestion.
Peter
-----------------------------------------
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermPositions;
import org.apache.lucene.search.*;
import java.io.IOException;
/**
* Copyright 2004 The Apache Software Foundation
* <p/>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* The BoostingTermQuery is very similar to the [EMAIL PROTECTED]
org.apache.lucene.search.spans.SpanTermQuery} except
* that it factors in the value of the payload located at each of the
positions where the
* [EMAIL PROTECTED] org.apache.lucene.index.Term} occurs.
* <p>
* In order to take advantage of this, you must override [EMAIL PROTECTED]
org.apache.lucene.search.Similarity#scorePayload(byte[],int,int)}
* which returns 1 by default.
* <p>
* Payload scores are averaged across term occurrences in the
document.
*
* <p><font color="#FF0000">
* WARNING: The status of the <b>Payloads</b> feature is experimental.
* The APIs introduced here might change in the future and will not be
* supported anymore in such a case.</font>
*
* @see org.apache.lucene.search.Similarity#scorePayload(byte[],
int, int)
*/
public class BoostingTermQuery extends TermQuery{
Term term;
Similarity similarity;
public BoostingTermQuery(Term term) {
super(term);
this.term = term;
}
protected Weight createWeight(Searcher searcher) throws
IOException {
this.similarity = getSimilarity(searcher);
return new BoostingTermWeight(this, searcher);
}
protected class BoostingTermWeight extends TermWeight implements
Weight {
public BoostingTermWeight(BoostingTermQuery query, Searcher
searcher)
throws IOException {
super(searcher);
}
public Scorer scorer(IndexReader reader) throws IOException {
return new BoostingTermScorer(reader.termDocs(term),
reader.termPositions(term), this, similarity,
reader.norms(term.field()));
}
class BoostingTermScorer extends TermScorer {
//TODO: is this the best way to allocate this?
byte[] payload = new byte[256];
private TermPositions positions;
protected float payloadScore;
private int payloadsSeen;
public BoostingTermScorer(TermDocs termDocs, TermPositions
termPositions, Weight weight,
Similarity similarity, byte[]
norms) throws
IOException {
super(weight, termDocs, similarity, norms);
positions = termPositions;
}
/**
* Go to the next document
*
*/
public boolean next() throws IOException {
boolean result = super.next();
//set the payload. super.next() properly increments the term
positions
if (result) {
if (positions.skipTo(super.doc())) {
positions.nextPosition();
processPayload(similarity);
}
}
return result;
}
public boolean skipTo(int target) throws IOException {
boolean result = super.skipTo(target);
if (result) {
if (positions.skipTo(target)) {
positions.nextPosition();
processPayload(similarity);
}
}
return result;
}
// protected boolean setFreqCurrentDoc() throws IOException {
// if (!more) {
// return false;
// }
// doc = spans.doc();
// freq = 0.0f;
// payloadScore = 0;
// payloadsSeen = 0;
// Similarity similarity1 = getSimilarity();
// while (more && doc == spans.doc()) {
// int matchLength = spans.end() - spans.start();
//
// freq += similarity1.sloppyFreq(matchLength);
// processPayload(similarity1);
//
// more = spans.next();//this moves positions to the next
match in
this document
// }
// return more || (freq != 0);
// }
protected void processPayload(Similarity similarity) throws
IOException {
if (positions.isPayloadAvailable()) {
payload = positions.getPayload(payload, 0);
payloadScore += similarity.scorePayload(payload, 0,
positions.getPayloadLength());
payloadsSeen++;
} else {
//zero out the payload?
}
}
public float score() {
return super.score() * (payloadsSeen > 0 ? (payloadScore /
payloadsSeen) : 1);
}
public Explanation explain(final int doc) throws IOException {
Explanation result = new Explanation();
Explanation nonPayloadExpl = super.explain(doc);
result.addDetail(nonPayloadExpl);
//QUESTION: Is there a wau to avoid this skipTo call? We
need to
know whether to load the payload or not
Explanation payloadBoost = new Explanation();
result.addDetail(payloadBoost);
/*
if (skipTo(doc) == true) {
processPayload();
}
*/
float avgPayloadScore = (payloadsSeen > 0 ? (payloadScore /
payloadsSeen) : 1);
payloadBoost.setValue(avgPayloadScore);
//GSI: I suppose we could toString the payload, but I don't
think
that would be a good idea
payloadBoost.setDescription("scorePayload(...)");
result.setValue(nonPayloadExpl.getValue() * avgPayloadScore);
result.setDescription("btq, product of:");
return result;
}
}
}
public boolean equals(Object o) {
if (!(o instanceof BoostingTermQuery))
return false;
BoostingTermQuery other = (BoostingTermQuery) o;
return (this.getBoost() == other.getBoost())
&& this.term.equals(other.term);
}
}
Diffs for TermQuery, TermScorer:
Index: src/java/org/apache/lucene/search/TermQuery.java
===================================================================
--- src/java/org/apache/lucene/search/TermQuery.java (revision
581018)
+++ src/java/org/apache/lucene/search/TermQuery.java (working copy)
@@ -31,7 +31,7 @@
public class TermQuery extends Query {
private Term term;
- private class TermWeight implements Weight {
+ public class TermWeight implements Weight {
private Similarity similarity;
private float value;
private float idf;
Index: src/java/org/apache/lucene/search/TermScorer.java
===================================================================
--- src/java/org/apache/lucene/search/TermScorer.java (revision
581018)
+++ src/java/org/apache/lucene/search/TermScorer.java (working
copy)
@@ -23,7 +23,7 @@
/** Expert: A <code>Scorer</code> for documents matching a
<code>Term</code>.
*/
-final class TermScorer extends Scorer {
+public class TermScorer extends Scorer {
private Weight weight;
private TermDocs termDocs;
private byte[] norms;
@@ -44,7 +44,7 @@
* @param similarity The </code>Similarity</code> implementation
to be
used for score computations.
* @param norms The field norms of the document fields for the
<code>Term</code>.
*/
- TermScorer(Weight weight, TermDocs td, Similarity similarity,
+ public TermScorer(Weight weight, TermDocs td, Similarity
similarity,
byte[] norms) {
super(similarity);
this.weight = weight;
Peter
