/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.spatial.prefix;
import java.io.IOException;
import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
Computes facets on cells for PrefixTreeStrategy.
NOTE: If for a given document and a given field using RecursivePrefixTreeStrategy multiple values are indexed (i.e. multi-valued) and at least one of them is a non-point, then there is a possibility of double-counting the document in the facet results. Since each shape is independently turned into grid cells at a resolution chosen by the shape's size, it's possible they will be indexed at different resolutions. This means the document could be present in BOTH the postings for a cell in both its prefix and leaf variants. To avoid this, use a single valued field with a ShapeCollection (or WKT equivalent). Or calculate a suitable level/distErr to index both and call PrefixTreeStrategy.createIndexableFields(Shape, int) with the same value for all shapes for a given document/field.
@lucene.experimental
/**
* Computes facets on cells for {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy}.
* <p>
* <em>NOTE:</em> If for a given document and a given field using
* {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy}
* multiple values are indexed (i.e. multi-valued) and at least one of them is a non-point, then there is a possibility
* of double-counting the document in the facet results. Since each shape is independently turned into grid cells at
* a resolution chosen by the shape's size, it's possible they will be indexed at different resolutions. This means
* the document could be present in BOTH the postings for a cell in both its prefix and leaf variants. To avoid this,
* use a single valued field with a {@link org.locationtech.spatial4j.shape.ShapeCollection} (or WKT equivalent). Or
* calculate a suitable level/distErr to index both and call
* {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy#createIndexableFields(org.locationtech.spatial4j.shape.Shape, int)}
* with the same value for all shapes for a given document/field.
*
* @lucene.experimental
*/
public class PrefixTreeFacetCounter {
A callback/visitor of facet counts. /** A callback/visitor of facet counts. */
public static abstract class FacetVisitor {
Called at the start of the segment, if there is indexed data. /** Called at the start of the segment, if there is indexed data. */
public void startOfSegment() {}
Called for cells with a leaf, or cells at the target facet level. count is greater than zero. When an ancestor cell is given with non-zero count, the count can be considered to be added to all cells below. You won't necessarily get a cell at level facetLevel if the indexed data is courser (bigger). /** Called for cells with a leaf, or cells at the target facet level. {@code count} is greater than zero.
* When an ancestor cell is given with non-zero count, the count can be considered to be added to all cells
* below. You won't necessarily get a cell at level {@code facetLevel} if the indexed data is courser (bigger).
*/
public abstract void visit(Cell cell, int count);
}
private PrefixTreeFacetCounter() {
}
Computes facets using a callback/visitor style design, allowing flexibility for the caller to determine what to do
with each underlying count.
Params: - strategy – the prefix tree strategy (contains the field reference, grid, max levels)
- context – the IndexReader's context
- topAcceptDocs – a Bits to limit counted docs. If null, live docs are counted.
- queryShape – the shape to limit the range of facet counts to
- facetLevel – the maximum depth (detail) of faceted cells
- facetVisitor – the visitor/callback to receive the counts
/**
* Computes facets using a callback/visitor style design, allowing flexibility for the caller to determine what to do
* with each underlying count.
* @param strategy the prefix tree strategy (contains the field reference, grid, max levels)
* @param context the IndexReader's context
* @param topAcceptDocs a Bits to limit counted docs. If null, live docs are counted.
* @param queryShape the shape to limit the range of facet counts to
* @param facetLevel the maximum depth (detail) of faceted cells
* @param facetVisitor the visitor/callback to receive the counts
*/
public static void compute(PrefixTreeStrategy strategy, IndexReaderContext context, Bits topAcceptDocs,
Shape queryShape, int facetLevel, FacetVisitor facetVisitor)
throws IOException {
//We collect per-leaf
for (final LeafReaderContext leafCtx : context.leaves()) {
//determine leaf acceptDocs Bits
Bits leafAcceptDocs;
if (topAcceptDocs == null) {
leafAcceptDocs = leafCtx.reader().getLiveDocs();//filter deleted
} else {
leafAcceptDocs = new Bits() {
@Override
public boolean get(int index) {
return topAcceptDocs.get(leafCtx.docBase + index);
}
@Override
public int length() {
return leafCtx.reader().maxDoc();
}
};
}
compute(strategy, leafCtx, leafAcceptDocs, queryShape, facetLevel, facetVisitor);
}
}
Lower-level per-leaf segment method. /** Lower-level per-leaf segment method. */
public static void compute(final PrefixTreeStrategy strategy, final LeafReaderContext context, final Bits acceptDocs,
final Shape queryShape, final int facetLevel, final FacetVisitor facetVisitor)
throws IOException {
if (acceptDocs != null && acceptDocs.length() != context.reader().maxDoc()) {
throw new IllegalArgumentException(
"acceptDocs bits length " + acceptDocs.length() +" != leaf maxdoc " + context.reader().maxDoc());
}
final SpatialPrefixTree tree = strategy.getGrid();
//scanLevel is an optimization knob of AbstractVisitingPrefixTreeFilter. It's unlikely
// another scanLevel would be much faster and it tends to be a risky knob (can help a little, can hurt a ton).
// TODO use RPT's configured scan level? Do we know better here? Hard to say.
final int scanLevel = tree.getMaxLevels();
//AbstractVisitingPrefixTreeFilter is a Lucene Filter. We don't need a filter; we use it for its great prefix-tree
// traversal code. TODO consider refactoring if/when it makes sense (more use cases than this)
new AbstractVisitingPrefixTreeQuery(queryShape, strategy.getFieldName(), tree, facetLevel, scanLevel) {
@Override
public String toString(String field) {
return "anonPrefixTreeQuery";//un-used
}
@Override
public DocIdSet getDocIdSet(LeafReaderContext contexts) throws IOException {
assert facetLevel == super.detailLevel;//same thing, FYI. (constant)
return new VisitorTemplate(context) {
@Override
protected void start() throws IOException {
facetVisitor.startOfSegment();
}
@Override
protected DocIdSet finish() throws IOException {
return null;//unused;
}
@Override
protected boolean visitPrefix(Cell cell) throws IOException {
// At facetLevel...
if (cell.getLevel() == facetLevel) {
// Count docs
visitLeaf(cell);//we're not a leaf but we treat it as such at facet level
return false;//don't descend further; this is enough detail
}
// We optimize for discriminating filters (reflected in acceptDocs) and short-circuit if no
// matching docs. We could do this at all levels or never but the closer we get to the facet level, the
// higher the probability this is worthwhile. We do when docFreq == 1 because it's a cheap check, especially
// due to "pulsing" in the codec.
//TODO this opt should move to VisitorTemplate (which contains an optimization TODO to this effect)
if (cell.getLevel() == facetLevel - 1 || termsEnum.docFreq() == 1) {
if (!hasDocsAtThisTerm()) {
return false;
}
}
return true;
}
@Override
protected void visitLeaf(Cell cell) throws IOException {
final int count = countDocsAtThisTerm();
if (count > 0) {
facetVisitor.visit(cell, count);
}
}
private int countDocsAtThisTerm() throws IOException {
if (acceptDocs == null) {
return termsEnum.docFreq();
}
int count = 0;
postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
if (acceptDocs.get(postingsEnum.docID()) == false) {
continue;
}
count++;
}
return count;
}
private boolean hasDocsAtThisTerm() throws IOException {
if (acceptDocs == null) {
return true;
}
postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
int nextDoc = postingsEnum.nextDoc();
while (nextDoc != DocIdSetIterator.NO_MORE_DOCS && acceptDocs.get(nextDoc) == false) {
nextDoc = postingsEnum.nextDoc();
}
return nextDoc != DocIdSetIterator.NO_MORE_DOCS;
}
}.getDocIdSet();
}
}.getDocIdSet(context);
}
}