/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;


import java.io.Closeable;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FutureArrays;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.CompiledAutomaton;

import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;

Basic tool and API to check the health of an index and write a new segments file that removes reference to problematic segments.

As this tool checks every byte in the index, on a large index it can take quite a long time to run.

@lucene.experimentalPlease make a complete backup of your index before using this to exorcise corrupted documents from your index!
/** * Basic tool and API to check the health of an index and * write a new segments file that removes reference to * problematic segments. * * <p>As this tool checks every byte in the index, on a large * index it can take quite a long time to run. * * @lucene.experimental Please make a complete backup of your * index before using this to exorcise corrupted documents from your index! */
public final class CheckIndex implements Closeable { private PrintStream infoStream; private Directory dir; private Lock writeLock; private volatile boolean closed;
Returned from CheckIndex.checkIndex() detailing the health and status of the index.
@lucene.experimental
/** * Returned from {@link #checkIndex()} detailing the health and status of the index. * * @lucene.experimental **/
public static class Status { Status() { }
True if no problems were found with the index.
/** True if no problems were found with the index. */
public boolean clean;
True if we were unable to locate and load the segments_N file.
/** True if we were unable to locate and load the segments_N file. */
public boolean missingSegments;
Name of latest segments_N file in the index.
/** Name of latest segments_N file in the index. */
public String segmentsFileName;
Number of segments in the index.
/** Number of segments in the index. */
public int numSegments;
Empty unless you passed specific segments list to check as optional 3rd argument. @see CheckIndex#checkIndex(List)
/** Empty unless you passed specific segments list to check as optional 3rd argument. * @see CheckIndex#checkIndex(List) */
public List<String> segmentsChecked = new ArrayList<>();
True if the index was created with a newer version of Lucene than the CheckIndex tool.
/** True if the index was created with a newer version of Lucene than the CheckIndex tool. */
public boolean toolOutOfDate;
List of SegmentInfoStatus instances, detailing status of each segment.
/** List of {@link SegmentInfoStatus} instances, detailing status of each segment. */
public List<SegmentInfoStatus> segmentInfos = new ArrayList<>();
Directory index is in.
/** Directory index is in. */
public Directory dir;
SegmentInfos instance containing only segments that had no problems (this is used with the CheckIndex.exorciseIndex method to repair the index.
/** * SegmentInfos instance containing only segments that * had no problems (this is used with the {@link CheckIndex#exorciseIndex} * method to repair the index. */
SegmentInfos newSegments;
How many documents will be lost to bad segments.
/** How many documents will be lost to bad segments. */
public int totLoseDocCount;
How many bad segments were found.
/** How many bad segments were found. */
public int numBadSegments;
True if we checked only specific segments (CheckIndex.checkIndex(List<String>)) was called with non-null argument).
/** True if we checked only specific segments ({@link * #checkIndex(List)}) was called with non-null * argument). */
public boolean partial;
The greatest segment name.
/** The greatest segment name. */
public long maxSegmentName;
Whether the SegmentInfos.counter is greater than any of the segments' names.
/** Whether the SegmentInfos.counter is greater than any of the segments' names. */
public boolean validCounter;
Holds the userData of the last commit in the index
/** Holds the userData of the last commit in the index */
public Map<String, String> userData;
Holds the status of each segment in the index. See Status.segmentInfos.
@lucene.experimental
/** Holds the status of each segment in the index. * See {@link #segmentInfos}. * * @lucene.experimental */
public static class SegmentInfoStatus { SegmentInfoStatus() { }
Name of the segment.
/** Name of the segment. */
public String name;
Codec used to read this segment.
/** Codec used to read this segment. */
public Codec codec;
Document count (does not take deletions into account).
/** Document count (does not take deletions into account). */
public int maxDoc;
True if segment is compound file format.
/** True if segment is compound file format. */
public boolean compound;
Number of files referenced by this segment.
/** Number of files referenced by this segment. */
public int numFiles;
Net size (MB) of the files referenced by this segment.
/** Net size (MB) of the files referenced by this * segment. */
public double sizeMB;
True if this segment has pending deletions.
/** True if this segment has pending deletions. */
public boolean hasDeletions;
Current deletions generation.
/** Current deletions generation. */
public long deletionsGen;
True if we were able to open a CodecReader on this segment.
/** True if we were able to open a CodecReader on this * segment. */
public boolean openReaderPassed;
Map that includes certain debugging details that IndexWriter records into each segment it creates
/** Map that includes certain * debugging details that IndexWriter records into * each segment it creates */
public Map<String,String> diagnostics;
Status for testing of livedocs
/** Status for testing of livedocs */
public LiveDocStatus liveDocStatus;
Status for testing of field infos
/** Status for testing of field infos */
public FieldInfoStatus fieldInfoStatus;
Status for testing of field norms (null if field norms could not be tested).
/** Status for testing of field norms (null if field norms could not be tested). */
public FieldNormStatus fieldNormStatus;
Status for testing of indexed terms (null if indexed terms could not be tested).
/** Status for testing of indexed terms (null if indexed terms could not be tested). */
public TermIndexStatus termIndexStatus;
Status for testing of stored fields (null if stored fields could not be tested).
/** Status for testing of stored fields (null if stored fields could not be tested). */
public StoredFieldStatus storedFieldStatus;
Status for testing of term vectors (null if term vectors could not be tested).
/** Status for testing of term vectors (null if term vectors could not be tested). */
public TermVectorStatus termVectorStatus;
Status for testing of DocValues (null if DocValues could not be tested).
/** Status for testing of DocValues (null if DocValues could not be tested). */
public DocValuesStatus docValuesStatus;
Status for testing of PointValues (null if PointValues could not be tested).
/** Status for testing of PointValues (null if PointValues could not be tested). */
public PointsStatus pointsStatus;
Status of index sort
/** Status of index sort */
public IndexSortStatus indexSortStatus; }
Status from testing livedocs
/** * Status from testing livedocs */
public static final class LiveDocStatus { private LiveDocStatus() { }
Number of deleted documents.
/** Number of deleted documents. */
public int numDeleted;
Exception thrown during term index test (null on success)
/** Exception thrown during term index test (null on success) */
public Throwable error = null; }
Status from testing field infos.
/** * Status from testing field infos. */
public static final class FieldInfoStatus { private FieldInfoStatus() { }
Number of fields successfully tested
/** Number of fields successfully tested */
public long totFields = 0L;
Exception thrown during term index test (null on success)
/** Exception thrown during term index test (null on success) */
public Throwable error = null; }
Status from testing field norms.
/** * Status from testing field norms. */
public static final class FieldNormStatus { private FieldNormStatus() { }
Number of fields successfully tested
/** Number of fields successfully tested */
public long totFields = 0L;
Exception thrown during term index test (null on success)
/** Exception thrown during term index test (null on success) */
public Throwable error = null; }
Status from testing term index.
/** * Status from testing term index. */
public static final class TermIndexStatus { TermIndexStatus() { }
Number of terms with at least one live doc.
/** Number of terms with at least one live doc. */
public long termCount = 0L;
Number of terms with zero live docs docs.
/** Number of terms with zero live docs docs. */
public long delTermCount = 0L;
Total frequency across all terms.
/** Total frequency across all terms. */
public long totFreq = 0L;
Total number of positions.
/** Total number of positions. */
public long totPos = 0L;
Exception thrown during term index test (null on success)
/** Exception thrown during term index test (null on success) */
public Throwable error = null;
Holds details of block allocations in the block tree terms dictionary (this is only set if the PostingsFormat for this segment uses block tree.
/** Holds details of block allocations in the block * tree terms dictionary (this is only set if the * {@link PostingsFormat} for this segment uses block * tree. */
public Map<String,Object> blockTreeStats = null; }
Status from testing stored fields.
/** * Status from testing stored fields. */
public static final class StoredFieldStatus { StoredFieldStatus() { }
Number of documents tested.
/** Number of documents tested. */
public int docCount = 0;
Total number of stored fields tested.
/** Total number of stored fields tested. */
public long totFields = 0;
Exception thrown during stored fields test (null on success)
/** Exception thrown during stored fields test (null on success) */
public Throwable error = null; }
Status from testing stored fields.
/** * Status from testing stored fields. */
public static final class TermVectorStatus { TermVectorStatus() { }
Number of documents tested.
/** Number of documents tested. */
public int docCount = 0;
Total number of term vectors tested.
/** Total number of term vectors tested. */
public long totVectors = 0;
Exception thrown during term vector test (null on success)
/** Exception thrown during term vector test (null on success) */
public Throwable error = null; }
Status from testing DocValues
/** * Status from testing DocValues */
public static final class DocValuesStatus { DocValuesStatus() { }
Total number of docValues tested.
/** Total number of docValues tested. */
public long totalValueFields;
Total number of numeric fields
/** Total number of numeric fields */
public long totalNumericFields;
Total number of binary fields
/** Total number of binary fields */
public long totalBinaryFields;
Total number of sorted fields
/** Total number of sorted fields */
public long totalSortedFields;
Total number of sortednumeric fields
/** Total number of sortednumeric fields */
public long totalSortedNumericFields;
Total number of sortedset fields
/** Total number of sortedset fields */
public long totalSortedSetFields;
Exception thrown during doc values test (null on success)
/** Exception thrown during doc values test (null on success) */
public Throwable error = null; }
Status from testing PointValues
/** * Status from testing PointValues */
public static final class PointsStatus { PointsStatus() { }
Total number of values points tested.
/** Total number of values points tested. */
public long totalValuePoints;
Total number of fields with points.
/** Total number of fields with points. */
public int totalValueFields;
Exception thrown during doc values test (null on success)
/** Exception thrown during doc values test (null on success) */
public Throwable error = null; }
Status from testing index sort
/** * Status from testing index sort */
public static final class IndexSortStatus { IndexSortStatus() { }
Exception thrown during term index test (null on success)
/** Exception thrown during term index test (null on success) */
public Throwable error = null; } }
Create a new CheckIndex on the directory.
/** Create a new CheckIndex on the directory. */
public CheckIndex(Directory dir) throws IOException { this(dir, dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)); }
Expert: create a directory with the specified lock. This should really not be used except for unit tests!!!! It exists only to support special tests (such as TestIndexWriterExceptions*), that would otherwise be more complicated to debug if they had to close the writer for each check.
/** * Expert: create a directory with the specified lock. * This should really not be used except for unit tests!!!! * It exists only to support special tests (such as TestIndexWriterExceptions*), * that would otherwise be more complicated to debug if they had to close the writer * for each check. */
public CheckIndex(Directory dir, Lock writeLock) { this.dir = dir; this.writeLock = writeLock; this.infoStream = null; } private void ensureOpen() { if (closed) { throw new AlreadyClosedException("this instance is closed"); } } @Override public void close() throws IOException { closed = true; IOUtils.close(writeLock); } private boolean doSlowChecks;
If true, additional slow checks are performed. This will likely drastically increase time it takes to run CheckIndex!
/** If true, additional slow checks are performed. This will likely * drastically increase time it takes to run CheckIndex! */
public void setDoSlowChecks(boolean v) { doSlowChecks = v; } /** See {@link #setDoSlowChecks}. */ public boolean doSlowChecks() { return doSlowChecks; } private boolean failFast;
If true, just throw the original exception immediately when corruption is detected, rather than continuing to iterate to other segments looking for more corruption.
/** If true, just throw the original exception immediately when * corruption is detected, rather than continuing to iterate to other * segments looking for more corruption. */
public void setFailFast(boolean v) { failFast = v; } /** See {@link #setFailFast}. */ public boolean getFailFast() { return failFast; } private boolean verbose; /** See {@link #getChecksumsOnly}. */ public boolean getChecksumsOnly() { return checksumsOnly; }
If true, only validate physical integrity for all files. Note that the returned nested status objects (e.g. storedFieldStatus) will be null.
/** * If true, only validate physical integrity for all files. * Note that the returned nested status objects (e.g. storedFieldStatus) will be null. */
public void setChecksumsOnly(boolean v) { checksumsOnly = v; } private boolean checksumsOnly;
Set infoStream where messages should go. If null, no messages are printed. If verbose is true then more details are printed.
/** Set infoStream where messages should go. If null, no * messages are printed. If verbose is true then more * details are printed. */
public void setInfoStream(PrintStream out, boolean verbose) { infoStream = out; this.verbose = verbose; }
Set infoStream where messages should go. See setInfoStream(PrintStream, boolean).
/** Set infoStream where messages should go. See {@link #setInfoStream(PrintStream,boolean)}. */
public void setInfoStream(PrintStream out) { setInfoStream(out, false); } private static void msg(PrintStream out, String msg) { if (out != null) out.println(msg); }
Returns a Status instance detailing the state of the index.

As this method checks every byte in the index, on a large index it can take quite a long time to run.

WARNING: make sure you only call this when the index is not opened by any writer.

/** Returns a {@link Status} instance detailing * the state of the index. * * <p>As this method checks every byte in the index, on a large * index it can take quite a long time to run. * * <p><b>WARNING</b>: make sure * you only call this when the index is not opened by any * writer. */
public Status checkIndex() throws IOException { return checkIndex(null); }
Returns a Status instance detailing the state of the index. @param onlySegments list of specific segment names to check

As this method checks every byte in the specified segments, on a large index it can take quite a long time to run.

/** Returns a {@link Status} instance detailing * the state of the index. * * @param onlySegments list of specific segment names to check * * <p>As this method checks every byte in the specified * segments, on a large index it can take quite a long * time to run. */
public Status checkIndex(List<String> onlySegments) throws IOException { ensureOpen(); long startNS = System.nanoTime(); NumberFormat nf = NumberFormat.getInstance(Locale.ROOT); SegmentInfos sis = null; Status result = new Status(); result.dir = dir; String[] files = dir.listAll(); String lastSegmentsFile = SegmentInfos.getLastCommitSegmentsFileName(files); if (lastSegmentsFile == null) { throw new IndexNotFoundException("no segments* file found in " + dir + ": files: " + Arrays.toString(files)); } try { // Do not use SegmentInfos.read(Directory) since the spooky // retrying it does is not necessary here (we hold the write lock): sis = SegmentInfos.readCommit(dir, lastSegmentsFile); } catch (Throwable t) { if (failFast) { throw IOUtils.rethrowAlways(t); } msg(infoStream, "ERROR: could not read any segments file in directory"); result.missingSegments = true; if (infoStream != null) t.printStackTrace(infoStream); return result; } if (infoStream != null) { int maxDoc = 0; int delCount = 0; for (SegmentCommitInfo info : sis) { maxDoc += info.info.maxDoc(); delCount += info.getDelCount(); } infoStream.println(String.format(Locale.ROOT, "%.2f%% total deletions; %d documents; %d deleteions", 100.*delCount/maxDoc, maxDoc, delCount)); } // find the oldest and newest segment versions Version oldest = null; Version newest = null; String oldSegs = null; for (SegmentCommitInfo si : sis) { Version version = si.info.getVersion(); if (version == null) { // pre-3.1 segment oldSegs = "pre-3.1"; } else { if (oldest == null || version.onOrAfter(oldest) == false) { oldest = version; } if (newest == null || version.onOrAfter(newest)) { newest = version; } } } final int numSegments = sis.size(); final String segmentsFileName = sis.getSegmentsFileName(); result.segmentsFileName = segmentsFileName; result.numSegments = numSegments; result.userData = sis.getUserData(); String userDataString; if (sis.getUserData().size() > 0) { userDataString = " userData=" + sis.getUserData(); } else { userDataString = ""; } String versionString = ""; if (oldSegs != null) { if (newest != null) { versionString = "versions=[" + oldSegs + " .. " + newest + "]"; } else { versionString = "version=" + oldSegs; } } else if (newest != null) { // implies oldest != null versionString = oldest.equals(newest) ? ( "version=" + oldest ) : ("versions=[" + oldest + " .. " + newest + "]"); } msg(infoStream, "Segments file=" + segmentsFileName + " numSegments=" + numSegments + " " + versionString + " id=" + StringHelper.idToString(sis.getId()) + userDataString); if (onlySegments != null) { result.partial = true; if (infoStream != null) { infoStream.print("\nChecking only these segments:"); for (String s : onlySegments) { infoStream.print(" " + s); } } result.segmentsChecked.addAll(onlySegments); msg(infoStream, ":"); } result.newSegments = sis.clone(); result.newSegments.clear(); result.maxSegmentName = -1; for(int i=0;i<numSegments;i++) { final SegmentCommitInfo info = sis.info(i); long segmentName = Long.parseLong(info.info.name.substring(1), Character.MAX_RADIX); if (segmentName > result.maxSegmentName) { result.maxSegmentName = segmentName; } if (onlySegments != null && !onlySegments.contains(info.info.name)) { continue; } Status.SegmentInfoStatus segInfoStat = new Status.SegmentInfoStatus(); result.segmentInfos.add(segInfoStat); msg(infoStream, " " + (1+i) + " of " + numSegments + ": name=" + info.info.name + " maxDoc=" + info.info.maxDoc()); segInfoStat.name = info.info.name; segInfoStat.maxDoc = info.info.maxDoc(); final Version version = info.info.getVersion(); if (info.info.maxDoc() <= 0) { throw new RuntimeException("illegal number of documents: maxDoc=" + info.info.maxDoc()); } int toLoseDocCount = info.info.maxDoc(); SegmentReader reader = null; Sort previousIndexSort = null; try { msg(infoStream, " version=" + (version == null ? "3.0" : version)); msg(infoStream, " id=" + StringHelper.idToString(info.info.getId())); final Codec codec = info.info.getCodec(); msg(infoStream, " codec=" + codec); segInfoStat.codec = codec; msg(infoStream, " compound=" + info.info.getUseCompoundFile()); segInfoStat.compound = info.info.getUseCompoundFile(); msg(infoStream, " numFiles=" + info.files().size()); Sort indexSort = info.info.getIndexSort(); if (indexSort != null) { msg(infoStream, " sort=" + indexSort); if (previousIndexSort != null) { if (previousIndexSort.equals(indexSort) == false) { throw new RuntimeException("index sort changed from " + previousIndexSort + " to " + indexSort); } } else { previousIndexSort = indexSort; } } segInfoStat.numFiles = info.files().size(); segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.); msg(infoStream, " size (MB)=" + nf.format(segInfoStat.sizeMB)); Map<String,String> diagnostics = info.info.getDiagnostics(); segInfoStat.diagnostics = diagnostics; if (diagnostics.size() > 0) { msg(infoStream, " diagnostics = " + diagnostics); } if (!info.hasDeletions()) { msg(infoStream, " no deletions"); segInfoStat.hasDeletions = false; } else { msg(infoStream, " has deletions [delGen=" + info.getDelGen() + "]"); segInfoStat.hasDeletions = true; segInfoStat.deletionsGen = info.getDelGen(); } long startOpenReaderNS = System.nanoTime(); if (infoStream != null) infoStream.print(" test: open reader........."); reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT); msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startOpenReaderNS))); segInfoStat.openReaderPassed = true; long startIntegrityNS = System.nanoTime(); if (infoStream != null) infoStream.print(" test: check integrity....."); reader.checkIntegrity(); msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startIntegrityNS))); if (reader.maxDoc() != info.info.maxDoc()) { throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfo.maxDoc " + info.info.maxDoc()); } final int numDocs = reader.numDocs(); toLoseDocCount = numDocs; if (reader.hasDeletions()) { if (reader.numDocs() != info.info.maxDoc() - info.getDelCount()) { throw new RuntimeException("delete count mismatch: info=" + (info.info.maxDoc() - info.getDelCount()) + " vs reader=" + reader.numDocs()); } if ((info.info.maxDoc() - reader.numDocs()) > reader.maxDoc()) { throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.info.maxDoc() - reader.numDocs())); } if (info.info.maxDoc() - reader.numDocs() != info.getDelCount()) { throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.maxDoc() - reader.numDocs())); } } else { if (info.getDelCount() != 0) { throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.maxDoc() - reader.numDocs())); } } if (checksumsOnly == false) { // Test Livedocs segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast); // Test Fieldinfos segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast); // Test Field Norms segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast); // Test the Term Index segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, doSlowChecks, failFast); // Test Stored Fields segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast); // Test Term Vectors segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, doSlowChecks, failFast); // Test Docvalues segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast); // Test PointValues segInfoStat.pointsStatus = testPoints(reader, infoStream, failFast); // Test index sort segInfoStat.indexSortStatus = testSort(reader, indexSort, infoStream, failFast); // Rethrow the first exception we encountered // This will cause stats for failed segments to be incremented properly if (segInfoStat.liveDocStatus.error != null) { throw new RuntimeException("Live docs test failed"); } else if (segInfoStat.fieldInfoStatus.error != null) { throw new RuntimeException("Field Info test failed"); } else if (segInfoStat.fieldNormStatus.error != null) { throw new RuntimeException("Field Norm test failed"); } else if (segInfoStat.termIndexStatus.error != null) { throw new RuntimeException("Term Index test failed"); } else if (segInfoStat.storedFieldStatus.error != null) { throw new RuntimeException("Stored Field test failed"); } else if (segInfoStat.termVectorStatus.error != null) { throw new RuntimeException("Term Vector test failed"); } else if (segInfoStat.docValuesStatus.error != null) { throw new RuntimeException("DocValues test failed"); } else if (segInfoStat.pointsStatus.error != null) { throw new RuntimeException("Points test failed"); } } final String softDeletesField = reader.getFieldInfos().getSoftDeletesField(); if (softDeletesField != null) { checkSoftDeletes(softDeletesField, info, reader, infoStream, failFast); } msg(infoStream, ""); if (verbose) { msg(infoStream, "detailed segment RAM usage: "); msg(infoStream, Accountables.toString(reader)); } } catch (Throwable t) { if (failFast) { throw IOUtils.rethrowAlways(t); } msg(infoStream, "FAILED"); String comment; comment = "exorciseIndex() would remove reference to this segment"; msg(infoStream, " WARNING: " + comment + "; full exception:"); if (infoStream != null) t.printStackTrace(infoStream); msg(infoStream, ""); result.totLoseDocCount += toLoseDocCount; result.numBadSegments++; continue; } finally { if (reader != null) reader.close(); } // Keeper result.newSegments.add(info.clone()); } if (0 == result.numBadSegments) { result.clean = true; } else msg(infoStream, "WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected"); if ( ! (result.validCounter = (result.maxSegmentName < sis.counter))) { result.clean = false; result.newSegments.counter = result.maxSegmentName + 1; msg(infoStream, "ERROR: Next segment name counter " + sis.counter + " is not greater than max segment name " + result.maxSegmentName); } if (result.clean) { msg(infoStream, "No problems were detected with this index.\n"); } msg(infoStream, String.format(Locale.ROOT, "Took %.3f sec total.", nsToSec(System.nanoTime()-startNS))); return result; }
Tests index sort order.
@lucene.experimental
/** * Tests index sort order. * @lucene.experimental */
public static Status.IndexSortStatus testSort(CodecReader reader, Sort sort, PrintStream infoStream, boolean failFast) throws IOException { // This segment claims its documents are sorted according to the incoming sort ... let's make sure: long startNS = System.nanoTime(); Status.IndexSortStatus status = new Status.IndexSortStatus(); if (sort != null) { if (infoStream != null) { infoStream.print(" test: index sort.........."); } SortField fields[] = sort.getSort(); final int reverseMul[] = new int[fields.length]; final LeafFieldComparator comparators[] = new LeafFieldComparator[fields.length]; LeafReaderContext readerContext = new LeafReaderContext(reader); for (int i = 0; i < fields.length; i++) { reverseMul[i] = fields[i].getReverse() ? -1 : 1; comparators[i] = fields[i].getComparator(1, i).getLeafComparator(readerContext); } int maxDoc = reader.maxDoc(); try { for(int docID=1;docID < maxDoc;docID++) { int cmp = 0; for (int i = 0; i < comparators.length; i++) { // TODO: would be better if copy() didnt cause a term lookup in TermOrdVal & co, // the segments are always the same here... comparators[i].copy(0, docID-1); comparators[i].setBottom(0); cmp = reverseMul[i] * comparators[i].compareBottom(docID); if (cmp != 0) { break; } } if (cmp > 0) { throw new RuntimeException("segment has indexSort=" + sort + " but docID=" + (docID-1) + " sorts after docID=" + docID); } } msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } } return status; }
Test live docs.
@lucene.experimental
/** * Test live docs. * @lucene.experimental */
public static Status.LiveDocStatus testLiveDocs(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.LiveDocStatus status = new Status.LiveDocStatus(); try { if (infoStream != null) infoStream.print(" test: check live docs....."); final int numDocs = reader.numDocs(); if (reader.hasDeletions()) { Bits liveDocs = reader.getLiveDocs(); if (liveDocs == null) { throw new RuntimeException("segment should have deletions, but liveDocs is null"); } else { int numLive = 0; for (int j = 0; j < liveDocs.length(); j++) { if (liveDocs.get(j)) { numLive++; } } if (numLive != numDocs) { throw new RuntimeException("liveDocs count mismatch: info=" + numDocs + ", vs bits=" + numLive); } } status.numDeleted = reader.numDeletedDocs(); msg(infoStream, String.format(Locale.ROOT, "OK [%d deleted docs] [took %.3f sec]", status.numDeleted, nsToSec(System.nanoTime()-startNS))); } else { Bits liveDocs = reader.getLiveDocs(); if (liveDocs != null) { // it's ok for it to be non-null here, as long as none are set right? for (int j = 0; j < liveDocs.length(); j++) { if (!liveDocs.get(j)) { throw new RuntimeException("liveDocs mismatch: info says no deletions but doc " + j + " is deleted."); } } } msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", (nsToSec(System.nanoTime()-startNS)))); } } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Test field infos.
@lucene.experimental
/** * Test field infos. * @lucene.experimental */
public static Status.FieldInfoStatus testFieldInfos(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.FieldInfoStatus status = new Status.FieldInfoStatus(); try { // Test Field Infos if (infoStream != null) { infoStream.print(" test: field infos........."); } FieldInfos fieldInfos = reader.getFieldInfos(); for (FieldInfo f : fieldInfos) { f.checkConsistency(); } msg(infoStream, String.format(Locale.ROOT, "OK [%d fields] [took %.3f sec]", fieldInfos.size(), nsToSec(System.nanoTime()-startNS))); status.totFields = fieldInfos.size(); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Test field norms.
@lucene.experimental
/** * Test field norms. * @lucene.experimental */
public static Status.FieldNormStatus testFieldNorms(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.FieldNormStatus status = new Status.FieldNormStatus(); try { // Test Field Norms if (infoStream != null) { infoStream.print(" test: field norms........."); } NormsProducer normsReader = reader.getNormsReader(); if (normsReader != null) { normsReader = normsReader.getMergeInstance(); } for (FieldInfo info : reader.getFieldInfos()) { if (info.hasNorms()) { checkNumericDocValues(info.name, normsReader.getNorms(info), normsReader.getNorms(info)); ++status.totFields; } } msg(infoStream, String.format(Locale.ROOT, "OK [%d fields] [took %.3f sec]", status.totFields, nsToSec(System.nanoTime()-startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Visits all terms in the range minTerm (inclusive) to maxTerm (exclusive), marking all doc IDs encountered into allDocsSeen, and returning the total number of terms visited.
/** Visits all terms in the range minTerm (inclusive) to maxTerm (exclusive), marking all doc IDs encountered into allDocsSeen, and * returning the total number of terms visited. */
private static long getDocsFromTermRange(String field, int maxDoc, TermsEnum termsEnum, FixedBitSet docsSeen, BytesRef minTerm, BytesRef maxTerm, boolean isIntersect) throws IOException { docsSeen.clear(0, docsSeen.length()); long termCount = 0; PostingsEnum postingsEnum = null; BytesRefBuilder lastTerm = null; while (true) { BytesRef term; // Kinda messy: for intersect, we must first next(), but for "normal", we are already on our first term: if (isIntersect || termCount != 0) { term = termsEnum.next(); } else { term = termsEnum.term(); } if (term == null) { if (isIntersect == false) { throw new RuntimeException("didn't see max term field=" + field + " term=" + maxTerm); } //System.out.println(" terms=" + termCount); return termCount; } assert term.isValid(); if (lastTerm == null) { lastTerm = new BytesRefBuilder(); lastTerm.copyBytes(term); } else { if (lastTerm.get().compareTo(term) >= 0) { throw new RuntimeException("terms out of order: lastTerm=" + lastTerm.get() + " term=" + term); } lastTerm.copyBytes(term); } //System.out.println(" term=" + term); // Caller already ensured terms enum positioned >= minTerm: if (term.compareTo(minTerm) < 0) { throw new RuntimeException("saw term before min term field=" + field + " term=" + minTerm); } if (isIntersect == false) { int cmp = term.compareTo(maxTerm); if (cmp == 0) { // Done! //System.out.println(" terms=" + termCount); return termCount; } else if (cmp > 0) { throw new RuntimeException("didn't see end term field=" + field + " term=" + maxTerm); } } postingsEnum = termsEnum.postings(postingsEnum, 0); int lastDoc = -1; while (true) { int doc = postingsEnum.nextDoc(); if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } if (doc <= lastDoc) { throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc); } if (doc >= maxDoc) { throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc); } //System.out.println(" doc=" + doc); docsSeen.set(doc); lastDoc = doc; } termCount++; } }
Test Terms.intersect on this range, and validates that it returns the same doc ids as using non-intersect TermsEnum. Returns true if any fake terms were seen.
/** Test Terms.intersect on this range, and validates that it returns the same doc ids as using non-intersect TermsEnum. Returns true if * any fake terms were seen. */
private static boolean checkSingleTermRange(String field, int maxDoc, Terms terms, BytesRef minTerm, BytesRef maxTerm, FixedBitSet normalDocs, FixedBitSet intersectDocs) throws IOException { //System.out.println(" check minTerm=" + minTerm.utf8ToString() + " maxTerm=" + maxTerm.utf8ToString()); assert minTerm.compareTo(maxTerm) <= 0; TermsEnum termsEnum = terms.iterator(); TermsEnum.SeekStatus status = termsEnum.seekCeil(minTerm); if (status != TermsEnum.SeekStatus.FOUND) { throw new RuntimeException("failed to seek to existing term field=" + field + " term=" + minTerm); } // Do "dumb" iteration to visit all terms in the range: long normalTermCount = getDocsFromTermRange(field, maxDoc, termsEnum, normalDocs, minTerm, maxTerm, false); // Now do the same operation using intersect: long intersectTermCount = getDocsFromTermRange(field, maxDoc, terms.intersect(new CompiledAutomaton(Automata.makeBinaryInterval(minTerm, true, maxTerm, false), true, false, Integer.MAX_VALUE, true), null), intersectDocs, minTerm, maxTerm, true); if (intersectTermCount > normalTermCount) { throw new RuntimeException("intersect returned too many terms: field=" + field + " intersectTermCount=" + intersectTermCount + " normalTermCount=" + normalTermCount); } if (normalDocs.equals(intersectDocs) == false) { throw new RuntimeException("intersect visited different docs than straight terms enum: " + normalDocs.cardinality() + " for straight enum, vs " + intersectDocs.cardinality() + " for intersect, minTerm=" + minTerm + " maxTerm=" + maxTerm); } //System.out.println(" docs=" + normalTermCount); //System.out.println(" " + intersectTermCount + " vs " + normalTermCount); return intersectTermCount != normalTermCount; }
checks Fields api is consistent with itself. searcher is optional, to verify with queries. Can be null.
/** * checks Fields api is consistent with itself. * searcher is optional, to verify with queries. Can be null. */
private static Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, NormsProducer normsProducer, boolean doPrint, boolean isVectors, PrintStream infoStream, boolean verbose, boolean doSlowChecks) throws IOException { // TODO: we should probably return our own stats thing...?! long startNS; if (doPrint) { startNS = System.nanoTime(); } else { startNS = 0; } final Status.TermIndexStatus status = new Status.TermIndexStatus(); int computedFieldCount = 0; PostingsEnum postings = null; String lastField = null; for (String field : fields) { // MultiFieldsEnum relies upon this order... if (lastField != null && field.compareTo(lastField) <= 0) { throw new RuntimeException("fields out of order: lastField=" + lastField + " field=" + field); } lastField = field; // check that the field is in fieldinfos, and is indexed. // TODO: add a separate test to check this for different reader impls FieldInfo fieldInfo = fieldInfos.fieldInfo(field); if (fieldInfo == null) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field); } if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field); } // TODO: really the codec should not return a field // from FieldsEnum if it has no Terms... but we do // this today: // assert fields.terms(field) != null; computedFieldCount++; final Terms terms = fields.terms(field); if (terms == null) { continue; } if (terms.getDocCount() > maxDoc) { throw new RuntimeException("docCount > maxDoc for field: " + field + ", docCount=" + terms.getDocCount() + ", maxDoc=" + maxDoc); } final boolean hasFreqs = terms.hasFreqs(); final boolean hasPositions = terms.hasPositions(); final boolean hasPayloads = terms.hasPayloads(); final boolean hasOffsets = terms.hasOffsets(); BytesRef maxTerm; BytesRef minTerm; if (isVectors) { // Term vectors impls can be very slow for getMax maxTerm = null; minTerm = null; } else { BytesRef bb = terms.getMin(); if (bb != null) { assert bb.isValid(); minTerm = BytesRef.deepCopyOf(bb); } else { minTerm = null; } bb = terms.getMax(); if (bb != null) { assert bb.isValid(); maxTerm = BytesRef.deepCopyOf(bb); if (minTerm == null) { throw new RuntimeException("field \"" + field + "\" has null minTerm but non-null maxTerm"); } } else { maxTerm = null; if (minTerm != null) { throw new RuntimeException("field \"" + field + "\" has non-null minTerm but null maxTerm"); } } } // term vectors cannot omit TF: final boolean expectedHasFreqs = (isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0); if (hasFreqs != expectedHasFreqs) { throw new RuntimeException("field \"" + field + "\" should have hasFreqs=" + expectedHasFreqs + " but got " + hasFreqs); } if (!isVectors) { final boolean expectedHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; if (hasPositions != expectedHasPositions) { throw new RuntimeException("field \"" + field + "\" should have hasPositions=" + expectedHasPositions + " but got " + hasPositions); } final boolean expectedHasPayloads = fieldInfo.hasPayloads(); if (hasPayloads != expectedHasPayloads) { throw new RuntimeException("field \"" + field + "\" should have hasPayloads=" + expectedHasPayloads + " but got " + hasPayloads); } final boolean expectedHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; if (hasOffsets != expectedHasOffsets) { throw new RuntimeException("field \"" + field + "\" should have hasOffsets=" + expectedHasOffsets + " but got " + hasOffsets); } } final TermsEnum termsEnum = terms.iterator(); boolean hasOrd = true; final long termCountStart = status.delTermCount + status.termCount; BytesRefBuilder lastTerm = null; long sumTotalTermFreq = 0; long sumDocFreq = 0; FixedBitSet visitedDocs = new FixedBitSet(maxDoc); while(true) { final BytesRef term = termsEnum.next(); if (term == null) { break; } // System.out.println("CI: field=" + field + " check term=" + term + " docFreq=" + termsEnum.docFreq()); assert term.isValid(); // make sure terms arrive in order according to // the comp if (lastTerm == null) { lastTerm = new BytesRefBuilder(); lastTerm.copyBytes(term); } else { if (lastTerm.get().compareTo(term) >= 0) { throw new RuntimeException("terms out of order: lastTerm=" + lastTerm.get() + " term=" + term); } lastTerm.copyBytes(term); } if (isVectors == false) { if (minTerm == null) { // We checked this above: assert maxTerm == null; throw new RuntimeException("field=\"" + field + "\": invalid term: term=" + term + ", minTerm=" + minTerm); } if (term.compareTo(minTerm) < 0) { throw new RuntimeException("field=\"" + field + "\": invalid term: term=" + term + ", minTerm=" + minTerm); } if (term.compareTo(maxTerm) > 0) { throw new RuntimeException("field=\"" + field + "\": invalid term: term=" + term + ", maxTerm=" + maxTerm); } } final int docFreq = termsEnum.docFreq(); if (docFreq <= 0) { throw new RuntimeException("docfreq: " + docFreq + " is out of bounds"); } sumDocFreq += docFreq; postings = termsEnum.postings(postings, PostingsEnum.ALL); if (hasFreqs == false) { if (termsEnum.totalTermFreq() != termsEnum.docFreq()) { throw new RuntimeException("field \"" + field + "\" hasFreqs is false, but TermsEnum.totalTermFreq()=" + termsEnum.totalTermFreq() + " (should be " + termsEnum.docFreq() + ")"); } } if (hasOrd) { long ord = -1; try { ord = termsEnum.ord(); } catch (UnsupportedOperationException uoe) { hasOrd = false; } if (hasOrd) { final long ordExpected = status.delTermCount + status.termCount - termCountStart; if (ord != ordExpected) { throw new RuntimeException("ord mismatch: TermsEnum has ord=" + ord + " vs actual=" + ordExpected); } } } int lastDoc = -1; int docCount = 0; boolean hasNonDeletedDocs = false; long totalTermFreq = 0; while(true) { final int doc = postings.nextDoc(); if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } visitedDocs.set(doc); int freq = postings.freq(); if (freq <= 0) { throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds"); } if (hasFreqs == false) { // When a field didn't index freq, it must // consistently "lie" and pretend that freq was // 1: if (postings.freq() != 1) { throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " != 1 when Terms.hasFreqs() is false"); } } totalTermFreq += freq; if (liveDocs == null || liveDocs.get(doc)) { hasNonDeletedDocs = true; status.totFreq++; if (freq >= 0) { status.totPos += freq; } } docCount++; if (doc <= lastDoc) { throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc); } if (doc >= maxDoc) { throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc); } lastDoc = doc; int lastPos = -1; int lastOffset = 0; if (hasPositions) { for(int j=0;j<freq;j++) { final int pos = postings.nextPosition(); if (pos < 0) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds"); } if (pos > IndexWriter.MAX_POSITION) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " > IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION); } if (pos < lastPos) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos); } lastPos = pos; BytesRef payload = postings.getPayload(); if (payload != null) { assert payload.isValid(); } if (payload != null && payload.length < 1) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length); } if (hasOffsets) { int startOffset = postings.startOffset(); int endOffset = postings.endOffset(); if (startOffset < 0) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds"); } if (startOffset < lastOffset) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset + "; consider using the FixBrokenOffsets tool in Lucene's backward-codecs module to correct your index"); } if (endOffset < 0) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds"); } if (endOffset < startOffset) { throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset); } lastOffset = startOffset; } } } } if (hasNonDeletedDocs) { status.termCount++; } else { status.delTermCount++; } final long totalTermFreq2 = termsEnum.totalTermFreq(); if (docCount != docFreq) { throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != tot docs w/o deletions " + docCount); } if (docFreq > terms.getDocCount()) { throw new RuntimeException("term " + term + " docFreq=" + docFreq + " > docCount=" + terms.getDocCount()); } if (totalTermFreq2 <= 0) { throw new RuntimeException("totalTermFreq: " + totalTermFreq2 + " is out of bounds"); } sumTotalTermFreq += totalTermFreq; if (totalTermFreq != totalTermFreq2) { throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq2 + " != recomputed totalTermFreq=" + totalTermFreq); } if (totalTermFreq2 < docFreq) { throw new RuntimeException("totalTermFreq: " + totalTermFreq2 + " is out of bounds, docFreq=" + docFreq); } if (hasFreqs == false && totalTermFreq != docFreq) { throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq + " != docFreq=" + docFreq); } // Test skipping if (hasPositions) { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); postings = termsEnum.postings(postings, PostingsEnum.ALL); final int docID = postings.advance(skipDocID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } else { if (docID < skipDocID) { throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID); } final int freq = postings.freq(); if (freq <= 0) { throw new RuntimeException("termFreq " + freq + " is out of bounds"); } int lastPosition = -1; int lastOffset = 0; for(int posUpto=0;posUpto<freq;posUpto++) { final int pos = postings.nextPosition(); if (pos < 0) { throw new RuntimeException("position " + pos + " is out of bounds"); } if (pos < lastPosition) { throw new RuntimeException("position " + pos + " is < lastPosition " + lastPosition); } lastPosition = pos; if (hasOffsets) { int startOffset = postings.startOffset(); int endOffset = postings.endOffset(); // NOTE: we cannot enforce any bounds whatsoever on vectors... they were a free-for-all before? // but for offsets in the postings lists these checks are fine: they were always enforced by IndexWriter if (!isVectors) { if (startOffset < 0) { throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds"); } if (startOffset < lastOffset) { throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset); } if (endOffset < 0) { throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds"); } if (endOffset < startOffset) { throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset); } } lastOffset = startOffset; } } final int nextDocID = postings.nextDoc(); if (nextDocID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (nextDocID <= docID) { throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID); } } if (isVectors) { // Only 1 doc in the postings for term vectors, so we only test 1 advance: break; } } } else { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); postings = termsEnum.postings(postings, PostingsEnum.NONE); final int docID = postings.advance(skipDocID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } else { if (docID < skipDocID) { throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID); } final int nextDocID = postings.nextDoc(); if (nextDocID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (nextDocID <= docID) { throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID); } } if (isVectors) { // Only 1 doc in the postings for term vectors, so we only test 1 advance: break; } } } // Checking score blocks is heavy, we only do it on long postings lists, on every 1024th term // or if slow checks are enabled. if (doSlowChecks || docFreq > 1024 || (status.termCount + status.delTermCount) % 1024 == 0) { // First check max scores and block uptos // But only if slok checks are enabled since we visit all docs if (doSlowChecks) { int max = -1; int maxFreq = 0; ImpactsEnum impactsEnum = termsEnum.impacts(PostingsEnum.FREQS); postings = termsEnum.postings(postings, PostingsEnum.FREQS); for (int doc = impactsEnum.nextDoc(); ; doc = impactsEnum.nextDoc()) { if (postings.nextDoc() != doc) { throw new RuntimeException("Wrong next doc: " + doc + ", expected " + postings.docID()); } if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } if (postings.freq() != impactsEnum.freq()) { throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impactsEnum.freq()); } if (doc > max) { impactsEnum.advanceShallow(doc); Impacts impacts = impactsEnum.getImpacts(); checkImpacts(impacts, doc); max = impacts.getDocIdUpTo(0); List<Impact> impacts0 = impacts.getImpacts(0); maxFreq = impacts0.get(impacts0.size() - 1).freq; } if (impactsEnum.freq() > maxFreq) { throw new RuntimeException("freq " + impactsEnum.freq() + " is greater than the max freq according to impacts " + maxFreq); } } } // Now check advancing ImpactsEnum impactsEnum = termsEnum.impacts(PostingsEnum.FREQS); postings = termsEnum.postings(postings, PostingsEnum.FREQS); int max = -1; int maxFreq = 0; while (true) { int doc = impactsEnum.docID(); boolean advance; int target; if (((field.hashCode() + doc) & 1) == 1) { advance = false; target = doc + 1; } else { advance = true; int delta = Math.min(1 + ((31 * field.hashCode() + doc) & 0x1ff), DocIdSetIterator.NO_MORE_DOCS - doc); target = impactsEnum.docID() + delta; } if (target > max && target % 2 == 1) { int delta = Math.min((31 * field.hashCode() + target) & 0x1ff, DocIdSetIterator.NO_MORE_DOCS - target); max = target + delta; impactsEnum.advanceShallow(target); Impacts impacts = impactsEnum.getImpacts(); checkImpacts(impacts, doc); maxFreq = Integer.MAX_VALUE; for (int level = 0; level < impacts.numLevels(); ++level) { if (impacts.getDocIdUpTo(level) >= max) { List<Impact> perLevelImpacts = impacts.getImpacts(level); maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq; break; } } } if (advance) { doc = impactsEnum.advance(target); } else { doc = impactsEnum.nextDoc(); } if (postings.advance(target) != doc) { throw new RuntimeException("Impacts do not advance to the same document as postings for target " + target + ", postings: " + postings.docID() + ", impacts: " + doc); } if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } if (postings.freq() != impactsEnum.freq()) { throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impactsEnum.freq()); } if (doc >= max) { int delta = Math.min((31 * field.hashCode() + target & 0x1ff), DocIdSetIterator.NO_MORE_DOCS - doc); max = doc + delta; impactsEnum.advanceShallow(doc); Impacts impacts = impactsEnum.getImpacts(); checkImpacts(impacts, doc); maxFreq = Integer.MAX_VALUE; for (int level = 0; level < impacts.numLevels(); ++level) { if (impacts.getDocIdUpTo(level) >= max) { List<Impact> perLevelImpacts = impacts.getImpacts(level); maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq; break; } } } if (impactsEnum.freq() > maxFreq) { throw new RuntimeException("Term frequency " + impactsEnum.freq() + " is greater than the max freq according to impacts " + maxFreq); } } } } if (minTerm != null && status.termCount + status.delTermCount == 0) { throw new RuntimeException("field=\"" + field + "\": minTerm is non-null yet we saw no terms: " + minTerm); } final Terms fieldTerms = fields.terms(field); if (fieldTerms == null) { // Unusual: the FieldsEnum returned a field but // the Terms for that field is null; this should // only happen if it's a ghost field (field with // no terms, eg there used to be terms but all // docs got deleted and then merged away): } else { long fieldTermCount = (status.delTermCount+status.termCount)-termCountStart; final Object stats = fieldTerms.getStats(); assert stats != null; if (status.blockTreeStats == null) { status.blockTreeStats = new HashMap<>(); } status.blockTreeStats.put(field, stats); final long actualSumDocFreq = fields.terms(field).getSumDocFreq(); if (sumDocFreq != actualSumDocFreq) { throw new RuntimeException("sumDocFreq for field " + field + "=" + actualSumDocFreq + " != recomputed sumDocFreq=" + sumDocFreq); } final long actualSumTotalTermFreq = fields.terms(field).getSumTotalTermFreq(); if (sumTotalTermFreq != actualSumTotalTermFreq) { throw new RuntimeException("sumTotalTermFreq for field " + field + "=" + actualSumTotalTermFreq + " != recomputed sumTotalTermFreq=" + sumTotalTermFreq); } if (hasFreqs == false && sumTotalTermFreq != sumDocFreq) { throw new RuntimeException("sumTotalTermFreq for field " + field + " should be " + sumDocFreq + ", got sumTotalTermFreq=" + sumTotalTermFreq); } final int v = fieldTerms.getDocCount(); if (visitedDocs.cardinality() != v) { throw new RuntimeException("docCount for field " + field + "=" + v + " != recomputed docCount=" + visitedDocs.cardinality()); } if (fieldInfo.hasNorms() && isVectors == false) { final NumericDocValues norms = normsProducer.getNorms(fieldInfo); // Cross-check terms with norms for (int doc = norms.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = norms.nextDoc()) { if (liveDocs != null && liveDocs.get(doc) == false) { // Norms may only be out of sync with terms on deleted documents. // This happens when a document fails indexing and in that case it // should be immediately marked as deleted by the IndexWriter. continue; } final long norm = norms.longValue(); if (norm != 0 && visitedDocs.get(doc) == false) { throw new RuntimeException("Document " + doc + " doesn't have terms according to postings but has a norm value that is not zero: " + Long.toUnsignedString(norm)); } else if (norm == 0 && visitedDocs.get(doc)) { throw new RuntimeException("Document " + doc + " has terms according to postings but its norm value is 0, which may only be used on documents that have no terms"); } } } // Test seek to last term: if (lastTerm != null) { if (termsEnum.seekCeil(lastTerm.get()) != TermsEnum.SeekStatus.FOUND) { throw new RuntimeException("seek to last term " + lastTerm.get() + " failed"); } if (termsEnum.term().equals(lastTerm.get()) == false) { throw new RuntimeException("seek to last term " + lastTerm.get() + " returned FOUND but seeked to the wrong term " + termsEnum.term()); } int expectedDocFreq = termsEnum.docFreq(); PostingsEnum d = termsEnum.postings(null, PostingsEnum.NONE); int docFreq = 0; while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { docFreq++; } if (docFreq != expectedDocFreq) { throw new RuntimeException("docFreq for last term " + lastTerm.get() + "=" + expectedDocFreq + " != recomputed docFreq=" + docFreq); } } // check unique term count long termCount = -1; if (fieldTermCount > 0) { termCount = fields.terms(field).size(); if (termCount != -1 && termCount != fieldTermCount) { throw new RuntimeException("termCount mismatch " + termCount + " vs " + fieldTermCount); } } // Test seeking by ord if (hasOrd && status.termCount-termCountStart > 0) { int seekCount = (int) Math.min(10000L, termCount); if (seekCount > 0) { BytesRef[] seekTerms = new BytesRef[seekCount]; // Seek by ord for(int i=seekCount-1;i>=0;i--) { long ord = i*(termCount/seekCount); termsEnum.seekExact(ord); long actualOrd = termsEnum.ord(); if (actualOrd != ord) { throw new RuntimeException("seek to ord " + ord + " returned ord " + actualOrd); } seekTerms[i] = BytesRef.deepCopyOf(termsEnum.term()); } // Seek by term for(int i=seekCount-1;i>=0;i--) { if (termsEnum.seekCeil(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) { throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed"); } if (termsEnum.term().equals(seekTerms[i]) == false) { throw new RuntimeException("seek to existing term " + seekTerms[i] + " returned FOUND but seeked to the wrong term " + termsEnum.term()); } postings = termsEnum.postings(postings, PostingsEnum.NONE); if (postings == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } } } } } } int fieldCount = fields.size(); if (fieldCount != -1) { if (fieldCount < 0) { throw new RuntimeException("invalid fieldCount: " + fieldCount); } if (fieldCount != computedFieldCount) { throw new RuntimeException("fieldCount mismatch " + fieldCount + " vs recomputed field count " + computedFieldCount); } } if (doPrint) { msg(infoStream, String.format(Locale.ROOT, "OK [%d terms; %d terms/docs pairs; %d tokens] [took %.3f sec]", status.termCount, status.totFreq, status.totPos, nsToSec(System.nanoTime()-startNS))); } if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) { for(Map.Entry<String, Object> ent : status.blockTreeStats.entrySet()) { infoStream.println(" field \"" + ent.getKey() + "\":"); infoStream.println(" " + ent.getValue().toString().replace("\n", "\n ")); } } return status; } static void checkImpacts(Impacts impacts, int lastTarget) { final int numLevels = impacts.numLevels(); if (numLevels < 1) { throw new RuntimeException("The number of levels must be >= 1, got " + numLevels); } int docIdUpTo0 = impacts.getDocIdUpTo(0); if (docIdUpTo0 < lastTarget) { throw new RuntimeException("getDocIdUpTo returned " + docIdUpTo0 + " on level 0, which is less than the target " + lastTarget); } for (int level = 1; level < numLevels; ++level) { int docIdUpTo = impacts.getDocIdUpTo(level); int previousDocIdUpTo = impacts.getDocIdUpTo(level - 1); if (docIdUpTo < previousDocIdUpTo) { throw new RuntimeException("Decreasing return for getDocIdUpTo: level " + (level-1) + " returned " + previousDocIdUpTo + " but level " + level + " returned " + docIdUpTo + " for target " + lastTarget); } } for (int level = 0; level < numLevels; ++level) { List<Impact> perLevelImpacts = impacts.getImpacts(level); if (perLevelImpacts.isEmpty()) { throw new RuntimeException("Got empty list of impacts on level " + level); } Impact first = perLevelImpacts.get(0); if (first.freq < 1) { throw new RuntimeException("First impact had a freq <= 0: " + first); } if (first.norm == 0) { throw new RuntimeException("First impact had a norm == 0: " + first); } // Impacts must be in increasing order of norm AND freq Impact previous = first; for (int i = 1; i < perLevelImpacts.size(); ++i) { Impact impact = perLevelImpacts.get(i); if (impact.freq <= previous.freq || Long.compareUnsigned(impact.norm, previous.norm) <= 0) { throw new RuntimeException("Impacts are not ordered or contain dups, got " + previous + " then " + impact); } } if (level > 0) { // Make sure that impacts at level N trigger better scores than an level N-1 Iterator<Impact> previousIt = impacts.getImpacts(level-1).iterator(); previous = previousIt.next(); Iterator<Impact> it = perLevelImpacts.iterator(); Impact impact = it.next(); while (previousIt.hasNext()) { previous = previousIt.next(); if (previous.freq <= impact.freq && Long.compareUnsigned(previous.norm, impact.norm) >= 0) { // previous triggers a lower score than the current impact, all good continue; } if (it.hasNext() == false) { throw new RuntimeException("Found impact " + previous + " on level " + (level-1) + " but no impact on level " + level + " triggers a better score: " + perLevelImpacts); } impact = it.next(); } } } }
Test the term index.
@lucene.experimental
/** * Test the term index. * @lucene.experimental */
public static Status.TermIndexStatus testPostings(CodecReader reader, PrintStream infoStream) throws IOException { return testPostings(reader, infoStream, false, true, false); }
Test the term index.
@lucene.experimental
/** * Test the term index. * @lucene.experimental */
public static Status.TermIndexStatus testPostings(CodecReader reader, PrintStream infoStream, boolean verbose, boolean doSlowChecks, boolean failFast) throws IOException { // TODO: we should go and verify term vectors match, if // doSlowChecks is on... Status.TermIndexStatus status; final int maxDoc = reader.maxDoc(); try { if (infoStream != null) { infoStream.print(" test: terms, freq, prox..."); } final Fields fields = reader.getPostingsReader().getMergeInstance(); final FieldInfos fieldInfos = reader.getFieldInfos(); NormsProducer normsProducer = reader.getNormsReader(); if (normsProducer != null) { normsProducer = normsProducer.getMergeInstance(); } status = checkFields(fields, reader.getLiveDocs(), maxDoc, fieldInfos, normsProducer, true, false, infoStream, verbose, doSlowChecks); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR: " + e); status = new Status.TermIndexStatus(); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Test the points index
@lucene.experimental
/** * Test the points index * @lucene.experimental */
public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { if (infoStream != null) { infoStream.print(" test: points.............."); } long startNS = System.nanoTime(); FieldInfos fieldInfos = reader.getFieldInfos(); Status.PointsStatus status = new Status.PointsStatus(); try { if (fieldInfos.hasPointValues()) { PointsReader pointsReader = reader.getPointsReader(); if (pointsReader == null) { throw new RuntimeException("there are fields with points, but reader.getPointsReader() is null"); } for (FieldInfo fieldInfo : fieldInfos) { if (fieldInfo.getPointDimensionCount() > 0) { PointValues values = pointsReader.getValues(fieldInfo.name); if (values == null) { continue; } status.totalValueFields++; long size = values.size(); int docCount = values.getDocCount(); final long crossCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_CROSSES_QUERY)); if (crossCost < size / 2) { throw new RuntimeException("estimatePointCount should return >= size/2 when all cells match"); } final long insideCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_INSIDE_QUERY)); if (insideCost < size) { throw new RuntimeException("estimatePointCount should return >= size when all cells fully match"); } final long outsideCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_OUTSIDE_QUERY)); if (outsideCost != 0) { throw new RuntimeException("estimatePointCount should return 0 when no cells match"); } VerifyPointsVisitor visitor = new VerifyPointsVisitor(fieldInfo.name, reader.maxDoc(), values); values.intersect(visitor); if (visitor.getPointCountSeen() != size) { throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have size=" + size + " points, but in fact has " + visitor.getPointCountSeen()); } if (visitor.getDocCountSeen() != docCount) { throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have docCount=" + docCount + " but in fact has " + visitor.getDocCountSeen()); } status.totalValuePoints += visitor.getPointCountSeen(); } } } msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR: " + e); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Walks the entire N-dimensional points space, verifying that all points fall within the last cell's boundaries.
@lucene.internal
/** Walks the entire N-dimensional points space, verifying that all points fall within the last cell's boundaries. * * @lucene.internal */
public static class VerifyPointsVisitor implements PointValues.IntersectVisitor { private long pointCountSeen; private int lastDocID = -1; private final int maxDoc; private final FixedBitSet docsSeen; private final byte[] lastMinPackedValue; private final byte[] lastMaxPackedValue; private final byte[] lastPackedValue; private final byte[] globalMinPackedValue; private final byte[] globalMaxPackedValue; private final int packedBytesCount; private final int packedIndexBytesCount; private final int numDataDims; private final int numIndexDims; private final int bytesPerDim; private final String fieldName;
Sole constructor
/** Sole constructor */
public VerifyPointsVisitor(String fieldName, int maxDoc, PointValues values) throws IOException { this.maxDoc = maxDoc; this.fieldName = fieldName; numDataDims = values.getNumDimensions(); numIndexDims = values.getNumIndexDimensions(); bytesPerDim = values.getBytesPerDimension(); packedBytesCount = numDataDims * bytesPerDim; packedIndexBytesCount = numIndexDims * bytesPerDim; globalMinPackedValue = values.getMinPackedValue(); globalMaxPackedValue = values.getMaxPackedValue(); docsSeen = new FixedBitSet(maxDoc); lastMinPackedValue = new byte[packedIndexBytesCount]; lastMaxPackedValue = new byte[packedIndexBytesCount]; lastPackedValue = new byte[packedBytesCount]; if (values.getDocCount() > values.size()) { throw new RuntimeException("point values for field \"" + fieldName + "\" claims to have size=" + values.size() + " points and inconsistent docCount=" + values.getDocCount()); } if (values.getDocCount() > maxDoc) { throw new RuntimeException("point values for field \"" + fieldName + "\" claims to have docCount=" + values.getDocCount() + " but that's greater than maxDoc=" + maxDoc); } if (globalMinPackedValue == null) { if (values.size() != 0) { throw new RuntimeException("getMinPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size()); } } else if (globalMinPackedValue.length != packedIndexBytesCount) { throw new RuntimeException("getMinPackedValue for field \"" + fieldName + "\" return length=" + globalMinPackedValue.length + " array, but should be " + packedBytesCount); } if (globalMaxPackedValue == null) { if (values.size() != 0) { throw new RuntimeException("getMaxPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size()); } } else if (globalMaxPackedValue.length != packedIndexBytesCount) { throw new RuntimeException("getMaxPackedValue for field \"" + fieldName + "\" return length=" + globalMaxPackedValue.length + " array, but should be " + packedBytesCount); } }
Returns total number of points in this BKD tree
/** Returns total number of points in this BKD tree */
public long getPointCountSeen() { return pointCountSeen; }
Returns total number of unique docIDs in this BKD tree
/** Returns total number of unique docIDs in this BKD tree */
public long getDocCountSeen() { return docsSeen.cardinality(); } @Override public void visit(int docID) { throw new RuntimeException("codec called IntersectVisitor.visit without a packed value for docID=" + docID); } @Override public void visit(int docID, byte[] packedValue) { checkPackedValue("packed value", packedValue, docID); pointCountSeen++; docsSeen.set(docID); for(int dim=0;dim<numIndexDims;dim++) { int offset = bytesPerDim * dim; // Compare to last cell: if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, lastMinPackedValue, offset, offset + bytesPerDim) < 0) { // This doc's point, in this dimension, is lower than the minimum value of the last cell checked: throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for field=\"" + fieldName + "\", docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); } if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, lastMaxPackedValue, offset, offset + bytesPerDim) > 0) { // This doc's point, in this dimension, is greater than the maximum value of the last cell checked: throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for field=\"" + fieldName + "\", docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); } } // In the 1D data case, PointValues must make a single in-order sweep through all values, and tie-break by // increasing docID: // for data dimension > 1, leaves are sorted by the dimension with the lowest cardinality to improve block compression if (numDataDims == 1) { int cmp = FutureArrays.compareUnsigned(lastPackedValue, 0, bytesPerDim, packedValue, 0, bytesPerDim); if (cmp > 0) { throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for field=\"" + fieldName + "\", for docID=" + docID + " is out-of-order vs the previous document's value " + Arrays.toString(lastPackedValue)); } else if (cmp == 0) { if (docID < lastDocID) { throw new RuntimeException("packed points value is the same, but docID=" + docID + " is out of order vs previous docID=" + lastDocID + ", field=\"" + fieldName + "\""); } } System.arraycopy(packedValue, 0, lastPackedValue, 0, bytesPerDim); lastDocID = docID; } } @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { checkPackedValue("min packed value", minPackedValue, -1); System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedIndexBytesCount); checkPackedValue("max packed value", maxPackedValue, -1); System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedIndexBytesCount); for(int dim=0;dim<numIndexDims;dim++) { int offset = bytesPerDim * dim; if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) > 0) { throw new RuntimeException("packed points cell minPackedValue " + Arrays.toString(minPackedValue) + " is out-of-bounds of the cell's maxPackedValue " + Arrays.toString(maxPackedValue) + " dim=" + dim + " field=\"" + fieldName + "\""); } // Make sure this cell is not outside of the global min/max: if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, globalMinPackedValue, offset, offset + bytesPerDim) < 0) { throw new RuntimeException("packed points cell minPackedValue " + Arrays.toString(minPackedValue) + " is out-of-bounds of the global minimum " + Arrays.toString(globalMinPackedValue) + " dim=" + dim + " field=\"" + fieldName + "\""); } if (FutureArrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, globalMinPackedValue, offset, offset + bytesPerDim) < 0) { throw new RuntimeException("packed points cell maxPackedValue " + Arrays.toString(maxPackedValue) + " is out-of-bounds of the global minimum " + Arrays.toString(globalMinPackedValue) + " dim=" + dim + " field=\"" + fieldName + "\""); } if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, globalMaxPackedValue, offset, offset + bytesPerDim) > 0) { throw new RuntimeException("packed points cell minPackedValue " + Arrays.toString(minPackedValue) + " is out-of-bounds of the global maximum " + Arrays.toString(globalMaxPackedValue) + " dim=" + dim + " field=\"" + fieldName + "\""); } if (FutureArrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, globalMaxPackedValue, offset, offset + bytesPerDim) > 0) { throw new RuntimeException("packed points cell maxPackedValue " + Arrays.toString(maxPackedValue) + " is out-of-bounds of the global maximum " + Arrays.toString(globalMaxPackedValue) + " dim=" + dim + " field=\"" + fieldName + "\""); } } // We always pretend the query shape is so complex that it crosses every cell, so // that packedValue is passed for every document return PointValues.Relation.CELL_CROSSES_QUERY; } private void checkPackedValue(String desc, byte[] packedValue, int docID) { if (packedValue == null) { throw new RuntimeException(desc + " is null for docID=" + docID + " field=\"" + fieldName + "\""); } if (packedValue.length != (docID < 0 ? packedIndexBytesCount : packedBytesCount)) { throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedIndexBytesCount + " for docID=" + docID + " field=\"" + fieldName + "\""); } } } private static class ConstantRelationIntersectVisitor implements IntersectVisitor { private final Relation relation; ConstantRelationIntersectVisitor(Relation relation) { this.relation = relation; } @Override public void visit(int docID) throws IOException { throw new UnsupportedOperationException(); } @Override public void visit(int docID, byte[] packedValue) throws IOException { throw new UnsupportedOperationException(); } @Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return relation; } }
Test stored fields.
@lucene.experimental
/** * Test stored fields. * @lucene.experimental */
public static Status.StoredFieldStatus testStoredFields(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.StoredFieldStatus status = new Status.StoredFieldStatus(); try { if (infoStream != null) { infoStream.print(" test: stored fields......."); } // Scan stored fields for all documents final Bits liveDocs = reader.getLiveDocs(); StoredFieldsReader storedFields = reader.getFieldsReader().getMergeInstance(); for (int j = 0; j < reader.maxDoc(); ++j) { // Intentionally pull even deleted documents to // make sure they too are not corrupt: DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(); storedFields.visitDocument(j, visitor); Document doc = visitor.getDocument(); if (liveDocs == null || liveDocs.get(j)) { status.docCount++; status.totFields += doc.getFields().size(); } } // Validate docCount if (status.docCount != reader.numDocs()) { throw new RuntimeException("docCount=" + status.docCount + " but saw " + status.docCount + " undeleted docs"); } msg(infoStream, String.format(Locale.ROOT, "OK [%d total field count; avg %.1f fields per doc] [took %.3f sec]", status.totFields, (((float) status.totFields)/status.docCount), nsToSec(System.nanoTime() - startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Test docvalues.
@lucene.experimental
/** * Test docvalues. * @lucene.experimental */
public static Status.DocValuesStatus testDocValues(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.DocValuesStatus status = new Status.DocValuesStatus(); try { if (infoStream != null) { infoStream.print(" test: docvalues..........."); } DocValuesProducer dvReader = reader.getDocValuesReader(); if (dvReader != null) { dvReader = dvReader.getMergeInstance(); } for (FieldInfo fieldInfo : reader.getFieldInfos()) { if (fieldInfo.getDocValuesType() != DocValuesType.NONE) { status.totalValueFields++; checkDocValues(fieldInfo, dvReader, reader.maxDoc(), infoStream, status); } } msg(infoStream, String.format(Locale.ROOT, "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET] [took %.3f sec]", status.totalValueFields, status.totalBinaryFields, status.totalNumericFields, status.totalSortedFields, status.totalSortedNumericFields, status.totalSortedSetFields, nsToSec(System.nanoTime()-startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; } @FunctionalInterface private static interface DocValuesIteratorSupplier { DocValuesIterator get(FieldInfo fi) throws IOException; } private static void checkDVIterator(FieldInfo fi, int maxDoc, DocValuesIteratorSupplier producer) throws IOException { String field = fi.name; // Check advance DocValuesIterator it1 = producer.get(fi); DocValuesIterator it2 = producer.get(fi); int i = 0; for (int doc = it1.nextDoc(); ; doc = it1.nextDoc()) { if (i++ % 10 == 1) { int doc2 = it2.advance(doc - 1); if (doc2 < doc - 1) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + (doc-1) + " went backwords (got: " + doc2 + ")"); } if (doc2 == doc - 1) { doc2 = it2.nextDoc(); } if (doc2 != doc) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + doc + " was not found through advance() (got: " + doc2 + ")"); } if (it2.docID() != doc) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + doc + " reports wrong doc ID (got: " + it2.docID() + ")"); } } if (doc == NO_MORE_DOCS) { break; } } // Check advanceExact it1 = producer.get(fi); it2 = producer.get(fi); i = 0; int lastDoc = -1; for (int doc = it1.nextDoc(); doc != NO_MORE_DOCS ; doc = it1.nextDoc()) { if (i++ % 13 == 1) { boolean found = it2.advanceExact(doc - 1); if ((doc - 1 == lastDoc) != found) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + (doc-1) + " disagrees about whether document exists (got: " + found + ")"); } if (it2.docID() != doc - 1) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + (doc-1) + " reports wrong doc ID (got: " + it2.docID() + ")"); } boolean found2 = it2.advanceExact(doc - 1); if (found != found2) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + (doc-1) + " has unstable advanceExact"); } if (i % 2 == 0) { int doc2 = it2.nextDoc(); if (doc != doc2) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + doc + " was not found through advance() (got: " + doc2 + ")"); } if (it2.docID() != doc) { throw new RuntimeException("dv iterator field=" + field + ": doc=" + doc + " reports wrong doc ID (got: " + it2.docID() + ")"); } } } lastDoc = doc; } } private static void checkBinaryDocValues(String fieldName, int maxDoc, BinaryDocValues bdv, BinaryDocValues bdv2) throws IOException { if (bdv.docID() != -1) { throw new RuntimeException("binary dv iterator for field: " + fieldName + " should start at docID=-1, but got " + bdv.docID()); } // TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field for (int doc = bdv.nextDoc(); doc != NO_MORE_DOCS; doc = bdv.nextDoc()) { BytesRef value = bdv.binaryValue(); value.isValid(); if (bdv2.advanceExact(doc) == false) { throw new RuntimeException("advanceExact did not find matching doc ID: " + doc); } BytesRef value2 = bdv2.binaryValue(); if (value.equals(value2) == false) { throw new RuntimeException("nextDoc and advanceExact report different values: " + value + " != " + value2); } } } private static void checkSortedDocValues(String fieldName, int maxDoc, SortedDocValues dv, SortedDocValues dv2) throws IOException { if (dv.docID() != -1) { throw new RuntimeException("sorted dv iterator for field: " + fieldName + " should start at docID=-1, but got " + dv.docID()); } final int maxOrd = dv.getValueCount()-1; FixedBitSet seenOrds = new FixedBitSet(dv.getValueCount()); int maxOrd2 = -1; for (int doc = dv.nextDoc(); doc != NO_MORE_DOCS; doc = dv.nextDoc()) { int ord = dv.ordValue(); if (ord == -1) { throw new RuntimeException("dv for field: " + fieldName + " has -1 ord"); } else if (ord < -1 || ord > maxOrd) { throw new RuntimeException("ord out of bounds: " + ord); } else { maxOrd2 = Math.max(maxOrd2, ord); seenOrds.set(ord); } if (dv2.advanceExact(doc) == false) { throw new RuntimeException("advanceExact did not find matching doc ID: " + doc); } int ord2 = dv2.ordValue(); if (ord != ord2) { throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2); } } if (maxOrd != maxOrd2) { throw new RuntimeException("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2); } if (seenOrds.cardinality() != dv.getValueCount()) { throw new RuntimeException("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.getValueCount() + " but only used: " + seenOrds.cardinality()); } BytesRef lastValue = null; for (int i = 0; i <= maxOrd; i++) { final BytesRef term = dv.lookupOrd(i); term.isValid(); if (lastValue != null) { if (term.compareTo(lastValue) <= 0) { throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + term); } } lastValue = BytesRef.deepCopyOf(term); } } private static void checkSortedSetDocValues(String fieldName, int maxDoc, SortedSetDocValues dv, SortedSetDocValues dv2) throws IOException { final long maxOrd = dv.getValueCount()-1; LongBitSet seenOrds = new LongBitSet(dv.getValueCount()); long maxOrd2 = -1; for (int docID = dv.nextDoc(); docID != NO_MORE_DOCS; docID = dv.nextDoc()) { if (dv2.advanceExact(docID) == false) { throw new RuntimeException("advanceExact did not find matching doc ID: " + docID); } long lastOrd = -1; long ord; int ordCount = 0; while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { long ord2 = dv2.nextOrd(); if (ord != ord2) { throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2); } if (ord <= lastOrd) { throw new RuntimeException("ords out of order: " + ord + " <= " + lastOrd + " for doc: " + docID); } if (ord < 0 || ord > maxOrd) { throw new RuntimeException("ord out of bounds: " + ord); } lastOrd = ord; maxOrd2 = Math.max(maxOrd2, ord); seenOrds.set(ord); ordCount++; } if (ordCount == 0) { throw new RuntimeException("dv for field: " + fieldName + " returned docID=" + docID + " yet has no ordinals"); } long ord2 = dv2.nextOrd(); if (ord != ord2) { throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2); } } if (maxOrd != maxOrd2) { throw new RuntimeException("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2); } if (seenOrds.cardinality() != dv.getValueCount()) { throw new RuntimeException("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.getValueCount() + " but only used: " + seenOrds.cardinality()); } BytesRef lastValue = null; for (long i = 0; i <= maxOrd; i++) { final BytesRef term = dv.lookupOrd(i); assert term.isValid(); if (lastValue != null) { if (term.compareTo(lastValue) <= 0) { throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + term); } } lastValue = BytesRef.deepCopyOf(term); } } private static void checkSortedNumericDocValues(String fieldName, int maxDoc, SortedNumericDocValues ndv, SortedNumericDocValues ndv2) throws IOException { if (ndv.docID() != -1) { throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID()); } for (int docID = ndv.nextDoc(); docID != NO_MORE_DOCS; docID = ndv.nextDoc()) { int count = ndv.docValueCount(); if (count == 0) { throw new RuntimeException("sorted numeric dv for field: " + fieldName + " returned docValueCount=0 for docID=" + docID); } if (ndv2.advanceExact(docID) == false) { throw new RuntimeException("advanceExact did not find matching doc ID: " + docID); } int count2 = ndv2.docValueCount(); if (count != count2) { throw new RuntimeException("advanceExact reports different value count: " + count + " != " + count2); } long previous = Long.MIN_VALUE; for (int j = 0; j < count; j++) { long value = ndv.nextValue(); if (value < previous) { throw new RuntimeException("values out of order: " + value + " < " + previous + " for doc: " + docID); } previous = value; long value2 = ndv2.nextValue(); if (value != value2) { throw new RuntimeException("advanceExact reports different value: " + value + " != " + value2); } } } } private static void checkNumericDocValues(String fieldName, NumericDocValues ndv, NumericDocValues ndv2) throws IOException { if (ndv.docID() != -1) { throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID()); } // TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field for (int doc = ndv.nextDoc(); doc != NO_MORE_DOCS; doc = ndv.nextDoc()) { long value = ndv.longValue(); if (ndv2.advanceExact(doc) == false) { throw new RuntimeException("advanceExact did not find matching doc ID: " + doc); } long value2 = ndv2.longValue(); if (value != value2) { throw new RuntimeException("advanceExact reports different value: " + value + " != " + value2); } } } private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, int maxDoc, PrintStream infoStream, DocValuesStatus status) throws Exception { switch(fi.getDocValuesType()) { case SORTED: status.totalSortedFields++; checkDVIterator(fi, maxDoc, dvReader::getSorted); checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi)); checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi)); break; case SORTED_NUMERIC: status.totalSortedNumericFields++; checkDVIterator(fi, maxDoc, dvReader::getSortedNumeric); checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi), dvReader.getSortedNumeric(fi)); break; case SORTED_SET: status.totalSortedSetFields++; checkDVIterator(fi, maxDoc, dvReader::getSortedSet); checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi), dvReader.getSortedSet(fi)); break; case BINARY: status.totalBinaryFields++; checkDVIterator(fi, maxDoc, dvReader::getBinary); checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi), dvReader.getBinary(fi)); break; case NUMERIC: status.totalNumericFields++; checkDVIterator(fi, maxDoc, dvReader::getNumeric); checkNumericDocValues(fi.name, dvReader.getNumeric(fi), dvReader.getNumeric(fi)); break; default: throw new AssertionError(); } }
Test term vectors.
@lucene.experimental
/** * Test term vectors. * @lucene.experimental */
public static Status.TermVectorStatus testTermVectors(CodecReader reader, PrintStream infoStream) throws IOException { return testTermVectors(reader, infoStream, false, false, false); }
Test term vectors.
@lucene.experimental
/** * Test term vectors. * @lucene.experimental */
public static Status.TermVectorStatus testTermVectors(CodecReader reader, PrintStream infoStream, boolean verbose, boolean doSlowChecks, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.TermVectorStatus status = new Status.TermVectorStatus(); final FieldInfos fieldInfos = reader.getFieldInfos(); try { if (infoStream != null) { infoStream.print(" test: term vectors........"); } PostingsEnum postings = null; // Only used if doSlowChecks is true: PostingsEnum postingsDocs = null; final Bits liveDocs = reader.getLiveDocs(); final Fields postingsFields; // TODO: testTermsIndex if (doSlowChecks) { postingsFields = reader.getPostingsReader().getMergeInstance(); } else { postingsFields = null; } TermVectorsReader vectorsReader = reader.getTermVectorsReader(); if (vectorsReader != null) { vectorsReader = vectorsReader.getMergeInstance(); for (int j = 0; j < reader.maxDoc(); ++j) { // Intentionally pull/visit (but don't count in // stats) deleted documents to make sure they too // are not corrupt: Fields tfv = vectorsReader.get(j); // TODO: can we make a IS(FIR) that searches just // this term vector... to pass for searcher? if (tfv != null) { // First run with no deletions: checkFields(tfv, null, 1, fieldInfos, null, false, true, infoStream, verbose, doSlowChecks); // Only agg stats if the doc is live: final boolean doStats = liveDocs == null || liveDocs.get(j); if (doStats) { status.docCount++; } for(String field : tfv) { if (doStats) { status.totVectors++; } // Make sure FieldInfo thinks this field is vector'd: final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); if (!fieldInfo.hasVectors()) { throw new RuntimeException("docID=" + j + " has term vectors for field=" + field + " but FieldInfo has storeTermVector=false"); } if (doSlowChecks) { Terms terms = tfv.terms(field); TermsEnum termsEnum = terms.iterator(); final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; final boolean postingsHasPayload = fieldInfo.hasPayloads(); final boolean vectorsHasPayload = terms.hasPayloads(); Terms postingsTerms = postingsFields.terms(field); if (postingsTerms == null) { throw new RuntimeException("vector field=" + field + " does not exist in postings; doc=" + j); } TermsEnum postingsTermsEnum = postingsTerms.iterator(); final boolean hasProx = terms.hasOffsets() || terms.hasPositions(); BytesRef term = null; while ((term = termsEnum.next()) != null) { // This is the term vectors: postings = termsEnum.postings(postings, PostingsEnum.ALL); assert postings != null; if (!postingsTermsEnum.seekExact(term)) { throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j); } // This is the inverted index ("real" postings): postingsDocs = postingsTermsEnum.postings(postingsDocs, PostingsEnum.ALL); assert postingsDocs != null; final int advanceDoc = postingsDocs.advance(j); if (advanceDoc != j) { throw new RuntimeException("vector term=" + term + " field=" + field + ": doc=" + j + " was not found in postings (got: " + advanceDoc + ")"); } final int doc = postings.nextDoc(); if (doc != 0) { throw new RuntimeException("vector for doc " + j + " didn't return docID=0: got docID=" + doc); } if (postingsHasFreq) { final int tf = postings.freq(); if (postingsHasFreq && postingsDocs.freq() != tf) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": freq=" + tf + " differs from postings freq=" + postingsDocs.freq()); } // Term vectors has prox? if (hasProx) { for (int i = 0; i < tf; i++) { int pos = postings.nextPosition(); if (postingsTerms.hasPositions()) { int postingsPos = postingsDocs.nextPosition(); if (terms.hasPositions() && pos != postingsPos) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": pos=" + pos + " differs from postings pos=" + postingsPos); } } // Call the methods to at least make // sure they don't throw exc: final int startOffset = postings.startOffset(); final int endOffset = postings.endOffset(); // TODO: these are too anal...? /* if (endOffset < startOffset) { throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset); } if (startOffset < lastStartOffset) { throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset); } lastStartOffset = startOffset; */ if (startOffset != -1 && endOffset != -1 && postingsTerms.hasOffsets()) { int postingsStartOffset = postingsDocs.startOffset(); int postingsEndOffset = postingsDocs.endOffset(); if (startOffset != postingsStartOffset) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset); } if (endOffset != postingsEndOffset) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset); } } BytesRef payload = postings.getPayload(); if (payload != null) { assert vectorsHasPayload; } if (postingsHasPayload && vectorsHasPayload) { if (payload == null) { // we have payloads, but not at this position. // postings has payloads too, it should not have one at this position if (postingsDocs.getPayload() != null) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsDocs.getPayload()); } } else { // we have payloads, and one at this position // postings should also have one at this position, with the same bytes. if (postingsDocs.getPayload() == null) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not."); } BytesRef postingsPayload = postingsDocs.getPayload(); if (!payload.equals(postingsPayload)) { throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but differs from postings payload=" + postingsPayload); } } } } } } } } } } } } float vectorAvg = status.docCount == 0 ? 0 : status.totVectors / (float)status.docCount; msg(infoStream, String.format(Locale.ROOT, "OK [%d total term vector count; avg %.1f term/freq vector fields per doc] [took %.3f sec]", status.totVectors, vectorAvg, nsToSec(System.nanoTime() - startNS))); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); status.error = e; if (infoStream != null) { e.printStackTrace(infoStream); } } return status; }
Repairs the index using previously returned result from checkIndex. Note that this does not remove any of the unreferenced files after it's done; you must separately open an IndexWriter, which deletes unreferenced files when it's created.

WARNING: this writes a new segments file into the index, effectively removing all documents in broken segments from the index. BE CAREFUL.

/** Repairs the index using previously returned result * from {@link #checkIndex}. Note that this does not * remove any of the unreferenced files after it's done; * you must separately open an {@link IndexWriter}, which * deletes unreferenced files when it's created. * * <p><b>WARNING</b>: this writes a * new segments file into the index, effectively removing * all documents in broken segments from the index. * BE CAREFUL. */
public void exorciseIndex(Status result) throws IOException { ensureOpen(); if (result.partial) { throw new IllegalArgumentException("can only exorcise an index that was fully checked (this status checked a subset of segments)"); } result.newSegments.changed(); result.newSegments.commit(result.dir); } private static boolean assertsOn; private static boolean testAsserts() { assertsOn = true; return true; }
Check whether asserts are enabled or not.
Returns:true iff asserts are enabled
/** * Check whether asserts are enabled or not. * @return true iff asserts are enabled */
public static boolean assertsOn() { assert testAsserts(); return assertsOn; }
Command-line interface to check and exorcise corrupt segments from an index.

Run it like this:

java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-verbose] [-segment X] [-segment Y]
  • -exorcise: actually write a new segments_N file, removing any problematic segments. *LOSES DATA*
  • -segment X: only check the specified segment(s). This can be specified multiple times, to check more than one segment, eg -segment _2 -segment _a. You can't use this with the -exorcise option.

WARNING: -exorcise should only be used on an emergency basis as it will cause documents (perhaps many) to be permanently removed from the index. Always make a backup copy of your index before running this! Do not run this tool on an index that is actively being written to. You have been warned!

Run without -exorcise, this tool will open the index, report version information and report any exceptions it hits and what action it would take if -exorcise were specified. With -exorcise, this tool will remove any segments that have issues and write a new segments_N file. This means all documents contained in the affected segments will be removed.

This tool exits with exit code 1 if the index cannot be opened or has any corruption, else 0.

/** Command-line interface to check and exorcise corrupt segments from an index. <p> Run it like this: <pre> java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-verbose] [-segment X] [-segment Y] </pre> <ul> <li><code>-exorcise</code>: actually write a new segments_N file, removing any problematic segments. *LOSES DATA* <li><code>-segment X</code>: only check the specified segment(s). This can be specified multiple times, to check more than one segment, eg <code>-segment _2 -segment _a</code>. You can't use this with the -exorcise option. </ul> <p><b>WARNING</b>: <code>-exorcise</code> should only be used on an emergency basis as it will cause documents (perhaps many) to be permanently removed from the index. Always make a backup copy of your index before running this! Do not run this tool on an index that is actively being written to. You have been warned! <p> Run without -exorcise, this tool will open the index, report version information and report any exceptions it hits and what action it would take if -exorcise were specified. With -exorcise, this tool will remove any segments that have issues and write a new segments_N file. This means all documents contained in the affected segments will be removed. <p> This tool exits with exit code 1 if the index cannot be opened or has any corruption, else 0. */
public static void main(String[] args) throws IOException, InterruptedException { int exitCode = doMain(args); System.exit(exitCode); }
Run-time configuration options for CheckIndex commands.
/** * Run-time configuration options for CheckIndex commands. */
public static class Options { boolean doExorcise = false; boolean doSlowChecks = false; boolean verbose = false; boolean doChecksumsOnly = false; List<String> onlySegments = new ArrayList<>(); String indexPath = null; String dirImpl = null; PrintStream out = null;
Sole constructor.
/** Sole constructor. */
public Options() {}
Get the name of the FSDirectory implementation class to use.
/** * Get the name of the FSDirectory implementation class to use. */
public String getDirImpl() { return dirImpl; }
Get the directory containing the index.
/** * Get the directory containing the index. */
public String getIndexPath() { return indexPath; }
Set the PrintStream to use for reporting results.
/** * Set the PrintStream to use for reporting results. */
public void setOut(PrintStream out) { this.out = out; } } // actual main: returns exit code instead of terminating JVM (for easy testing) @SuppressForbidden(reason = "System.out required: command line tool") private static int doMain(String args[]) throws IOException, InterruptedException { Options opts; try { opts = parseOptions(args); } catch (IllegalArgumentException e) { System.out.println(e.getMessage()); return 1; } if (!assertsOn()) System.out.println("\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene...', so assertions are enabled"); System.out.println("\nOpening index @ " + opts.indexPath + "\n"); Directory directory = null; Path path = Paths.get(opts.indexPath); try { if (opts.dirImpl == null) { directory = FSDirectory.open(path); } else { directory = CommandLineUtil.newFSDirectory(opts.dirImpl, path); } } catch (Throwable t) { System.out.println("ERROR: could not open directory \"" + opts.indexPath + "\"; exiting"); t.printStackTrace(System.out); return 1; } try (Directory dir = directory; CheckIndex checker = new CheckIndex(dir)) { opts.out = System.out; return checker.doCheck(opts); } }
Parse command line args into fields
Params:
  • args – The command line arguments
Throws:
Returns:An Options struct
/** * Parse command line args into fields * @param args The command line arguments * @return An Options struct * @throws IllegalArgumentException if any of the CLI args are invalid */
@SuppressForbidden(reason = "System.err required: command line tool") public static Options parseOptions(String[] args) { Options opts = new Options(); int i = 0; while(i < args.length) { String arg = args[i]; if ("-fast".equals(arg)) { opts.doChecksumsOnly = true; } else if ("-exorcise".equals(arg)) { opts.doExorcise = true; } else if ("-crossCheckTermVectors".equals(arg)) { System.err.println("-crossCheckTermVectors is deprecated, use -slow instead"); opts.doSlowChecks = true; } else if ("-slow".equals(arg)) { opts.doSlowChecks = true; } else if (arg.equals("-verbose")) { opts.verbose = true; } else if (arg.equals("-segment")) { if (i == args.length-1) { throw new IllegalArgumentException("ERROR: missing name for -segment option"); } i++; opts.onlySegments.add(args[i]); } else if ("-dir-impl".equals(arg)) { if (i == args.length - 1) { throw new IllegalArgumentException("ERROR: missing value for -dir-impl option"); } i++; opts.dirImpl = args[i]; } else { if (opts.indexPath != null) { throw new IllegalArgumentException("ERROR: unexpected extra argument '" + args[i] + "'"); } opts.indexPath = args[i]; } i++; } if (opts.indexPath == null) { throw new IllegalArgumentException("\nERROR: index path not specified" + "\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-slow] [-segment X] [-segment Y] [-dir-impl X]\n" + "\n" + " -exorcise: actually write a new segments_N file, removing any problematic segments\n" + " -fast: just verify file checksums, omitting logical integrity checks\n" + " -slow: do additional slow checks; THIS IS VERY SLOW!\n" + " -codec X: when exorcising, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -exorcise option\n" + " -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " + "If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" + "\n" + "**WARNING**: -exorcise *LOSES DATA*. This should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -exorcise, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -exorcise were\n" + "specified. With -exorcise, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n"); } if (opts.onlySegments.size() == 0) { opts.onlySegments = null; } else if (opts.doExorcise) { throw new IllegalArgumentException("ERROR: cannot specify both -exorcise and -segment"); } if (opts.doChecksumsOnly && opts.doSlowChecks) { throw new IllegalArgumentException("ERROR: cannot specify both -fast and -slow"); } return opts; }
Actually perform the index check
Params:
  • opts – The options to use for this check
Returns:0 iff the index is clean, 1 otherwise
/** * Actually perform the index check * @param opts The options to use for this check * @return 0 iff the index is clean, 1 otherwise */
public int doCheck(Options opts) throws IOException, InterruptedException { setDoSlowChecks(opts.doSlowChecks); setChecksumsOnly(opts.doChecksumsOnly); setInfoStream(opts.out, opts.verbose); Status result = checkIndex(opts.onlySegments); if (result.missingSegments) { return 1; } if (!result.clean) { if (!opts.doExorcise) { opts.out.println("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -exorcise were specified\n"); } else { opts.out.println("WARNING: " + result.totLoseDocCount + " documents will be lost\n"); opts.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. YOU WILL LOSE DATA. THIS IS YOUR LAST CHANCE TO CTRL+C!"); for(int s=0;s<5;s++) { Thread.sleep(1000); opts.out.println(" " + (5-s) + "..."); } opts.out.println("Writing..."); exorciseIndex(result); opts.out.println("OK"); opts.out.println("Wrote new segments file \"" + result.newSegments.getSegmentsFileName() + "\""); } } opts.out.println(""); if (result.clean == true) { return 0; } else { return 1; } } private static void checkSoftDeletes(String softDeletesField, SegmentCommitInfo info, SegmentReader reader, PrintStream infoStream, boolean failFast) throws IOException { if (infoStream != null) infoStream.print(" test: check soft deletes....."); try { int softDeletes = PendingSoftDeletes.countSoftDeletes(DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(softDeletesField, reader), reader.getLiveDocs()); if (softDeletes != info.getSoftDelCount()) { throw new RuntimeException("actual soft deletes: " + softDeletes + " but expected: " +info.getSoftDelCount()); } } catch (Exception e) { if (failFast) { throw IOUtils.rethrowAlways(e); } msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]"); if (infoStream != null) { e.printStackTrace(infoStream); } } } private static double nsToSec(long ns) { return ns/1000000000.0; } }