Module org.apache.lucene.codecs

Package org.apache.lucene.codecs.simpletext

Class SimpleTextBKDWriter

java.lang.Object

org.apache.lucene.codecs.simpletext.SimpleTextBKDWriter

All Implemented Interfaces:

Closeable, AutoCloseable

final class SimpleTextBKDWriter
extends Object
implements Closeable

Forked from BKDWriter and simplified/specialized for SimpleText's usage

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private class	SimpleTextBKDWriter.OneDimensionBKDWriter

Field Summary

Fields

Modifier and Type	Field	Description
static final String	CODEC_NAME
(package private) final int[]	commonPrefixLengths
protected final BKDConfig	config	How many dimensions we are storing at the leaf (data) nodes
static final float	DEFAULT_MAX_MB_SORT_IN_HEAP	Default maximum heap to use, before spilling to (slower) disk
protected final FixedBitSet	docsSeen
private boolean	finished
private final int	maxDoc
(package private) final double	maxMBSortInHeap
protected final byte[]	maxPackedValue	Maximum per-dim values, packed
private final int	maxPointsSortInHeap
protected final byte[]	minPackedValue	Minimum per-dim values, packed
protected long	pointCount
private PointWriter	pointWriter
(package private) final BytesRefBuilder	scratch
(package private) final byte[]	scratch1
(package private) final byte[]	scratch2
(package private) final BytesRef	scratchBytesRef1
(package private) final BytesRef	scratchBytesRef2
(package private) final byte[]	scratchDiff
(package private) final TrackingDirectoryWrapper	tempDir
(package private) final String	tempFileNamePrefix
private IndexOutput	tempInput
private final long	totalPointCount	An upper bound on how many points the caller will add (includes deletions)
static final int	VERSION_COMPRESSED_DOC_IDS
static final int	VERSION_COMPRESSED_VALUES
static final int	VERSION_CURRENT
static final int	VERSION_IMPLICIT_SPLIT_DIM_1D
static final int	VERSION_START

Constructor Summary

Constructors

Constructor	Description
SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, BKDConfig config, double maxMBSortInHeap, long totalPointCount)

Method Summary

Modifier and Type	Method	Description
void	add(byte[] packedValue, int docID)
private void	build(int nodeID, int leafNodeOffset, MutablePointTree reader, int from, int to, IndexOutput out, byte[] minPackedValue, byte[] maxPackedValue, byte[] splitPackedValues, long[] leafBlockFPs, int[] spareDocIds)
private void	build(int nodeID, int leafNodeOffset, BKDRadixSelector.PathSlice points, IndexOutput out, BKDRadixSelector radixSelector, byte[] minPackedValue, byte[] maxPackedValue, byte[] splitPackedValues, long[] leafBlockFPs, int[] spareDocIds)	The array (sized numDims) of PathSlice describe the cell we have currently recursed to.
private void	checkMaxLeafNodeCount(int numLeaves)
void	close()
private void	computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] commonPrefix)
long	finish(IndexOutput out)	Writes the BKD tree to the provided IndexOutput and returns the file offset where index was written.
long	getPointCount()	How many points have been added so far
private void	newline(IndexOutput out)
private void	rotateToTree(int nodeID, int offset, int count, byte[] index, List<byte[]> leafBlockStartValues)
protected int	split(byte[] minPackedValue, byte[] maxPackedValue)
private HeapPointWriter	switchToHeap(PointWriter source)	Pull a partition back into heap once the point count is low enough while recursing.
private boolean	valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue)	Called only in assert
private boolean	valueInOrder(long ord, int sortedDim, byte[] lastPackedValue, byte[] packedValue, int packedValueOffset, int doc, int lastDoc)
private boolean	valuesInOrderAndBounds(int count, int sortedDim, byte[] minPackedValue, byte[] maxPackedValue, IntFunction<BytesRef> values, int[] docs, int docsOffset)
private Error	verifyChecksum(Throwable priorException, PointWriter writer)	Called on exception, to check whether the checksum is also corrupt in this source, and add that information (checksum matched or didn't) as a suppressed exception.
static void	verifyParams(double maxMBSortInHeap, long totalPointCount)
private void	write(IndexOutput out, String s)
private void	write(IndexOutput out, BytesRef b)
long	writeField(IndexOutput out, String fieldName, MutablePointTree reader)	Write a field from a MutablePointTree.
private long	writeField1Dim(IndexOutput out, String fieldName, MutablePointTree reader)
private long	writeFieldNDims(IndexOutput out, String fieldName, MutablePointTree values)
private void	writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues, int maxPointsInLeafNode)	Subclass can change how it writes the index.
private void	writeInt(IndexOutput out, int x)
protected void	writeLeafBlockDocs(IndexOutput out, int[] docIDs, int start, int count)
protected void	writeLeafBlockPackedValues(IndexOutput out, int[] commonPrefixLengths, int count, int sortedDim, IntFunction<BytesRef> packedValues)
private void	writeLong(IndexOutput out, long x)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CODEC_NAME

public static final String CODEC_NAME

See Also:

• Constant Field Values

VERSION_START

public static final int VERSION_START

See Also:

• Constant Field Values

VERSION_COMPRESSED_DOC_IDS

public static final int VERSION_COMPRESSED_DOC_IDS

See Also:

• Constant Field Values

VERSION_COMPRESSED_VALUES

public static final int VERSION_COMPRESSED_VALUES

See Also:

• Constant Field Values

VERSION_IMPLICIT_SPLIT_DIM_1D

public static final int VERSION_IMPLICIT_SPLIT_DIM_1D

See Also:

• Constant Field Values

VERSION_CURRENT

public static final int VERSION_CURRENT

See Also:

• Constant Field Values

DEFAULT_MAX_MB_SORT_IN_HEAP

public static final float DEFAULT_MAX_MB_SORT_IN_HEAP

Default maximum heap to use, before spilling to (slower) disk

See Also:

• Constant Field Values

config

protected final BKDConfig config

How many dimensions we are storing at the leaf (data) nodes

scratch

final BytesRefBuilder scratch

tempDir

final TrackingDirectoryWrapper tempDir

tempFileNamePrefix

final String tempFileNamePrefix

maxMBSortInHeap

final double maxMBSortInHeap

scratchDiff

final byte[] scratchDiff

scratch1

final byte[] scratch1

scratch2

final byte[] scratch2

scratchBytesRef1

final BytesRef scratchBytesRef1

scratchBytesRef2

final BytesRef scratchBytesRef2

commonPrefixLengths

final int[] commonPrefixLengths

docsSeen

protected final FixedBitSet docsSeen

pointWriter

private PointWriter pointWriter

finished

private boolean finished

tempInput

private IndexOutput tempInput

maxPointsSortInHeap

private final int maxPointsSortInHeap

minPackedValue

protected final byte[] minPackedValue

Minimum per-dim values, packed

maxPackedValue

protected final byte[] maxPackedValue

Maximum per-dim values, packed

pointCount

protected long pointCount

totalPointCount

private final long totalPointCount

An upper bound on how many points the caller will add (includes deletions)

maxDoc

private final int maxDoc

Constructor Details

SimpleTextBKDWriter

public SimpleTextBKDWriter(int maxDoc,
 Directory tempDir,
 String tempFileNamePrefix,
 BKDConfig config,
 double maxMBSortInHeap,
 long totalPointCount)
                    throws IOException

Throws:

IOException

Method Details

verifyParams

public static void verifyParams(double maxMBSortInHeap,
 long totalPointCount)

add

public void add(byte[] packedValue,
 int docID)
         throws IOException

Throws:

IOException

getPointCount

public long getPointCount()

How many points have been added so far

writeField

public long writeField(IndexOutput out,
 String fieldName,
 MutablePointTree reader)
                throws IOException

Write a field from a MutablePointTree. This way of writing points is faster than regular writes with BKDWriter.add(byte[], int) since there is opportunity for reordering points before writing them to disk. This method does not use transient disk in order to reorder points.

Throws:

IOException

writeFieldNDims

private long writeFieldNDims(IndexOutput out,
 String fieldName,
 MutablePointTree values)
                      throws IOException

Throws:

IOException

writeField1Dim

private long writeField1Dim(IndexOutput out,
 String fieldName,
 MutablePointTree reader)
                     throws IOException

Throws:

IOException

rotateToTree

private void rotateToTree(int nodeID,
 int offset,
 int count,
 byte[] index,
 List<byte[]> leafBlockStartValues)

checkMaxLeafNodeCount

private void checkMaxLeafNodeCount(int numLeaves)

finish

public long finish(IndexOutput out)
            throws IOException

Writes the BKD tree to the provided IndexOutput and returns the file offset where index was written.

Throws:

IOException

writeIndex

private void writeIndex(IndexOutput out,
 long[] leafBlockFPs,
 byte[] splitPackedValues,
 int maxPointsInLeafNode)
                 throws IOException

Subclass can change how it writes the index.

Throws:

IOException

writeLeafBlockDocs

protected void writeLeafBlockDocs(IndexOutput out,
 int[] docIDs,
 int start,
 int count)
                           throws IOException

Throws:

IOException

writeLeafBlockPackedValues

protected void writeLeafBlockPackedValues(IndexOutput out,
 int[] commonPrefixLengths,
 int count,
 int sortedDim,
 IntFunction<BytesRef> packedValues)
                                   throws IOException

Throws:

IOException

close

public void close()
           throws IOException

Specified by:

close in interface AutoCloseable

Specified by:

close in interface Closeable

Throws:

IOException

verifyChecksum

private Error verifyChecksum(Throwable priorException,
 PointWriter writer)
                      throws IOException

Called on exception, to check whether the checksum is also corrupt in this source, and add that information (checksum matched or didn't) as a suppressed exception.

Throws:

IOException

valueInBounds

private boolean valueInBounds(BytesRef packedValue,
 byte[] minPackedValue,
 byte[] maxPackedValue)

Called only in assert

split

protected int split(byte[] minPackedValue,
 byte[] maxPackedValue)

switchToHeap

private HeapPointWriter switchToHeap(PointWriter source)
                              throws IOException

Pull a partition back into heap once the point count is low enough while recursing.

Throws:

IOException

build

private void build(int nodeID,
 int leafNodeOffset,
 MutablePointTree reader,
 int from,
 int to,
 IndexOutput out,
 byte[] minPackedValue,
 byte[] maxPackedValue,
 byte[] splitPackedValues,
 long[] leafBlockFPs,
 int[] spareDocIds)
            throws IOException

Throws:

IOException

build

private void build(int nodeID,
 int leafNodeOffset,
 BKDRadixSelector.PathSlice points,
 IndexOutput out,
 BKDRadixSelector radixSelector,
 byte[] minPackedValue,
 byte[] maxPackedValue,
 byte[] splitPackedValues,
 long[] leafBlockFPs,
 int[] spareDocIds)
            throws IOException

The array (sized numDims) of PathSlice describe the cell we have currently recursed to.

Throws:

IOException

computeCommonPrefixLength

private void computeCommonPrefixLength(HeapPointWriter heapPointWriter,
 byte[] commonPrefix)

valuesInOrderAndBounds

private boolean valuesInOrderAndBounds(int count,
 int sortedDim,
 byte[] minPackedValue,
 byte[] maxPackedValue,
 IntFunction<BytesRef> values,
 int[] docs,
 int docsOffset)
                                throws IOException

Throws:

IOException

valueInOrder

private boolean valueInOrder(long ord,
 int sortedDim,
 byte[] lastPackedValue,
 byte[] packedValue,
 int packedValueOffset,
 int doc,
 int lastDoc)

write

private void write(IndexOutput out,
 String s)
            throws IOException

Throws:

IOException

writeInt

private void writeInt(IndexOutput out,
 int x)
               throws IOException

Throws:

IOException

writeLong

private void writeLong(IndexOutput out,
 long x)
                throws IOException

Throws:

IOException

write

private void write(IndexOutput out,
 BytesRef b)
            throws IOException

Throws:

IOException

newline

private void newline(IndexOutput out)
              throws IOException

Throws:

IOException