LcsClusterAppendExecStream Class Reference

Given a stream of tuples corresponding to the column values in a cluster, loads the cluster pages. More...

#include <LcsClusterAppendExecStream.h>

Inheritance diagram for LcsClusterAppendExecStream:

List of all members.

Public Member Functions
virtual void	prepare (LcsClusterAppendExecStreamParams const &params)
virtual void	open (bool restart)
	Opens this stream, acquiring any resources needed in order to be able to fetch data.
virtual ExecStreamResult	execute (ExecStreamQuantum const &quantum)
	Executes this stream.
virtual void	getResourceRequirements (ExecStreamResourceQuantity &minQuantity, ExecStreamResourceQuantity &optQuantity)
virtual void	closeImpl ()
	Implements ClosableObject.
virtual ExecStreamBufProvision	getOutputBufProvision () const
	Queries the BufferProvision which this stream is capable of when producing tuples.
virtual void	prepare (BTreeExecStreamParams const &params)
virtual void	prepare (SingleOutputExecStreamParams const &params)
virtual void	prepare (ExecStreamParams const &params)
	Prepares this stream for execution.
virtual void	setInputBufAccessors (std::vector< SharedExecStreamBufAccessor > const &inAccessors)
	Initializes the buffer accessors for inputs to this stream.
virtual void	setOutputBufAccessors (std::vector< SharedExecStreamBufAccessor > const &outAccessors)
	Initializes the buffer accessors for outputs from this stream.
virtual bool	canEarlyClose ()
	Returns: true if the stream can be closed early
ExecStreamGraph &	getGraph () const
	Returns: reference to containing graph
ExecStreamId	getStreamId () const
	Returns: the identifier for this stream within containing graph
virtual void	getResourceRequirements (ExecStreamResourceQuantity &minQuantity, ExecStreamResourceQuantity &optQuantity, ExecStreamResourceSettingType &optType)
	Determines resource requirements for this stream.
virtual void	setResourceAllocation (ExecStreamResourceQuantity &quantity)
	Sets current resource allocation for this stream.
virtual void	setName (std::string const &)
	Sets unique name of this stream.
virtual std::string const &	getName () const
	Returns: the name of this stream, as known by the optimizer
virtual bool	mayBlock () const
	Queries whether this stream's implementation may block when execute() is called.
virtual void	checkAbort () const
	Checks whether there is an abort request for this stream's scheduler.
virtual ExecStreamBufProvision	getOutputBufConversion () const
	Queries the BufferProvision to which this stream needs its output to be converted, if any.
virtual ExecStreamBufProvision	getInputBufProvision () const
	Queries the BufferProvision which this stream requires of its inputs when consuming their tuples.
bool	isClosed () const
	Returns: whether the object has been closed
virtual void	initTraceSource (SharedTraceTarget pTraceTarget, std::string name)
	For use when initialization has to be deferred until after construction.
void	trace (TraceLevel level, std::string message) const
	Records a trace message.
bool	isTracing () const
	Returns: true iff tracing is enabled for this source
bool	isTracingLevel (TraceLevel level) const
	Determines whether a particular level is being traced.
TraceTarget &	getTraceTarget () const
	Returns: the TraceTarget for this source
SharedTraceTarget	getSharedTraceTarget () const
	Returns: the SharedTraceTarget for this source
std::string	getTraceSourceName () const
	Gets the name of this source.
void	setTraceSourceName (std::string const &n)
	Sets the name of this source.
TraceLevel	getMinimumTraceLevel () const
void	disableTracing ()
virtual void	initErrorSource (SharedErrorTarget pErrorTarget, const std::string &name)
	For use when initialization has to be deferred until after construction.
void	postError (ErrorLevel level, const std::string &message, void *address, long capacity, int index)
	Posts an exception, such as a row exception.
void	postError (ErrorLevel level, const std::string &message, const TupleDescriptor &errorDesc, const TupleData &errorTuple, int index)
	Posts an exception, such as a row exception.
bool	hasTarget () const
	Returns: true iff an error target has been set
ErrorTarget &	getErrorTarget () const
	Returns: the ErrorTarget for this source
SharedErrorTarget	getSharedErrorTarget () const
	Returns: the SharedErrorTarget for this source
std::string	getErrorSourceName () const
	Gets the name of this source.
void	setErrorSourceName (std::string const &n)
	Sets the name of this source.
void	disableTarget ()
virtual void	setOutputBufAccessors (std::vector< SharedExecStreamBufAccessor > const &outAccessors)
	Initializes the buffer accessors for outputs from this stream.
virtual void	setInputBufAccessors (std::vector< SharedExecStreamBufAccessor > const &inAccessors)
	Initializes the buffer accessors for inputs to this stream.
virtual void	prepare (ConduitExecStreamParams const &params)
virtual void	prepare (SingleInputExecStreamParams const &params)
virtual ExecStreamBufProvision	getInputBufProvision () const
	Queries the BufferProvision which this stream requires of its inputs when consuming their tuples.
Static Public Member Functions
static SharedBTreeWriter	newWriter (BTreeExecStreamParams const &params)
static void	copyParamsToDescriptor (BTreeDescriptor &, BTreeParams const &, SharedCacheAccessor const &)
Protected Member Functions
void	allocArrays ()
	Allocate memory for arrays.
void	initLoad ()
	Initializes the load.
void	loadExistingBlock ()
	Populates row and hash arrays from existing index block.
void	startNewBlock ()
	Prepare to write a fresh block.
void	convertTuplesToCols ()
	Given a TupleData representing all columns in a cluster, converts each column into its own TupleData.
void	addValueOrdinal (uint column, uint16_t vOrd)
	Adds value ordinal to row array for new row.
bool	isRowArrayFull ()
	True if row array is full.
void	writeBatch (bool lastBatch)
	Writes a batch(run) to index block.
void	writeBlock ()
	Writes block to index when the block is full or this is the last block in the load.
bool	getLastBlock (PLcsClusterNode &pBlock)
	Gets last block written to disk so we can append to it, reading in the first rid value stored on the page.
void	init ()
	Initializes and sets up object with content specific to the load that will be carried out.
ExecStreamResult	compress (ExecStreamQuantum const &quantum)
	Processes rows for loading.
virtual void	close ()
	Writes out the last pending batches and btree pages.
virtual void	initTupleLoadParams (const TupleProjection &inputProj)
	Initializes member fields corresponding to the data to be loaded.
virtual ExecStreamResult	getTupleForLoad ()
	Retrieves the tuple that will be loaded into the cluster.
virtual void	postProcessTuple ()
	Performs post-processing after a tuple has been loaded.
virtual SharedBTreeReader	newReader ()
SharedBTreeWriter	newWriter (bool monotonic=false)
virtual void	endSearch ()
	Forgets the current reader or writer's search, releasing any page locks.
ExecStreamResult	precheckConduitBuffers ()
	Checks the state of the input and output buffers.
Protected Attributes
uint	blockSize
	Space available on page blocks for writing cluster data.
TupleDescriptor	tableColsTupleDesc
	Tuple descriptor for the tuple representing all cluster columns across the table that this cluster is a part of.
TupleData	clusterColsTupleData
	Tuple data for the tuple datums representing only this cluster.
TupleDescriptor	clusterColsTupleDesc
	Tuple descriptors for the columns that are part of this cluster.
boost::scoped_array< TupleDescriptor >	colTupleDesc
	Individual tuple descriptors for each column in the cluster.
SegmentAccessor	scratchAccessor
	Scratch accessor for allocating large buffer pages.
ClusterPageLock	bufferLock
	Lock on scratch page.
bool	overwrite
	True if overwriting all existing data.
bool	isDone
	Whether row count has been produced.
TupleData	outputTuple
	Output tuple containing count of number of rows loaded.
TupleAccessor *	outputTupleAccessor
	A reference to the output accessor contained in SingleOutputExecStream::pOutAccessor.
boost::scoped_array< FixedBuffer >	outputTupleBuffer
	buffer holding the outputTuple to provide to the consumers
bool	compressCalled
	True if execute has been called at least once.
boost::scoped_array< LcsHash >	hash
	Array of hashes, one per cluster column.
uint	numColumns
	Number of columns in the cluster.
boost::scoped_array< PBuffer >	rowBlock
	Array of temporary blocks for row array.
uint	nRowsMax
	Maximum number of values that can be stored in m_rowBlock.
boost::scoped_array< PBuffer >	hashBlock
	Array of temporary blocks for hash table.
boost::scoped_array< PBuffer >	builderBlock
	Array of temporary blocks used by ClusterNodeWriter.
uint	rowCnt
	Number of rows loaded into the current set of batches.
bool	indexBlockDirty
	True if index blocks need to be written to disk.
LcsRid	firstRow
	Starting rowid in a cluster page.
LcsRid	lastRow
	Last rowid in the last batch.
LcsRid	startRow
SharedLcsClusterNodeWriter	lcsBlockBuilder
	Page builder object.
boost::scoped_array< LcsHashValOrd >	hashValOrd
	Row value ordinal returned from hash, one per cluster column.
boost::scoped_array< boost::scoped_array< FixedBuffer > >	tempBuf
	Temporary buffers used by WriteBatch.
boost::scoped_array< uint >	maxValueSize
	Max size for each column cluster used by WriteBatch.
bool	arraysAlloced
	Indicates where or not we have already allocated arrays.
PLcsClusterNode	pIndexBlock
	Buffer pointing to cluster page that will actually be written.
RecordNum	numRowCompressed
	Total number of rows loaded by this object.
BTreeDescriptor	treeDescriptor
BTreeOwnerRootMap *	pRootMap
SharedBTreeAccessBase	pBTreeAccessBase
SharedBTreeReader	pBTreeReader
DynamicParamId	rootPageIdParamId
SharedExecStreamBufAccessor	pOutAccessor
bool	isOpen
	Whether this stream is currently open.
ExecStreamGraph *	pGraph
	Dataflow graph containing this stream.
ExecStreamId	id
	Identifier for this stream; local to its containing graph.
std::string	name
	Name of stream, as known by optimizer.
SharedDynamicParamManager	pDynamicParamManager
	The dynamic parameter manager available to this stream.
SharedLogicalTxn	pTxn
	The transaction embracing the stream.
ExecStreamResourceQuantity	resourceAllocation
	Resource quantities currently allocated to this stream.
SharedCacheAccessor	pQuotaAccessor
	CacheAccessor used for quota tracking.
SharedCacheAccessor	pScratchQuotaAccessor
	CacheAccessor used for scratch page quota tracking.
bool	needsClose
SharedExecStreamBufAccessor	pInAccessor

---

Detailed Description

Given a stream of tuples corresponding to the column values in a cluster, loads the cluster pages.

Definition at line 50 of file LcsClusterAppendExecStream.h.

---

Member Function Documentation

void LcsClusterAppendExecStream::allocArrays

(

)

[protected]

Allocate memory for arrays.

Definition at line 714 of file LcsClusterAppendExecStream.cpp.

References arraysAlloced, builderBlock, hash, hashBlock, hashValOrd, maxValueSize, numColumns, rowBlock, and tempBuf.

Referenced by initLoad().

{
    // allocate arrays only if they have not been allocated already
    if (arraysAlloced) {
        return;
    }
    arraysAlloced = true;

    // instantiate hashes
    hash.reset(new LcsHash[numColumns]);

    // allocate pointers for row, hash blocks, other arrays
    rowBlock.reset(new PBuffer[numColumns]);
    hashBlock.reset(new PBuffer[numColumns]);

    builderBlock.reset(new PBuffer[numColumns]);

    hashValOrd.reset(new LcsHashValOrd[numColumns]);
    tempBuf.reset(new boost::scoped_array<FixedBuffer>[numColumns]);
    maxValueSize.reset(new uint[numColumns]);
}

void LcsClusterAppendExecStream::initLoad

(

)

[protected]

Initializes the load.

This method should only be called when the input stream has data available to read.

Definition at line 285 of file LcsClusterAppendExecStream.cpp.

References allocArrays(), SegNodeLock< Node >::allocatePage(), blockSize, bufferLock, builderBlock, clusterColsTupleDesc, colTupleDesc, compressCalled, getLastBlock(), SegPageLock::getPage(), TraceSource::getSharedTraceTarget(), TraceSource::getTraceSourceName(), CachePage::getWritableData(), hash, hashBlock, lcsBlockBuilder, loadExistingBlock(), nRowsMax, numColumns, pIndexBlock, rowBlock, scratchAccessor, startNewBlock(), startRow, BTreeExecStream::treeDescriptor, and SegPageLock::unlock().

Referenced by LcsClusterReplaceExecStream::getTupleForLoad(), and getTupleForLoad().

{
    // If this is the first time this method is called, then
    // start a new block (for the new table), or load the last block
    // (of the existing table).  We do this here rather than in
    // init() because for INSERT into T as SELECT * from T,
    // we need to make sure that we extract all the data from
    // T before modifying the blocks there; hence that's why this
    // method should not be called until there is input available.
    // Use the boolean to ensure that initialization of cluster page
    // is only done once.

    if (!compressCalled) {
        compressCalled = true;

        // The dynamic allocated memory in lcsBlockBuilder is allocated for
        // every LcsClusterAppendExecStream.open() and deallocated for every
        // LcsClusterAppendExecStream.closeImpl(). The dynamic memory is not
        // reused across calls(e.g. when issueing the same statement twice).
        lcsBlockBuilder = SharedLcsClusterNodeWriter(
            new LcsClusterNodeWriter(
                treeDescriptor,
                scratchAccessor,
                clusterColsTupleDesc,
                getSharedTraceTarget(),
                getTraceSourceName()));

        allocArrays();

        // get blocks from cache to use as temporary space and initialize arrays
        for (uint i = 0; i < numColumns; i++) {
            bufferLock.allocatePage();
            rowBlock[i] = bufferLock.getPage().getWritableData();
            bufferLock.unlock();

            bufferLock.allocatePage();
            hashBlock[i] = bufferLock.getPage().getWritableData();
            bufferLock.unlock();

            bufferLock.allocatePage();
            builderBlock[i] = bufferLock.getPage().getWritableData();
            bufferLock.unlock();

            hash[i].init(
                hashBlock[i], lcsBlockBuilder, colTupleDesc[i], i, blockSize);
        }

        nRowsMax = blockSize / sizeof(uint16_t);

        // if the index exists, get last block written

        PLcsClusterNode pExistingIndexBlock;

        bool found = getLastBlock(pExistingIndexBlock);
        if (found) {
            // indicate we are updating a leaf
            pIndexBlock = pExistingIndexBlock;

            // extract rows and values from last batch so we can
            // add to it.
            loadExistingBlock();
        } else {
            // Start writing a new block
            startNewBlock();
            startRow = LcsRid(0);
        }
    }
}

void LcsClusterAppendExecStream::loadExistingBlock

(

)

[protected]

Populates row and hash arrays from existing index block.

Definition at line 418 of file LcsClusterAppendExecStream.cpp.

References addValueOrdinal(), blockSize, builderBlock, firstRow, FixedBuffer, LcsHashValOrd::getValOrd(), hash, indexBlockDirty, lastRow, lcsBlockBuilder, numColumns, pIndexBlock, rowCnt, startNewBlock(), startRow, and writeBlock().

Referenced by initLoad().

{
    boost::scoped_array<uint> numVals;      // number of values in block
    boost::scoped_array<uint16_t> lastValOff;
    boost::scoped_array<boost::scoped_array<FixedBuffer> > aLeftOverBufs;
                                            // array of buffers to hold
                                            // rolled back data for each
                                            // column
    uint anLeftOvers;                       // number of leftover rows for
                                            // each col; since the value is
                                            // same for every column, no need
                                            // for this to be an array
    boost::scoped_array<uint> aiFixedSize;  // how much space was used for
                                            // each column; should be
                                            // equal for each value in a column
    LcsHashValOrd vOrd;

    uint i, j;
    RecordNum startRowCnt;
    RecordNum nrows;

    lcsBlockBuilder->init(
        numColumns, reinterpret_cast<uint8_t *> (pIndexBlock),
        builderBlock.get(), blockSize);

    lastValOff.reset(new uint16_t[numColumns]);
    numVals.reset(new uint[numColumns]);

    // REVIEW jvs 28-Nov-2005:  A simpler approach to this whole problem
    // might be to pretend we were starting an entirely new block,
    // use an LcsClusterReader to read the old one logically and append
    // the old rows into the new block, and then carry on from there with
    // the new rows.

    // Append to an existing cluster page.  Set the last rowid based on
    // the first rowid and the number of rows currently on the page.
    // As rows are "rolled back", lastRow is decremented accordingly

    bool bStartNewBlock =
        lcsBlockBuilder->openAppend(numVals.get(), lastValOff.get(), nrows);
    lastRow = firstRow + nrows;
    startRow = lastRow;

    // Setup structures to hold rolled back information
    aiFixedSize.reset(new uint[numColumns]);
    aLeftOverBufs.reset(new boost::scoped_array<FixedBuffer>[numColumns]);

    startRowCnt = rowCnt;

    // Rollback the final batch for each column
    // We need to rollback all
    // the batches before we can start the new batches because
    //  1) in openAppend() we adjust m_szLeft to not include space
    //     for numColumns * sizeof(RIBatch).  So if the
    //     block was full, then m_szLeft would be negative,
    //     since we decreased it by numColumns * sizeof(LcsBatch)
    //  2) the rollback code will add sizeof(LcsBatch) to szLeft
    //                      for each batch it rolls back
    //  3) the code to add values to a batch gets upset if
    //                      szLeft < 0
    for (i = 0; i < numColumns; i++) {
        //reset everytime through loop
        rowCnt = startRowCnt;
        lcsBlockBuilder->describeLastBatch(i, anLeftOvers, aiFixedSize[i]);

        // if we have left overs from the last batch (ie. batch did not end on
        // an 8 boundary), rollback and store in temporary mem
        // aLeftOverBufs[i]
        if (anLeftOvers > 0) {
            aLeftOverBufs[i].reset(
                new FixedBuffer[anLeftOvers * aiFixedSize[i]]);
            lcsBlockBuilder->rollBackLastBatch(i, aLeftOverBufs[i].get());
            indexBlockDirty = true;
        }
    }

    // Decrement lastRow if there was a rollback of the last batch
    lastRow -= anLeftOvers;

    // If the last page is already full, then write it out and start a new one
    if (bStartNewBlock) {
        writeBlock();
        startNewBlock();
    }

    // Start a new batch for each column.
    for (i = 0; i < numColumns; i++) {
        //reset everytime through loop
        rowCnt = startRowCnt;

        if (!bStartNewBlock) {
            // Repopulate the hash table with the values already in the
            // data segment at the bottom of the block (because we didn't
            // roll back these values, we only roll back the pointers to
            // these values).  But only do this if we haven't started a new
            // block.
            hash[i].restore(numVals[i], lastValOff[i]);
        }

        // if we had left overs from the last batch, start a new batch
        // NOTE: we are guaranteed to be able to add these values back
        // to the current block
        if (anLeftOvers > 0) {
            uint8_t *val;
            bool undoInsert = false;

            // There is a very small probability that when the hash is
            // restored, using the existing values in the block, that the
            // hash will be full and some of the left over values
            // can not be stored in the hash.  If this is true then clear
            // the hash.
            if (hash[i].isHashFull(anLeftOvers)) {
                hash[i].startNewBatch(anLeftOvers);
            }

            for (j = 0, val = aLeftOverBufs[i].get();
                j < anLeftOvers;
                j++, val += aiFixedSize[i])
            {
                hash[i].insert(val, &vOrd, &undoInsert);

                //If we have left overs they should fit in the block
                assert(!undoInsert);
                addValueOrdinal(i, vOrd.getValOrd());
                rowCnt++;
            }
        }
    }
}

void LcsClusterAppendExecStream::startNewBlock

(

)

[protected]

Prepare to write a fresh block.

Definition at line 378 of file LcsClusterAppendExecStream.cpp.

References blockSize, builderBlock, colTupleDesc, firstRow, hash, hashBlock, indexBlockDirty, init(), lastRow, lcsBlockBuilder, numColumns, pIndexBlock, and rowCnt.

Referenced by initLoad(), loadExistingBlock(), and writeBatch().

{
    firstRow = lastRow;

    // Get a new cluster page from the btree segment
    pIndexBlock = lcsBlockBuilder->allocateClusterPage(firstRow);

    // Reset index block and block builder.
    lcsBlockBuilder->init(
        numColumns, reinterpret_cast<uint8_t *> (pIndexBlock),
        builderBlock.get(), blockSize);

    // reset Hashes
    for (uint i = 0; i < numColumns; i++) {
        hash[i].init(
            hashBlock[i], lcsBlockBuilder, colTupleDesc[i], i, blockSize);
    }

    // reset row count
    // NOTE:  if the rowCnt is less than eight then we know we are carrying
    //        over rows from previous block because the count did not end
    //        on a boundary of 8
    if (rowCnt >= 8) {
        rowCnt = 0;
    }
    indexBlockDirty = false;

    // Start writing a new block.
    lcsBlockBuilder->openNew(firstRow);
}

void LcsClusterAppendExecStream::convertTuplesToCols

(

)

[protected]

Given a TupleData representing all columns in a cluster, converts each column into its own TupleData.

void LcsClusterAppendExecStream::addValueOrdinal	(	uint	column,
		uint16_t	vOrd
	)			[protected]

Adds value ordinal to row array for new row.

Definition at line 548 of file LcsClusterAppendExecStream.cpp.

References indexBlockDirty, rowBlock, and rowCnt.

Referenced by compress(), loadExistingBlock(), and writeBatch().

{
    uint16_t *rowWordArray = (uint16_t *) rowBlock[column];
    rowWordArray[rowCnt] = vOrd;

    // since we added a row mark block as dirty
    indexBlockDirty = true;
}

bool LcsClusterAppendExecStream::isRowArrayFull

(

)

[protected]

True if row array is full.

Definition at line 557 of file LcsClusterAppendExecStream.cpp.

References nRowsMax, and rowCnt.

Referenced by compress().

{
    if (rowCnt >= nRowsMax) {
        return true;
    } else {
        return false;
    }
}

void LcsClusterAppendExecStream::writeBatch

(

bool

lastBatch

)

[protected]

Writes a batch(run) to index block.

Batches have a multiple of 8 rows.

Parameters:

lastBatch

true if last batch

Definition at line 566 of file LcsClusterAppendExecStream.cpp.

References addValueOrdinal(), count(), FixedBuffer, hash, lastRow, LCS_FIXED, LCS_VARIABLE, lcsBlockBuilder, maxValueSize, numColumns, rowBlock, rowCnt, startNewBlock(), tempBuf, and writeBlock().

Referenced by compress(), and writeBlock().

{
    uint16_t *oVals;
    uint leftOvers;
    PBuffer val;
    LcsBatchMode mode;
    uint i, j;
    uint origRowCnt, count = 0;

    lastRow += rowCnt;

    for (origRowCnt = rowCnt, i = 0; i < numColumns; i++) {
        rowCnt = origRowCnt;

        // save max value size so we can read leftovers
        maxValueSize[i] = hash[i].getMaxValueSize();

        // Pick which compression mode to use (fixed, variable, or compressed)
        lcsBlockBuilder->pickCompressionMode(
            i, maxValueSize[i], rowCnt, &oVals, mode);
        leftOvers = rowCnt > 8 ? rowCnt % 8 : 0;

        // all batches must end on an eight boundary so we move
        // values over eight boundary to the next batch.
        // if there are leftOvers or if the there are less than
        // eight values in this batch allocate buffer to store
        // values to be written to next batch
        if (leftOvers) {
            tempBuf[i].reset(new FixedBuffer[leftOvers * maxValueSize[i]]);
            count = leftOvers;

        } else if (origRowCnt < 8) {
            tempBuf[i].reset(new FixedBuffer[origRowCnt * maxValueSize[i]]);
            count = origRowCnt;
        } else {
            // no values to write to next batch (ie on boundary of 8)
            tempBuf[i].reset();
        }

        // Write out the batch and collect the leftover rows in tempBuf
        if (LCS_FIXED == mode || LCS_VARIABLE == mode) {
            hash[i].prepareFixedOrVariableBatch(
                (PBuffer) rowBlock[i], rowCnt);
            lcsBlockBuilder->putFixedVarBatch(
                i, (uint16_t *) rowBlock[i], tempBuf[i].get());
            if (mode == LCS_FIXED) {
                hash[i].clearFixedEntries();
            }

        } else {
            uint16_t numVals;

            // write orderVals to oVals and remap val ords in row array
            hash[i].prepareCompressedBatch(
                (PBuffer) rowBlock[i], rowCnt, (uint16_t *) &numVals, oVals);
            lcsBlockBuilder->putCompressedBatch(
                i, (PBuffer) rowBlock[i], tempBuf[i].get());
        }

        // setup next batch
        rowCnt = 0;
        hash[i].startNewBatch(!lastBatch ? count : 0);
    }

    //compensate for left over and rolled back rows
    if (!lastBatch) {
        lastRow -= count;
    }
    bool bStartNewBlock;
    bStartNewBlock = false;

    // If we couldn't even fit 8 values into the batch (and this is not the
    // final batch), then the block must be full.  putCompressedBatch()/
    // putFixedVarBatch() assumed that this was the last batch, so they wrote
    // out these rows in a small batch.  Roll back the entire batch (putting
    // rolled back results in tempBuf) and move to next block
    if (!lastBatch && origRowCnt < 8) {
        // rollback each batch
        for (i = 0; i < numColumns; i++) {
            lcsBlockBuilder->rollBackLastBatch(i, tempBuf[i].get());
        }
        bStartNewBlock = true;
    }

    // Should we move to a new block?  Move if
    //    (a) bStartNewBlock (we need to move just to write the current batch)
    // or (b) lcsBlockBuilder->isEndOfBlock() (there isn't room to even start
    // the next batch)
    if (bStartNewBlock || (!lastBatch && lcsBlockBuilder->isEndOfBlock())) {
        writeBlock();
        startNewBlock();
    }

    // Add leftOvers or rolled back values to new batch
    if (!lastBatch) {
        for (i = 0; i < numColumns; i++) {
            rowCnt = 0;
            for (j = 0, val = tempBuf[i].get(); j < count; j++) {
                LcsHashValOrd vOrd;
                bool undoInsert = false;

                hash[i].insert(val, &vOrd, &undoInsert);

                // If we have leftovers they should fit in the current block
                // (because we moved to a new block above, if it was necessary)
                assert(!undoInsert);
                addValueOrdinal(i, vOrd.getValOrd());
                rowCnt++;
                val += maxValueSize[i];
            }
        }
    }

    for (i = 0; i < numColumns; i++) {
        if (tempBuf[i].get()) {
            tempBuf[i].reset();
        }
    }
}

void LcsClusterAppendExecStream::writeBlock

(

)

[protected]

Writes block to index when the block is full or this is the last block in the load.

Definition at line 686 of file LcsClusterAppendExecStream.cpp.

References indexBlockDirty, lcsBlockBuilder, rowCnt, and writeBatch().

Referenced by compress(), loadExistingBlock(), and writeBatch().

{
    if (indexBlockDirty) {
        // If the rowCnt is not zero, then the last batch was not on
        // a boundary of 8 so we need to write the last batch
        if (rowCnt) {
            writeBatch(true);

            // REVIEW jvs 28-Nov-2005:  it must be possible to eliminate
            // this circularity between writeBlock and writeBatch.

            // Handle corner case. writeBatch may have written this block
            // to the btree
            if (!indexBlockDirty) {
                return;
            }
        }

        // Tell block builder we are done so it can wrap up writing to the
        // index block
        lcsBlockBuilder->endBlock();

        // Dump out the page contents to trace if appropriate

        indexBlockDirty = false;
    }
}

bool LcsClusterAppendExecStream::getLastBlock

(

PLcsClusterNode &

pBlock

)

[protected]

Gets last block written to disk so we can append to it, reading in the first rid value stored on the page.

Parameters:

pBlock

returns pointer to last cluster block

Returns:

true if cluster is non-empty

Definition at line 409 of file LcsClusterAppendExecStream.cpp.

References firstRow, and lcsBlockBuilder.

Referenced by initLoad().

{
    if (!lcsBlockBuilder->getLastClusterPageForWrite(pBlock, firstRow)) {
        return false;
    } else {
        return true;
    }
}

void LcsClusterAppendExecStream::init

(

)

[protected]

Initializes and sets up object with content specific to the load that will be carried out.

Definition at line 131 of file LcsClusterAppendExecStream.cpp.

References arraysAlloced, compressCalled, firstRow, indexBlockDirty, lastRow, numRowCompressed, pIndexBlock, rowCnt, and startRow.

Referenced by open(), and startNewBlock().

{
    pIndexBlock = 0;
    firstRow = LcsRid(0);
    lastRow = LcsRid(0);
    startRow = LcsRid(0);
    rowCnt = 0;
    indexBlockDirty = false;
    arraysAlloced = false;
    compressCalled = false;
    numRowCompressed = 0;
}

ExecStreamResult LcsClusterAppendExecStream::compress

(

ExecStreamQuantum const &

quantum

)

[protected]

Processes rows for loading.

Calls WriteBatch once values cannot fit into a page

Parameters:

quantum

ExecStream quantum

Returns:

ExecStreamResult value

Definition at line 144 of file LcsClusterAppendExecStream.cpp.

References addValueOrdinal(), close(), clusterColsTupleData, EXECRC_BUF_OVERFLOW, EXECRC_EOS, EXECRC_QUANTUM_EXPIRED, EXECRC_YIELD, TupleAccessor::getCurrentByteCount(), getTupleForLoad(), hash, hashValOrd, isDone, isRowArrayFull(), lcsBlockBuilder, TupleAccessor::marshal(), ExecStreamQuantum::nTuplesMax, numColumns, outputTuple, outputTupleAccessor, outputTupleBuffer, postProcessTuple(), SingleOutputExecStream::pOutAccessor, rowCnt, writeBatch(), and writeBlock().

Referenced by execute().

{
    uint i, j, k;
    bool canFit = false;
    bool undoInsert = false;

    if (isDone) {
        // already returned final result
        pOutAccessor->markEOS();
        return EXECRC_EOS;
    }

    for (i = 0; i < quantum.nTuplesMax; i++) {
        // if we have finished processing the previous row, retrieve
        // the next cluster tuple and then convert the columns in the
        // cluster into individual tuples, one per cluster column
        ExecStreamResult rc = getTupleForLoad();

        // no more input; produce final row count
        if (rc == EXECRC_EOS) {
            // since we're done adding rows to the index, write the last batch
            // and block
            if (rowCnt) {
                // if rowCnt < 8 or a multiple of 8, force writeBatch to
                // treat this as the last batch
                if (rowCnt < 8 || (rowCnt % 8) == 0) {
                    writeBatch(true);
                } else {
                    writeBatch(false);
                }
            }

            // Write out the last block and then free up resources
            // rather than waiting until stream close. This will keep
            // resource usage window smaller and avoid interference with
            // downstream processing such as writing to unclustered indexes.
            writeBlock();
            if (lcsBlockBuilder) {
                lcsBlockBuilder->close();
            }
            close();

            // outputTuple was already initialized to point to numRowCompressed/
            // startRow in prepare()
            // Write a single outputTuple(numRowCompressed, [startRow])
            // and indicate OVERFLOW.

            outputTupleAccessor->marshal(outputTuple, outputTupleBuffer.get());
            pOutAccessor->provideBufferForConsumption(
                outputTupleBuffer.get(),
                outputTupleBuffer.get() +
                    outputTupleAccessor->getCurrentByteCount());

            isDone = true;
            return EXECRC_BUF_OVERFLOW;
        } else if (rc != EXECRC_YIELD) {
            return rc;
        }

        // Go through each column value for current row and insert it.
        // If we run out of space then rollback all the columns that
        // I already inserted.
        undoInsert = false;

        for (j = 0; j < numColumns; j++) {
            hash[j].insert(
                clusterColsTupleData[j], &hashValOrd[j], &undoInsert);

            if (undoInsert) {
                // rollback cluster columns already inserted
                // j has not been incremented yet, so the condition should be
                //     k <= j
                for (k = 0; k <= j; k++) {
                    hash[k].undoInsert(clusterColsTupleData[k]);
                }
                break;
            }
        }

        // Was there enough space to add this row?  Note that the Insert()
        // calls above accounted for the space needed by addValueOrdinal()
        // below, so we don't have to worry about addValueOrdinal() running
        // out of space
        if (!undoInsert) {
            canFit = true;
        } else {
            canFit = false;
        }

        if (canFit) {
            // Add the pointers from the batch to the data values
            for (j = 0; j < numColumns; j++) {
                addValueOrdinal(j, hashValOrd[j].getValOrd());
            }

            rowCnt++;

            // if reach max rows that can fit in row array then write batch
            if (isRowArrayFull()) {
                writeBatch(false);
            }
        } else {
            // since we can't fit anymore values write out current batch
            writeBatch(false);

            // restart using last value retrieved from stream because it
            // could not fit in the batch; by continuing we can avoid
            // a goto to jump back to the top of this for loop at the
            // expense of a harmless increment of the quantum
            continue;
        }

        // only consume the tuple after we know the row can fit
        // on the current page
        postProcessTuple();
    }

    return EXECRC_QUANTUM_EXPIRED;
}

void LcsClusterAppendExecStream::close

(

)

[protected, virtual]

Writes out the last pending batches and btree pages.

Deallocates temporary memory and buffer pages. Allows resources to be freed before the execution stream is actually closed.

Reimplemented from ClosableObject.

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 360 of file LcsClusterAppendExecStream.cpp.

References builderBlock, hash, hashBlock, hashValOrd, lcsBlockBuilder, maxValueSize, NULL_PAGE_ID, SegmentAccessor::pSegment, rowBlock, scratchAccessor, and tempBuf.

Referenced by LcsClusterReplaceExecStream::close(), closeImpl(), and compress().

{
    if (scratchAccessor.pSegment) {
        scratchAccessor.pSegment->deallocatePageRange(
            NULL_PAGE_ID, NULL_PAGE_ID);
    }
    rowBlock.reset();
    hashBlock.reset();
    builderBlock.reset();

    hash.reset();
    hashValOrd.reset();
    tempBuf.reset();
    maxValueSize.reset();

    lcsBlockBuilder.reset();
}

void LcsClusterAppendExecStream::initTupleLoadParams

(

const TupleProjection &

inputProj

)

[protected, virtual]

Initializes member fields corresponding to the data to be loaded.

Parameters:

inputProj

projection of the input tuple that's relevant to this cluster append

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 70 of file LcsClusterAppendExecStream.cpp.

References clusterColsTupleData, clusterColsTupleDesc, colTupleDesc, TupleData::compute(), numColumns, TupleDescriptor::projectFrom(), and tableColsTupleDesc.

Referenced by prepare().

{
    numColumns = inputProj.size();
    clusterColsTupleDesc.projectFrom(tableColsTupleDesc, inputProj);
    clusterColsTupleData.compute(clusterColsTupleDesc);

    // setup one tuple descriptor per cluster column
    colTupleDesc.reset(new TupleDescriptor[numColumns]);
    for (int i = 0; i < numColumns; i++) {
        colTupleDesc[i].push_back(tableColsTupleDesc[inputProj[i]]);
    }
}

ExecStreamResult LcsClusterAppendExecStream::getTupleForLoad

(

)

[protected, virtual]

Retrieves the tuple that will be loaded into the cluster.

Returns:

EXECRC_BUF_UNDERFLOW if the input buffer needs to be replenished; EXECRC_EOS if there are no more tuples to load; else, EXECRC_YIELD

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 265 of file LcsClusterAppendExecStream.cpp.

References clusterColsTupleData, EXECBUF_EOS, EXECRC_BUF_UNDERFLOW, EXECRC_EOS, EXECRC_YIELD, initLoad(), and SingleInputExecStream::pInAccessor.

Referenced by compress().

{
    if (pInAccessor->getState() == EXECBUF_EOS) {
        return EXECRC_EOS;
    }

    if (!pInAccessor->demandData()) {
        return EXECRC_BUF_UNDERFLOW;
    }

    // Initialize the load, only after we know we have input available
    initLoad();

    if (!pInAccessor->isTupleConsumptionPending()) {
        pInAccessor->unmarshalProjectedTuple(clusterColsTupleData);
    }

    return EXECRC_YIELD;
}

void LcsClusterAppendExecStream::postProcessTuple

(

)

[protected, virtual]

Performs post-processing after a tuple has been loaded.

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 354 of file LcsClusterAppendExecStream.cpp.

References numRowCompressed, and SingleInputExecStream::pInAccessor.

Referenced by compress(), and LcsClusterReplaceExecStream::postProcessTuple().

{
    pInAccessor->consumeTuple();
    numRowCompressed++;
}

void LcsClusterAppendExecStream::prepare

(

LcsClusterAppendExecStreamParams const &

params

)

[virtual]

Definition at line 31 of file LcsClusterAppendExecStream.cpp.

References SegPageLock::accessSegment(), blockSize, bufferLock, TupleData::compute(), initTupleLoadParams(), LcsClusterAppendExecStreamParams::inputProj, numRowCompressed, outputTuple, outputTupleAccessor, SingleInputExecStream::pInAccessor, SingleOutputExecStream::pOutAccessor, ConduitExecStream::prepare(), BTreeExecStream::prepare(), SegmentAccessor::pSegment, ExecStreamParams::scratchAccessor, scratchAccessor, BTreeDescriptor::segmentAccessor, startRow, tableColsTupleDesc, and BTreeExecStream::treeDescriptor.

Referenced by LcsClusterReplaceExecStream::prepare().

{
    BTreeExecStream::prepare(params);
    ConduitExecStream::prepare(params);

    tableColsTupleDesc = pInAccessor->getTupleDesc();
    initTupleLoadParams(params.inputProj);

    // setup descriptors, accessors and data to access only the columns
    // for this cluster, based on the input projection

    pInAccessor->bindProjection(params.inputProj);

    // setup bufferLock to access temporary large page blocks

    scratchAccessor = params.scratchAccessor;
    bufferLock.accessSegment(scratchAccessor);

    // The output stream from the loader is either a single column representing
    // the number of rows loaded or two columns -- number of rows loaded and
    // starting rid value.  The latter applies when there are
    // downstream create indexes

    TupleDescriptor outputTupleDesc;

    outputTupleDesc = pOutAccessor->getTupleDesc();
    outputTuple.compute(outputTupleDesc);
    outputTuple[0].pData = (PConstBuffer) &numRowCompressed;
    if (outputTupleDesc.size() > 1) {
        outputTuple[1].pData = (PConstBuffer) &startRow;
    }

    outputTupleAccessor = & pOutAccessor->getScratchTupleAccessor();

    blockSize = treeDescriptor.segmentAccessor.pSegment->getUsablePageSize();

}

void LcsClusterAppendExecStream::open

(

bool

restart

)

[virtual]

Opens this stream, acquiring any resources needed in order to be able to fetch data.

A precondition is that input streams must already be opened. A stream can be closed and reopened.

Parameters:

restart

if true, the stream must be already open, and should reset itself to start from the beginning of its result set

Reimplemented from ConduitExecStream.

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 103 of file LcsClusterAppendExecStream.cpp.

References FixedBuffer, TupleAccessor::getMaxByteCount(), init(), isDone, ConduitExecStream::open(), BTreeExecStream::open(), outputTupleAccessor, and outputTupleBuffer.

Referenced by LcsClusterReplaceExecStream::open().

{
    BTreeExecStream::open(restart);
    ConduitExecStream::open(restart);

    if (!restart) {
        outputTupleBuffer.reset(
            new FixedBuffer[outputTupleAccessor->getMaxByteCount()]);
    }

    init();
    isDone = false;
}

ExecStreamResult LcsClusterAppendExecStream::execute

(

ExecStreamQuantum const &

quantum

)

[virtual]

Executes this stream.

Parameters:

quantum

governs the maximum amount of execution to perform

Returns:

code indicating reason execution ceased

Implements ExecStream.

Definition at line 117 of file LcsClusterAppendExecStream.cpp.

References compress().

{
    return compress(quantum);
}

void LcsClusterAppendExecStream::getResourceRequirements	(	ExecStreamResourceQuantity &	minQuantity,
		ExecStreamResourceQuantity &	optQuantity
	)			[virtual]

Reimplemented from ExecStream.

Reimplemented in LcsClusterReplaceExecStream.

Definition at line 84 of file LcsClusterAppendExecStream.cpp.

References ExecStream::getResourceRequirements(), ExecStreamResourceQuantity::nCachePages, and numColumns.

Referenced by LcsClusterReplaceExecStream::getResourceRequirements().

{
    ConduitExecStream::getResourceRequirements(minQuantity,optQuantity);

    // REVIEW --
    // 4 pages per cluster column
    // - 1 for indexBlock
    // - 1 for rowBlock
    // - 1 for hash,
    // - 1 for value bank
    // 5 pages for btree and then 1 for cluster page
    minQuantity.nCachePages += (numColumns * 4) + 6;

    // TODO
    optQuantity = minQuantity;
}

void LcsClusterAppendExecStream::closeImpl

(

)

[virtual]

Implements ClosableObject.

ExecStream implementations may override this to release any resources acquired while open.

Reimplemented from BTreeExecStream.

Definition at line 123 of file LcsClusterAppendExecStream.cpp.

References close(), ExecStream::closeImpl(), BTreeExecStream::closeImpl(), and outputTupleBuffer.

{
    BTreeExecStream::closeImpl();
    ConduitExecStream::closeImpl();
    outputTupleBuffer.reset();
    close();
}

ExecStreamBufProvision LcsClusterAppendExecStream::getOutputBufProvision

(

)

const [virtual]

Queries the BufferProvision which this stream is capable of when producing tuples.

Returns:

supported model; default is BUFPROV_NONE

Reimplemented from SingleOutputExecStream.

Definition at line 737 of file LcsClusterAppendExecStream.cpp.

References BUFPROV_PRODUCER.

{
    return BUFPROV_PRODUCER;
}

SharedBTreeReader BTreeExecStream::newReader

(

)

[protected, virtual, inherited]

Reimplemented in BTreePrefetchSearchExecStream.

Definition at line 67 of file BTreeExecStream.cpp.

References BTreeExecStream::pBTreeAccessBase, BTreeExecStream::pBTreeReader, and BTreeExecStream::treeDescriptor.

Referenced by BTreeReadExecStream::open().

{
    SharedBTreeReader pReader = SharedBTreeReader(
        new BTreeReader(treeDescriptor));
    pBTreeAccessBase = pBTreeReader = pReader;
    return pReader;
}

SharedBTreeWriter BTreeExecStream::newWriter

(

bool

monotonic = false

)

[protected, inherited]

Definition at line 75 of file BTreeExecStream.cpp.

References BTreeExecStream::pBTreeAccessBase, BTreeExecStream::pBTreeReader, BTreeExecStream::scratchAccessor, and BTreeExecStream::treeDescriptor.

Referenced by FtrsTableWriter::createIndexWriter(), and BTreeInsertExecStream::open().

{
    SharedBTreeWriter pWriter = SharedBTreeWriter(
        new BTreeWriter(treeDescriptor,scratchAccessor,monotonic));
    pBTreeAccessBase = pBTreeReader = pWriter;
    return pWriter;
}

SharedBTreeWriter BTreeExecStream::newWriter

(

BTreeExecStreamParams const &

params

)

[static, inherited]

Definition at line 83 of file BTreeExecStream.cpp.

References BTreeExecStream::copyParamsToDescriptor(), ExecStreamParams::pCacheAccessor, ExecStreamParams::scratchAccessor, and BTreeExecStream::treeDescriptor.

{
    BTreeDescriptor treeDescriptor;
    copyParamsToDescriptor(treeDescriptor,params,params.pCacheAccessor);
    return SharedBTreeWriter(
        new BTreeWriter(
            treeDescriptor,params.scratchAccessor));
}

void BTreeExecStream::endSearch

(

)

[protected, virtual, inherited]

Forgets the current reader or writer's search, releasing any page locks.

Definition at line 107 of file BTreeExecStream.cpp.

References BTreeExecStream::pBTreeReader.

Referenced by BTreeExecStream::closeImpl(), and BTreeExecStream::open().

{
    if (pBTreeReader && pBTreeReader->isSingular() == false) {
        pBTreeReader->endSearch();
    }
}

void BTreeExecStream::prepare

(

BTreeExecStreamParams const &

params

)

[virtual, inherited]

Definition at line 30 of file BTreeExecStream.cpp.

References BTreeExecStream::copyParamsToDescriptor(), ExecStreamParams::pCacheAccessor, SingleOutputExecStream::prepare(), BTreeParams::pRootMap, BTreeExecStream::pRootMap, BTreeParams::rootPageIdParamId, BTreeExecStream::rootPageIdParamId, ExecStreamParams::scratchAccessor, BTreeExecStream::scratchAccessor, and BTreeExecStream::treeDescriptor.

Referenced by prepare(), LbmGeneratorExecStream::prepare(), BTreeReadExecStream::prepare(), and BTreeInsertExecStream::prepare().

{
    SingleOutputExecStream::prepare(params);

    copyParamsToDescriptor(treeDescriptor,params,params.pCacheAccessor);
    scratchAccessor = params.scratchAccessor;
    pRootMap = params.pRootMap;
    rootPageIdParamId = params.rootPageIdParamId;
}

void SingleOutputExecStream::prepare

(

SingleOutputExecStreamParams const &

params

)

[virtual, inherited]

Definition at line 48 of file SingleOutputExecStream.cpp.

References SingleOutputExecStream::getOutputBufProvision(), SingleOutputExecStreamParams::outputTupleDesc, SingleOutputExecStreamParams::outputTupleFormat, SingleOutputExecStream::pOutAccessor, and ExecStream::prepare().

Referenced by BTreeExecStream::prepare(), FlatFileExecStreamImpl::prepare(), ValuesExecStream::prepare(), MockResourceExecStream::prepare(), MockProducerExecStream::prepare(), ConfluenceExecStream::prepare(), and ConduitExecStream::prepare().

{
    ExecStream::prepare(params);
    assert(pOutAccessor);
    assert(pOutAccessor->getProvision() == getOutputBufProvision());
    if (pOutAccessor->getTupleDesc().empty()) {
        assert(!params.outputTupleDesc.empty());
        pOutAccessor->setTupleShape(
            params.outputTupleDesc,
            params.outputTupleFormat);
    }
}

void ExecStream::prepare

(

ExecStreamParams const &

params

)

[virtual, inherited]

Prepares this stream for execution.

A precondition is that input streams must already be defined and prepared. As an effect of this call, the tuple shape should be defined for all output buffers and remain unchanged for the lifetime of the stream. This method is only ever called once, before the first open. Although this method is virtual, derived classes may choose to define an overloaded version instead with a specialized covariant parameter class.

Parameters:

params

instance of stream parameterization class which should be used to prepare this stream

Definition at line 84 of file ExecStream.cpp.

References ExecStreamGraph::getDynamicParamManager(), SegmentAccessor::pCacheAccessor, ExecStreamParams::pCacheAccessor, ExecStream::pDynamicParamManager, ExecStream::pGraph, ExecStream::pQuotaAccessor, ExecStream::pScratchQuotaAccessor, and ExecStreamParams::scratchAccessor.

Referenced by JavaTransformExecStream::prepare(), SingleOutputExecStream::prepare(), and SingleInputExecStream::prepare().

{
    if (pGraph) {
        pDynamicParamManager = pGraph->getDynamicParamManager();
    }
    pQuotaAccessor = params.pCacheAccessor;
    pScratchQuotaAccessor = params.scratchAccessor.pCacheAccessor;
}

void BTreeExecStream::copyParamsToDescriptor	(	BTreeDescriptor &	,
		BTreeParams const &	,
		SharedCacheAccessor const &
	)			[static, inherited]

Definition at line 93 of file BTreeExecStream.cpp.

References BTreeParams::keyProj, BTreeDescriptor::keyProjection, BTreeParams::pageOwnerId, BTreeDescriptor::pageOwnerId, SegmentAccessor::pCacheAccessor, BTreeParams::pSegment, SegmentAccessor::pSegment, BTreeParams::rootPageId, BTreeDescriptor::rootPageId, BTreeDescriptor::segmentAccessor, BTreeParams::segmentId, BTreeDescriptor::segmentId, BTreeExecStream::treeDescriptor, BTreeParams::tupleDesc, and BTreeDescriptor::tupleDescriptor.

Referenced by BTreeExecStream::newWriter(), LbmSplicerExecStream::prepare(), and BTreeExecStream::prepare().

{
    treeDescriptor.segmentAccessor.pSegment = params.pSegment;
    treeDescriptor.segmentAccessor.pCacheAccessor = pCacheAccessor;
    treeDescriptor.tupleDescriptor = params.tupleDesc;
    treeDescriptor.keyProjection = params.keyProj;
    treeDescriptor.rootPageId = params.rootPageId;
    treeDescriptor.segmentId = params.segmentId;
    treeDescriptor.pageOwnerId = params.pageOwnerId;
}

void SingleOutputExecStream::setInputBufAccessors

(

std::vector< SharedExecStreamBufAccessor > const &

inAccessors

)

[virtual, inherited]

Initializes the buffer accessors for inputs to this stream.

This method is only ever called once, before prepare.

Parameters:

inAccessors

buffer accessors ordered by input stream

Implements ExecStream.

Reimplemented in ConduitExecStream, and ConfluenceExecStream.

Definition at line 35 of file SingleOutputExecStream.cpp.

{
    assert(inAccessors.size() == 0);
}

void SingleOutputExecStream::setOutputBufAccessors

(

std::vector< SharedExecStreamBufAccessor > const &

outAccessors

)

[virtual, inherited]

Initializes the buffer accessors for outputs from this stream.

This method is only ever called once, before prepare.

Parameters:

outAccessors

buffer accessors ordered by output stream

Implements ExecStream.

Reimplemented in ConduitExecStream.

Definition at line 41 of file SingleOutputExecStream.cpp.

References SingleOutputExecStream::pOutAccessor.

Referenced by ConduitExecStream::setOutputBufAccessors().

{
    assert(outAccessors.size() == 1);
    pOutAccessor = outAccessors[0];
}

bool ExecStream::canEarlyClose

(

)

[virtual, inherited]

Returns:

true if the stream can be closed early

Reimplemented in SegBufferWriterExecStream.

Definition at line 49 of file ExecStream.cpp.

{
    return true;
}

ExecStreamGraph & ExecStream::getGraph

(

)

const [inline, inherited]

Returns:

reference to containing graph

Definition at line 293 of file ExecStream.h.

References ExecStream::pGraph.

Referenced by ExternalSortExecStreamImpl::execute(), JavaSinkExecStream::execute(), SegBufferWriterExecStream::execute(), SegBufferExecStream::execute(), BarrierExecStream::execute(), Java_net_sf_farrago_fennel_FennelStorage_tupleStreamFetch(), Java_net_sf_farrago_fennel_FennelStorage_tupleStreamTransformFetch(), JavaSinkExecStream::stuffByteBuffer(), and ExecStreamScheduler::traceStreamBuffers().

{
    assert(pGraph);
    return *pGraph;
}

ExecStreamId ExecStream::getStreamId

(

)

const [inline, inherited]

Returns:

the identifier for this stream within containing graph

Definition at line 288 of file ExecStream.h.

References ExecStream::id.

Referenced by ExternalSortExecStreamImpl::execute(), SegBufferWriterExecStream::execute(), SegBufferExecStream::execute(), CorrelationJoinExecStream::execute(), BarrierExecStream::execute(), Java_net_sf_farrago_fennel_FennelStorage_tupleStreamTransformFetch(), JavaTransformExecStream::open(), SingleInputExecStream::open(), ConfluenceExecStream::open(), CartesianJoinExecStream::prepare(), ParallelExecStreamScheduler::readStream(), DfsTreeExecStreamScheduler::readStream(), LbmMinusExecStream::restartSubtrahends(), ExecStreamScheduler::tracePostExecution(), ExecStreamScheduler::tracePreExecution(), ExecStreamScheduler::traceStreamBuffers(), and ParallelExecStreamScheduler::tryExecuteTask().

{
    return id;
}

void ExecStream::getResourceRequirements	(	ExecStreamResourceQuantity &	minQuantity,
		ExecStreamResourceQuantity &	optQuantity,
		ExecStreamResourceSettingType &	optType
	)			[virtual, inherited]

Determines resource requirements for this stream.

Default implementation declares zero resource requirements.

Parameters:

	minQuantity	receives the minimum resource quantity needed by this stream in order to execute
	optQuantity	receives the resource quantity needed by this stream in order to execute optimally
	optType	Receives the value indicating the accuracy of the optQuantity parameter. This parameter is optional and defaults to EXEC_RESOURCE_ACCURATE if omitted. If the optimum setting is an estimate or no value can be specified (e.g., due to lack of statistics), then this parameter needs to be used to indicate a non-accurate optimum resource setting.

Reimplemented in MockResourceExecStream, BTreePrefetchSearchExecStream, LhxAggExecStream, LhxJoinExecStream, LbmGeneratorExecStream, LbmUnionExecStream, and ExternalSortExecStreamImpl.

Definition at line 93 of file ExecStream.cpp.

References EXEC_RESOURCE_ACCURATE.

Referenced by ExternalSortExecStreamImpl::getResourceRequirements(), LcsRowScanBaseExecStream::getResourceRequirements(), getResourceRequirements(), LbmUnionExecStream::getResourceRequirements(), LbmSplicerExecStream::getResourceRequirements(), LbmGeneratorExecStream::getResourceRequirements(), LbmChopperExecStream::getResourceRequirements(), LhxJoinExecStream::getResourceRequirements(), LhxAggExecStream::getResourceRequirements(), FtrsTableWriterExecStream::getResourceRequirements(), BTreeReadExecStream::getResourceRequirements(), BTreeInsertExecStream::getResourceRequirements(), FlatFileExecStreamImpl::getResourceRequirements(), SegBufferWriterExecStream::getResourceRequirements(), SegBufferReaderExecStream::getResourceRequirements(), SegBufferExecStream::getResourceRequirements(), ScratchBufferExecStream::getResourceRequirements(), and DoubleBufferExecStream::getResourceRequirements().

{
    getResourceRequirements(minQuantity, optQuantity);
    optType = EXEC_RESOURCE_ACCURATE;
}

void ExecStream::setResourceAllocation

(

ExecStreamResourceQuantity &

quantity

)

[virtual, inherited]

Sets current resource allocation for this stream.

If called while the stream is open, this indicates a request for the stream to dynamically adjust its memory usage. If the stream is incapable of honoring the request, it should update quantity with the actual amounts still in use.

Parameters:

quantity

allocated resource quantity

Reimplemented in MockResourceExecStream, BTreePrefetchSearchExecStream, LhxAggExecStream, LhxJoinExecStream, LbmGeneratorExecStream, LbmUnionExecStream, and ExternalSortExecStreamImpl.

Definition at line 111 of file ExecStream.cpp.

References ExecStreamResourceQuantity::nCachePages, ExecStream::pQuotaAccessor, ExecStream::pScratchQuotaAccessor, and ExecStream::resourceAllocation.

Referenced by ExternalSortExecStreamImpl::setResourceAllocation(), LbmUnionExecStream::setResourceAllocation(), LbmGeneratorExecStream::setResourceAllocation(), LhxJoinExecStream::setResourceAllocation(), LhxAggExecStream::setResourceAllocation(), and BTreePrefetchSearchExecStream::setResourceAllocation().

{
    resourceAllocation = quantity;
    if (pQuotaAccessor) {
        pQuotaAccessor->setMaxLockedPages(quantity.nCachePages);
    }
    if (pScratchQuotaAccessor) {
        pScratchQuotaAccessor->setMaxLockedPages(quantity.nCachePages);
    }
}

void ExecStream::setName

(

std::string const &

)

[virtual, inherited]

Sets unique name of this stream.

Definition at line 157 of file ExecStream.cpp.

References ExecStream::name.

{
    name = nameInit;
}

std::string const & ExecStream::getName

(

)

const [virtual, inherited]

Returns:

the name of this stream, as known by the optimizer

Definition at line 162 of file ExecStream.cpp.

References ExecStream::name.

Referenced by DfsTreeExecStreamScheduler::findNextConsumer(), ParallelExecStreamScheduler::readStream(), DfsTreeExecStreamScheduler::readStream(), ExecStreamScheduler::tracePostExecution(), and ExecStreamScheduler::tracePreExecution().

{
    return name;
}

bool ExecStream::mayBlock

(

)

const [virtual, inherited]

Queries whether this stream's implementation may block when execute() is called.

For accurate scheduling, non-blocking implementations are preferred; the scheduler must be aware of the potential for blocking so that it can allocate extra threads accordingly.

Returns:

whether stream may block; default is false

Definition at line 167 of file ExecStream.cpp.

{
    return false;
}

void ExecStream::checkAbort

(

)

const [virtual, inherited]

Checks whether there is an abort request for this stream's scheduler.

Normally, streams don't need to check this, since the scheduler services abort requests in between quanta. However, streams which enter long-running loops need to check for themselves. If an abort is scheduled, this method will throw an AbortExcn automatically.

Definition at line 72 of file ExecStream.cpp.

References ExecStreamScheduler::checkAbort(), ExecStreamGraph::getScheduler(), and ExecStream::pGraph.

Referenced by LhxJoinExecStream::execute(), LhxAggExecStream::execute(), ExternalSortRunAccessor::fetch(), and ExternalSortMerger::fetch().

{
    if (!pGraph) {
        return;
    }
    ExecStreamScheduler *pScheduler = pGraph->getScheduler();
    if (!pScheduler) {
        return;
    }
    pScheduler->checkAbort();
}

ExecStreamBufProvision ExecStream::getOutputBufConversion

(

)

const [virtual, inherited]

Queries the BufferProvision to which this stream needs its output to be converted, if any.

Returns:

required conversion; default is BUFPROV_NONE

Reimplemented in JavaTransformExecStream.

Definition at line 177 of file ExecStream.cpp.

References BUFPROV_NONE.

{
    return BUFPROV_NONE;
}

ExecStreamBufProvision ExecStream::getInputBufProvision

(

)

const [virtual, inherited]

Queries the BufferProvision which this stream requires of its inputs when consuming their tuples.

Returns:

required model; default is BUFPROV_NONE

Reimplemented in ConfluenceExecStream, DoubleBufferExecStream, ScratchBufferExecStream, SegBufferExecStream, SegBufferWriterExecStream, SingleInputExecStream, and JavaTransformExecStream.

Definition at line 182 of file ExecStream.cpp.

References BUFPROV_NONE.

{
    return BUFPROV_NONE;
}

bool ClosableObject::isClosed

(

)

const [inline, inherited]

Returns:

whether the object has been closed

Definition at line 58 of file ClosableObject.h.

    {
        return !needsClose;
    }

void TraceSource::initTraceSource	(	SharedTraceTarget	pTraceTarget,
		std::string	name
	)			[virtual, inherited]

For use when initialization has to be deferred until after construction.

Parameters:

	pTraceTarget	the TraceTarget to which messages will be sent
	name	the name of this source

Definition at line 46 of file TraceSource.cpp.

References TraceSource::isTracing(), TraceSource::minimumLevel, TraceSource::name, TraceSource::pTraceTarget, and TRACE_OFF.

Referenced by TestBase::beforeTestCase(), TestBase::TestBase(), and TraceSource::TraceSource().

{
    assert(!pTraceTarget.get());

    pTraceTarget = pTraceTargetInit;
    name = nameInit;
    if (isTracing()) {
        minimumLevel = pTraceTarget->getSourceTraceLevel(name);
    } else {
        minimumLevel = TRACE_OFF;
    }
}

void TraceSource::trace	(	TraceLevel	level,
		std::string	message
	)			const [inherited]

Records a trace message.

Normally only called via FENNEL_TRACE.

Parameters:

	level	severity level of event being trace
	message	the text of the message

Definition at line 61 of file TraceSource.cpp.

References TraceSource::getTraceTarget(), TraceSource::isTracing(), TraceSource::name, and TraceTarget::notifyTrace().

Referenced by Calculator::exec(), and ExecStreamScheduler::traceStreamBufferContents().

{
    if (isTracing()) {
        getTraceTarget().notifyTrace(name,level,message);
    }
}

bool TraceSource::isTracing

(

)

const [inline, inherited]

Returns:

true iff tracing is enabled for this source

Definition at line 88 of file TraceSource.h.

Referenced by TraceSource::initTraceSource(), CalcExecStream::prepare(), and TraceSource::trace().

    {
        return pTraceTarget.get() ? true : false;
    }

bool TraceSource::isTracingLevel

(

TraceLevel

level

)

const [inline, inherited]

Determines whether a particular level is being traced.

Parameters:

level

trace level to test

Returns:

true iff tracing is enabled for the given level

Definition at line 100 of file TraceSource.h.

Referenced by ExecStreamScheduler::addGraph(), SimpleExecStreamGovernor::assignCachePages(), SimpleExecStreamGovernor::distributeCachePages(), Calculator::exec(), ExecStreamScheduler::ExecStreamScheduler(), LcsClusterNodeWriter::getLastClusterPageForWrite(), LcsClusterNodeWriter::moveFromTempToIndex(), JavaSinkExecStream::stuffByteBuffer(), and ExecStreamScheduler::traceStreamBuffers().

    {
        return level >= minimumLevel;
    }

TraceTarget& TraceSource::getTraceTarget

(

)

const [inline, inherited]

Returns:

the TraceTarget for this source

Definition at line 108 of file TraceSource.h.

Referenced by TraceSource::trace().

    {
        assert(isTracing());
        return *(pTraceTarget.get());
    }

SharedTraceTarget TraceSource::getSharedTraceTarget

(

)

const [inline, inherited]

Returns:

the SharedTraceTarget for this source

Definition at line 117 of file TraceSource.h.

Referenced by Database::init(), initLoad(), and CalcExecStream::prepare().

    {
        return pTraceTarget;
    }

std::string TraceSource::getTraceSourceName

(

)

const [inline, inherited]

Gets the name of this source.

Useful to construct nested names for subcomponents that are also TraceSources.

Returns:

the name

Definition at line 127 of file TraceSource.h.

Referenced by initLoad().

    {
        return name;
    }

void TraceSource::setTraceSourceName

(

std::string const &

n

)

[inline, inherited]

Sets the name of this source.

Useful to construct dynamic names for fine-grained filtering.

Definition at line 136 of file TraceSource.h.

    {
        name = n;
    }

TraceLevel TraceSource::getMinimumTraceLevel

(

)

const [inline, inherited]

Definition at line 141 of file TraceSource.h.

    {
        return minimumLevel;
    }

void TraceSource::disableTracing

(

)

[inherited]

Definition at line 68 of file TraceSource.cpp.

References TraceSource::minimumLevel, TraceSource::pTraceTarget, and TRACE_OFF.

Referenced by TestBase::afterTestCase().

{
    pTraceTarget.reset();
    minimumLevel = TRACE_OFF;
}

void ErrorSource::initErrorSource	(	SharedErrorTarget	pErrorTarget,
		const std::string &	name
	)			[virtual, inherited]

For use when initialization has to be deferred until after construction.

Parameters:

	pErrorTarget	the ErrorTarget to which errors will be posted
	name	the name of this source

Definition at line 47 of file ErrorSource.cpp.

References ErrorSource::name, and ErrorSource::pErrorTarget.

Referenced by ErrorSource::ErrorSource().

{
    pErrorTarget = pErrorTargetInit;
    name = nameInit;
}

void ErrorSource::postError	(	ErrorLevel	level,
		const std::string &	message,
		void *	address,
		long	capacity,
		int	index
	)			[inherited]

Posts an exception, such as a row exception.

See also:

ErrorTarget for a description of the parameters

Definition at line 55 of file ErrorSource.cpp.

References ErrorSource::getErrorTarget(), ErrorSource::hasTarget(), ErrorSource::name, and ErrorTarget::notifyError().

Referenced by FlatFileExecStreamImpl::logError(), ErrorSource::postError(), and LbmSplicerExecStream::postViolation().

{
    if (hasTarget()) {
        getErrorTarget().notifyError(
            name, level, message, address, capacity, index);
    }
}

void ErrorSource::postError	(	ErrorLevel	level,
		const std::string &	message,
		const TupleDescriptor &	errorDesc,
		const TupleData &	errorTuple,
		int	index
	)			[inherited]

Posts an exception, such as a row exception.

See also:

ErrorTarget for a description of the parameters

Definition at line 65 of file ErrorSource.cpp.

References TupleAccessor::compute(), ErrorSource::errorAccessor, FixedBuffer, TupleAccessor::getByteCount(), TupleAccessor::getMaxByteCount(), ErrorSource::hasTarget(), TupleAccessor::marshal(), ErrorSource::pErrorBuf, and ErrorSource::postError().

{
    if (!hasTarget()) {
        return;
    }

    if (!pErrorBuf) {
        errorAccessor.compute(errorDesc);
        uint cbMax = errorAccessor.getMaxByteCount();
        pErrorBuf.reset(new FixedBuffer[cbMax]);
    }

    uint cbTuple = errorAccessor.getByteCount(errorTuple);
    errorAccessor.marshal(errorTuple, pErrorBuf.get());
    postError(level, message, pErrorBuf.get(), cbTuple, index);
}

bool ErrorSource::hasTarget

(

)

const [inline, inherited]

Returns:

true iff an error target has been set

Definition at line 112 of file ErrorSource.h.

Referenced by ErrorSource::postError().

    {
        return pErrorTarget.get() ? true : false;
    }

ErrorTarget& ErrorSource::getErrorTarget

(

)

const [inline, inherited]

Returns:

the ErrorTarget for this source

Definition at line 120 of file ErrorSource.h.

Referenced by ErrorSource::postError().

    {
        assert(hasTarget());
        return *(pErrorTarget.get());
    }

SharedErrorTarget ErrorSource::getSharedErrorTarget

(

)

const [inline, inherited]

Returns:

the SharedErrorTarget for this source

Definition at line 129 of file ErrorSource.h.

    {
        return pErrorTarget;
    }

std::string ErrorSource::getErrorSourceName

(

)

const [inline, inherited]

Gets the name of this source.

Useful to construct nested names for subcomponents that are also ErrorSources.

Returns:

the name

Definition at line 139 of file ErrorSource.h.

    {
        return name;
    }

void ErrorSource::setErrorSourceName

(

std::string const &

n

)

[inline, inherited]

Sets the name of this source.

Useful to construct dynamic names for fine-grained filtering.

Definition at line 148 of file ErrorSource.h.

    {
        name = n;
    }

void ErrorSource::disableTarget

(

)

[inherited]

Definition at line 84 of file ErrorSource.cpp.

References ErrorSource::pErrorTarget.

{
    pErrorTarget.reset();
}

ExecStreamResult ConduitExecStream::precheckConduitBuffers

(

)

[protected, inherited]

Checks the state of the input and output buffers.

If input empty, requests production. If input EOS, propagates that to output buffer. If output full, returns EXECRC_OVERFLOW.

Returns:

result of precheck; anything but EXECRC_YIELD indicates that execution should terminate immediately with returned code

Definition at line 61 of file ConduitExecStream.cpp.

References EXECBUF_EMPTY, EXECBUF_EOS, EXECBUF_NONEMPTY, EXECBUF_OVERFLOW, EXECBUF_UNDERFLOW, EXECRC_BUF_OVERFLOW, EXECRC_BUF_UNDERFLOW, EXECRC_EOS, EXECRC_YIELD, SingleInputExecStream::pInAccessor, and SingleOutputExecStream::pOutAccessor.

Referenced by ExternalSortExecStreamImpl::execute(), FtrsTableWriterExecStream::execute(), BTreeSearchUniqueExecStream::execute(), BTreeSearchExecStream::execute(), BTreeInsertExecStream::execute(), UncollectExecStream::execute(), SortedAggExecStream::execute(), ReshapeExecStream::execute(), CopyExecStream::execute(), CollectExecStream::execute(), BernoulliSamplingExecStream::execute(), and CalcExecStream::execute().

{
    switch (pInAccessor->getState()) {
    case EXECBUF_EMPTY:
        pInAccessor->requestProduction();
        return EXECRC_BUF_UNDERFLOW;
    case EXECBUF_UNDERFLOW:
        return EXECRC_BUF_UNDERFLOW;
    case EXECBUF_EOS:
        pOutAccessor->markEOS();
        return EXECRC_EOS;
    case EXECBUF_NONEMPTY:
    case EXECBUF_OVERFLOW:
        break;
    default:
        permAssert(false);
    }
    if (pOutAccessor->getState() == EXECBUF_OVERFLOW) {
        return EXECRC_BUF_OVERFLOW;
    }
    return EXECRC_YIELD;
}

void ConduitExecStream::setOutputBufAccessors

(

std::vector< SharedExecStreamBufAccessor > const &

outAccessors

)

[virtual, inherited]

Initializes the buffer accessors for outputs from this stream.

This method is only ever called once, before prepare.

Parameters:

outAccessors

buffer accessors ordered by output stream

Reimplemented from SingleInputExecStream.

Definition at line 36 of file ConduitExecStream.cpp.

References SingleOutputExecStream::setOutputBufAccessors().

{
    SingleOutputExecStream::setOutputBufAccessors(outAccessors);
}

void ConduitExecStream::setInputBufAccessors

(

std::vector< SharedExecStreamBufAccessor > const &

inAccessors

)

[virtual, inherited]

Initializes the buffer accessors for inputs to this stream.

This method is only ever called once, before prepare.

Parameters:

inAccessors

buffer accessors ordered by input stream

Reimplemented from SingleInputExecStream.

Definition at line 30 of file ConduitExecStream.cpp.

References SingleInputExecStream::setInputBufAccessors().

{
    SingleInputExecStream::setInputBufAccessors(inAccessors);
}

void ConduitExecStream::prepare

(

ConduitExecStreamParams const &

params

)

[virtual, inherited]

Definition at line 42 of file ConduitExecStream.cpp.

References SingleInputExecStream::pInAccessor, SingleOutputExecStream::pOutAccessor, SingleOutputExecStream::prepare(), and SingleInputExecStream::prepare().

Referenced by ExternalSortExecStreamImpl::prepare(), prepare(), LbmNormalizerExecStream::prepare(), LhxAggExecStream::prepare(), FtrsTableWriterExecStream::prepare(), BTreeSearchExecStream::prepare(), BTreeInsertExecStream::prepare(), UncollectExecStream::prepare(), SortedAggExecStream::prepare(), SegBufferReaderExecStream::prepare(), SegBufferExecStream::prepare(), ScratchBufferExecStream::prepare(), ReshapeExecStream::prepare(), DoubleBufferExecStream::prepare(), CopyExecStream::prepare(), CollectExecStream::prepare(), BernoulliSamplingExecStream::prepare(), and CalcExecStream::prepare().

{
    SingleInputExecStream::prepare(params);

    if (params.outputTupleDesc.empty()) {
        pOutAccessor->setTupleShape(
            pInAccessor->getTupleDesc(),
            pInAccessor->getTupleFormat());
    }

    SingleOutputExecStream::prepare(params);
}

void SingleInputExecStream::prepare

(

SingleInputExecStreamParams const &

params

)

[virtual, inherited]

Definition at line 44 of file SingleInputExecStream.cpp.

References SingleInputExecStream::getInputBufProvision(), SingleInputExecStream::pInAccessor, and ExecStream::prepare().

Referenced by JavaSinkExecStream::prepare(), MockConsumerExecStream::prepare(), DiffluenceExecStream::prepare(), and ConduitExecStream::prepare().

{
    ExecStream::prepare(params);

    assert(pInAccessor);
    assert(pInAccessor->getProvision() == getInputBufProvision());
}

ExecStreamBufProvision SingleInputExecStream::getInputBufProvision

(

)

const [virtual, inherited]

Queries the BufferProvision which this stream requires of its inputs when consuming their tuples.

Returns:

required model; default is BUFPROV_NONE

Reimplemented from ExecStream.

Reimplemented in DoubleBufferExecStream, ScratchBufferExecStream, SegBufferExecStream, and SegBufferWriterExecStream.

Definition at line 62 of file SingleInputExecStream.cpp.

References BUFPROV_PRODUCER.

Referenced by SingleInputExecStream::prepare().

{
    return BUFPROV_PRODUCER;
}

---

Member Data Documentation

uint LcsClusterAppendExecStream::blockSize [protected]

Space available on page blocks for writing cluster data.

Definition at line 58 of file LcsClusterAppendExecStream.h.

Referenced by initLoad(), loadExistingBlock(), prepare(), and startNewBlock().

TupleDescriptor LcsClusterAppendExecStream::tableColsTupleDesc [protected]

Tuple descriptor for the tuple representing all cluster columns across the table that this cluster is a part of.

Definition at line 64 of file LcsClusterAppendExecStream.h.

Referenced by LcsClusterReplaceExecStream::initTupleLoadParams(), initTupleLoadParams(), and prepare().

TupleData LcsClusterAppendExecStream::clusterColsTupleData [protected]

Tuple data for the tuple datums representing only this cluster.

Definition at line 69 of file LcsClusterAppendExecStream.h.

Referenced by compress(), LcsClusterReplaceExecStream::getTupleForLoad(), getTupleForLoad(), LcsClusterReplaceExecStream::initTupleLoadParams(), initTupleLoadParams(), and LcsClusterReplaceExecStream::readOrigClusterRow().

TupleDescriptor LcsClusterAppendExecStream::clusterColsTupleDesc [protected]

Tuple descriptors for the columns that are part of this cluster.

Definition at line 74 of file LcsClusterAppendExecStream.h.

Referenced by initLoad(), LcsClusterReplaceExecStream::initTupleLoadParams(), and initTupleLoadParams().

boost::scoped_array<TupleDescriptor> LcsClusterAppendExecStream::colTupleDesc [protected]

Individual tuple descriptors for each column in the cluster.

Definition at line 79 of file LcsClusterAppendExecStream.h.

Referenced by initLoad(), LcsClusterReplaceExecStream::initTupleLoadParams(), initTupleLoadParams(), and startNewBlock().

SegmentAccessor LcsClusterAppendExecStream::scratchAccessor [protected]

Scratch accessor for allocating large buffer pages.

Reimplemented from BTreeExecStream.

Definition at line 84 of file LcsClusterAppendExecStream.h.

Referenced by close(), initLoad(), and prepare().

ClusterPageLock LcsClusterAppendExecStream::bufferLock [protected]

Lock on scratch page.

Definition at line 89 of file LcsClusterAppendExecStream.h.

Referenced by initLoad(), and prepare().

bool LcsClusterAppendExecStream::overwrite [protected]

True if overwriting all existing data.

Definition at line 94 of file LcsClusterAppendExecStream.h.

bool LcsClusterAppendExecStream::isDone [protected]

Whether row count has been produced.

Definition at line 99 of file LcsClusterAppendExecStream.h.

Referenced by compress(), and open().

TupleData LcsClusterAppendExecStream::outputTuple [protected]

Output tuple containing count of number of rows loaded.

Definition at line 104 of file LcsClusterAppendExecStream.h.

Referenced by compress(), and prepare().

TupleAccessor* LcsClusterAppendExecStream::outputTupleAccessor [protected]

A reference to the output accessor contained in SingleOutputExecStream::pOutAccessor.

Definition at line 110 of file LcsClusterAppendExecStream.h.

Referenced by compress(), open(), and prepare().

boost::scoped_array<FixedBuffer> LcsClusterAppendExecStream::outputTupleBuffer [protected]

buffer holding the outputTuple to provide to the consumers

Definition at line 115 of file LcsClusterAppendExecStream.h.

Referenced by closeImpl(), compress(), and open().

bool LcsClusterAppendExecStream::compressCalled [protected]

True if execute has been called at least once.

Definition at line 120 of file LcsClusterAppendExecStream.h.

Referenced by init(), and initLoad().

boost::scoped_array<LcsHash> LcsClusterAppendExecStream::hash [protected]

Array of hashes, one per cluster column.

Definition at line 125 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), compress(), initLoad(), loadExistingBlock(), startNewBlock(), and writeBatch().

uint LcsClusterAppendExecStream::numColumns [protected]

Number of columns in the cluster.

Definition at line 130 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), compress(), getResourceRequirements(), initLoad(), LcsClusterReplaceExecStream::initTupleLoadParams(), initTupleLoadParams(), loadExistingBlock(), startNewBlock(), and writeBatch().

boost::scoped_array<PBuffer> LcsClusterAppendExecStream::rowBlock [protected]

Array of temporary blocks for row array.

Definition at line 135 of file LcsClusterAppendExecStream.h.

Referenced by addValueOrdinal(), allocArrays(), close(), initLoad(), and writeBatch().

uint LcsClusterAppendExecStream::nRowsMax [protected]

Maximum number of values that can be stored in m_rowBlock.

Definition at line 140 of file LcsClusterAppendExecStream.h.

Referenced by initLoad(), and isRowArrayFull().

boost::scoped_array<PBuffer> LcsClusterAppendExecStream::hashBlock [protected]

Array of temporary blocks for hash table.

Definition at line 145 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), initLoad(), and startNewBlock().

boost::scoped_array<PBuffer> LcsClusterAppendExecStream::builderBlock [protected]

Array of temporary blocks used by ClusterNodeWriter.

Definition at line 150 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), initLoad(), loadExistingBlock(), and startNewBlock().

uint LcsClusterAppendExecStream::rowCnt [protected]

Number of rows loaded into the current set of batches.

Definition at line 155 of file LcsClusterAppendExecStream.h.

Referenced by addValueOrdinal(), compress(), init(), isRowArrayFull(), loadExistingBlock(), startNewBlock(), writeBatch(), and writeBlock().

bool LcsClusterAppendExecStream::indexBlockDirty [protected]

True if index blocks need to be written to disk.

Definition at line 160 of file LcsClusterAppendExecStream.h.

Referenced by addValueOrdinal(), init(), loadExistingBlock(), startNewBlock(), and writeBlock().

LcsRid LcsClusterAppendExecStream::firstRow [protected]

Starting rowid in a cluster page.

Definition at line 165 of file LcsClusterAppendExecStream.h.

Referenced by getLastBlock(), init(), loadExistingBlock(), and startNewBlock().

LcsRid LcsClusterAppendExecStream::lastRow [protected]

Last rowid in the last batch.

Definition at line 170 of file LcsClusterAppendExecStream.h.

Referenced by init(), loadExistingBlock(), startNewBlock(), and writeBatch().

LcsRid LcsClusterAppendExecStream::startRow [protected]

Definition at line 174 of file LcsClusterAppendExecStream.h.

Referenced by init(), initLoad(), loadExistingBlock(), and prepare().

SharedLcsClusterNodeWriter LcsClusterAppendExecStream::lcsBlockBuilder [protected]

Page builder object.

Definition at line 179 of file LcsClusterAppendExecStream.h.

Referenced by close(), compress(), getLastBlock(), initLoad(), loadExistingBlock(), startNewBlock(), writeBatch(), and writeBlock().

boost::scoped_array<LcsHashValOrd> LcsClusterAppendExecStream::hashValOrd [protected]

Row value ordinal returned from hash, one per cluster column.

Definition at line 184 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), and compress().

boost::scoped_array<boost::scoped_array<FixedBuffer> > LcsClusterAppendExecStream::tempBuf [protected]

Temporary buffers used by WriteBatch.

Definition at line 189 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), and writeBatch().

boost::scoped_array<uint> LcsClusterAppendExecStream::maxValueSize [protected]

Max size for each column cluster used by WriteBatch.

Definition at line 194 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), close(), and writeBatch().

bool LcsClusterAppendExecStream::arraysAlloced [protected]

Indicates where or not we have already allocated arrays.

Definition at line 199 of file LcsClusterAppendExecStream.h.

Referenced by allocArrays(), and init().

PLcsClusterNode LcsClusterAppendExecStream::pIndexBlock [protected]

Buffer pointing to cluster page that will actually be written.

Definition at line 204 of file LcsClusterAppendExecStream.h.

Referenced by init(), initLoad(), loadExistingBlock(), and startNewBlock().

RecordNum LcsClusterAppendExecStream::numRowCompressed [protected]

Total number of rows loaded by this object.

Definition at line 209 of file LcsClusterAppendExecStream.h.

Referenced by init(), postProcessTuple(), and prepare().

BTreeDescriptor BTreeExecStream::treeDescriptor [protected, inherited]

Definition at line 113 of file BTreeExecStream.h.

Referenced by BTreeInsertExecStream::buildTree(), BTreeExecStream::closeImpl(), BTreeExecStream::copyParamsToDescriptor(), LcsClusterReplaceExecStream::getTupleForLoad(), initLoad(), LcsClusterReplaceExecStream::initTupleLoadParams(), BTreePrefetchSearchExecStream::innerSearchLoop(), BTreePrefetchSearchExecStream::newReader(), BTreeExecStream::newReader(), BTreeExecStream::newWriter(), LcsClusterReplaceExecStream::open(), BTreeSearchExecStream::open(), BTreePrefetchSearchExecStream::open(), BTreeInsertExecStream::open(), BTreeExecStream::open(), LcsClusterReplaceExecStream::prepare(), prepare(), LbmSearchExecStream::prepare(), LbmGeneratorExecStream::prepare(), BTreeSearchExecStream::prepare(), BTreeInsertExecStream::prepare(), BTreeExecStream::prepare(), and BTreeInsertExecStream::truncateTree().

BTreeOwnerRootMap* BTreeExecStream::pRootMap [protected, inherited]

Definition at line 115 of file BTreeExecStream.h.

Referenced by BTreeExecStream::closeImpl(), BTreeExecStream::open(), and BTreeExecStream::prepare().

SharedBTreeAccessBase BTreeExecStream::pBTreeAccessBase [protected, inherited]

Definition at line 116 of file BTreeExecStream.h.

Referenced by BTreeInsertExecStream::closeImpl(), BTreeExecStream::closeImpl(), BTreePrefetchSearchExecStream::newReader(), BTreeExecStream::newReader(), BTreeExecStream::newWriter(), and BTreeExecStream::open().

SharedBTreeReader BTreeExecStream::pBTreeReader [protected, inherited]

Definition at line 117 of file BTreeExecStream.h.

Referenced by BTreeExecStream::endSearch(), BTreePrefetchSearchExecStream::newReader(), BTreeExecStream::newReader(), and BTreeExecStream::newWriter().

DynamicParamId BTreeExecStream::rootPageIdParamId [protected, inherited]

Definition at line 118 of file BTreeExecStream.h.

Referenced by BTreeSearchExecStream::open(), BTreeInsertExecStream::open(), BTreeInsertExecStream::prepare(), and BTreeExecStream::prepare().

SharedExecStreamBufAccessor SingleOutputExecStream::pOutAccessor [protected, inherited]

Definition at line 56 of file SingleOutputExecStream.h.

Referenced by compress(), ExternalSortExecStreamImpl::execute(), LcsRowScanExecStream::execute(), LbmUnionExecStream::execute(), LbmNormalizerExecStream::execute(), LbmGeneratorExecStream::execute(), LbmChopperExecStream::execute(), LhxJoinExecStream::execute(), LhxAggExecStream::execute(), FtrsTableWriterExecStream::execute(), BTreeSortExecStream::execute(), BTreeSearchUniqueExecStream::execute(), BTreeScanExecStream::execute(), BTreePrefetchSearchExecStream::execute(), BTreeInsertExecStream::execute(), FlatFileExecStreamImpl::execute(), ValuesExecStream::execute(), UncollectExecStream::execute(), SortedAggExecStream::execute(), SegBufferReaderExecStream::execute(), ScratchBufferExecStream::execute(), ReshapeExecStream::execute(), MockResourceExecStream::execute(), MockProducerExecStream::execute(), MergeExecStream::execute(), DoubleBufferExecStream::execute(), CorrelationJoinExecStream::execute(), CopyExecStream::execute(), CollectExecStream::execute(), CartesianJoinExecStream::execute(), BernoulliSamplingExecStream::execute(), BarrierExecStream::execute(), CalcExecStream::execute(), LbmGeneratorExecStream::flushEntry(), MockProducerExecStream::getProducedRowCount(), ExternalSortExecStreamImpl::getResourceRequirements(), BTreeSearchExecStream::innerFetchLoop(), LbmUnionExecStream::open(), LbmChopperExecStream::open(), LbmBitOpExecStream::open(), SingleOutputExecStream::open(), SegBufferReaderExecStream::open(), SegBufferExecStream::open(), ScratchBufferExecStream::open(), DoubleBufferExecStream::open(), CollectExecStream::open(), SegBufferExecStream::openBufferForRead(), ConduitExecStream::precheckConduitBuffers(), LcsRowScanExecStream::prepare(), LcsRowScanBaseExecStream::prepare(), prepare(), LbmUnionExecStream::prepare(), LbmGeneratorExecStream::prepare(), LhxJoinExecStream::prepare(), LhxAggExecStream::prepare(), FtrsTableWriterExecStream::prepare(), FlatFileExecStreamImpl::prepare(), UncollectExecStream::prepare(), SortedAggExecStream::prepare(), SingleOutputExecStream::prepare(), ReshapeExecStream::prepare(), MockResourceExecStream::prepare(), MockProducerExecStream::prepare(), MergeExecStream::prepare(), CorrelationJoinExecStream::prepare(), ConduitExecStream::prepare(), CollectExecStream::prepare(), CartesianJoinExecStream::prepare(), BernoulliSamplingExecStream::prepare(), BarrierExecStream::prepare(), CalcExecStream::prepare(), SortedAggExecStream::produce(), LbmBitOpExecStream::producePendingOutput(), LbmUnionExecStream::produceTuple(), LbmNormalizerExecStream::produceTuple(), LbmMinusExecStream::produceTuple(), LbmChopperExecStream::produceTuple(), LbmBitOpExecStream::produceTuple(), LbmBitOpExecStream::readInput(), LbmMinusExecStream::readMinuendInput(), and SingleOutputExecStream::setOutputBufAccessors().

bool ExecStream::isOpen [protected, inherited]

Whether this stream is currently open.

Note that this is not quite the opposite of the inherited ClosableObject.needsClose, since a stream needs to be closed before destruction if it has been prepared but never opened.

Definition at line 61 of file ExecStream.h.

Referenced by ExecStream::closeImpl(), ExecStream::ExecStream(), and ExecStream::open().

ExecStreamGraph* ExecStream::pGraph [protected, inherited]

Dataflow graph containing this stream.

Note that we don't use a weak_ptr for this because it needs to be accessed frequently during execution, and the extra locking overhead would be frivolous.

Definition at line 68 of file ExecStream.h.

Referenced by ExecStream::checkAbort(), ExecStream::ExecStream(), CorrelationJoinExecStream::execute(), ExecStream::getGraph(), JavaTransformExecStream::open(), SingleInputExecStream::open(), MergeExecStream::open(), ExecStream::open(), CorrelationJoinExecStream::open(), ConfluenceExecStream::open(), ExecStream::prepare(), CartesianJoinExecStream::prepare(), and LbmMinusExecStream::restartSubtrahends().

ExecStreamId ExecStream::id [protected, inherited]

Identifier for this stream; local to its containing graph.

Definition at line 73 of file ExecStream.h.

Referenced by ExecStream::getStreamId().

std::string ExecStream::name [protected, inherited]

Name of stream, as known by optimizer.

Reimplemented from TraceSource.

Definition at line 78 of file ExecStream.h.

Referenced by ExecStream::ExecStream(), ExecStream::getName(), FlatFileExecStreamImpl::open(), and ExecStream::setName().

SharedDynamicParamManager ExecStream::pDynamicParamManager [protected, inherited]

The dynamic parameter manager available to this stream.

(Obtained at prepare() time. Keep a shared pointer in case the stream is reassigned to another graph for execution; cf ExecStreamGraph::mergeFrom())

Definition at line 85 of file ExecStream.h.

Referenced by CorrelationJoinExecStream::close(), SegBufferReaderExecStream::closeImpl(), LbmUnionExecStream::execute(), LbmSplicerExecStream::execute(), LbmGeneratorExecStream::execute(), LbmChopperExecStream::execute(), CorrelationJoinExecStream::execute(), BarrierExecStream::execute(), LcsClusterReplaceExecStream::open(), LbmUnionExecStream::open(), LbmSplicerExecStream::open(), LbmGeneratorExecStream::open(), LbmBitOpExecStream::open(), BTreeSearchExecStream::open(), BTreeInsertExecStream::open(), SegBufferWriterExecStream::open(), SegBufferReaderExecStream::open(), NestedLoopJoinExecStream::open(), CorrelationJoinExecStream::open(), ExecStream::prepare(), CalcExecStream::prepare(), NestedLoopJoinExecStream::processLeftInput(), LbmSearchExecStream::reachedTupleLimit(), ReshapeExecStream::readDynamicParams(), SegBufferWriterExecStream::readReaderRefCount(), BTreeSearchExecStream::readSearchKey(), BTreeSearchExecStream::readUpperBoundKey(), LbmSearchExecStream::setAdditionalKeys(), and LbmBitOpExecStream::writeStartRidParamValue().

SharedLogicalTxn ExecStream::pTxn [protected, inherited]

The transaction embracing the stream.

Obtained at open() time; but not released at close() time, to allow TableWriters to replay a txn. Keep a shared pointer in case the stream is reassigned to another graph for execution; cf ExecStreamGraph::mergeFrom())

Definition at line 94 of file ExecStream.h.

Referenced by FtrsTableWriterExecStream::commitSavepoint(), FtrsTableWriterExecStream::createSavepoint(), FtrsTableWriterExecStream::open(), ExecStream::open(), and FtrsTableWriterExecStream::rollbackSavepoint().

ExecStreamResourceQuantity ExecStream::resourceAllocation [protected, inherited]

Resource quantities currently allocated to this stream.

Definition at line 100 of file ExecStream.h.

Referenced by ExecStream::setResourceAllocation().

SharedCacheAccessor ExecStream::pQuotaAccessor [protected, inherited]

CacheAccessor used for quota tracking.

Definition at line 105 of file ExecStream.h.

Referenced by ExecStream::open(), ExecStream::prepare(), and ExecStream::setResourceAllocation().

SharedCacheAccessor ExecStream::pScratchQuotaAccessor [protected, inherited]

CacheAccessor used for scratch page quota tracking.

Definition at line 110 of file ExecStream.h.

Referenced by ExecStream::open(), ExecStream::prepare(), and ExecStream::setResourceAllocation().

bool ClosableObject::needsClose [protected, inherited]

Definition at line 44 of file ClosableObject.h.

Referenced by SegStreamAllocation::beginWrite(), ExecStreamGraphImpl::clear(), ClosableObject::ClosableObject(), ClosableObject::close(), FlatFileBuffer::open(), ExecStreamGraphImpl::open(), ExecStream::open(), and ClosableObject::~ClosableObject().

SharedExecStreamBufAccessor SingleInputExecStream::pInAccessor [protected, inherited]

Definition at line 51 of file SingleInputExecStream.h.

Referenced by SortedAggExecStream::compareGroupByKeys(), ExternalSortExecStreamImpl::computeFirstResult(), ExternalSortExecStreamImpl::execute(), LbmSplicerExecStream::execute(), FtrsTableWriterExecStream::execute(), BTreeSortExecStream::execute(), BTreeSearchUniqueExecStream::execute(), BTreeSearchExecStream::execute(), BTreePrefetchSearchExecStream::execute(), BTreeInsertExecStream::execute(), JavaSinkExecStream::execute(), UncollectExecStream::execute(), SplitterExecStream::execute(), SortedAggExecStream::execute(), SegBufferWriterExecStream::execute(), SegBufferReaderExecStream::execute(), SegBufferExecStream::execute(), ScratchBufferExecStream::execute(), ReshapeExecStream::execute(), MockConsumerExecStream::execute(), DoubleBufferExecStream::execute(), CopyExecStream::execute(), CollectExecStream::execute(), BernoulliSamplingExecStream::execute(), CalcExecStream::execute(), BTreePrefetchSearchExecStream::getNextPageForPrefetch(), LcsClusterReplaceExecStream::getTupleForLoad(), getTupleForLoad(), LbmSplicerExecStream::getValidatedTuple(), LcsClusterReplaceExecStream::initTupleLoadParams(), BTreeSearchExecStream::innerSearchLoop(), BTreePrefetchSearchExecStream::innerSearchLoop(), LcsClusterReplaceExecStream::open(), LbmNormalizerExecStream::open(), SingleInputExecStream::open(), SegBufferWriterExecStream::open(), SegBufferExecStream::open(), ScratchBufferExecStream::open(), MockConsumerExecStream::open(), DoubleBufferExecStream::open(), CollectExecStream::open(), postProcessTuple(), ConduitExecStream::precheckConduitBuffers(), ExternalSortExecStreamImpl::prepare(), prepare(), LbmSplicerExecStream::prepare(), LbmNormalizerExecStream::prepare(), BTreeSearchExecStream::prepare(), BTreeInsertExecStream::prepare(), UncollectExecStream::prepare(), SortedAggExecStream::prepare(), SingleInputExecStream::prepare(), SegBufferReaderExecStream::prepare(), ReshapeExecStream::prepare(), DiffluenceExecStream::prepare(), ConduitExecStream::prepare(), CollectExecStream::prepare(), BernoulliSamplingExecStream::prepare(), CalcExecStream::prepare(), SortedAggExecStream::produce(), BTreeSearchExecStream::readSearchKey(), LhxAggExecStream::setAggComputers(), LhxAggExecStream::setHashInfo(), SingleInputExecStream::setInputBufAccessors(), and JavaSinkExecStream::stuffByteBuffer().

---

The documentation for this class was generated from the following files:

• /home/pub/open/dev/fennel/lucidera/colstore/LcsClusterAppendExecStream.h
• /home/pub/open/dev/fennel/lucidera/colstore/LcsClusterAppendExecStream.cpp

---