LbmExecStreamTestBase.cpp

Go to the documentation of this file.
00001 /*
00002 // $Id: //open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp#10 $
00003 // Fennel is a library of data storage and processing components.
00004 // Copyright (C) 2005-2009 LucidEra, Inc.
00005 // Copyright (C) 2005-2009 The Eigenbase Project
00006 //
00007 // This program is free software; you can redistribute it and/or modify it
00008 // under the terms of the GNU General Public License as published by the Free
00009 // Software Foundation; either version 2 of the License, or (at your option)
00010 // any later version approved by The Eigenbase Project.
00011 //
00012 // This program is distributed in the hope that it will be useful,
00013 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00014 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00015 // GNU General Public License for more details.
00016 //
00017 // You should have received a copy of the GNU General Public License
00018 // along with this program; if not, write to the Free Software
00019 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00020 */
00021 
00022 #include "fennel/common/CommonPreamble.h"
00023 #include "fennel/lucidera/test/LbmExecStreamTestBase.h"
00024 
00025 FENNEL_BEGIN_CPPFILE("$Id: //open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp#10 $");
00026 
00027 void LbmExecStreamTestBase::initBitmapInput(
00028     BitmapInput &bmInput, uint nRows, InputData const &inputData)
00029 {
00030     LbmNumberStreamInput input;
00031     SharedNumberStream pNumberStream(
00032         new SkipNumberStream(
00033             opaqueToInt(inputData.startRid),
00034             NumberStream::BIG_NUMBER, inputData.skipRows));
00035     input.pStream = pNumberStream;
00036     input.bitmapSize = inputData.bitmapSize;
00037 
00038     initBitmapInput(bmInput, nRows, input);
00039 }
00040 
00041 void LbmExecStreamTestBase::initBitmapInput(
00042     BitmapInput &bmInput, uint nRows, LbmNumberStreamInput input)
00043 {
00044     bmInput.fullBufSize = input.pStream->getMaxRowCount(nRows) * 16;
00045     bmInput.bufArray.reset(new FixedBuffer[bmInput.fullBufSize]);
00046     bmInput.nBitmaps = 0;
00047     bmInput.currBufSize = 0;
00048     generateBitmaps(nRows, input, bmInput);
00049 }
00050 
00051 void LbmExecStreamTestBase::generateBitmaps(
00052     uint nRows, LbmNumberStreamInput input, BitmapInput &bmInput)
00053 {
00054     LbmEntry lbmEntry;
00055     boost::scoped_array<FixedBuffer> entryBuf;
00056     LcsRid rid = LcsRid(input.pStream->getNext());
00057 
00058     // setup an LbmEntry with the initial rid value
00059     uint scratchBufSize = LbmEntry::getScratchBufferSize(bitmapColSize);
00060     entryBuf.reset(new FixedBuffer[scratchBufSize]);
00061     lbmEntry.init(entryBuf.get(), NULL, scratchBufSize, bitmapTupleDesc);
00062     bitmapTupleData[0].pData = (PConstBuffer) &rid;
00063     lbmEntry.setEntryTuple(bitmapTupleData);
00064     TraceLevel level = getSourceTraceLevel(getTraceName());
00065     if (level <= TRACE_FINER) {
00066         std::cout << "Set root: " << rid << std::endl;
00067     }
00068 
00069     // add on the remaining rids
00070     while (input.pStream->hasNext()) {
00071         rid = LcsRid(input.pStream->getNext());
00072         if (rid >= LcsRid(nRows)) {
00073             break;
00074         }
00075         if (level <= TRACE_FINER) {
00076             std::cout << "Set value: " << rid << std::endl;
00077         }
00078         if ((rid > LcsRid(0) &&
00079                 opaqueToInt(rid % (input.bitmapSize*8)) == 0) ||
00080             !lbmEntry.setRID(LcsRid(rid)))
00081         {
00082             // either hit desired number of rids per bitmap segment or
00083             // exhausted buffer space, so write the tuple to the output
00084             // buffer and reset LbmEntry
00085             produceEntry(lbmEntry, bitmapTupleAccessor, bmInput);
00086             lbmEntry.setEntryTuple(bitmapTupleData);
00087         }
00088     }
00089     // write out the last LbmEntry
00090     produceEntry(lbmEntry, bitmapTupleAccessor, bmInput);
00091 
00092     assert(bmInput.currBufSize <= bmInput.fullBufSize);
00093 }
00094 
00095 void LbmExecStreamTestBase::produceEntry(
00096     LbmEntry &lbmEntry, TupleAccessor &bitmapTupleAccessor,
00097     BitmapInput &bmInput)
00098 {
00099     TupleData bitmapTuple = lbmEntry.produceEntryTuple();
00100     bitmapTupleAccessor.marshal(
00101         bitmapTuple, bmInput.bufArray.get() + bmInput.currBufSize);
00102     bmInput.currBufSize += bitmapTupleAccessor.getCurrentByteCount();
00103     ++bmInput.nBitmaps;
00104 }
00105 
00106 void LbmExecStreamTestBase::initValuesExecStream(
00107     uint idx, ValuesExecStreamParams &valuesParams,
00108     ExecStreamEmbryo &valuesStreamEmbryo, BitmapInput &bmInput)
00109 {
00110     valuesParams.outputTupleDesc = bitmapTupleDesc;
00111     valuesParams.pTupleBuffer = bmInput.bufArray;
00112     valuesParams.bufSize = bmInput.currBufSize;
00113 
00114     valuesStreamEmbryo.init(new ValuesExecStream(), valuesParams);
00115     std::ostringstream oss;
00116     oss << "InputValuesExecStream" << "#" << idx;
00117     valuesStreamEmbryo.getStream()->setName(oss.str());
00118 }
00119 
00120 void LbmExecStreamTestBase::initSorterExecStream(
00121     ExternalSortExecStreamParams &params,
00122     ExecStreamEmbryo &embryo,
00123     TupleDescriptor const &outputDesc,
00124     uint nKeys)
00125 {
00126     params.outputTupleDesc = outputDesc;
00127     params.distinctness = DUP_ALLOW;
00128     params.pTempSegment = pRandomSegment;
00129     params.pCacheAccessor = pCache;
00130     params.scratchAccessor =
00131         pSegmentFactory->newScratchSegment(pCache, 10);
00132     params.keyProj.clear();
00133     for (uint i = 0; i < nKeys; i++) {
00134         params.keyProj.push_back(i);
00135     }
00136     params.storeFinalRun = false;
00137     params.estimatedNumRows = MAXU;
00138     params.earlyClose = false;
00139 
00140     embryo.init(ExternalSortExecStream::newExternalSortExecStream(), params);
00141     embryo.getStream()->setName("SorterExecStream");
00142 }
00143 
00144 void LbmExecStreamTestBase::initNormalizerExecStream(
00145     LbmNormalizerExecStreamParams &params,
00146     ExecStreamEmbryo &embryo,
00147     uint nKeys)
00148 {
00149     TupleProjection keyProj;
00150     for (int i = 0; i < nKeys; i++) {
00151         keyProj.push_back(i);
00152     }
00153     params.keyProj = keyProj;
00154 
00155     TupleDescriptor keyDesc;
00156     keyDesc.projectFrom(keyBitmapTupleDesc, keyProj);
00157     params.outputTupleDesc = keyDesc;
00158 
00159     embryo.init(new LbmNormalizerExecStream(), params);
00160     embryo.getStream()->setName("Normalizer");
00161 }
00162 
00163 void LbmExecStreamTestBase::generateBitmaps(
00164     uint nRows, uint start, uint skipRows, PBuffer pBuf, uint &bufSize,
00165     uint fullBufSize, uint &nBitmaps, bool includeKeys)
00166 {
00167     LbmEntry lbmEntry;
00168     boost::scoped_array<FixedBuffer> entryBuf;
00169     LcsRid rid = LcsRid(start);
00170 
00171     // setup an LbmEntry with the initial rid value
00172     uint scratchBufSize = LbmEntry::getScratchBufferSize(bitmapColSize);
00173     entryBuf.reset(new FixedBuffer[scratchBufSize]);
00174     lbmEntry.init(entryBuf.get(), NULL, scratchBufSize, bitmapTupleDesc);
00175     bitmapTupleData[0].pData = (PConstBuffer) &rid;
00176     lbmEntry.setEntryTuple(bitmapTupleData);
00177 
00178     // add on the remaining rids
00179     for (rid = LcsRid(start + skipRows); rid < LcsRid(nRows); rid += skipRows) {
00180         if (!lbmEntry.setRID(LcsRid(rid))) {
00181             // exhausted buffer space, so write the tuple to the output
00182             // buffer and reset LbmEntry
00183             produceEntry(
00184                 lbmEntry, bitmapTupleAccessor,
00185                 pBuf, bufSize, nBitmaps, includeKeys);
00186             lbmEntry.setEntryTuple(bitmapTupleData);
00187         }
00188     }
00189     // write out the last LbmEntry
00190     produceEntry(
00191         lbmEntry, bitmapTupleAccessor, pBuf, bufSize, nBitmaps, includeKeys);
00192 
00193     assert(bufSize <= fullBufSize);
00194 }
00195 
00196 void LbmExecStreamTestBase::produceEntry(
00197     LbmEntry &lbmEntry, TupleAccessor &bitmapTupleAccessor, PBuffer pBuf,
00198     uint &bufSize, uint &nBitmaps, bool includeKeys)
00199 {
00200     TupleData bitmapTuple = lbmEntry.produceEntryTuple();
00201     if (includeKeys) {
00202         int nKeys = keyBitmapTupleData.size() - bitmapTuple.size();
00203         assert(nKeys > 0);
00204         for (uint i = 0; i < bitmapTupleData.size(); i++) {
00205             keyBitmapTupleData[nKeys + i] = bitmapTuple[i];
00206         }
00207         keyBitmapTupleAccessor.marshal(keyBitmapTupleData, pBuf + bufSize);
00208         bufSize += keyBitmapTupleAccessor.getCurrentByteCount();
00209     } else {
00210         bitmapTupleAccessor.marshal(bitmapTuple, pBuf + bufSize);
00211         bufSize += bitmapTupleAccessor.getCurrentByteCount();
00212     }
00213     ++nBitmaps;
00214 }
00215 
00216 void LbmExecStreamTestBase::initKeyBitmap(
00217     uint nRows,
00218     std::vector<int> const &repeatSeqValues)
00219 {
00220     // find the interval for which the entire tuple's sequence repeats
00221     uint skipRows = getTupleInterval(repeatSeqValues);
00222 
00223     // generate a key bitmap for each distinct input value...
00224     // configure descriptor
00225     uint nInputKeys = repeatSeqValues.size();
00226     keyBitmapTupleDesc.clear();
00227     for (uint i = 0; i < nInputKeys; i++) {
00228         keyBitmapTupleDesc.push_back(attrDesc_int64);
00229     }
00230     for (uint i = 0; i < bitmapTupleDesc.size(); i++) {
00231         keyBitmapTupleDesc.push_back(bitmapTupleDesc[i]);
00232     }
00233 
00234     // configure accessor and key data (bitmap data handled elsewhere)
00235     keyBitmapTupleAccessor.compute(keyBitmapTupleDesc);
00236     keyBitmapTupleData.compute(keyBitmapTupleDesc);
00237     boost::scoped_array<uint64_t> vals(new uint64_t[nInputKeys]);
00238     for (uint i = 0; i < nInputKeys; i++) {
00239         keyBitmapTupleData[i].pData = (PConstBuffer) &vals[i];
00240     }
00241 
00242     uint fullBufSize = nRows * keyBitmapTupleAccessor.getMaxByteCount();
00243     keyBitmapBuf.reset(new FixedBuffer[fullBufSize]);
00244     PBuffer pBuf = keyBitmapBuf.get();
00245     keyBitmapBufSize = 0;
00246     uint nBitmaps = 0;
00247     for (uint i = 0; i < skipRows; i++) {
00248         // generate input keys
00249         for (uint j = 0; j < nInputKeys; j++) {
00250             vals[j] = i % repeatSeqValues[j];
00251         }
00252         generateBitmaps(
00253             nRows, i, skipRows, pBuf, keyBitmapBufSize,
00254             fullBufSize, nBitmaps, true);
00255     }
00256 }
00257 
00261 uint LbmExecStreamTestBase::getTupleInterval(
00262     std::vector<int> const &repeatSeqValues, uint nKeys)
00263 {
00264     if (nKeys == 0) {
00265         nKeys = repeatSeqValues.size();
00266     }
00267     uint interval = 1;
00268     for (uint i = 0; i < nKeys; i++) {
00269         interval *= repeatSeqValues[i];
00270     }
00271     return interval;
00272 }
00273 
00274 void LbmExecStreamTestBase::testCaseSetUp()
00275 {
00276     ExecStreamUnitTestBase::testCaseSetUp();
00277 
00278     bitmapColSize = pRandomSegment->getUsablePageSize() / 8;
00279     attrDesc_bitmap = TupleAttributeDescriptor(
00280         stdTypeFactory.newDataType(STANDARD_TYPE_VARBINARY),
00281         true, bitmapColSize);
00282     attrDesc_int64 = TupleAttributeDescriptor(
00283         stdTypeFactory.newDataType(STANDARD_TYPE_INT_64));
00284 
00285     bitmapTupleDesc.clear();
00286     bitmapTupleDesc.push_back(attrDesc_int64);
00287     bitmapTupleDesc.push_back(attrDesc_bitmap);
00288     bitmapTupleDesc.push_back(attrDesc_bitmap);
00289 
00290     bitmapTupleData.compute(bitmapTupleDesc);
00291     bitmapTupleData[1].pData = NULL;
00292     bitmapTupleData[1].cbData = 0;
00293     bitmapTupleData[2].pData = NULL;
00294     bitmapTupleData[2].cbData = 0;
00295 
00296     bitmapTupleAccessor.compute(bitmapTupleDesc);
00297 }
00298 
00299 FENNEL_END_CPPFILE("$Id: //open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp#10 $");
00300 
00301 // End LbmExecStreamTestBase.cpp

Generated on Mon Jun 22 04:00:19 2009 for Fennel by  doxygen 1.5.1