LbmSplicerExecStreamTest.cpp

Go to the documentation of this file.
00001 /*
00002 // $Id: //open/dev/fennel/lucidera/test/LbmSplicerExecStreamTest.cpp#7 $
00003 // Fennel is a library of data storage and processing components.
00004 // Copyright (C) 2007-2009 LucidEra, Inc.
00005 // Copyright (C) 2007-2009 The Eigenbase Project
00006 //
00007 // This program is free software; you can redistribute it and/or modify it
00008 // under the terms of the GNU General Public License as published by the Free
00009 // Software Foundation; either version 2 of the License, or (at your option)
00010 // any later version approved by The Eigenbase Project.
00011 //
00012 // This program is distributed in the hope that it will be useful,
00013 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00014 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00015 // GNU General Public License for more details.
00016 //
00017 // You should have received a copy of the GNU General Public License
00018 // along with this program; if not, write to the Free Software
00019 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00020 */
00021 
00022 #include "fennel/common/CommonPreamble.h"
00023 #include "fennel/test/ExecStreamUnitTestBase.h"
00024 #include "fennel/lucidera/bitmap/LbmSplicerExecStream.h"
00025 #include "fennel/btree/BTreeBuilder.h"
00026 #include "fennel/tuple/StandardTypeDescriptor.h"
00027 #include "fennel/tuple/TupleDescriptor.h"
00028 #include "fennel/exec/ExecStreamEmbryo.h"
00029 #include "fennel/exec/ExecStreamGraph.h"
00030 #include "fennel/exec/ValuesExecStream.h"
00031 #include "fennel/cache/Cache.h"
00032 #include <stdarg.h>
00033 #include <hash_set>
00034 
00035 #include <boost/test/test_tools.hpp>
00036 
00037 using namespace fennel;
00038 
00042 class LbmSplicerExecStreamTest : public ExecStreamUnitTestBase
00043 {
00044 protected:
00045     StandardTypeDescriptorFactory stdTypeFactory;
00046     TupleAttributeDescriptor attrDesc_int64;
00047     TupleAttributeDescriptor attrDesc_bitmap;
00048 
00056     void testSpliceRids(uint numSets, uint numRidsPerSet);
00057 
00066     void generateSeqRidSets(
00067         std::vector<std::vector<uint64_t> > &ridSets,
00068         uint numSets,
00069         uint numRidsPerSet);
00070 
00079     void generateRidInput(
00080         std::vector<uint64_t> const &rids,
00081         PBuffer ridBuffer,
00082         uint &bufferSize);
00083 
00098     void testSpliceWithKeys(
00099         uint numRows,
00100         uint factor,
00101         uint nKeys,
00102         bool multipleSplices);
00103 
00112     void generateRandomRids(
00113         std::vector<uint64_t> &rids,
00114         uint nRids,
00115         uint factor);
00116 
00130     void generateTupleInput(
00131         std::vector<uint64_t> const &rids,
00132         uint &currRidIdx,
00133         PBuffer buffer,
00134         uint &bufferSize,
00135         uint nKeys,
00136         bool oneTuple);
00137 
00147     void spliceInput(
00148         boost::shared_array<FixedBuffer> &inputBuffer,
00149         uint inputBufSize,
00150         uint numRows,
00151         BTreeDescriptor const &bTreeDesc);
00152 
00159     void initBTreeParam(BTreeParams &param, BTreeDescriptor const &bTreeDesc);
00160 
00168     void createBTree(BTreeDescriptor &bTreeDesc, uint nKeys);
00169 
00176     void initBTreeTupleDesc(TupleDescriptor &tupleDesc, uint nKeys);
00177 
00178 public:
00179     explicit LbmSplicerExecStreamTest()
00180     {
00181         FENNEL_UNIT_TEST_CASE(LbmSplicerExecStreamTest, testSpliceRids50);
00182         FENNEL_UNIT_TEST_CASE(
00183             LbmSplicerExecStreamTest, testSpliceRidsLargeSets);
00184         FENNEL_UNIT_TEST_CASE(
00185             LbmSplicerExecStreamTest, testSpliceRidsSmallSets);
00186         FENNEL_UNIT_TEST_CASE(
00187             LbmSplicerExecStreamTest, testSpliceWithKeys50);
00188         FENNEL_UNIT_TEST_CASE(
00189             LbmSplicerExecStreamTest, testSpliceWithKeysSmallSpread);
00190         FENNEL_UNIT_TEST_CASE(
00191             LbmSplicerExecStreamTest, testSpliceWithKeysLargeSpread);
00192         FENNEL_UNIT_TEST_CASE(
00193             LbmSplicerExecStreamTest, testMultipleSpliceWithKeysSmallSpread);
00194         FENNEL_UNIT_TEST_CASE(
00195             LbmSplicerExecStreamTest, testMultipleSpliceWithKeysLargeSpread);
00196         FENNEL_UNIT_TEST_CASE(LbmSplicerExecStreamTest, testLER5968);
00197         FENNEL_UNIT_TEST_CASE(LbmSplicerExecStreamTest, testLER6473);
00198     }
00199 
00200     void testCaseSetUp();
00201 
00202     void testSpliceRids50();
00203     void testSpliceRidsLargeSets();
00204     void testSpliceRidsSmallSets();
00205     void testSpliceWithKeys50();
00206     void testSpliceWithKeysSmallSpread();
00207     void testSpliceWithKeysLargeSpread();
00208     void testMultipleSpliceWithKeysSmallSpread();
00209     void testMultipleSpliceWithKeysLargeSpread();
00210     void testLER5968();
00211     void testLER6473();
00212 };
00213 
00214 void LbmSplicerExecStreamTest::testSpliceRids50()
00215 {
00216     testSpliceRids(5, 10);
00217 }
00218 
00219 void LbmSplicerExecStreamTest::testSpliceRidsLargeSets()
00220 {
00221     testSpliceRids(100, 1000);
00222 }
00223 
00224 void LbmSplicerExecStreamTest::testSpliceRidsSmallSets()
00225 {
00226     testSpliceRids(1000, 100);
00227 }
00228 
00229 void LbmSplicerExecStreamTest::testSpliceWithKeys50()
00230 {
00231     testSpliceWithKeys(50, 16, 1, false);
00232 }
00233 
00234 void LbmSplicerExecStreamTest::testSpliceWithKeysSmallSpread()
00235 {
00236     testSpliceWithKeys(50000, 8, 1, false);
00237 }
00238 
00239 void LbmSplicerExecStreamTest::testSpliceWithKeysLargeSpread()
00240 {
00241     testSpliceWithKeys(50000, 24, 1, false);
00242 }
00243 
00244 void LbmSplicerExecStreamTest::testMultipleSpliceWithKeysSmallSpread()
00245 {
00246     testSpliceWithKeys(5000, 8, 1, true);
00247 }
00248 
00249 void LbmSplicerExecStreamTest::testMultipleSpliceWithKeysLargeSpread()
00250 {
00251     testSpliceWithKeys(10000, 24, 1, true);
00252 }
00253 
00254 void LbmSplicerExecStreamTest::testSpliceRids(
00255     uint numSets,
00256     uint numRidsPerSet)
00257 {
00258     // Create the btree that the splicer will write into
00259     BTreeDescriptor bTreeDesc;
00260     createBTree(bTreeDesc, 0);
00261 
00262     // Generate the rid sets
00263     uint totalRids = numSets * numRidsPerSet;
00264     std::vector<std::vector<uint64_t> > ridSets;
00265     generateSeqRidSets(ridSets, numSets, numRidsPerSet);
00266 
00267     // Splice each set of rids, one per stream graph execution
00268     for (uint i = 0; i < numSets; i++) {
00269         // Generate the sequence of specified rids.  Splicer handles
00270         // individual rid values as input.
00271         boost::shared_array<FixedBuffer> ridBuffer;
00272         ridBuffer.reset(new FixedBuffer[numRidsPerSet * 8]);
00273         uint bufferSize = 0;
00274         generateRidInput(
00275             ridSets[i],
00276             ridBuffer.get(),
00277             bufferSize);
00278 
00279         spliceInput(
00280             ridBuffer,
00281             bufferSize,
00282             numRidsPerSet,
00283             bTreeDesc);
00284 
00285         resetExecStreamTest();
00286     }
00287 
00288     // Read the btree bitmap entries and confirm that they contain all
00289     // of the rids that were inserted.  The rids should be in a sequence
00290     // from 0 to totalRids - 1.
00291     BTreeReader reader(bTreeDesc);
00292     bool rc = reader.searchFirst();
00293     BOOST_REQUIRE(rc);
00294     TupleData tupleData;
00295     tupleData.compute(bTreeDesc.tupleDescriptor);
00296     LcsRid startRid = LcsRid(0);
00297     while (rc) {
00298         reader.getTupleAccessorForRead().unmarshal(tupleData);
00299         std::vector<LcsRid> ridsRead;
00300         LbmEntry::generateRIDs(tupleData, ridsRead);
00301         for (uint i = 0; i < ridsRead.size(); i++) {
00302             BOOST_CHECK_EQUAL(
00303                 opaqueToInt(ridsRead[i]),
00304                 opaqueToInt(startRid) + i);
00305         }
00306         startRid += ridsRead.size();
00307         rc = reader.searchNext();
00308     }
00309     BOOST_CHECK_EQUAL(opaqueToInt(startRid), totalRids);
00310 }
00311 
00312 void LbmSplicerExecStreamTest::generateSeqRidSets(
00313     std::vector<std::vector<uint64_t> > &ridSets,
00314     uint numSets,
00315     uint numRidsPerSet)
00316 {
00317     // Generate a sequence of rids from 0 to N.  Random shuffle the sequence.
00318     // Divide them into equal portions.  Then sort within each portion,
00319     // since splicer expects its rid input in sort order.
00320     uint totalRids = numSets * numRidsPerSet;
00321     std::vector<int> rids;
00322     rids.resize(totalRids);
00323     for (uint i = 0; i < totalRids; i++) {
00324         rids[i] = i;
00325     }
00326 
00327     std::random_shuffle(rids.begin(), rids.end());
00328 
00329     for (uint i = 0; i < numSets; i++) {
00330         std::vector<uint64_t> ridSet;
00331         ridSet.resize(numRidsPerSet);
00332         std::copy(
00333             rids.begin() + i * numRidsPerSet,
00334             rids.begin() + (i + 1) * numRidsPerSet,
00335             ridSet.begin());
00336         std::sort(ridSet.begin(), ridSet.end());
00337         ridSets.push_back(ridSet);
00338     }
00339 }
00340 
00341 void LbmSplicerExecStreamTest::generateRidInput(
00342     std::vector<uint64_t> const &rids,
00343     PBuffer ridBuffer,
00344     uint &bufferSize)
00345 {
00346     TupleData ridTupleData;
00347     TupleDescriptor ridTupleDesc;
00348     ridTupleDesc.push_back(attrDesc_int64);
00349     ridTupleData.compute(ridTupleDesc);
00350 
00351     TupleAccessor ridTupleAccessor;
00352     ridTupleAccessor.compute(ridTupleDesc);
00353 
00354     for (uint i = 0; i < rids.size(); i++) {
00355         ridTupleData[0].pData = (PConstBuffer) &rids[i];
00356         ridTupleAccessor.marshal(ridTupleData, ridBuffer + bufferSize);
00357         bufferSize += ridTupleAccessor.getCurrentByteCount();
00358     }
00359 }
00360 
00361 void LbmSplicerExecStreamTest::testSpliceWithKeys(
00362     uint numRows,
00363     uint factor,
00364     uint nKeys,
00365     bool multipleSplices)
00366 {
00367     // Create the btree that the splicer will write into
00368     BTreeDescriptor bTreeDesc;
00369     createBTree(bTreeDesc, nKeys);
00370 
00371     // Generate the random rids
00372     std::vector<uint64_t> rids;
00373     generateRandomRids(rids, numRows, factor);
00374 
00375     // Generate tuples containing the rids generated
00376     boost::shared_array<FixedBuffer> buffer;
00377     buffer.reset(new FixedBuffer[(nKeys + 2) * numRows * 8]);
00378     uint currRidIdx = 0;
00379     do {
00380         uint bufferSize = 0;
00381         generateTupleInput(
00382             rids,
00383             currRidIdx,
00384             buffer.get(),
00385             bufferSize,
00386             nKeys,
00387             multipleSplices);
00388         assert(bufferSize <= (nKeys + 2) * numRows * 8);
00389 
00390         spliceInput(
00391             buffer,
00392             bufferSize,
00393             numRows,
00394             bTreeDesc);
00395 
00396         resetExecStreamTest();
00397     } while (currRidIdx < rids.size());
00398 
00399     // Read the btree bitmap entries and confirm that they contain all
00400     // of the rids that were inserted.
00401     BTreeReader reader(bTreeDesc);
00402     bool rc = reader.searchFirst();
00403     BOOST_REQUIRE(rc);
00404     TupleData tupleData;
00405     tupleData.compute(bTreeDesc.tupleDescriptor);
00406     uint currIdx = 0;
00407     while (rc) {
00408         reader.getTupleAccessorForRead().unmarshal(tupleData);
00409         std::vector<LcsRid> ridsRead;
00410         LbmEntry::generateRIDs(tupleData, ridsRead);
00411         for (uint i = 0; i < ridsRead.size(); i++) {
00412             BOOST_CHECK_EQUAL(
00413                 opaqueToInt(ridsRead[i]),
00414                 opaqueToInt(rids[currIdx]));
00415             currIdx++;
00416         }
00417         rc = reader.searchNext();
00418     }
00419     BOOST_CHECK_EQUAL(currIdx, numRows);
00420 }
00421 
00422 void LbmSplicerExecStreamTest::testLER5968()
00423 {
00424     TupleDescriptor tupleDesc;
00425     initBTreeTupleDesc(tupleDesc, 1);
00426 
00427     // Create the btree that the splicer will write into
00428     BTreeDescriptor bTreeDesc;
00429     createBTree(bTreeDesc, 1);
00430 
00431     TupleAccessor tupleAccessor;
00432     tupleAccessor.compute(tupleDesc);
00433 
00434     // Key values are fixed
00435     uint64_t keyVal = 1;
00436     TupleData tupleData;
00437     tupleData.compute(bTreeDesc.tupleDescriptor);
00438     tupleData[0].pData = (PConstBuffer) &keyVal;
00439 
00440     LbmEntry lbmEntry;
00441     boost::scoped_array<FixedBuffer> entryBuf;
00442     uint entryBufSize = tupleDesc[2].cbStorage + 8 + 8;
00443     entryBuf.reset(new FixedBuffer[entryBufSize]);
00444     lbmEntry.init(entryBuf.get(), NULL, entryBufSize, tupleDesc);
00445 
00446     std::vector<uint64_t> rids;
00447 
00448     // First create a singleton entry
00449     uint64_t rid = 0;
00450     rids.push_back(rid);
00451     tupleData[1].pData = (PConstBuffer) &rid;
00452     tupleData[2].pData = NULL;
00453     tupleData[2].cbData = 0;
00454     tupleData[3].pData = NULL;
00455     tupleData[3].cbData = 0;
00456     lbmEntry.setEntryTuple(tupleData);
00457     tupleData = lbmEntry.produceEntryTuple();
00458     boost::shared_array<FixedBuffer> buffer;
00459     buffer.reset(new FixedBuffer[1024]);
00460     uint bufferSize = 0;
00461     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00462     bufferSize += tupleAccessor.getCurrentByteCount();
00463 
00464     // Next figure out how many rids are needed to create a full entry
00465     rid = 5984;
00466     rids.push_back(rid);
00467     tupleData[1].pData = (PConstBuffer) &rid;
00468     tupleData[2].pData = NULL;
00469     tupleData[2].cbData = 0;
00470     tupleData[3].pData = NULL;
00471     tupleData[3].cbData = 0;
00472     lbmEntry.setEntryTuple(tupleData);
00473     uint numRids = 1;
00474     do {
00475         bool rc = lbmEntry.setRID(LcsRid(rid + numRids * 16));
00476         if (!rc) {
00477             break;
00478         }
00479         numRids++;
00480     } while (true);
00481 
00482     // Now, create the actual second entry with a rid count based the count
00483     // determined above, but with two fewer, so this entry can be spliced with
00484     // the first.  Note that its startRID is 2 zero-length bytes away from the
00485     // initial singleton.
00486     tupleData[1].pData = (PConstBuffer) &rid;
00487     tupleData[2].pData = NULL;
00488     tupleData[2].cbData = 0;
00489     tupleData[3].pData = NULL;
00490     tupleData[3].cbData = 0;
00491     lbmEntry.setEntryTuple(tupleData);
00492     for (int i = 0; i < numRids - 2; i++) {
00493         rid += 16;
00494         bool rc = lbmEntry.setRID(LcsRid(rid));
00495         BOOST_REQUIRE(rc);
00496         rids.push_back(rid);
00497     }
00498     tupleData = lbmEntry.produceEntryTuple();
00499     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00500     bufferSize += tupleAccessor.getCurrentByteCount();
00501 
00502     // Create the third entry, again a singleton.
00503     rid = 10000;
00504     rids.push_back(rid);
00505     tupleData[1].pData = (PConstBuffer) &rid;
00506     tupleData[2].pData = NULL;
00507     tupleData[2].cbData = 0;
00508     tupleData[3].pData = NULL;
00509     tupleData[3].cbData = 0;
00510     lbmEntry.setEntryTuple(tupleData);
00511     tupleData = lbmEntry.produceEntryTuple();
00512     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00513     bufferSize += tupleAccessor.getCurrentByteCount();
00514 
00515     // Create the fourth entry with a rid count based the count determined
00516     // above, but with the last rid in a contiguous segment.  This entry,
00517     // when spliced with the singleton, should result in, an overflow.
00518     rid = 16984;
00519     rids.push_back(rid);
00520     tupleData[1].pData = (PConstBuffer) &rid;
00521     tupleData[2].pData = NULL;
00522     tupleData[2].cbData = 0;
00523     tupleData[3].pData = NULL;
00524     tupleData[3].cbData = 0;
00525     lbmEntry.setEntryTuple(tupleData);
00526     for (int i = 0; i < numRids - 1; i++) {
00527         rid += 16;
00528         bool rc = lbmEntry.setRID(LcsRid(rid));
00529         BOOST_REQUIRE(rc);
00530         rids.push_back(rid);
00531     }
00532     rid += 8;
00533     bool rc = lbmEntry.setRID(LcsRid(rid));
00534     rids.push_back(rid);
00535     tupleData = lbmEntry.produceEntryTuple();
00536     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00537     bufferSize += tupleAccessor.getCurrentByteCount();
00538 
00539     // Splice the four entries.  The splice of the first two entries should
00540     // fit, creating a combined entry.  Then, a singleton should be created.
00541     // An attempt to splice the fourth entry into the singleton will overflow
00542     // and therefore create a third entry.
00543     spliceInput(
00544         buffer,
00545         bufferSize,
00546         rids.size(),
00547         bTreeDesc);
00548     resetExecStreamTest();
00549 
00550     // Read the btree bitmap entries and confirm that they contain all
00551     // of the rids that were inserted.  Explicitly make sure there are
00552     // three btree entries.
00553     BTreeReader reader(bTreeDesc);
00554     rc = reader.searchFirst();
00555     BOOST_REQUIRE(rc);
00556     uint currIdx = 0;
00557     uint numEntries = 0;
00558     while (rc) {
00559         numEntries++;
00560         reader.getTupleAccessorForRead().unmarshal(tupleData);
00561         std::vector<LcsRid> ridsRead;
00562         LbmEntry::generateRIDs(tupleData, ridsRead);
00563         for (uint i = 0; i < ridsRead.size(); i++) {
00564             BOOST_CHECK_EQUAL(
00565                 opaqueToInt(ridsRead[i]),
00566                 opaqueToInt(rids[currIdx]));
00567             currIdx++;
00568         }
00569         rc = reader.searchNext();
00570     }
00571     BOOST_CHECK_EQUAL(currIdx, rids.size());
00572     BOOST_REQUIRE(numEntries == 3);
00573 }
00574 
00575 void LbmSplicerExecStreamTest::testLER6473()
00576 {
00577     TupleDescriptor tupleDesc;
00578     initBTreeTupleDesc(tupleDesc, 1);
00579 
00580     // Create the btree that the splicer will write into
00581     BTreeDescriptor bTreeDesc;
00582     createBTree(bTreeDesc, 1);
00583 
00584     TupleAccessor tupleAccessor;
00585     tupleAccessor.compute(tupleDesc);
00586 
00587     // Key values are fixed
00588     uint64_t keyVal = 1;
00589     TupleData tupleData;
00590     tupleData.compute(bTreeDesc.tupleDescriptor);
00591     tupleData[0].pData = (PConstBuffer) &keyVal;
00592 
00593     LbmEntry lbmEntry;
00594     boost::scoped_array<FixedBuffer> entryBuf;
00595     uint entryBufSize = tupleDesc[2].cbStorage + 8 + 8;
00596     entryBuf.reset(new FixedBuffer[entryBufSize]);
00597     lbmEntry.init(entryBuf.get(), NULL, entryBufSize, tupleDesc);
00598 
00599     std::vector<uint64_t> rids;
00600 
00601     // First create a bitmap entry at max capacity
00602     uint64_t rid = 0;
00603     tupleData[1].pData = (PConstBuffer) &rid;
00604     tupleData[2].pData = NULL;
00605     tupleData[2].cbData = 0;
00606     tupleData[3].pData = NULL;
00607     tupleData[3].cbData = 0;
00608     lbmEntry.setEntryTuple(tupleData);
00609     for (;; rid += 16) {
00610         bool rc = lbmEntry.setRID(LcsRid(rid));
00611         if (!rc) {
00612             break;
00613         }
00614         rids.push_back(rid);
00615     }
00616     tupleData = lbmEntry.produceEntryTuple();
00617 
00618     boost::shared_array<FixedBuffer> buffer;
00619     buffer.reset(new FixedBuffer[1024]);
00620     uint bufferSize = 0;
00621     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00622     bufferSize += tupleAccessor.getCurrentByteCount();
00623 
00624     // Then create a singleton that's further away
00625     rid += 64 + 1;
00626     rids.push_back(rid);
00627     tupleData[1].pData = (PConstBuffer) &rid;
00628     tupleData[2].pData = NULL;
00629     tupleData[2].cbData = 0;
00630     tupleData[3].pData = NULL;
00631     tupleData[3].cbData = 0;
00632     lbmEntry.setEntryTuple(tupleData);
00633     tupleData = lbmEntry.produceEntryTuple();
00634     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00635     bufferSize += tupleAccessor.getCurrentByteCount();
00636 
00637     // Insert the two bitmap entries we've just created into the btree
00638     spliceInput(
00639         buffer,
00640         bufferSize,
00641         rids.size(),
00642         bTreeDesc);
00643     resetExecStreamTest();
00644 
00645     // Now create a bitmap entry that overlaps with the singleton
00646     rid += 1;
00647     rids.push_back(rid);
00648     tupleData[1].pData = (PConstBuffer) &rid;
00649     tupleData[2].pData = NULL;
00650     tupleData[2].cbData = 0;
00651     tupleData[3].pData = NULL;
00652     tupleData[3].cbData = 0;
00653     lbmEntry.setEntryTuple(tupleData);
00654     for (uint i = 0; i < 5; i++) {
00655         rid += 8;
00656         rids.push_back(rid);
00657         bool rc = lbmEntry.setRID(LcsRid(rid));
00658         BOOST_REQUIRE(rc);
00659     }
00660     tupleData = lbmEntry.produceEntryTuple();
00661     bufferSize = 0;
00662     tupleAccessor.marshal(tupleData, buffer.get() + bufferSize);
00663     bufferSize += tupleAccessor.getCurrentByteCount();
00664 
00665     // Splice that into the existing btree
00666     spliceInput(
00667         buffer,
00668         bufferSize,
00669         5,
00670         bTreeDesc);
00671 
00672     // Read the btree bitmap entries and confirm that they contain all
00673     // of the rids that were inserted.
00674     BTreeReader reader(bTreeDesc);
00675     bool rc = reader.searchFirst();
00676     BOOST_REQUIRE(rc);
00677     uint currIdx = 0;
00678     while (rc) {
00679         reader.getTupleAccessorForRead().unmarshal(tupleData);
00680         std::vector<LcsRid> ridsRead;
00681         LbmEntry::generateRIDs(tupleData, ridsRead);
00682         for (uint i = 0; i < ridsRead.size(); i++) {
00683             BOOST_CHECK_EQUAL(
00684                 opaqueToInt(ridsRead[i]),
00685                 opaqueToInt(rids[currIdx]));
00686             currIdx++;
00687         }
00688         rc = reader.searchNext();
00689     }
00690     BOOST_CHECK_EQUAL(currIdx, rids.size());
00691 }
00692 
00693 void LbmSplicerExecStreamTest::generateRandomRids(
00694     std::vector<uint64_t> &rids,
00695     uint nRids,
00696     uint factor)
00697 {
00698     // Generate "nRids" unique, random rids between 0 and nRids * factor
00699     std::hash_set<uint64_t> ridsGenerated;
00700     uint numGenerated = 0;
00701     while (numGenerated < nRids) {
00702         uint64_t rid = uint64_t(((double) rand() / RAND_MAX) * nRids * factor);
00703         if (ridsGenerated.find(rid) == ridsGenerated.end()) {
00704             rids.push_back(rid);
00705             ridsGenerated.insert(rid);
00706             numGenerated++;
00707         }
00708     }
00709     std::sort(rids.begin(), rids.end());
00710 }
00711 
00712 void LbmSplicerExecStreamTest::generateTupleInput(
00713     std::vector<uint64_t> const &rids,
00714     uint &currRidIdx,
00715     PBuffer buffer,
00716     uint &bufferSize,
00717     uint nKeys,
00718     bool oneTuple)
00719 {
00720     TupleDescriptor tupleDesc;
00721     initBTreeTupleDesc(tupleDesc, nKeys);
00722 
00723     TupleAccessor tupleAccessor;
00724     tupleAccessor.compute(tupleDesc);
00725 
00726     TupleData tupleData;
00727     tupleData.compute(tupleDesc);
00728 
00729     // Key values are fixed
00730     for (uint i = 0; i < nKeys; i++) {
00731         uint64_t keyVal = i;
00732         tupleData[i].pData = (PConstBuffer) &keyVal;
00733     }
00734 
00735     LbmEntry lbmEntry;
00736     boost::scoped_array<FixedBuffer> entryBuf;
00737     uint entryBufSize = (nKeys + 1) * 8 + 4 * 3;
00738     entryBuf.reset(new FixedBuffer[entryBufSize]);
00739     lbmEntry.init(entryBuf.get(), NULL, entryBufSize, tupleDesc);
00740 
00741     uint numRids = rids.size();
00742     while (currRidIdx < numRids) {
00743         tupleData[nKeys].pData = (PConstBuffer) &rids[currRidIdx];
00744         tupleData[nKeys + 1].pData = NULL;
00745         tupleData[nKeys + 1].cbData = 0;
00746         tupleData[nKeys + 2].pData = NULL;
00747         tupleData[nKeys + 2].cbData = 0;
00748 
00749         // Randomly generate either singletons or bitmaps with 3 rids, except
00750         // when we're at the end of our rid list
00751         uint bitmapType;
00752         if (currRidIdx >= numRids - 3) {
00753             bitmapType = 0;
00754         } else {
00755             bitmapType = rand() % 2;
00756         }
00757         if (bitmapType == 0) {
00758             currRidIdx++;
00759         } else {
00760             lbmEntry.setEntryTuple(tupleData);
00761             bool rc = lbmEntry.setRID(LcsRid(rids[currRidIdx + 1]));
00762             BOOST_REQUIRE(rc);
00763             rc = lbmEntry.setRID(LcsRid(rids[currRidIdx + 2]));
00764             BOOST_REQUIRE(rc);
00765             tupleData = lbmEntry.produceEntryTuple();
00766             currRidIdx += 3;
00767         }
00768 
00769         tupleAccessor.marshal(tupleData, buffer + bufferSize);
00770         bufferSize += tupleAccessor.getCurrentByteCount();
00771         if (oneTuple) {
00772             break;
00773         }
00774     }
00775 }
00776 
00777 void LbmSplicerExecStreamTest::spliceInput(
00778     boost::shared_array<FixedBuffer> &inputBuffer,
00779     uint inputBufSize,
00780     uint numRows,
00781     BTreeDescriptor const &bTreeDesc)
00782 {
00783     // Create a ValuesExecStream that provides input rows to the splicer.
00784     // In the case of splicing only rids, i.e., no keys, the rids don't need
00785     // to be passed in as bitmap tuples
00786     uint nKeys = bTreeDesc.keyProjection.size();
00787     ValuesExecStreamParams valuesParams;
00788     for (uint i = 0; i < nKeys; i++) {
00789         valuesParams.outputTupleDesc.push_back(attrDesc_int64);
00790     }
00791     if (nKeys > 1) {
00792         valuesParams.outputTupleDesc.push_back(attrDesc_bitmap);
00793         valuesParams.outputTupleDesc.push_back(attrDesc_bitmap);
00794     }
00795     valuesParams.pTupleBuffer = inputBuffer,
00796     valuesParams.bufSize = inputBufSize;
00797 
00798     ExecStreamEmbryo valuesStreamEmbryo;
00799     valuesStreamEmbryo.init(new ValuesExecStream(), valuesParams);
00800     valuesStreamEmbryo.getStream()->setName("ValuesExecStream");
00801 
00802     // Create the splicer stream
00803     LbmSplicerExecStreamParams splicerParams;
00804     splicerParams.createNewIndex = false;
00805     splicerParams.scratchAccessor =
00806         pSegmentFactory->newScratchSegment(pCache, 15);
00807     splicerParams.pCacheAccessor = pCache;
00808     BTreeParams bTreeParams;
00809     initBTreeParam(bTreeParams, bTreeDesc);
00810     splicerParams.bTreeParams.push_back(bTreeParams);
00811     splicerParams.outputTupleDesc.push_back(attrDesc_int64);
00812     splicerParams.writeRowCountParamId = DynamicParamId(0);
00813 
00814     // In the case where there are index keys, we need to create a dynamic
00815     // parameter that passes in the rowcount of the number of tuples to
00816     // be spliced
00817     if (nKeys == 1) {
00818         splicerParams.insertRowCountParamId = DynamicParamId(0);
00819     } else {
00820         splicerParams.insertRowCountParamId = DynamicParamId(1);
00821         SharedDynamicParamManager pDynamicParamManager =
00822             pGraph->getDynamicParamManager();
00823         pDynamicParamManager->createParam(DynamicParamId(1), attrDesc_int64);
00824         TupleDatum paramValDatum;
00825         uint64_t rowCount = numRows;
00826         paramValDatum.pData = (PConstBuffer) &rowCount;
00827         paramValDatum.cbData = 8;
00828         pDynamicParamManager->writeParam(DynamicParamId(1), paramValDatum);
00829     }
00830 
00831     ExecStreamEmbryo splicerStreamEmbryo;
00832     splicerStreamEmbryo.init(new LbmSplicerExecStream(), splicerParams);
00833     splicerStreamEmbryo.getStream()->setName("LbmSplicerExecStream");
00834 
00835     SharedExecStream pOutputStream =
00836         prepareTransformGraph(valuesStreamEmbryo, splicerStreamEmbryo);
00837 
00838     // Setup a generator which expects the number of rows spliced
00839     RampExecStreamGenerator expectedResultGenerator(numRows);
00840 
00841     verifyOutput(*pOutputStream, 1, expectedResultGenerator);
00842 }
00843 
00844 void LbmSplicerExecStreamTest::initBTreeParam(
00845     BTreeParams &param,
00846     BTreeDescriptor const &bTreeDesc)
00847 {
00848     param.pSegment = pRandomSegment;
00849     param.pRootMap = 0;
00850     param.tupleDesc = bTreeDesc.tupleDescriptor;
00851     param.keyProj = bTreeDesc.keyProjection;
00852     param.pageOwnerId = bTreeDesc.pageOwnerId;
00853     param.segmentId = bTreeDesc.segmentId;
00854     param.rootPageId = bTreeDesc.rootPageId;
00855 }
00856 
00857 void LbmSplicerExecStreamTest::createBTree(
00858     BTreeDescriptor &bTreeDesc,
00859     uint nKeys)
00860 {
00861     initBTreeTupleDesc(bTreeDesc.tupleDescriptor, nKeys);
00862 
00863     for (uint j = 0; j < nKeys + 1; j++) {
00864         bTreeDesc.keyProjection.push_back(j);
00865     }
00866     bTreeDesc.rootPageId = NULL_PAGE_ID;
00867     bTreeDesc.segmentAccessor.pSegment = pRandomSegment;
00868     bTreeDesc.segmentAccessor.pCacheAccessor = pCache;
00869     BTreeBuilder builder(bTreeDesc, pRandomSegment);
00870     builder.createEmptyRoot();
00871     bTreeDesc.rootPageId = builder.getRootPageId();
00872 }
00873 
00874 void LbmSplicerExecStreamTest::initBTreeTupleDesc(
00875     TupleDescriptor &tupleDesc,
00876     uint nKeys)
00877 {
00878     for (uint i = 0; i < nKeys + 1; i++) {
00879         tupleDesc.push_back(attrDesc_int64);
00880     }
00881     tupleDesc.push_back(attrDesc_bitmap);
00882     tupleDesc.push_back(attrDesc_bitmap);
00883 }
00884 
00885 void LbmSplicerExecStreamTest::testCaseSetUp()
00886 {
00887     ExecStreamUnitTestBase::testCaseSetUp();
00888 
00889     attrDesc_int64 = TupleAttributeDescriptor(
00890         stdTypeFactory.newDataType(STANDARD_TYPE_INT_64));
00891 
00892     // Set the bitmap lengths to a smaller value to force more bitmap splits
00893     uint varColSize = 64;
00894     attrDesc_bitmap =
00895         TupleAttributeDescriptor(
00896             stdTypeFactory.newDataType(STANDARD_TYPE_VARBINARY), true,
00897             varColSize);
00898 }
00899 
00900 FENNEL_UNIT_TEST_SUITE(LbmSplicerExecStreamTest);
00901 
00902 // End LbmSplicerExecStreamTest.cpp

Generated on Mon Jun 22 04:00:20 2009 for Fennel by  doxygen 1.5.1