BackupRestoreTest.cpp

Go to the documentation of this file.
00001 /*
00002 // $Id: //open/dev/fennel/test/BackupRestoreTest.cpp#7 $
00003 // Fennel is a library of data storage and processing components.
00004 // Copyright (C) 2005-2009 The Eigenbase Project
00005 // Copyright (C) 2005-2009 SQLstream, Inc.
00006 // Copyright (C) 2005-2009 LucidEra, Inc.
00007 // Portions Copyright (C) 1999-2009 John V. Sichi
00008 //
00009 // This program is free software; you can redistribute it and/or modify it
00010 // under the terms of the GNU General Public License as published by the Free
00011 // Software Foundation; either version 2 of the License, or (at your option)
00012 // any later version approved by The Eigenbase Project.
00013 //
00014 // This program is distributed in the hope that it will be useful,
00015 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00016 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00017 // GNU General Public License for more details.
00018 //
00019 // You should have received a copy of the GNU General Public License
00020 // along with this program; if not, write to the Free Software
00021 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00022 */
00023 
00024 #include "fennel/common/CommonPreamble.h"
00025 #include "fennel/common/FileSystem.h"
00026 #include "fennel/test/SnapshotSegmentTestBase.h"
00027 #include "fennel/cache/PagePredicate.h"
00028 #include "fennel/txn/LogicalTxn.h"
00029 #include "fennel/txn/LogicalTxnLog.h"
00030 #include "fennel/segment/VersionedRandomAllocationSegment.h"
00031 #include "fennel/segment/SnapshotRandomAllocationSegment.h"
00032 #include "fennel/segment/SegPageBackupRestoreDevice.h"
00033 #include "fennel/db/Database.h"
00034 
00035 #include <boost/test/test_tools.hpp>
00036 
00037 using namespace fennel;
00038 
00043 class BackupRestoreTest : virtual public SnapshotSegmentTestBase
00044 {
00045     struct TestNode : public StoredNode
00046     {
00047         static const MagicNumber MAGIC_NUMBER = 0xa496c71bff0d41bdLL;
00048 
00049         uint x;
00050     };
00051 
00052     typedef SegNodeLock<TestNode> TestPageLock;
00053 
00054     SharedDatabase pDatabase;
00055     PageId persistentPageId;
00056 
00057     void createSnapshotData();
00058     void executeSnapshotTxn(int i);
00059     void verifySnapshotData(uint x);
00060 
00061     void testBackupRestore(bool isCompressed);
00062     void backup(
00063         std::string backupFileName,
00064         TxnId lowerBoundCsn,
00065         TxnId upperBoundCsn,
00066         bool isCompressed);
00067     void restore(
00068         std::string backupFileName,
00069         TxnId lowerBoundCsn,
00070         TxnId upperBoundCsn,
00071         bool isCompressed);
00072     std::string getCompressionProgram(bool isCompressed);
00073     void verifyData();
00074 
00075 public:
00076     explicit BackupRestoreTest()
00077     {
00078         FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testHeaderBackupRestore);
00079         FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupCleanup);
00080         FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupRestoreUncompressed);
00081         FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupRestoreCompressed);
00082     }
00083 
00087     void testHeaderBackupRestore();
00088 
00092     void testBackupCleanup();
00093 
00097     void testBackupRestoreUncompressed();
00098 
00102     void testBackupRestoreCompressed();
00103 };
00104 
00105 void BackupRestoreTest::testBackupRestoreUncompressed()
00106 {
00107     testBackupRestore(false);
00108 }
00109 
00110 void BackupRestoreTest::testBackupRestoreCompressed()
00111 {
00112     testBackupRestore(true);
00113 }
00114 
00115 void BackupRestoreTest::createSnapshotData()
00116 {
00117     // Create a database with a single data page with an initial value of 0.
00118     pDatabase = Database::newDatabase(
00119         pCache,
00120         configMap,
00121         DeviceMode::createNew,
00122         shared_from_this());
00123 
00124     SharedLogicalTxn pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00125     SharedSegment pSegment =
00126         pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00127             pDatabase->getDataSegment(),
00128             pDatabase->getDataSegment(),
00129             pTxn->getTxnId());
00130     SnapshotRandomAllocationSegment *pSnapshotSegment =
00131         SegmentFactory::dynamicCast<SnapshotRandomAllocationSegment *>(
00132             pSegment);
00133     SegmentAccessor segmentAccessor(pSegment,pCache);
00134 
00135     TestPageLock pageLock(segmentAccessor);
00136     persistentPageId = pageLock.allocatePage();
00137     pageLock.getNodeForWrite().x = 0;
00138     pageLock.unlock();
00139     pTxn->commit();
00140     pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00141     pSnapshotSegment->commitChanges(pTxn->getTxnId());
00142     pSnapshotSegment->checkpoint(CHECKPOINT_FLUSH_ALL);
00143     pTxn->commit();
00144     pDatabase->checkpointImpl();
00145 
00146     // Update the value to 5.
00147     executeSnapshotTxn(5);
00148 }
00149 
00150 void BackupRestoreTest::testHeaderBackupRestore()
00151 {
00152     configMap.setStringParam(
00153         Database::paramDatabaseDir,".");
00154     configMap.setStringParam(
00155         "databaseInitSize","1000");
00156     configMap.setStringParam(
00157         "tempInitSize","1000");
00158     configMap.setStringParam(
00159         "databaseShadowLogInitSize","1000");
00160     configMap.setStringParam(
00161         "databaseTxnLogInitSize","1000");
00162     configMap.setStringParam(
00163         "forceTxns","true");
00164     configMap.setStringParam(
00165         "disableSnapshots","false");
00166 
00167     CacheParams cacheParams;
00168     cacheParams.readConfig(configMap);
00169     pCache = Cache::newCache(cacheParams);
00170 
00171     // Create a single data page in the database with an initial value,
00172     // and then update the value to 5.  Doing the update will version
00173     // the data page.
00174     createSnapshotData();
00175     pDatabase->checkpointImpl();
00176     verifySnapshotData(5);
00177 
00178     // Backup the data with value 5.
00179     std::string fullBackup = "fullBackup.dat";
00180     FileSize dataDeviceSize;
00181     bool aborted = false;
00182     TxnId fullTxnId =
00183         pDatabase->initiateBackup(
00184             fullBackup,
00185             false,
00186             0,
00187             NULL_TXN_ID,
00188             "",
00189             dataDeviceSize,
00190             aborted);
00191     pDatabase->completeBackup(NULL_TXN_ID, fullTxnId, aborted);
00192 
00193     // Update the value to 15.
00194     executeSnapshotTxn(10);
00195     verifySnapshotData(15);
00196 
00197     // Do an incremental backup of value 15.
00198     std::string incrBackup1 = "incrBackup1.dat";
00199     TxnId incrTxnId1 =
00200         pDatabase->initiateBackup(
00201             incrBackup1,
00202             true,
00203             0,
00204             fullTxnId,
00205             "",
00206             dataDeviceSize,
00207             aborted);
00208 
00209     // Make sure ALTER SYSTEM DEALLOCATE OLD is disabled
00210     uint nPagesBefore =
00211         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00212     pDatabase->deallocateOldPages(incrTxnId1);
00213     uint nPagesAfter =
00214         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00215     BOOST_REQUIRE(nPagesBefore == nPagesAfter);
00216 
00217     pDatabase->completeBackup(fullTxnId, incrTxnId1, aborted);
00218 
00219     // Make sure ALTER SYSTEM DEALLOCATE OLD is reenabled
00220     pDatabase->deallocateOldPages(incrTxnId1);
00221     nPagesAfter = pDatabase->getDataSegment()->getAllocatedSizeInPages();
00222     BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00223 
00224     // Update the value to 35.
00225     executeSnapshotTxn(20);
00226     verifySnapshotData(35);
00227 
00228     // Do a second incremental backup with value 35.
00229     std::string incrBackup2 = "incrBackup2.dat";
00230     TxnId incrTxnId2 =
00231         pDatabase->initiateBackup(
00232             incrBackup2,
00233             true,
00234             4096,
00235             incrTxnId1,
00236             "",
00237             dataDeviceSize,
00238             aborted);
00239     pDatabase->completeBackup(incrTxnId1, incrTxnId2, aborted);
00240 
00241     // Restore the full backup.  The value should be 5.
00242     pDatabase->restoreFromBackup(
00243         fullBackup,
00244         1002 * pCache->getPageSize(),
00245         "",
00246         NULL_TXN_ID,
00247         fullTxnId,
00248         aborted);
00249     verifySnapshotData(5);
00250 
00251     // Restore the first incremental backup.  The value should be 15.
00252     pDatabase->restoreFromBackup(
00253         incrBackup1,
00254         1002 * pCache->getPageSize(),
00255         "",
00256         fullTxnId,
00257         incrTxnId1,
00258         aborted);
00259     verifySnapshotData(15);
00260 
00261     // Restore the second incremental backup.  The value should be 35.
00262     pDatabase->restoreFromBackup(
00263         incrBackup2,
00264         1002 * pCache->getPageSize(),
00265         "",
00266         incrTxnId1,
00267         incrTxnId2,
00268         aborted);
00269     verifySnapshotData(35);
00270 
00271     pDatabase.reset();
00272 }
00273 
00274 void BackupRestoreTest::testBackupCleanup()
00275 {
00276     configMap.setStringParam(
00277         Database::paramDatabaseDir,".");
00278     configMap.setStringParam(
00279         "databaseInitSize","1000");
00280     configMap.setStringParam(
00281         "tempInitSize","1000");
00282     configMap.setStringParam(
00283         "databaseShadowLogInitSize","1000");
00284     configMap.setStringParam(
00285         "databaseTxnLogInitSize","1000");
00286     configMap.setStringParam(
00287         "forceTxns","true");
00288     configMap.setStringParam(
00289         "disableSnapshots","false");
00290 
00291     CacheParams cacheParams;
00292     cacheParams.readConfig(configMap);
00293     pCache = Cache::newCache(cacheParams);
00294 
00295     // Create a single data page in the database with an initial value,
00296     // and then update the value to 5.  Doing the update will version
00297     // the data page.
00298     createSnapshotData();
00299     pDatabase->checkpointImpl();
00300     verifySnapshotData(5);
00301 
00302     // Update the value to 15.
00303     executeSnapshotTxn(10);
00304     verifySnapshotData(15);
00305 
00306     std::string fullBackup = "fullBackup.dat";
00307 
00308     // Set the space padding to the amount of space currently available
00309     // multipled by 1000.  This should result in a failure unless some other
00310     // user is using this filesystem and either:
00311     // A) that user is freeing up a large amount of space, or
00312     // B) that user is freeing up space and the filesystem is short on space,
00313     //    in which case, multiplying by 1000 doesn't yield a large enough value
00314     //    to offset the amount that the user has freed
00315     FileSystem::remove(fullBackup.c_str());
00316     FileSize spaceAvailable;
00317     FileSystem::getDiskFreeSpace(".", spaceAvailable);
00318     FileSize dataDeviceSize;
00319     bool aborted = false;
00320     try {
00321         pDatabase->initiateBackup(
00322             fullBackup,
00323             true,
00324             spaceAvailable * 1000,
00325             NULL_TXN_ID,
00326             "",
00327             dataDeviceSize,
00328             aborted);
00329         BOOST_FAIL("Out of space exception not returned");
00330     } catch (FennelExcn &ex) {
00331         std::string errMsg = ex.getMessage();
00332         if (errMsg.find("Insufficient space") != 0) {
00333             BOOST_FAIL("Wrong exception returned");
00334         }
00335     }
00336 
00337     // Make sure ALTER SYSTEM DEALLOCATE OLD is enabled, even after the
00338     // exception
00339     uint nPagesBefore =
00340         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00341     pDatabase->deallocateOldPages(pDatabase->getLastCommittedTxnId());
00342     uint nPagesAfter =
00343         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00344     BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00345 
00346     // Do another update so a new page is allocated
00347     executeSnapshotTxn(20);
00348     verifySnapshotData(35);
00349 
00350     // Initiate a new backup and then abort it.
00351     pDatabase->initiateBackup(
00352         fullBackup,
00353         false,
00354         0,
00355         NULL_TXN_ID,
00356         getCompressionProgram(true),
00357         dataDeviceSize,
00358         aborted);
00359     pDatabase->abortBackup();
00360 
00361     // Make sure ALTER SYSTEM DEALLOCATE OLD is enabled, even after the
00362     // backup was aborted
00363     nPagesBefore =
00364         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00365     pDatabase->deallocateOldPages(pDatabase->getLastCommittedTxnId());
00366     nPagesAfter =
00367         pDatabase->getDataSegment()->getAllocatedSizeInPages();
00368     BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00369 
00370     // Abort a backup that was never initiated; should be a no-op
00371     pDatabase->abortBackup();
00372 
00373     pDatabase.reset();
00374 }
00375 
00376 void BackupRestoreTest::executeSnapshotTxn(int i)
00377 {
00378     SharedLogicalTxn pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00379     SharedSegment pSegment =
00380         pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00381             pDatabase->getDataSegment(),
00382             pDatabase->getDataSegment(),
00383             pTxn->getTxnId());
00384     SnapshotRandomAllocationSegment *pSnapshotSegment =
00385         SegmentFactory::dynamicCast<SnapshotRandomAllocationSegment *>(
00386             pSegment);
00387 
00388     // Update the value, which will version the page.
00389     SegmentAccessor segmentAccessor(pSegment, pCache);
00390     TestPageLock pageLock(segmentAccessor);
00391     pageLock.lockExclusive(persistentPageId);
00392     pageLock.getNodeForWrite().x += i;
00393     pageLock.unlock();
00394     pTxn->commit();
00395 
00396     // Commit the changes through the snapshot segment.
00397     pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00398     pSnapshotSegment->commitChanges(pTxn->getTxnId());
00399     pSnapshotSegment->checkpoint(CHECKPOINT_FLUSH_ALL);
00400     pTxn->commit();
00401     pDatabase->checkpointImpl();
00402 }
00403 
00404 void BackupRestoreTest::verifySnapshotData(uint x)
00405 {
00406     // Lock the original page, but because we're accessing the segment
00407     // through a snapshot segment with the csn set to that of the last
00408     // committed transaction, we'll pick up the latest version of the page.
00409     SharedSegment pSegment =
00410         pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00411             pDatabase->getDataSegment(),
00412             pDatabase->getDataSegment(),
00413             pDatabase->getLastCommittedTxnId());
00414     SegmentAccessor segmentAccessor(pSegment,pCache);
00415     TestPageLock pageLock(segmentAccessor);
00416     pageLock.lockShared(persistentPageId);
00417     BOOST_CHECK_EQUAL(pageLock.getNodeForRead().x,x);
00418 }
00419 
00420 void BackupRestoreTest::testBackupRestore(bool isCompressed)
00421 {
00422     // Create and initialize a VersionedRandomAllocationSegment using
00423     // a snapshot with TxnId(0)
00424     currCsn = TxnId(0);
00425     openStorage(DeviceMode::createNew);
00426     testAllocateAll();
00427     closeStorage();
00428 
00429     // Update every 5 pages
00430     currCsn = TxnId(5);
00431     updatedCsns.push_back(currCsn);
00432     openStorage(DeviceMode::load);
00433     testSkipWrite(5);
00434     closeStorage();
00435 
00436     // Do a backup of all pages as of TxnId 5
00437     std::string backup5 = "backupTxnId5.dat";
00438     backup(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00439 
00440     // Update every 7 pages
00441     currCsn = TxnId(7);
00442     updatedCsns.push_back(currCsn);
00443     openStorage(DeviceMode::load);
00444     testSkipWrite(7);
00445     closeStorage();
00446 
00447     // Do another backup of all pages between TxnId's 5 and 7
00448     std::string backup5to7 = "backupTxnId5to7.dat";
00449     backup(backup5to7, TxnId(5), TxnId(7), isCompressed);
00450 
00451     // Update every 11 pages
00452     currCsn = TxnId(11);
00453     updatedCsns.push_back(currCsn);
00454     openStorage(DeviceMode::load);
00455     testSkipWrite(11);
00456     closeStorage();
00457 
00458     // Do a backup of all pages between TxnId's 7 and 11
00459     std::string backup7to11 = "backupTxnId7to11.dat";
00460     backup(backup7to11, TxnId(7), TxnId(11), isCompressed);
00461 
00462     // Do a backup of all pages between TxnId's 5 and 11
00463     std::string backup5to11 = "backupTxnId5to11.dat";
00464     backup(backup5to11, TxnId(5), TxnId(11), isCompressed);
00465 
00466     // Restore the backup as of TxnId 5
00467     restore(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00468 
00469     // Make sure the data only reflects up to TxnId 5.  Leave the currCsn
00470     // set to the larger value, but have updatedCsn's reflect only the
00471     // updates up to TxnId 5.
00472     updatedCsns.clear();
00473     updatedCsns.push_back(TxnId(5));
00474     verifyData();
00475 
00476     // Restore the backup as of TxnId 7 and verify that it only includes
00477     // updates up to TxnId 7
00478     restore(backup5to7, TxnId(5), TxnId(7), isCompressed);
00479     updatedCsns.push_back(TxnId(7));
00480     verifyData();
00481 
00482     // Restore the backup as of TxnId 11 and verify that it includes all
00483     // updates
00484     restore(backup7to11, TxnId(7), TxnId(11), isCompressed);
00485     updatedCsns.push_back(TxnId(11));
00486     verifyData();
00487 
00488     // Go back and do restores of the backup at TxnId5 followed by the
00489     // backup of TxnId's 5 through 11.
00490     restore(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00491     updatedCsns.clear();
00492     updatedCsns.push_back(TxnId(5));
00493     verifyData();
00494     restore(backup5to11, TxnId(5), TxnId(11), isCompressed);
00495     updatedCsns.push_back(TxnId(7));
00496     updatedCsns.push_back(TxnId(11));
00497     verifyData();
00498 }
00499 
00500 void BackupRestoreTest::backup(
00501     std::string backupFileName,
00502     TxnId lowerBoundCsn,
00503     TxnId upperBoundCsn,
00504     bool isCompressed)
00505 {
00506     openStorage(DeviceMode::load);
00507     VersionedRandomAllocationSegment *pVRSegment =
00508         SegmentFactory::dynamicCast<VersionedRandomAllocationSegment *>(
00509             pVersionedRandomSegment);
00510     SegmentAccessor scratchAccessor =
00511         pSegmentFactory->newScratchSegment(pCache, 12);
00512 
00513     if (isCompressed) {
00514         backupFileName.append(".gz");
00515     }
00516     SharedSegPageBackupRestoreDevice pBackupDevice =
00517         SegPageBackupRestoreDevice::newSegPageBackupRestoreDevice(
00518             backupFileName,
00519             "w",
00520             getCompressionProgram(isCompressed),
00521             10,
00522             2,
00523             scratchAccessor,
00524             pCache->getDeviceAccessScheduler(*pRandomAccessDevice),
00525             pRandomAccessDevice);
00526     bool abortFlag = false;
00527     pVRSegment->backupAllocationNodes(
00528         pBackupDevice,
00529         false,
00530         lowerBoundCsn,
00531         upperBoundCsn,
00532         abortFlag);
00533     pVRSegment->backupDataPages(
00534         pBackupDevice,
00535         lowerBoundCsn,
00536         upperBoundCsn,
00537         abortFlag);
00538 
00539     assert(pBackupDevice.unique());
00540     pBackupDevice.reset();
00541 
00542     scratchAccessor.reset();
00543     closeStorage();
00544 }
00545 
00546 void BackupRestoreTest::restore(
00547     std::string backupFileName,
00548     TxnId lowerBoundCsn,
00549     TxnId upperBoundCsn,
00550     bool isCompressed)
00551 {
00552     openStorage(DeviceMode::load);
00553     VersionedRandomAllocationSegment *pVRSegment =
00554         SegmentFactory::dynamicCast<VersionedRandomAllocationSegment *>(
00555             pVersionedRandomSegment);
00556     SegmentAccessor scratchAccessor =
00557         pSegmentFactory->newScratchSegment(pCache, 10);
00558 
00559     // Flush and unmap any pages currently in the cache so the restore
00560     // avoids reading out-of-date pages
00561     MappedPageListenerPredicate pagePredicate(*pVRSegment);
00562     pCache->checkpointPages(pagePredicate, CHECKPOINT_FLUSH_AND_UNMAP);
00563 
00564     if (isCompressed) {
00565         backupFileName.append(".gz");
00566     }
00567     SharedSegPageBackupRestoreDevice pBackupDevice =
00568         SegPageBackupRestoreDevice::newSegPageBackupRestoreDevice(
00569             backupFileName,
00570             "r",
00571             getCompressionProgram(isCompressed),
00572             10,
00573             0,
00574             scratchAccessor,
00575             pCache->getDeviceAccessScheduler(*pRandomAccessDevice),
00576             pRandomAccessDevice);
00577     bool abortFlag = false;
00578     pVRSegment->restoreFromBackup(
00579         pBackupDevice,
00580         lowerBoundCsn,
00581         upperBoundCsn,
00582         abortFlag);
00583 
00584     assert(pBackupDevice.unique());
00585     pBackupDevice.reset();
00586 
00587     scratchAccessor.reset();
00588     closeStorage();
00589 }
00590 
00591 std::string BackupRestoreTest::getCompressionProgram(bool isCompressed)
00592 {
00593     if (!isCompressed) {
00594         return "";
00595     } else {
00596         return "gzip";
00597     }
00598 }
00599 
00600 void BackupRestoreTest::verifyData()
00601 {
00602     openStorage(DeviceMode::load);
00603     testSequentialRead();
00604     closeStorage();
00605 }
00606 
00607 FENNEL_UNIT_TEST_SUITE(BackupRestoreTest);
00608 
00609 // End BackupRestoreTest.cpp

Generated on Mon Jun 22 04:00:20 2009 for Fennel by  doxygen 1.5.1