AioLinuxScheduler.cpp

Go to the documentation of this file.
00001 /*
00002 // $Id: //open/dev/fennel/device/AioLinuxScheduler.cpp#9 $
00003 // Fennel is a library of data storage and processing components.
00004 // Copyright (C) 2005-2009 The Eigenbase Project
00005 // Copyright (C) 2005-2009 SQLstream, Inc.
00006 // Copyright (C) 2005-2009 LucidEra, Inc.
00007 //
00008 // This program is free software; you can redistribute it and/or modify it
00009 // under the terms of the GNU General Public License as published by the Free
00010 // Software Foundation; either version 2 of the License, or (at your option)
00011 // any later version approved by The Eigenbase Project.
00012 //
00013 // This program is distributed in the hope that it will be useful,
00014 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00015 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00016 // GNU General Public License for more details.
00017 //
00018 // You should have received a copy of the GNU General Public License
00019 // along with this program; if not, write to the Free Software
00020 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00021 */
00022 
00023 #include "fennel/common/CommonPreamble.h"
00024 
00025 #ifdef USE_LIBAIO_H
00026 
00027 #include "fennel/device/AioLinuxScheduler.h"
00028 #include "fennel/device/RandomAccessDevice.h"
00029 #include "fennel/device/DeviceAccessSchedulerParams.h"
00030 #include "fennel/common/SysCallExcn.h"
00031 #include <errno.h>
00032 
00033 FENNEL_BEGIN_CPPFILE("$Id: //open/dev/fennel/device/AioLinuxScheduler.cpp#9 $");
00034 
00035 extern "C"
00036 DeviceAccessScheduler *newAioLinuxScheduler(
00037     DeviceAccessSchedulerParams const &params)
00038 {
00039     return new AioLinuxScheduler(params);
00040 }
00041 
00042 AioLinuxScheduler::AioLinuxScheduler(
00043     DeviceAccessSchedulerParams const &params)
00044 {
00045     quit = false;
00046     nRequestsOutstanding.clear();
00047     context = NULL;
00048     int rc = io_queue_init(params.maxRequests, &context);
00049     if (rc) {
00050         throw SysCallExcn("io_queue_init failed");
00051     }
00052 
00053     // NOTE jvs 29-Oct-2005:  we ignore params.nThreads because
00054     // no more than one thread is needed for io_getevents
00055     Thread::start();
00056 }
00057 
00058 inline bool AioLinuxScheduler::isStarted() const
00059 {
00060     return Thread::isStarted();
00061 }
00062 
00063 AioLinuxScheduler::~AioLinuxScheduler()
00064 {
00065     assert(!isStarted());
00066     assert(!nRequestsOutstanding);
00067     int rc = io_queue_release(context);
00068     if (rc) {
00069         throw SysCallExcn("io_queue_release failed");
00070     }
00071 }
00072 
00073 void AioLinuxScheduler::registerDevice(
00074     SharedRandomAccessDevice pDevice)
00075 {
00076     int hFile = pDevice->getHandle();
00077 
00078     // Linux requires files accessed via libaio to be opened with O_DIRECT,
00079     // so force that now.
00080     int flags = fcntl(hFile, F_GETFL);
00081     fcntl(hFile, F_SETFL, flags | O_DIRECT);
00082 }
00083 
00084 bool AioLinuxScheduler::schedule(RandomAccessRequest &request)
00085 {
00086     assert(isStarted());
00087     request.pDevice->prepareTransfer(request);
00088     return submitRequests(request.bindingList);
00089 }
00090 
00091 bool AioLinuxScheduler::submitRequests(
00092     RandomAccessRequest::BindingList &bindingList)
00093 {
00094     iocb *requestsArray[bindingList.size()];
00095     iocb **requests = requestsArray;
00096 
00097     // convert list to array
00098     int n = 0;
00099     RandomAccessRequest::BindingListMutator bindingMutator(bindingList);
00100     for (; bindingMutator; ++n) {
00101         RandomAccessRequestBinding *pBinding = bindingMutator.detach();
00102         requests[n] = pBinding;
00103     }
00104 
00105     if (n == 0) {
00106         // just in case someone asks for a nop
00107         return true;
00108     }
00109 
00110     // submit array
00111     int rc = io_submit(context, n, requests);
00112     if (rc == -EAGAIN) {
00113         rc = 0;
00114     }
00115 
00116     if (rc < 0) {
00117         // hard error
00118         throw SysCallExcn("io_submit failed");
00119     }
00120 
00121     // keep track of the number successfully submitted
00122     // (can't use += because nRequestsOutstanding is
00123     // an AtomicCounter)
00124     for (int i = 0; i < rc; ++i) {
00125         ++nRequestsOutstanding;
00126     }
00127 
00128     if (rc == n) {
00129         // we're done
00130         return true;
00131     } else {
00132         // io_submit is allowed to do less than we asked for, so
00133         // we need to resubmit some leftovers
00134         requests += rc;
00135         n -= rc;
00136         deferLeftoverRequests(requests, n);
00137         return false;
00138     }
00139 }
00140 
00141 void AioLinuxScheduler::deferLeftoverRequests(
00142     iocb **ppLeftovers,
00143     uint nLeftovers)
00144 {
00145     assert(nLeftovers > 0);
00146 
00147     // convert array back to list
00148     RandomAccessRequest::BindingList bindingList;
00149 
00150     for (uint i = 0; i < nLeftovers; ++i) {
00151         RandomAccessRequestBinding *pBinding =
00152             static_cast<RandomAccessRequestBinding *>(ppLeftovers[i]);
00153         bindingList.push_back(*pBinding);
00154     }
00155 
00156     StrictMutexGuard deferredQueueGuard(deferredQueueMutex);
00157     deferredQueue.push_back(bindingList);
00158 }
00159 
00160 bool AioLinuxScheduler::retryDeferredRequests()
00161 {
00162     for (;;) {
00163         StrictMutexGuard deferredQueueGuard(deferredQueueMutex);
00164         if (deferredQueue.empty()) {
00165             // all resubmitted successfully (or none to begin with)
00166             return true;
00167         }
00168         RandomAccessRequest::BindingList bindingList = deferredQueue.front();
00169         deferredQueue.pop_front();
00170         // release mutex now to avoid potential deadlocks
00171         deferredQueueGuard.unlock();
00172 
00173         bool success = submitRequests(bindingList);
00174         if (!success) {
00175             // at least one failed
00176             return false;
00177         }
00178     }
00179 }
00180 
00181 void AioLinuxScheduler::stop()
00182 {
00183     assert(isStarted());
00184     quit = true;
00185 
00186     Thread::join();
00187 }
00188 
00189 void AioLinuxScheduler::run()
00190 {
00191     while (nRequestsOutstanding || !quit) {
00192         io_event event;
00193         timespec ts;
00194 
00195         // Check the deferred request queue before entering wait state.
00196         if (retryDeferredRequests()) {
00197             // If we retried any requests, they all succeeded, so we're in our
00198             // normal wait state: timeout every second to check the quit flag.
00199             ts.tv_sec = 1;
00200             ts.tv_nsec = 0;
00201         } else {
00202             // At least one retry just failed, so during wait, timeout in a
00203             // millisecond so we can retry the failed requests.
00204             ts.tv_sec = 0;
00205             ts.tv_nsec = 1000000;
00206         }
00207 
00208         long rc = io_getevents(context, 1, 1, &event, &ts);
00209 
00210         // NOTE jvs 20-Jan-2008:  Docs don't mention the possibility of
00211         // spurious interrupts, but they can occur, at least while
00212         // debugging with gdb, so treat them as timeout.
00213         if ((rc == 0) || (rc == -EINTR)) {
00214             // timed out
00215             continue;
00216         }
00217 
00218         if (rc != 1) {
00219             throw SysCallExcn("io_getevents failed");
00220         }
00221         RandomAccessRequestBinding *pBinding =
00222             static_cast<RandomAccessRequestBinding *>(event.obj);
00223         bool success = (pBinding->getBufferSize() == event.res)
00224             && !event.res2;
00225         pBinding->notifyTransferCompletion(success);
00226         --nRequestsOutstanding;
00227     }
00228 }
00229 
00230 FENNEL_END_CPPFILE("$Id: //open/dev/fennel/device/AioLinuxScheduler.cpp#9 $");
00231 
00232 #endif
00233 
00234 // End AioLinuxScheduler.cpp

Generated on Mon Jun 22 04:00:18 2009 for Fennel by  doxygen 1.5.1