FlatFileExecStreamImpl.h

Go to the documentation of this file.

/*
// $Id: //open/dev/fennel/flatfile/FlatFileExecStreamImpl.h#2 $
// Fennel is a library of data storage and processing components.
// Copyright (C) 2005-2009 The Eigenbase Project
// Copyright (C) 2009-2009 SQLstream, Inc.
// Copyright (C) 2004-2009 LucidEra, Inc.
// Portions Copyright (C) 2004-2009 John V. Sichi
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation; either version 2 of the License, or (at your option)
// any later version approved by The Eigenbase Project.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#ifndef Fennel_FlatFileExecStreamImpl_Included
#define Fennel_FlatFileExecStreamImpl_Included

#include "fennel/flatfile/FlatFileBuffer.h"
#include "fennel/flatfile/FlatFileExecStream.h"
#include "fennel/flatfile/FlatFileParser.h"
#include "fennel/segment/SegmentAccessor.h"
#include "fennel/segment/SegPageLock.h"
#include "fennel/tuple/TupleData.h"

#include <boost/scoped_ptr.hpp>

FENNEL_BEGIN_NAMESPACE

const int FLAT_FILE_MAX_COLUMN_NAME_LEN = 255;

const int FLAT_FILE_MAX_NON_CHAR_VALUE_LEN = 255;

class FENNEL_FLATFILE_EXPORT FlatFileExecStreamImpl
    : public FlatFileExecStream
{
    // max length of text for a row when signalling an error
    static const uint MAX_ROW_ERROR_TEXT_WIDTH;

    // parameters
    std::string dataFilePath;
    bool header;
    bool lenient;
    bool trim;
    bool mapped;
    std::vector<std::string> columnNames;

    FlatFileRowDescriptor rowDesc;
    SharedFlatFileBuffer pBuffer;
    PBuffer pBufferStorage;
    char *next;
    SharedFlatFileParser pParser;
    FlatFileRowParseResult lastResult;
    TupleDescriptor textDesc;
    TupleData textTuple, dataTuple;
    bool isRowPending;

    // for sampling/describe mode
    FlatFileMode mode;
    int numRowsScan;
    bool done;
    VectorOfUint fieldSizes;
    std::string describeResult;

    SegPageLock bufferLock;
    SegmentAccessor scratchAccessor;

    // error handling
    uint nRowsOutput, nRowErrors;
    std::string reason;
    TupleDescriptor errorDesc;
    TupleData errorTuple;

    // implement ExecStream
    virtual void closeImpl();

    void releaseResources();

    uint findField(const std::string &name);

    FlatFileRowDescriptor readTupleDescriptor(
        const TupleDescriptor &tupleDesc);

    void handleTuple(
        FlatFileRowParseResult &result,
        TupleData &tuple);

    void describeStream(TupleData &tupleData);

    void logError(const FlatFileRowParseResult &result);

    void logError(
        const std::string reason,
        const FlatFileRowParseResult &result);

    void checkRowDelimiter();

public:
    // implement ExecStream
    virtual void prepare(FlatFileExecStreamParams const &params);
    virtual void getResourceRequirements(
        ExecStreamResourceQuantity &minQuantity,
        ExecStreamResourceQuantity &optQuantity);
    virtual void open(bool restart);
    virtual ExecStreamResult execute(ExecStreamQuantum const &quantum);
};

FENNEL_END_NAMESPACE

#endif

// End FlatFileExecStreamImpl.h

---