FlatFileParser.h

Go to the documentation of this file.

/*
// $Id: //open/dev/fennel/flatfile/FlatFileParser.h#2 $
// Fennel is a library of data storage and processing components.
// Copyright (C) 2005-2009 The Eigenbase Project
// Copyright (C) 2009-2009 SQLstream, Inc.
// Copyright (C) 2005-2009 LucidEra, Inc.
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation; either version 2 of the License, or (at your option)
// any later version approved by The Eigenbase Project.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#ifndef Fennel_FlatFileParser_Included
#define Fennel_FlatFileParser_Included


#include <vector>

FENNEL_BEGIN_NAMESPACE

class FlatFileParser;
typedef boost::shared_ptr<FlatFileParser> SharedFlatFileParser;

class FENNEL_FLATFILE_EXPORT FlatFileColumnParseResult
{
public:
    enum DelimiterType {
        NO_DELIM = 0,
        FIELD_DELIM,
        ROW_DELIM,
        MAX_LENGTH
    };

    DelimiterType type;

    uint size;

    char *next;

    void setResult(DelimiterType type, char *buffer, uint size);
};

class FENNEL_FLATFILE_EXPORT FlatFileRowParseResult
{
public:
    enum RowStatus {
        NO_STATUS = 0,
        INCOMPLETE_COLUMN,
        ROW_TOO_LARGE,
        NO_COLUMN_DELIM,
        TOO_FEW_COLUMNS,
        TOO_MANY_COLUMNS
    };

    explicit FlatFileRowParseResult();
    void reset();

    RowStatus status;

    VectorOfUint offsets;

    VectorOfUint sizes;

    VectorOfUint strippedSizes;

    char *current;

    char *next;

    uint nRowDelimsRead;

    uint getReadCount()
    {
        return offsets.size();
    }

    char *getColumn(uint iColumn)
    {
        if (sizes[iColumn] == 0) {
            return NULL;
        }
        return current + offsets[iColumn];
    }

    uint getRawColumnSize(uint iColumn)
    {
        return sizes[iColumn];
    }

    uint getColumnSize(uint iColumn)
    {
        return strippedSizes[iColumn];
    }

    void clear()
    {
        offsets.clear();
        sizes.clear();
    }

    void resize(uint nColumns)
    {
        offsets.resize(nColumns);
        sizes.resize(nColumns);
    }

    void setColumn(uint iColumn, uint offset, uint size)
    {
        offsets[iColumn] = offset;
        sizes[iColumn] = size;
    }

    void setNull(uint iColumn)
    {
        setColumn(iColumn, 0, 0);
    }

    void addColumn(uint offset, uint size)
    {
        offsets.push_back(offset);
        sizes.push_back(size);
    }
};

class FENNEL_FLATFILE_EXPORT FlatFileColumnDescriptor
{
public:
    uint maxLength;

#ifdef __MSVC__
    explicit FlatFileColumnDescriptor()
    {
        maxLength = 0;
    }
#endif

    explicit FlatFileColumnDescriptor(uint maxLengthInit)
    {
        maxLength = maxLengthInit;
    }
};

class FENNEL_FLATFILE_EXPORT FlatFileRowDescriptor
    : public std::vector<FlatFileColumnDescriptor>
{
    bool bounded;
    bool lenient;

    VectorOfUint columnMap;

public:
    static const int MAX_COLUMNS = 1024;

    static const int MAX_COLUMN_LENGTH = 65535;

    FlatFileRowDescriptor();

    void setUnbounded();

    bool isBounded() const;

    void setMap(VectorOfUint map)
    {
        columnMap = map;
    }

    bool isMapped() const
    {
        return columnMap.size() > 0;
    }

    int getMap(uint iSource) const
    {
        if (iSource >= columnMap.size()) {
            return -1;
        }
        return columnMap[iSource];
    }

    void setLenient(bool lenientIn)
    {
        lenient = lenientIn;
    }

    bool isLenient() const
    {
        return lenient;
    }

    uint getMaxColumns() const
    {
        if (!bounded) {
            return MAX_COLUMNS;
        } else if (isMapped()) {
            return columnMap.size();
        } else {
            return size();
        }
    }

    uint getMaxLength(uint i) const
    {
        uint realIndex = 0;
        if (!bounded) {
            return MAX_COLUMN_LENGTH;
        } else if (isMapped()) {
            realIndex = getMap(i);
        } else {
            realIndex = i;
        }
        if (realIndex < 0 || realIndex >= size()) {
            return MAX_COLUMN_LENGTH;
        } else {
            return (*this)[realIndex].maxLength;
        }
    }
};

class FENNEL_FLATFILE_EXPORT FlatFileParser
{
    char fieldDelim;
    char rowDelim;
    char quote;
    char escape;
    bool doTrim;

    bool fixed;

    const char *scanRowEnd(
        const char *buffer,
        int size,
        bool rowDelim,
        FlatFileRowParseResult &result);

    const char *scanRowDelim(
        const char *buffer,
        int size,
        bool search);

    bool isRowDelim(char c);

public:
    FlatFileParser(
        const char fieldDelim,
        const char rowDelim,
        const char quote,
        const char escape,
        bool doTrim = false);

    void scanRow(
        const char *buffer,
        int size,
        const FlatFileRowDescriptor &columns,
        FlatFileRowParseResult &result);

    void scanColumn(
        const char *buffer,
        uint size,
        uint maxLength,
        FlatFileColumnParseResult &result);


    void scanFixedColumn(
        const char *buffer,
        uint size,
        uint maxLength,
        FlatFileColumnParseResult &result);

    void stripQuoting(
        FlatFileRowParseResult &rowResult,
        bool trim);

    uint stripQuoting(char *buffer, uint size, bool untrimmed);

    uint trim(char *buffer, uint size);
};

FENNEL_END_NAMESPACE

#endif

// End FlatFileParser.h

---