Subversion Repositories SmartDukaan

/************************************************************************
The zlib/libpng License

Copyright (c) 2006 Joerg Wiedenmann

This software is provided 'as-is', without any express or implied warranty.
In no event will the authors be held liable for any damages arising from
the use of this software.

Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:

1. The origin of this software must not be misrepresented;
you must not claim that you wrote the original software.
If you use this software in a product, an acknowledgment
in the product documentation would be appreciated but is
not required.

2. Altered source versions must be plainly marked as such,
and must not be misrepresented as being the original software.

3. This notice may not be removed or altered from any source distribution.

***********************************************************************/

/********************************************************************
        created:        2006-01-28
        filename:       tokenizer.cpp
        author:         Jörg Wiedenmann
        
        purpose:        A tokenizer function which provides a very
                                customizable way of breaking up strings.

        history:        2006-01-28, Original version
                                2006-03-04, Fixed a small parsing bug, thanks Elias.
*********************************************************************/

#include "tokenizer.h"

using namespace std;

void tokenize ( const string& str, vector<string>& result,
                           const string& delimiters, const string& delimiters_preserve,
                           const string& quote, const string& esc )
{
        // clear the vector
        if ( false == result.empty() )
        {
                result.clear();
        }

        string::size_type pos = 0; // the current position (char) in the string
        char ch = 0; // buffer for the current character
        char delimiter = 0;     // the buffer for the delimiter char which
                                                        // will be added to the tokens if the delimiter
                                                        // is preserved
        char current_quote = 0; // the char of the current open quote
        bool quoted = false; // indicator if there is an open quote
        string token;  // string buffer for the token
        bool token_complete = false; // indicates if the current token is
                                                                 // read to be added to the result vector
        string::size_type len = str.length();  // length of the input-string

        // for every char in the input-string
        while ( len > pos )
        {
                // get the character of the string and reset the delimiter buffer
                ch = str.at(pos);
                delimiter = 0;

                // assume ch isn't a delimiter
                bool add_char = true;

                // check ...

                // ... if the delimiter is an escaped character
                bool escaped = false; // indicates if the next char is protected
                if ( false == esc.empty() ) // check if esc-chars are  provided
                {
                        if ( string::npos != esc.find_first_of(ch) )
                        {
                                // get the escaped char
                                ++pos;
                                if ( pos < len ) // if there are more chars left
                                {
                                        // get the next one
                                        ch = str.at(pos);

                                        // add the escaped character to the token
                                        add_char = true;
                                }
                                else // cannot get any more characters
                                {
                                        // don't add the esc-char
                                        add_char = false;
                                }

                                // ignore the remaining delimiter checks
                                escaped = true;
                        }
                }

                // ... if the delimiter is a quote
                if ( false == quote.empty() && false == escaped )
                {
                        // if quote chars are provided and the char isn't protected
                        if ( string::npos != quote.find_first_of(ch) )
                        {
                                // if not quoted, set state to open quote and set
                                // the quote character
                                if ( false == quoted )
                                {
                                        quoted = true;
                                        current_quote = ch;

                                        // don't add the quote-char to the token
                                        add_char = false;
                                }
                                else // if quote is open already
                                {
                                        // check if it is the matching character to close it
                                        if ( current_quote == ch )
                                        {
                                                // close quote and reset the quote character
                                                quoted = false;
                                                current_quote = 0;

                                                // don't add the quote-char to the token
                                                add_char = false;
                                        }
                                } // else
                        }
                }

                // ... if the delimiter isn't preserved
                if ( false == delimiters.empty() && false == escaped &&
                         false == quoted )
                {
                        // if a delimiter is provided and the char isn't protected by
                        // quote or escape char
                        if ( string::npos != delimiters.find_first_of(ch) )
                        {
                                // if ch is a delimiter and the token string isn't empty
                                // the token is complete
                                if ( false == token.empty() ) // BUGFIX: 2006-03-04
                                {
                                        token_complete = true;
                                }

                                // don't add the delimiter to the token
                                add_char = false;
                        }
                }

                // ... if the delimiter is preserved - add it as a token
                bool add_delimiter = false;
                if ( false == delimiters_preserve.empty() && false == escaped &&
                         false == quoted )
                {
                        // if a delimiter which will be preserved is provided and the
                        // char isn't protected by quote or escape char
                        if ( string::npos != delimiters_preserve.find_first_of(ch) )
                        {
                                // if ch is a delimiter and the token string isn't empty
                                // the token is complete
                                if ( false == token.empty() ) // BUGFIX: 2006-03-04
                                {
                                        token_complete = true;
                                }

                                // don't add the delimiter to the token
                                add_char = false;

                                // add the delimiter
                                delimiter = ch;
                                add_delimiter = true;
                        }
                }


                // add the character to the token
                if ( true == add_char )
                {
                        // add the current char
                        token.push_back( ch );
                }

                // add the token if it is complete
                if ( true == token_complete && false == token.empty() )
                {
                        // add the token string
                        result.push_back( token );

                        // clear the contents
                        token.clear();

                        // build the next token
                        token_complete = false;
                }

                // add the delimiter
                if ( true == add_delimiter )
                {
                        // the next token is the delimiter
                        string delim_token;
                        delim_token.push_back( delimiter );
                        result.push_back( delim_token );

                        // REMOVED: 2006-03-04, Bugfix
                }

                // repeat for the next character
                ++pos;
        } // while

        // add the final token
        if ( false == token.empty() )
        {
                result.push_back( token );
        }
}