uhferret-gem/ext/tokenset.h

#if !defined tokenset_h
#define tokenset_h

/** 
 * This file is part of uhferret.
 *
 * Author::    Peter Lane
 * Copyright:: Copyright 2011, Peter Lane.
 * License::   GPLv3
 *
 * uhferret is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * uhferret is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with uhferret.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <map>
#include <string>
#include <vector>

/** A Token is a sequence of characters read in by a TokenReader
  * -- this class provides a dynamic storage for the token supporting
  *    addition of characters
  * -- when finished, the token can be queried for its length and made into a string
*/
class Token
{
	public:
		Token ();
		~Token ();
		void Erase ();
		void AddChar (char c);
    std::string GetString () const;
		int GetLength () const;
	private:
		void Grow ();
		char * _token; // storage for the token
		int	_capacity; // size of the stored token
		int 	_top;	// pointer to end of token
};

/** A TokenSet maps strings to token indices 
  * -- this is for memory efficiency, ensuring every token's string is 
  *    stored once within the application 
  */
class TokenSet
{
	public:
		TokenSet ();
		std::size_t GetIndexFor (std::string token);
		std::string GetStringFor (std::size_t token);
		void Clear ();
		void SetNextIndex (int index);
		void SetIndexString (std::string token, int index);
	private:
		std::map<std::string, std::size_t> _tokens;
		std::map<std::string, std::size_t>::const_iterator _tokens_it;
		std::size_t _nextindex; // next free index for new string
		std::map<std::size_t, std::string> _strings;
		std::map<std::size_t, std::string>::const_iterator _strings_it;
};

#endif