#ifndef CPPTOKER_H  // compile if CPPTOKER_H is not in symbol table
#define CPPTOKER_H  // put CPPTOKER_H in symbol table
/////////////////////////////////////////////////////////////////////////
// CppToker.h - declares tokenizer which extracts words from a stream  //
// ver 2.6                                                             //
// Language:    Visual C++, ver 7.0                                    //
// Platform:    Dell Dimension 8300, Windows XP Prof, SP 1             //
// Application: CSE687 prototype for project #1, Spring 2004           //
// Author:      Jim Fawcett, CST 2-187, Syracuse University            //
//              (315) 443-3948, jfawcett@twcny.rr.com                  //
/////////////////////////////////////////////////////////////////////////
/*
 * Module Operations:
 * ==================
 * CppToker.h Declares a class CToker designed to extract words from a 
 * std stream.  Each word is either an identifier or punctuator token.
 * Identifiers consist of alphabetic characters, digits, and underscores.
 * Punctuators are composed of all other non-whitespace characters.  A
 * specific set of punctuation characters are designated as single character
 * tokens.  All other non-identifiers consist of multiple punctuators.
 *
 * CppToker objects cannot be copied or assigned.
 *
 * CToker accepts plug-in filters, using ITokenFilter interface.
 * Four useful sample filters are provided, e.g.:
 * CCppCommentFilter   - discards each C++ comment or returns as token 
 * CCCommentFilter     - discards each C comment or returns as token 
 * CDoubleQuoteFilter  - returns double quoted string as token
 * CSingleQuoteFilter  - returns quoted character as token
 *
 * Public Interface:
 * =================
 * std::fstream in(someFile);
 * CToker toker(&in);                           // create toker
 * CCppCommentFilter commFilt(&toker);          // create a C++ comment filter
 * toker.registerFilter(&commFilt);             // register comment filter
 * toker.registerDefaultFilters();              // registers all defaults
 * std::string tok;
 * while((tok = toker.getTok()) != "")          // read tokens
 *  // do something with tok
 * std::istream *pStr = toker.getStream();      // get pointer to stream
 * std::istream *pOld = toker.setStream(pNew);  // give toker new stream
 * std::string curr = toker.curr();             // get last token extracted
 * std::string prev = toker.prev();             // get one before that
 *
 * bool inThere = isInTokBuffer(endTok);        // is endTok in tokBuffer?
 * std::string tok = getSubStringFromTokBuffer(endTok);
 *                                              // extracts to end of endTok
 */
//
/*
 * Maintenance History:
 * ====================
 * ver 2.6 : 22 Jan 05
 * - minor change to test stub
 * ver 2.5 : 23 Mar 04
 * - Kiran Maiya and YogeshKumar Khatri identified another bug in CppToker.
 *   YogeshKumar even supplied a suggested change which I've used nearly 
 *   as he provided.  The problem has been verified and fixed.  The
 *   CSingleQuoteFilter::extract(...) function got confused by '\\' tokens
 *   A corresponding fix was also made to CDoubleQuoteFilter::extract(...).
 * ver 2.4 : 13 Mar 04
 * - fixed a bug in CSingleQuoteFilter, identified by Siddharth Mulchandani,
 *   that caused quoted characters seperated by commas to return the comma
 *   as part of the quote.  Made the same change to CDoubleQuoteFilter as
 *   the resulting code is simpler.
 * ver 2.3 : 11 Mar 04
 * - fixed bug, identified by Siddharth Mulchandani, with filtering double
 *   quotes containing escape chars.  Failure was caused positioning of 
 *   escapes.  The change affected CToker::getSubStringFromTokBuffer(...) 
 *   and CDoubleQuoteFilter::extract(...).
 * - Pritesh Velankar reported a similar problem with CSingleQuoteFilter.  I
 *   was unable to reproduce the problem, but made the same changes to 
 *   CSingleQuoteFilter::extract(...) as for CDoubleQuoteFilter::extract(...)
 * ver 2.2 : 09 Mar 04
 * - I removed an extraneous CToker:: from the isInTokBuffer(...) declaration
 *   within the CToker class declaration.  That was a cut and past error that
 *   caused no harm in the prototype but confuses the parser when templated.
 * - Also removed four pointers to filters from the CToker class declaration
 *   that were part of a design experiment that was not used.  These pointers
 *   and filter newing and deleting were also removed from the CToker
 *   constructor and destructor.  Thanks to Gang Cheng for pointing out both
 *   the issues corrected with this revision.
 * ver 2.1 : 01 Mar 04
 * - provided two public filter interface functions and moved low level
 *   functions into private region of class declaration
 * ver 2.0 : 29 Feb 04
 * - moved ITokenFilter interface declaration back into this file as it
 *   is no longer used by CSemiExp
 * - added clone function to all filters so registerDefaultFilters 
 *   function can be made to work without statics or memory leaks
 * - gave class default filter pointers so toker will own filter clones,
 *   used in toker constructor and destructor
 * - found and fixed bug in CDoubleQuoteFilter::extract() handling quoted
 *   strings immediately followed by punctuator character(s)
 * - moved some functions clients don't need into private area
 * - made the quotes filters friends so they can access tokBuffer
 * ver 1.2 : 17 Feb 04
 * - modified manual page and maintenance page comments only
 * ver 1.1 : 08 Feb 04
 * - added '~' to set of single character tokens in CTokBuffer constructor
 * ver 1.0 : 12 Jan 04
 * - first release
 *
 * Build Process:
 * ==============
 * Required Files:
 *   CppToker.h, CppToker.cpp
 *
 * Compiler Command:
 *   cl /DTest_CPPTOKER /EHsc CppToker.cpp;
 */
//
/*
 * Design Notes:
 * =============
 * This module illustrates many things we will be covering this semester, some
 * rather unusual things, for purposes of illustration:
 * - The CToker class has a private helper class, CTokBuffer, and declares a
 *   reference to that type.  This prevents CToker class from participating
 *   in copy or assignment operations.  That is consistent with my intended
 *   abstraction for this class.
 * - I wanted to keep CTokBuffer out of the abstraction of the CToker class
 *   For that reason its declaration is in the implementation file,
 *   CppToker.cpp, instead of in this header file.  A consequence of that is
 *   that the CToker class cannot hold an object of CTokBuffer, because its
 *   size is not known to the compiler yet.  It could hold either a pointer
 *   or a reference to its CTokBuffer instance.  I chose to use a reference
 *   so that I could use object semantics rather than pointer semantics in 
 *   the CToker member function implementations.
 */

#include <iostream>
#include <list>
#include <string>
#include <set>

namespace Scanner
{
  ///////////////////////////////////////////////////////////////////////
  // family of token filters

  class CToker;

  class ITokenFilter
  {
  public:
    ITokenFilter(CToker *pToker) : _pToker(pToker) {}
    virtual bool present()=0;
    virtual bool extract(std::string &token)=0;
    virtual ITokenFilter* clone() const=0;
    virtual void keep()=0;
    virtual bool discard() const=0;
  protected:
    CToker *_pToker;
  };

//
  class CCppCommentFilter : public ITokenFilter
  {
  public:
    CCppCommentFilter(CToker *pToker) 
      : ITokenFilter(pToker), _discard(true) {}
    virtual bool present();
    virtual bool extract(std::string &token);
    virtual ITokenFilter* clone() const;
    virtual void keep() { _discard = false; }
    virtual bool discard() const { return _discard; }
  private:
    std::string save;
    bool _discard;
  };

  class CCCommentFilter : public ITokenFilter
  {
  public:
    CCCommentFilter(CToker *pToker) 
      : ITokenFilter(pToker), _discard(true) {}
    virtual bool present();
    virtual bool extract(std::string &token);
    virtual ITokenFilter* clone() const;
    virtual void keep() { _discard = false; }
    virtual bool discard() const { return _discard; }
  private:
    std::string save;
    bool _discard;
  };

  class CDoubleQuoteFilter : public ITokenFilter
  {
  public:
    CDoubleQuoteFilter(CToker *pToker) : ITokenFilter(pToker) {}
    virtual bool present();
    virtual bool extract(std::string &token);
    virtual ITokenFilter* clone() const;
    virtual void keep() {}
    virtual bool discard() const { return false; }
  private:
    std::string save;
  };

  class CSingleQuoteFilter : public ITokenFilter
  {
  public:
    CSingleQuoteFilter(CToker *pToker) : ITokenFilter(pToker) {}
    virtual bool present();
    virtual bool extract(std::string &token);
    virtual ITokenFilter* clone() const;
    virtual void keep() {}
    virtual bool discard() const { return false; }
  private:
    std::string save;
  };

//
  ///////////////////////////////////////////////////////////////////////
  // class CToker - extracts tokens from a std stream
  //              - uses private CTokBuffer class to help build
  //                tokens from stream

  class CToker
  {
  public:
    CToker(std::basic_istream<char> *pStream=0);
    ~CToker(void);
    std::string getTok();
    std::istream* setStream(std::basic_istream<char> *pStream);
    std::istream* getStream();
    std::string prev();
    std::string curr();
    void registerFilter(ITokenFilter* pFilter);
    void registerDefaultFilters(bool Keep=false);

    // filter interface
    size_t tokBufferSize();
    bool isInTokBuffer(const std::string& token);
    std::string getSubStringFromTokBuffer(const std::string& end);
    int getNextCharFromTokBuffer();

  private:
    std::string getTokBuffer();
    void putTokBuffer(const std::string& str);
    void clearBuffer();
    bool filter(std::string &token);
    std::istream *_pStream;
   // forward reference to helper class
    class CTokBuffer;
   // reference type below will prevent CToker objects
   // being copied or assigned
    CTokBuffer &tokBuffer;
    std::string prevTok;
    std::string currTok;
    std::list<ITokenFilter*> _filters;
  };
}
#endif