/////////////////////////////////////////////////////////////////////////
// CppToker.cpp - defines tokenizer which extracts words from a stream //
// ver 2.6                                                             //
// Language:    Visual C++, ver 7.0                                    //
// Platform:    Dell Dimension 8300, Windows XP Prof, SP 1             //
// Application: CSE687 prototype for project #1, Spring 2004           //
// Author:      Jim Fawcett, CST 2-187, Syracuse University            //
//              (315) 443-3948, jfawcett@twcny.rr.com                  //
/////////////////////////////////////////////////////////////////////////

#include "cpptoker.h"
#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <ctype.h>

using namespace Scanner;

/////////////////////////////////////////////////////////////////////////
// private helper class CTokBuffer - manages token string processing
//
//   Its main job is to distinguish between identifiers and punctuator
//   sequences.

class CToker::CTokBuffer : public std::string
  {
  public:

    // default copy constructor and assignment operator
    // work correctly, so are not provided here

    CTokBuffer();
    CTokBuffer(const std::string &s);
    char operator[](unsigned int n);
    bool isIdentifierChar(int ch);
    std::string getIdentifier();
    bool isPunctuatorChar(int ch);
    std::string getPunctuator();
  private:
    std::set<char> singleCharToken;
  };

//
/////////////////////////////////////////////////////////////////////////
// CTokBuffer members

//----< construct CTokBuffer instance >----------------------------------
//
// Note: making '/', '\\', or '*' singles will affect processing in
//       many of the classes derived from ITokenFilter.
//
CToker::CTokBuffer::CTokBuffer()
{
  const int Max = 15;
  char single[] = {
    '\n', '(', ')', '{', '}', '[', ']', '<', '>', 
    '#', ';', ',', '\"', '\'', '~'
  };

  // save single character tokens in an STL set

  for(int i=0; i<Max; i++)
    singleCharToken.insert(single[i]);
}
//----< promotion constructor >------------------------------------------

CToker::CTokBuffer::CTokBuffer(const std::string &s) : std::string(s)
{
  const int Max = 15;
  char single[] = {
    '\n', '(', ')', '{', '}', '[', ']', '<', '>', 
    '#', ';', ',', '\"', '\'', '~'
  };

  // save single character tokens in an STL set

  for(int i=0; i<Max; i++)
    singleCharToken.insert(single[i]);
}
//----< index operator >-------------------------------------------------

char CToker::CTokBuffer::operator [](unsigned int n)
{
  if(n<0 || length()<=n)
    throw std::exception("index out of domain in CTokBuffer::operator[]()");
  return ((std::string)(*this))[n]; 
}
//----< is the nth character of this buffer an identifier? >-------------

bool CToker::CTokBuffer::isIdentifierChar(int ch)
{
  return (::isalnum(ch) || ::isdigit(ch) || '_'==ch);
}
//
//----< extract a consecutive sequence of identifier chars >-------------

std::string CToker::CTokBuffer::getIdentifier()
{
  std::string::iterator iter = begin();
  std::string tok;
  while(iter != end() && isIdentifierChar(*iter))
  {
    tok.push_back(*iter);
    iter++;
  }
  erase(begin(),iter);
  return tok;
}
//----< is the nth character of this buffer a punctuator? >--------------

bool CToker::CTokBuffer::isPunctuatorChar(int ch)
{
  return (bool)(!isIdentifierChar(ch));
}
//----< extract a consecutive sequence of punctuator chars >-------------
//
// certain punctuators, identified in CTokBuffer constructor,
// are extracted as single character tokens
//
std::string CToker::CTokBuffer::getPunctuator()
{
  // look for leading single char token

  std::string::iterator iter = begin();
  char ch = *iter;
  if(singleCharToken.find(ch) != singleCharToken.end())
  {
    std::string debug = *this;
    std::string token;
    token.push_back(ch);
    erase(iter);
    return token;
  }
  // start collecting consecutive punctuator characters

  size_type n = 0, len = length();
  std::string tok;
  while(iter != end() && isPunctuatorChar(*iter))
  {
    tok.push_back(*iter);
    iter++;
  
    // if next character is a single char token then quit

    if(iter != end())
    {
      if(singleCharToken.find(*iter) != singleCharToken.end())
        break;
    }
  }
  erase(begin(),iter);  // remove token from front of buffer
  return tok;           // return extracted token
}
//
/////////////////////////////////////////////////////////////////////////
// Filter Definitions

//----< is this a C language comment? >----------------------------------

bool CCCommentFilter::present()
{
  std::string temp = _pToker->curr();
  if(0 == temp.find("/*"))
  {
    save = temp;
    return true;
  }
  return false;
}

//----< return C language comment as token >---------------------------

bool CCCommentFilter::extract(std::string &token)
{
  save += _pToker->getSubStringFromTokBuffer("*/");
  if(save.find("*/") < save.size())
  {
    if(!_discard)
      token = save;
    return true;
  }

  // ok, pull rest out of stream

  std::string test;
  std::istream* in = _pToker->getStream();
  char prev = 0, curr = 0;
  while(true)
  {
    prev = curr;
    curr = in->get();
    if(!in->good())
      break;
    save += curr;
    if(prev == '\\' && curr == '*')
      curr = 0;
    if(prev == '*' && curr == '/')
      break;
  }
  if(!_discard)
    token = save;
  return true;
}
//----< return a pointer to clone >--------------------------------------

ITokenFilter* CCCommentFilter::clone() const
{
  CCCommentFilter* pFilt = new CCCommentFilter(_pToker);
  if(!discard())
    pFilt->keep();
  return pFilt;
}
//
//----< is this a C++ language comment? >--------------------------------

bool CCppCommentFilter::present()
{
  std::string temp = _pToker->curr();
  if(0 == temp.find("//"))
  {
    save = temp;
    return true;
  }
  return false;
}
//----< return C++ language comment as a single token >------------------

bool CCppCommentFilter::extract(std::string &token)
{
  save += _pToker->getSubStringFromTokBuffer("\n");
  if(save.find("\n") < save.size())
  {
    if(!_discard)
      token = save;
    return true;
  }

  // ok, pull rest out of stream

  std::istream* in = _pToker->getStream();
  char curr = 0;
  while(true)
  {
    curr = in->peek();
    if(curr == '\n')
      break;
    curr = in->get();
    if(!in->good())
      break;
    save += curr;
  }
  if(!_discard)
    token = save;
  return true;
}
//----< return a pointer to clone >--------------------------------------

ITokenFilter* CCppCommentFilter::clone() const
{
  CCppCommentFilter* pFilt = new CCppCommentFilter(_pToker);
  if(!discard())
    pFilt->keep();
  return pFilt;
}
//
//----< is this a quoted string? >---------------------------------------

bool CDoubleQuoteFilter::present()
{
  std::string temp = _pToker->curr();
  if(temp.length() > 0 && temp[0] == '\"')
  {
    save = temp;
    return true;
  }
  return false;
}
//----< collect quoted string as a token >-------------------------------

bool CDoubleQuoteFilter::extract(std::string &token)
{
  while(_pToker->tokBufferSize() > 0)
  {
    save += static_cast<char>(_pToker->getNextCharFromTokBuffer());
    size_t end = save.size() - 1;
    if(end > 0 && save[end-1] != '\\' && save[end] == '\"')
    {
      token = save;
      return true;
    }
    if(end > 2 && save[end-2] == '\\'
               && save[end-1] == '\\' && save[end] == '\"')
    {
      token = save;
      return true;
    }
  }

  // ok, pull rest out of stream

  std::istream* in = _pToker->getStream();
  char prev, curr = '\"';
  while(in->good())
  {
    prev = curr;
    curr = in->get();
    if(in->good())
      save += curr;
    else
      break;
    if(prev != '\\' && curr == '\"')
    {
      token = save;
      return true;
    }
  }
  token = save;
  return true;
}
//----< return a pointer to clone >--------------------------------------

ITokenFilter* CDoubleQuoteFilter::clone() const
{
  return new CDoubleQuoteFilter(_pToker);
}
//
//----< is this a quoted character? >------------------------------------

bool CSingleQuoteFilter::present()
{
  std::string temp = _pToker->curr();
  if(temp.length() > 0 && temp[0] == '\'')
  {
    save = temp;
    return true;
  }
  return false;
}
//----< collect quoted character as a token >----------------------------

bool CSingleQuoteFilter::extract(std::string &token)
{
  while(_pToker->tokBufferSize() > 0)
  {
    save += static_cast<char>(_pToker->getNextCharFromTokBuffer());
    size_t end = save.size() - 1;
    if(end > 0 && save[end-1] != '\\' && save[end] == '\'')
    {
      token = save;
      return true;
    }
    if(end > 2 && save[end-2] == '\\' 
               && save[end-1] == '\\' && save[end] == '\'')
    {
      token = save;
      return true;
    }
  }

  // ok, pull rest out of stream

  std::istream* in = _pToker->getStream();
  char prev, curr = '\'';
  while(in->good())
  {
    prev = curr;
    curr = in->get();
    if(in->good())
      save += curr;
    else
      break;
    if(prev != '\\' && curr == '\'')
    {
      token = save;
      return true;
    }
  }
  token = save;
  return true;
}
//----< return a pointer to clone >--------------------------------------

ITokenFilter* CSingleQuoteFilter::clone() const
{
  return new CSingleQuoteFilter(_pToker);
}
//
/////////////////////////////////////////////////////////////////////////
// CToker Definitions

//----< construct CToker instance >--------------------------------------

CToker::CToker(std::istream *pStream) 
      : _pStream(pStream), prevTok(""), currTok(""), 
        tokBuffer(*new CToker::CTokBuffer())
{}
//----< destructor >-----------------------------------------------------

CToker::~CToker(void)
{
  // CToker uses stream but does not create it, so don't delete it

  delete &tokBuffer;

  // remove filters held by this instance

  std::list<ITokenFilter*>::iterator iter;
  for(iter=_filters.begin(); iter != _filters.end(); iter++)
    delete *iter;
  _filters.clear();
}
//----< replace stream - returns old stream >----------------------------
//
//  user's responsibility to deallocate the old stream eventually
//
std::istream* CToker::setStream(std::istream *pStream)
{
  std::istream *temp = _pStream;
  _pStream = pStream;
  return temp;
}
//----< provides access to stream >--------------------------------------

std::istream* CToker::getStream()
{
  return _pStream;
}
//
/////////////////////////////////////////////////////////////////////////
// Filter interface

//----< is token in tokBuffer? >-----------------------------------------

bool CToker::isInTokBuffer(const std::string& token)
{
  size_t pos = tokBuffer.find(token);
  return (pos < tokBuffer.size());
}
//----< return length of tokBuffer string >------------------------------

size_t CToker::tokBufferSize()
{
  return tokBuffer.size();
}
//----< extract next character from tokBuffer >--------------------------

int CToker::getNextCharFromTokBuffer()
{
  if(tokBuffer.size() == 0)
    return EOF;
  CTokBuffer::iterator iter = tokBuffer.begin();
  int temp = *iter;
  tokBuffer.erase(iter);
  return temp;
}
//----< get string specified by end substring from tokBuffer >-----------
//
//  removes some or all of tokBuffer contents
//
std::string CToker::getSubStringFromTokBuffer(const std::string& end)
{
  std::string temp;
  size_t pos = tokBuffer.find(end,1);
  size_t delta = end.size();
  if(pos < tokBuffer.size() && 
     pos > 0 && 
     tokBuffer[(unsigned int)pos-1] != '\\')
  {
    for(size_t i = 0; i<pos+delta; ++i)
      temp.push_back(tokBuffer[(unsigned int)i]);
    std::string::iterator iter = tokBuffer.begin();
    tokBuffer.erase(iter,iter+pos+delta);
  }
  else
  {
    temp = tokBuffer;
    tokBuffer.clear();
  }
  return temp;
}
//
//----< return contents of tokBuffer >-----------------------------------
//
// empties tokBuffer
//
std::string CToker::getTokBuffer()
{
  std::string temp = tokBuffer;
  tokBuffer.clear();
  return temp;
}
//----< push string into tokBuffer >-------------------------------------
//
// assigns new string to tokBuffer, overwritting prev contents
//
void CToker::putTokBuffer(const std::string& str)
{
  tokBuffer = str;
}
//
//----< return previously extracted token >------------------------------

std::string CToker::prev()
{
  return prevTok;
}
//----< return last extracted token >------------------------------------

std::string CToker::curr()
{
  return currTok;
}
//----< register type derived from ITokenFilter >------------------------
//
// register creates clone of user's filter and so
// does not take ownership of user's filter
//
void CToker::registerFilter(ITokenFilter *pITokenFilter)
{
  _filters.push_back(pITokenFilter->clone());
}
//----< register default filters >---------------------------------------

void CToker::registerDefaultFilters(bool Keep)
{
  std::list<ITokenFilter*>::iterator iter;
  for(iter=_filters.begin(); iter != _filters.end(); iter++)
    delete *iter;
  _filters.clear();

  CCppCommentFilter CppCommentFilter(this);
  if(Keep)
    CppCommentFilter.keep();
  registerFilter(&CppCommentFilter);
  CCCommentFilter CCommentFilter(this);
  if(Keep)
    CCommentFilter.keep();
  registerFilter(&CCommentFilter);
  CDoubleQuoteFilter DoubleQuoteFilter(this);
  registerFilter(&DoubleQuoteFilter);
  CSingleQuoteFilter SingleQuoteFilter(this);
  registerFilter(&SingleQuoteFilter);
}
//
//----< remove or transform groups of tokens >---------------------------

bool CToker::filter(std::string &token)
{
//  std::cout << "-- in filter token = " << token << std::endl;
  std::string dummy;
  std::list<ITokenFilter*>::iterator iter;

  for(iter=_filters.begin(); iter!=_filters.end(); iter++)
  {
    if((*iter)->present())
    {
      dummy = token;
      if((*iter)->extract(dummy))
      {
        if(((*iter)->discard()) == true)
          token = getTok();
        else
          token = dummy;
        return true;
      }
      else
      {
        token = getTok();
        return false;   // don't return token
      }
    }
  }
  return true;
}
//----< clear tokBuffer contents >---------------------------------------

void CToker::clearBuffer() { tokBuffer.clear(); }
//
//----< extracts next token from stream >--------------------------------

std::string CToker::getTok()
{
  prevTok = currTok;
  if(!_pStream)   // no stream
    return "";
  if(tokBuffer.length() == 0)
  {
    while(_pStream->good())
    {
      int ch = _pStream->peek();
      if(ch == EOF)
        return "";
      if(ch == '\n')
      {
        _pStream->get();
        currTok = "\n";
        return currTok;
      }
      if(!::isspace(ch))
        break;
      _pStream->get(); // throw away white space that isn't newline
    }

    while(_pStream->good())
    {
      int ch = _pStream->peek();
      if(ch == EOF)
        break;
      if(::isspace(ch))
        break;
      ch = _pStream->get();
      if(_pStream->good())
        tokBuffer.push_back(ch);
    }
/////////////////////////////////////////////////////////////////////////
//  The preceding two while loops could be replaced with the following 
//  line if we didn't need to collect newlines as tokens
//
//    *_pStream >> tokBuffer;
  
    if(tokBuffer.length() == 0) // end of file
      return "";
  }
  if(tokBuffer.isPunctuatorChar(tokBuffer[0]))
  {
    currTok = tokBuffer.getPunctuator();
    filter(currTok);
    return currTok;
  }
  if(tokBuffer.isIdentifierChar(tokBuffer[0]))
    return (currTok = tokBuffer.getIdentifier());
  std::string err = "parse error: " + tokBuffer + " in CToker::getTok()";
  throw(std::exception(err.c_str()));
}
//
/////////////////////////////////////////////////////////////////////////
// Test Stub

#ifdef TEST_CPPTOKER  // compile only if TEST_CPPTOKER is in symbol table

int main(int argc, char *argv[])
{
  std::ifstream in;
  if(argc < 2)
  {
    std::cout << "\n  please enter name of file to process on command line\n";
    return 1;
  }
  in.open(argv[1]);
  if(in.good())
  {
    std::cout << "\n  processing file " << argv[1];
    std::cout << "\n -----------------" 
              << std::string(strlen(argv[1])+1,'-') << '\n';
    try
    {
      // tokenizing file stream

      CToker toker(&in);
   /////////////////////////////////////////////////////////////////
   // use the following statement to register the default filters,
   // e.g., C comments, C++ comments, double quotes, single quotes 
  
      toker.registerDefaultFilters();

      std::string tok;
      while((tok = toker.getTok()) != "")
      {
        if(tok == "\n")
          std::cout << "newline" << std::endl;
        else
          std::cout << tok << std::endl;
      }
      in.close();

      // tokenizing string stream

      std::istringstream sin(
        "I regret that I have but one life to give for my course"
      );
      toker.setStream(&sin);
      std::cout << std::endl << std::endl;
      while((tok = toker.getTok()) != "")
        if(tok == "\n")
          std::cout << "newline" << std::endl;
        else
          std::cout << tok << std::endl;
      in.close();
    }
    catch(std::exception e)
    {
      std::cout << "\n\n  " << e.what() << "\n\n";
    }
  }
  else
  {
    std::cout << "\n  Can't open file " << argv[1] << "\n\n";
  }
}

#endif