/////////////////////////////////////////////////////////////////////////
// CppToker.cpp - defines tokenizer which extracts words from a stream //
// ver 1.1                                                             //
// Language:    Visual C++, ver 7.0                                    //
// Platform:    Dell Dimension 8300, Windows XP Prof, SP 1             //
// Application: CSE687 prototype for project #1, Spring 2004           //
// Author:      Jim Fawcett, CST 2-187, Syracuse University            //
//              (315) 443-3948, jfawcett@twcny.rr.com                  //
/////////////////////////////////////////////////////////////////////////

#include "cpptoker.h"
#include <string>
#include <fstream>
#include <sstream>
#include <ctype.h>

using namespace Scanner;

/////////////////////////////////////////////////////////////////////////
// private helper class CTokBuffer - manages token string processing
//
//   Its main job is to distinguish between identifiers and punctuator
//   sequences.

class CToker::CTokBuffer : public std::string
  {
  public:

    // default copy constructor and assignment operator
    // work correctly, so are not provided here

    CTokBuffer();
    CTokBuffer(const std::string &s);
    char operator[](unsigned int n);
    bool isIdentifierChar(int ch);
    std::string getIdentifier();
    bool isPunctuatorChar(int ch);
    std::string getPunctuator();
  private:
    std::set<char> singleCharToken;
  };

/////////////////////////////////////////////////////////////////////////
// CTokBuffer members

//----< construct CTokBuffer instance >----------------------------------
//
// Note: making '/', '\\', or '*' singles will affect processing in
//       many of the classes derived from ITokenFilter.
//
CToker::CTokBuffer::CTokBuffer()
{
  const int Max = 15;
  char single[] = {
    '\n', '(', ')', '{', '}', '[', ']', '<', '>', 
    '#', ';', ',', '\"', '\'', '~'
  };

  // save single character tokens in an STL set

  for(int i=0; i<Max; i++)
    singleCharToken.insert(single[i]);
}
//
//----< promotion constructor >------------------------------------------

CToker::CTokBuffer::CTokBuffer(const std::string &s) : std::string(s)
{
  const int Max = 15;
  char single[] = {
    '\n', '(', ')', '{', '}', '[', ']', '<', '>', 
    '#', ';', ',', '\"', '\'', '~'
  };

  // save single character tokens in an STL set

  for(int i=0; i<Max; i++)
    singleCharToken.insert(single[i]);
}
//----< index operator >-------------------------------------------------

char CToker::CTokBuffer::operator [](unsigned int n)
{
  if(n<0 || length()<=n)
    throw std::exception("index out of domain in CTokBuffer::operator[]()");
  return ((std::string)(*this))[n]; 
}
//----< is the nth character of this buffer an identifier? >-------------

bool CToker::CTokBuffer::isIdentifierChar(int ch)
{
  return (::isalnum(ch) || ::isdigit(ch) || '_'==ch);
}
//----< extract a consecutive sequence of identifier chars >-------------

std::string CToker::CTokBuffer::getIdentifier()
{
  std::string::iterator iter = begin();
  std::string tok;
  while(iter != end() && isIdentifierChar(*iter))
  {
    tok.push_back(*iter);
    iter++;
  }
  erase(begin(),iter);
  return tok;
}
//----< is the nth character of this buffer a punctuator? >--------------

bool CToker::CTokBuffer::isPunctuatorChar(int ch)
{
  return (bool)(!isIdentifierChar(ch));
}
//
//----< extract a consecutive sequence of punctuator chars >-------------
//
// certain punctuators, identified in CTokBuffer constructor,
// are extracted as single character tokens
//
std::string CToker::CTokBuffer::getPunctuator()
{
  // look for leading single char token

  std::string::iterator iter = begin();
  char ch = *iter;
  if(singleCharToken.find(ch) != singleCharToken.end())
  {
    std::string token;
    token.push_back(ch);
    erase(iter);
    return token;
  }
  // start collecting consecutive punctuator characters

  size_type n = 0, len = length();
  std::string tok;
  while(iter != end() && isPunctuatorChar(*iter))
  {
    tok.push_back(*iter);
    iter++;
  
    // if next character is a single char token then quit

    if(iter != end())
    {
      if(singleCharToken.find(*iter) != singleCharToken.end())
        break;
    }
  }
  erase(begin(),iter);  // remove token from front of buffer
  return tok;           // return extracted token
}

//
//----< construct CToker instance >--------------------------------------

CToker::CToker(std::istream *pStream) 
      : _pStream(pStream), prevTok(""), currTok(""), 
        tokBuffer(*new CToker::CTokBuffer())
{
  // Note: Reference members MUST be created in initializer sequences
  //       as done with tokBuffer, above.
}
//----< destructor >-----------------------------------------------------

CToker::~CToker(void)
{
  // CToker uses stream but does not create it, so don't delete it

  delete &tokBuffer;
}
//----< replace stream - returns old stream >----------------------------
//
//  user's responsibility to deallocate the old stream eventually
//
std::istream* CToker::setStream(std::istream *pStream)
{
  std::istream *temp = _pStream;
  _pStream = pStream;
  return temp;
}
//----< provides access to stream >--------------------------------------

std::istream* CToker::getStream()
{
  return _pStream;
}
//----< return previously extracted token >------------------------------

std::string CToker::prev()
{
  return prevTok;
}
//----< return last extracted token >------------------------------------

std::string CToker::curr()
{
  return currTok;
}
//
//----< extracts next token from stream >--------------------------------

std::string CToker::getTok()
{
  prevTok = currTok;
  if(!_pStream)   // no stream
    return "";
  if(tokBuffer.length() == 0)
  {
    while(_pStream->good())
    {
      int ch = _pStream->peek();
      if(ch == EOF)
      {
        return "";
      }
      if(!::isspace(ch) || ch == '\n')
        break;
      _pStream->get(); // throw away white space that isn't newline
    }

    while(_pStream->good())
    {
      int ch = _pStream->peek();
      if(ch == EOF)
      {
        break;
      }
      if(::isspace(ch) && ch != '\n')
        break;
      ch = _pStream->get();
      if(_pStream->good())
        tokBuffer.push_back(ch);
    }
/////////////////////////////////////////////////////////////////////////
//  The preceding two while loops could be replaced with the following 
//  line if we didn't need to collect newlines as tokens
//
//    *_pStream >> tokBuffer;
  
    if(tokBuffer.length() == 0) // end of file
      return "";
  }
  if(tokBuffer.isPunctuatorChar(tokBuffer[0]))
    return (currTok = tokBuffer.getPunctuator());
  if(tokBuffer.isIdentifierChar(tokBuffer[0]))
    return (currTok = tokBuffer.getIdentifier());
  std::string err = "parse error: " + tokBuffer + " in CToker::getTok()";
  throw(std::exception(err.c_str()));
}

//
/////////////////////////////////////////////////////////////////////////
// Test Stub

#ifdef TEST_CPPTOKER  // compile only if TEST_CPPTOKER is in symbol table

int main()
{
  std::string file = "CppToker.cpp";

  std::ifstream in;
  in.open(file.c_str());
  if(in.good())
  {
    try
    {
      // tokenizing file stream

      CToker toker(&in);
      std::string tok;
      while((tok = toker.getTok()) != "")
      {
        if(tok == "\n")
          std::cout << "newline" << std::endl;
        else
          std::cout << tok << std::endl;
      }
      in.close();

      // tokenizing string stream

      std::istringstream sin(
        "I regret that I have but one life to give for my course"
      );
      toker.setStream(&sin);
      std::cout << std::endl << std::endl;
      while((tok = toker.getTok()) != "")
        if(tok == "\n")
          std::cout << "newline" << std::endl;
        else
          std::cout << tok << std::endl;
      in.close();
    }
    catch(std::exception e)
    {
      std::cout << "\n\n  " << e.what() << "\n\n";
    }
  }
  else
  {
    std::cout << "\n  Can't open file " << file << "\n\n";
  }
}

#endif