/////////////////////////////////////////////////////////////////////
// Tokenizer.cpp - read words from a std::stream                   //
// ver 3.8                                                         //
// Language:    C++, Visual Studio 2015                            //
// Application: Parser component, CSE687 - Object Oriented Design  //
// Author:      Jim Fawcett, Syracuse University, CST 4-187        //
//              jfawcett@twcny.rr.com                              //
/////////////////////////////////////////////////////////////////////

#include <iostream>
#include <cctype>
#include <string>
#include <vector>
#include "Tokenizer.h"
#include "..\Utilities\Utilities.h"

namespace Scanner
{
  ///////////////////////////////////////////////////////////////////
  // private ConsumeState worker class
  /*
   *   - At exit the eatChars() member should return with
   *     currChar set to the first non-state character.
   *   - Each derived class must insure that this condition
   *     is meet.
   */
  class ConsumeState
  {
    friend class Toker;
  public:
    using Token = std::string;
    ConsumeState();
    ConsumeState(const ConsumeState&) = delete;
    ConsumeState& operator=(const ConsumeState&) = delete;
    virtual ~ConsumeState();
    void attach(std::istream* pIn) { _pIn = pIn; }
    virtual void eatChars() = 0;
    void consumeChars() {
      _pState->eatChars();
      _pState = nextState();
    }
    bool canRead() { return _pIn->good(); }
    std::string getTok() { return token; }
    bool hasTok() { return token.size() > 0; }
    ConsumeState* nextState();
    void returnComments(bool doReturnComments = false);
    size_t currentLineCount();
    void setSpecialTokens(const std::string& commaSeparatedString);
    static void reset();
    static bool isActive();
  protected:
    bool collectChar();
    bool isOneCharToken(Token tok);
    bool isTwoCharToken(Token tok);
    Token makeString(int ch);
    static std::string token;
    static std::istream* _pIn;
    static std::vector<std::string> _oneCharTokens;
    static std::vector<std::string> _twoCharTokens;
    static int prevChar;
    static int currChar;
    static bool _doReturnComments;
    static size_t _lineCount;
    static bool first_ctor;
    static bool first_dtor;
    static ConsumeState* _pState;
    static ConsumeState* _pEatCppComment;
    static ConsumeState* _pEatCComment;
    static ConsumeState* _pEatWhitespace;
    static ConsumeState* _pEatPunctuator;
    static ConsumeState* _pEatAlphanum;
    static ConsumeState* _pEatSpecialCharacters;
    static ConsumeState* _pEatDQString;
    static ConsumeState* _pEatSQString;
  };
}

using namespace Scanner;
using Token = std::string;

// static data members must be initialized outside class declaration
// even if their values are not specified, i.e., token, etc.

std::string ConsumeState::token;
std::istream* ConsumeState::_pIn = nullptr;
int ConsumeState::prevChar;
int ConsumeState::currChar;
bool ConsumeState::_doReturnComments = false;
size_t ConsumeState::_lineCount = 1;
bool ConsumeState::first_ctor = true;
bool ConsumeState::first_dtor = true;
ConsumeState* ConsumeState::_pState = nullptr;
ConsumeState* ConsumeState::_pEatCppComment = nullptr;
ConsumeState* ConsumeState::_pEatCComment = nullptr;
ConsumeState* ConsumeState::_pEatWhitespace = nullptr;
ConsumeState* ConsumeState::_pEatPunctuator = nullptr;
ConsumeState* ConsumeState::_pEatAlphanum = nullptr;
ConsumeState* ConsumeState::_pEatSpecialCharacters = nullptr;
ConsumeState* ConsumeState::_pEatDQString = nullptr;
ConsumeState* ConsumeState::_pEatSQString = nullptr;
std::vector<std::string> ConsumeState::_oneCharTokens =
{
  "\n", "<", ">", "{", "}", "[", "]", "(", ")", ":", "=", "+", "-", "*", "."
};
std::vector<std::string> ConsumeState::_twoCharTokens =
{
  "<<", ">>", "::", "++", "--", "==", "+=", "-=", "*=", "/="
};

//----< called by Toker constructor >--------------------------------

void ConsumeState::reset()
{
  first_ctor = first_dtor = true;
}
//----< is a tokenizer running? >------------------------------------

bool ConsumeState::isActive()
{
  return (first_ctor == false);
}
//----< replace one and two char tokens >----------------------------

void ConsumeState::setSpecialTokens(const std::string& commaSeparatedTokens)
{
  std::vector<Token> result = Utilities::StringHelper::split(commaSeparatedTokens);
  _oneCharTokens.clear();
  _twoCharTokens.clear();
  for (std::string item : result)
  {
    if (item.size() == 1 || item == "\n")
      _oneCharTokens.push_back(item);
    if (item.size() >= 2)
      _twoCharTokens.push_back(item);
  }
}
//----< return number of newlines collected from stream >------------

size_t ConsumeState::currentLineCount() { return _lineCount; }

//----< collect character from stream >------------------------------

bool ConsumeState::collectChar()
{
  if (_pIn->good())
  {
    prevChar = currChar;
    currChar = _pIn->get();
    if (currChar == '\n')
      ++_lineCount;
    return true;
  }
  return false;
}

//----< logs to console if TEST_LOG is defined >---------------------

void testLog(const std::string& msg);  // forward declaration

//----< turn on\off returning comments as tokens >-------------------

void ConsumeState::returnComments(bool doReturnComments)
{
  _doReturnComments = doReturnComments;
}
//----< is tok one of the special one character tokens? >------------

bool ConsumeState::isOneCharToken(Token tok)
{
  for (size_t i = 0; i < _oneCharTokens.size(); ++i)
    if (_oneCharTokens[i] == tok)
      return true;
  return false;
}
//----< is tok one of the special two character tokens? >------------

bool ConsumeState::isTwoCharToken(Token tok)
{
  for (size_t i = 0; i < _twoCharTokens.size(); ++i)
    if (_twoCharTokens[i] == tok)
      return true;
  return false;
}
//----< make a string with this one integer >------------------------

ConsumeState::Token ConsumeState::makeString(int ch)
{
  Token temp;
  return temp += ch;
}
//----< decide which state to use next >-----------------------------

ConsumeState* ConsumeState::nextState()
{
  if (!(_pIn->good()))
  {
    return nullptr;
  }
  int chNext = _pIn->peek();
  if (chNext == EOF)
  {
    _pIn->clear();
    // if peek() reads end of file character, EOF, then eofbit is set and
    // _pIn->good() will return false.  clear() restores state to good
  }
  if (isOneCharToken(makeString(currChar)))
  {
    return _pEatSpecialCharacters;
  }
  if (std::isspace(currChar) && currChar != '\n')
  {
    testLog("state: eatWhitespace");
    return _pEatWhitespace;
  }
  if (currChar == '/' && chNext == '/')
  {
    testLog("state: eatCppComment");
    return _pEatCppComment;
  }
  if (currChar == '/' && chNext == '*')
  {
    testLog("state: eatCComment");
    return _pEatCComment;
  }
  if (std::isalnum(currChar) || currChar == '_')
  {
    testLog("state: eatAlphanum");
    return _pEatAlphanum;
  }
  if (currChar == '\"' && prevChar != '\\')
  {
    return _pEatDQString;
  }
  if (currChar == '\'' && prevChar != '\\')
  {
    return _pEatSQString;
  }
  if (ispunct(currChar))
  {
    testLog("state: eatPunctuator");
    return _pEatPunctuator;
  }
  if (!_pIn->good())
  {
    return _pEatWhitespace;
  }
  std::string error = "invalid type, currChar = " + Utilities::Converter<char>::toString(currChar);
  throw(std::logic_error(error.c_str()));
}
//----< class that consumes whitespace >-----------------------------

class EatWhitespace : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do {
      if (!collectChar())
        return;
    } while (std::isspace(currChar) && currChar != '\n');
  }
};
//----< class that consumes C++ comment >----------------------------

class EatCppComment : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do {
      if (_doReturnComments)
        token += currChar;
      if (!collectChar())
        return;
    } while (currChar != '\n');
  }
};
//----< class that consumes C comment >------------------------------

class EatCComment : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do {
      if (_doReturnComments)
        token += currChar;
      if (!collectChar())
        return;
    } while (currChar != '*' || _pIn->peek() != '/');
    if (_doReturnComments)
      token += currChar;
    if (!collectChar())      // get terminating '/'
      return;
    if (_doReturnComments)
      token += currChar;
    collectChar();           // get first non-comment char
  }
};
//----< class that eats punctuators >--------------------------------

class EatPunctuator : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do {
      if (currChar == '\"' && prevChar != '\\')    // start of double quoted string
      {
        return;
      }
      if (currChar == '\'' && prevChar != '\\')    // start of single quoted string
      {
        return;
      }
      Token temp = makeString(currChar);
      if (isOneCharToken(temp))
      {
        return;
      }
      token += currChar;
      if (!_pIn->good())  // end of stream
      {
        return;
      }
      if (!collectChar())
        return;
    } while (ispunct(currChar));
  }
};
//----< class that eats alphanumeric characters >--------------------
/*
 * treats underscore '_' as alphanumeric character
 */
class EatAlphanum : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do {
      token += currChar;
      if (!collectChar())
        return;
    } while (isalnum(currChar) || currChar == '_');
  }
};
//----< class that eats one and two character tokens >---------------

class EatSpecialCharacters : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    token += currChar;
    Token temp = makeString(currChar) += _pIn->peek();
    if (isTwoCharToken(temp))
    {
      collectChar();
      token += currChar;
    }
    collectChar();
  }
};
//----< class that eats double quoted string >-----------------------

class EatDQString : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do
    {
      token += currChar;
      if (!collectChar())
        return;
    } while (currChar != '\"' || prevChar == '\\');
    token += currChar;
    collectChar();
  }
};
//----< class that eats single quoted string >-----------------------

class EatSQString : public ConsumeState
{
public:
  virtual void eatChars()
  {
    token.clear();
    do
    {
      token += currChar;
      if (!collectChar())
        return;
    } while (currChar != '\'' || prevChar == '\\');
    token += currChar;
    collectChar();
  }
};
//----< constructor creates derived state instances only once >------

ConsumeState::ConsumeState()
{
  if (first_ctor)
  {
    first_ctor = false;
    _pEatAlphanum = new EatAlphanum();
    _pEatCComment = new EatCComment();
    _pEatCppComment = new EatCppComment();
    _pEatPunctuator = new EatPunctuator();
    _pEatWhitespace = new EatWhitespace();
    _pEatSpecialCharacters = new EatSpecialCharacters();
    _pEatDQString = new EatDQString();
    _pEatSQString = new EatSQString();
    _pState = _pEatWhitespace;
  }
}
//----< destructor deletes derived state instances only once >-------

ConsumeState::~ConsumeState()
{
  std::cout << "\n  ~ConsumeState() called";
  if (first_dtor)
  {
    first_dtor = false;
    delete _pEatAlphanum;
    delete _pEatCComment;
    delete _pEatCppComment;
    delete _pEatPunctuator;
    delete _pEatWhitespace;
    delete _pEatSpecialCharacters;
    delete _pEatDQString;
    delete _pEatSQString;
  }
  first_ctor = first_dtor = true;
  //_pState = nullptr;
}
//----< construct toker starting in state EatWhitespace >------------

Toker::Toker() : pConsumer(nullptr)
{
  ConsumeState::_lineCount = 1;
  if (ConsumeState::isActive())
    throw(std::exception("can't start Toker - another is active"));
  pConsumer = new EatWhitespace();
}

//----< destructor deletes the current state >-----------------------
/*
 * deleting any state deletes them all since derived destructor
 * implicitly invokes the base destructor
 */
Toker::~Toker() 
{
  std::cout << "\n  ~Toker() called";
  delete pConsumer->_pState;
  ConsumeState::reset();
}

//----< attach tokenizer to stream >---------------------------------
/*
 * stream can be either a std::fstream or std::stringstream
 */
bool Toker::attach(std::istream* pIn)
{
  if (pIn != nullptr && pIn->good())
  {
    pConsumer->attach(pIn);
    return true;
  }
  return false;
}
//----< collect token generated by ConsumeState >--------------------

std::string Toker::getTok()
{
  while(true) 
  {
    if (!pConsumer->canRead())
      return "";
    pConsumer->consumeChars();
    if (pConsumer->hasTok())
      break;
  }
  return pConsumer->getTok();
}
//----< has toker reached the end of its stream? >-------------------

bool Toker::canRead() { return pConsumer->canRead(); }

//----< start/stop returning comments as tokens >--------------------

void Toker::returnComments(bool doReturnComments)
{
  pConsumer->returnComments(doReturnComments);
}
//----< is this token a comment? >-----------------------------------

bool Toker::isComment(const std::string& tok)
{
  if (tok.find("//") < tok.size() || tok.find("/*") < tok.size())
    return true;
  return false;
}
//-----< return number of newlines pulled from stream >--------------

size_t Toker::currentLineCount()
{
  return pConsumer->currentLineCount();
}
//----< set one and two char tokens >--------------------------------

void Toker::setSpecialTokens(const std::string& commaSeparatedTokens)
{
  pConsumer->setSpecialTokens(commaSeparatedTokens);
}
//----< reset first_ctor and first_dtor >----------------------------

void Toker::reset()
{
  ConsumeState::reset();
}
//----< debugging output to console if TEST_LOG is #defined >--------

void testLog(const std::string& msg)
{
#ifdef TEST_LOG
  std::cout << "\n  " << msg;
#endif
}

//----< test stub >--------------------------------------------------

using Helper = Utilities::StringHelper;
using namespace Utilities;

#ifdef TEST_TOKENIZER

#include <fstream>
#include <exception>


int main()
{
  Helper::Title("Testing Tokenizer");

  //std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
  //std::string fileSpec = "../Tokenizer/Tokenizer.h";
  std::string fileSpec = "../Tokenizer/Test3.txt";

  try
  {
    std::ifstream in(fileSpec);
    if (!in.good())
    {
      std::cout << "\n  can't open " << fileSpec << "\n\n";
      return 1;
    }

    {
      Toker toker;
      toker.returnComments();
      toker.attach(&in);
      std::cout << "\n  current line count = " << toker.currentLineCount();
      do
      {
        std::string tok = toker.getTok();
        if (tok == "\n")
          tok = "newline";
        std::cout << "\n -- " << tok;
      } while (in.good());
      std::cout << "\n  current line count = " << toker.currentLineCount();
    }
 
    putline();
    Helper::title("Testing change of special characters");
    std::string newSpecialChars = "., :, +, +=, \n { }";

    /*
     * The following construction of Toker will throw an exception
     * if any other Toker object is active, e.g. has been constructed
     * but not yet destroyed.
     */

    Toker toker;
    toker.returnComments();
    toker.setSpecialTokens(newSpecialChars);
    in.clear();

    in.seekg(std::ios::beg);
    toker.attach(&in);
    std::cout << "\n  new special tokens: " << newSpecialChars;
    do
    {
      std::string tok = toker.getTok();
      if (tok == "\n")
        tok = "newline";
      std::cout << "\n -- " << tok;
    } while (in.good());
    std::cout << "\n  current line count = " << toker.currentLineCount();
  }
  catch (std::logic_error& ex)
  {
    std::cout << "\n  " << ex.what();
  }
  std::cout << "\n\n";
  return 0;
}
#endif