#include "Tokenizer.h"
#include <cctype>
#include <locale>
/*
   function   19  58
   do         22  56
   if         24  24
   if         28  42
   if         30  33
   if         36  36
   else       38  41
   if         44  52
   if         46  49
   if         55  55
*/
std::string tok;

std::string Toker::test()
{
  std::string tok;
  do
  {
    if(isFileEnd())
      return tok;

    getChar();
    if(isBeginComment())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatComment();
      if(doReturnComments)
        return tok;
      else
      {
        tok = "";
        continue;
      }
    }
    if(isBeginQuote())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatQuote();
      return tok;
    }
    std::locale loc;
    if(!isspace(currChar, loc))
      tok.append(1,currChar);
  }  while(!isTokEnd() || tok.length() == 0);
  return tok;
}