#include "Tokenizer.h"
#include <locale>
/*
   function   19  65
   if         23  27
   do         29  63
   if         31  31
   if         35  49
   if         37  40
   if         43  43
   else       45  48
   if         51  59
   if         53  56
   if         62  62
*/
//----< read token from attached file >------------------------

std::string Toker::getTok()
{
  std::string tok = "";
  stripWhiteSpace();
  if(isSingleCharTok(nextChar))
  {
    getChar();
    tok.append(1,currChar);
    return tok;
  }
  do
  {
    if(isFileEnd())
      return tok;

    getChar();
    if(isBeginComment())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatComment();
      if(doReturnComments)
        return tok;
      else
      {
        tok = "";
        continue;
      }
    }
    if(isBeginQuote())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatQuote();
      return tok;
    }
    std::locale loc;
    if(!isspace(currChar, loc))
      tok.append(1,currChar);
  }  while(!isTokEnd() || tok.length() == 0);
  return tok;
}