/////////////////////////////////////////////////////////////////
// simpleToker.cpp - Extracts tokens from a stream             //
// ver 1.0                                                     //
// Language:    Visual C++, ver 7.1                            //
// Platform:    Dell Dimension 8300, Windows XP Prof, SP 2     //
// Application: CSE687 prototype for project #1, Spring 2005   //
// Author:      Jim Fawcett, CST 2-187, Syracuse University    //
//              (315) 443-3948, jfawcett@twcny.rr.com          //
/////////////////////////////////////////////////////////////////

#include "simpleToker.h"
#include <cctype>

//----< constructor with initializer sequence >------------------

SimpleToker::SimpleToker(std::istream &input) : is_(input) {}

//
//----< returns tokens from stream >---------------------------
//
//  token is either a contiguous set of alphanumeric chars
//  or a single punctuator or a newline
//
std::string SimpleToker::getTok()
{
  // handle leading white space
  
  while(is_.good())
  {
    int ch = is_.get();

    // return newline as token else discard whitespace

    if(ch == static_cast<int>('\n'))
      return "\n";

    if(!isspace(ch))
    {
      is_.putback(ch);
      break;
    }
  }

  // collect non-white space characters

  std::string tok;
  while(is_.good())
  {
    int ch = is_.get();
    if(isspace(ch) && ch!=static_cast<int>('\n'))
      break;

    // return punctuators as single char tokens

    if(ispunct(ch) || ch == static_cast<int>('\n'))
    {
      if(tok.length() == 0)
      {
        tok += ch;
        return tok;
      }
      else
      {
        is_.putback(ch);
        return tok;
      }
    }
    tok += ch;
  }
  return tok;
}
//
//----< test stub >----------------------------------------------

#ifdef TEST_SIMPLETOKER

#include <fstream>

int main(int argc, char *argv[])
{
  if(argc < 2)
  {
    std::cout << "\n  please enter name of file to process\n\n";
    return 1;
  }
  std::ifstream in;
  try
  {
    in.open(argv[1]);
    if(!in.good())
    {
      std::cout << "\n  could not open file " << argv[1] << "\n\n";
      return 1;
    }
  }
  catch(std::exception e)
  {
    std::cout << "\n  " << e.what() << "\n\n";
    return 1;
  }
  SimpleToker toker(in);
  while(in.good())
  {
    std::string tok = toker.getTok();
    if(tok.compare("\n") == 0)
      tok = "newline";
    std::cout << tok.c_str() << std::endl;
  }
  in.clear();
  in.close();
}
#endif