///////////////////////////////////////////////////////////////
//  tok.cpp   --  read words from ASCII file                 //
//  ver 1.6                                                  //
//  Language:     Visual C++, ver 6.0                        //
//  Platform:     Micron Dual Pentium Pro 200, Win NT 4.0    //
//  Application:  CSE784 Example                             //
//  Author:       Jim Fawcett, CST 2-187, Syracuse Univ.     //
//                (315) 443-3948, fawcett@ecs.syr.edu        //
///////////////////////////////////////////////////////////////


#include <string>
#include <cctype>
#include "tok.h"

//----< constructor >------------------------------------------

token::token(istream &input) : in(input), _lines(0), _scopeLevel(0) { }

//----< find char type >---------------------------------------

token::charType token::type(char ch) {

  if(isalnum(ch) || ch == '_') return Char;
  if(ch == '\n')  return NewLine;
  if(isspace(ch)) return White;
  return Punct;
}

//----< is this char a delimiter? >----------------------------

int token::delim(char ch) {

  switch(ch) {
    case '[' : return 1;
    case ']' : return 1;
    case '{' : return 1;
    case '}' : return 1;
    case '(' : return 1;
    case ')' : return 1;
    case ';' : return 1;
    case ',' : return 1;
    default  : return 0;
  }
}
//
//----< is this a comment? >-----------------------------------

int token::isComment(const string &buffer) {

  int next = buffer.length();
  if(next < 2) return 0;
  if(buffer[next-2] == '/') {
    if(buffer[next-1] == '/') return 1;
    if(buffer[next-1] == '*') return 1;
  }
  return 0;
}

//----< is buffer filled only with whitespace chars? >---------

int token::isWhite(const string &buffer) {

  int i;
  int next = buffer.length();
  if(next == 0) return 1;
  for(i=0; i<next; i++)
    if(!isspace(buffer[i])) return 0;
  return 1;
}

//----< consume whitespace >-----------------------------------

int token::eatWhite(string &buffer) {

  char ch;
  while(isspace(ch = in.get()) && ch != EOF) {
    if(ch == '\n') {
      ++_lines;
      buffer += '\n';
      return buffer.length();
    }
  }
  if(!isspace(ch))
    in.putback(ch);
  return buffer.length();
}

//
//----< consume comment characters >---------------------------

int token::eatComment(string &buffer) {

  char ch = ' ', chPrev = ' ';
  if(buffer[buffer.length()-1] == '/') {       // eat C++ comment
    while(ch != '\n' && ch != EOF) {
      ch = in.get();
      if(ch == '\n')
        ++_lines;
    }
    buffer.erase(buffer.length()-2);
    return buffer.length();
  }
  int level = 1;
  while(ch != EOF) {   // eat nested C comments
    ch = in.get();
    if(ch == '\n')
      ++_lines;
    if(chPrev == '/' && ch == '*')
      level++;
    else if(chPrev == '*' && ch == '/') {
      level--;
      if(!_nestedComments)  // added 21 Apr 2000
        level = 0;          //   "    "  "    "
    }
    chPrev = ch;
    if(level == 0) {
      buffer.erase(buffer.length()-2);
      return buffer.length();
    }
  }
  // next two lines added 21 Apr 2000

  cout << "\n  ---- end of file while in comment               ----\n"
       <<   "  ---- probably due to improperly nested comments ----\n";
  return buffer.length();   // should never get here
}

//
//----< is this start of a quote? >----------------------------

int token::isQuote(const string &buffer) {

  int next = buffer.length();
  if(buffer[next-1] == '\"' ||buffer[next-1] == '\'') return 1;
  return 0;
}

//----< collect all chars in quote starting at buffer end >----

int token::collectQuote(string &buffer) {

  char ch, terminal = buffer[buffer.length()-1];
  while((ch = in.get()) != EOF) {
    buffer += ch;
    if(ch == '\n')
      ++_lines;
    if(ch == terminal)
      if(buffer[buffer.length()-2] != '\\' || 
         buffer[buffer.length()-3] == '\\')
        break;
  }
  return buffer.length();
}

//
//----< collect token from file >------------------------------

bool token::getTok(string &buffer) {

  char ch, chprev=' ';
  buffer.erase();
  if(eatWhite(buffer) > 0)
    return 1;
  while((ch = in.get()) != EOF) {
    switch(ch) {
      case '\n' : ++_lines;      break;
      case '{'  : ++_scopeLevel; break;
      case '}'  : --_scopeLevel; break;
    }
    if(buffer.length() == 0) {
      buffer += ch;
      if(delim(ch))
        break;
    }
    else if(type(ch) == type(chprev)) {
      if(!delim(ch)) {
        buffer += ch;
      }
      else {
        in.putback(ch);
        switch(ch) {
          case '\n' : --_lines;      break;
          case '{'  : --_scopeLevel; break;
          case '}'  : ++_scopeLevel; break;
        }
        break;
      }
    }
    else {    // either delimiter or change of charType 
              // so end this token collection           
      in.putback(ch);
      switch(ch) {
        case '\n' : --_lines;      break;
        case '{'  : --_scopeLevel; break;
        case '}'  : ++_scopeLevel; break;
      }
      break;
    }
    // don't collect C or C++ comments 

    if(isComment(buffer)) {
      eatComment(buffer);
      if(isWhite(buffer))
        eatWhite(buffer);
      if(buffer.length() > 0)
        break;
    }
    // collect quote as token

    if(isQuote(buffer)) {
      collectQuote(buffer);
      break;
    }
    chprev = ch;
  }
  return (ch != EOF);
}
//

//----< test stub >--------------------------------------------

#ifdef TEST_TOK

#include <fstream>

void main(int argc, char *argv[]) {

  ifstream in;
  token tok(in);
  string buffer;
  if(argc == 1) {
    cout << "\n  please enter file name on command line\n\n";
    return;
  }
  int i;
  for(i=1; i<argc; i++) {
    in.open(argv[i]);
    if(!in.good()) {
      cout << "\n  can't open file " << argv[i] << "\n\n";
      continue;
    }
    cout << "\n  processing file " << argv[i] << "\n\n";
    while(tok.getTok(buffer)) {
      if(buffer[0] == '\n')
        cout << "newline\n";
      else
        cout << buffer << '\n';
    }
    in.close();
    in.clear();
  }
  cout << "\n\n";
}

#endif