#ifndef TOK_H
#define TOK_H
/////////////////////////////////////////////////////////////// 
//  tok.h      -  read words from ASCII file                 //
//  ver 1.9                                                  //
//  Language:     Visual C, ver 6.0                          //
//  Platform:     Micron Dual Pentium Pro 200, Win NT 4.0    //
//  Application:  CSE784 Example                             //
//  Author:       Jim Fawcett, CST 2-187, Syracuse Univ.     //
//                (315) 443-3948, fawcett@ecs.syr.edu        //
///////////////////////////////////////////////////////////////
/*

    Module Operations
    =================
    This tok class provides one public function which extracts
    words from an existing file stream.  A word is a sequence of
    characters delimited by whitespace.  punctuator sequences are
    separated from alphanumberic characters; and certain punctuator
    characters are collected as single character tokens, e.g.:
    
        '{', '}', '(', ')', '[', ']', ';'

    The tokenizer is used in the following way:

      ifstream in(filename);
      toker Toker(in);
      string tok;
      while(getTok(tok)) {

          // do things with collected tokens

      }
      int nl = Toker.lines();       // number of '\n' chars
      int sl = Toker.scopeLevel();  // net number of '{' and '}' chars

    All newlines, '\n', are counted.  Braces, '{' and '}', are counted
    only if not in comments or quoted strings.

*/
//
/////////////////////////////////////////////////////////////// 
//  build process                                            //
///////////////////////////////////////////////////////////////
//                                                           //
//  Required Files:                                          //
//    tok.h, tok.c                                           //
//                                                           //
//  Compile Command:                                         //
//    cl -GX -DTEST_TOK tok.cpp                              //
//                                                           //
///////////////////////////////////////////////////////////////
/*
      Maintenance History
      ===================
      ver 1.9 : 24 Jan 2004
      - typename qualifier added for return type of toker<T>::type(char),
        now required as compiler has become more standard conforming.
      ver 1.8 : 10 Mar 2002
      - added tok<T>::collectSingleQuotes(bool pred=true) to allow
        user to turn off collection of single quotes as a token.
      ver 1.7 : 10 Mar 2002
      - tok::eatComment() modified to return the newline encountered
        at the end of a C++ comment by making the code substitution:
        ++_lines replaced by in.putback(ch).  This was done to fix
        a comment handling bug in semiExp.  It does not change the
        line count returned by the tokenizer.
      ver 1.6 : 04 Feb 2002
      - changed class name from token to toker - more accurate
        name
      - templatized class on T parameter which must be a functor
        providing the delim operation.  This makes the tokenizing
        rules more flexible.
      ver 1.5 : 30 Sep 2000
      - cosmetic changes to this header file and test stub
      ver 1.4 : 21 Apr 2000
      - tok now announces end of file while in comment
      - added option to terminate comment on * and / event
        if nested (like VC++ does by default)
      ver 1.3 : 10 Dec 1999
      - changed charType(ch) to return underscore as a char
        instead of a punctuator
      ver 1.2 : 11 Sep 1999
      - added counts for lines and braces
      ver 1.1 : 26 Apr 1999
      - fixed bug in test stub, e.g., added in.close() and
        in.clear() at end of file loop
      ver 1.0 : 20 Sep 1998
      - first release

*/

//

/////////////////////////////////////////////////////////////// 
//  declarations                                             //
/////////////////////////////////////////////////////////////// 

#include <iostream>
#include <string>
using namespace std;

///////////////////////////////////////////////////////////////
//  functor that defines delimiters needed for code analysis

class codeDelimiter {

  public:
    bool operator()(char ch);
};

//----< is this char a delimiter? >----------------------------

inline bool codeDelimiter::operator()(char ch) {

  switch(ch) {
    case '[' : return 1;
    case ']' : return 1;
    case '{' : return 1;
    case '}' : return 1;
    case '(' : return 1;
    case ')' : return 1;
    case ';' : return 1;
    case ',' : return 1;
    default  : return 0;
  }
}

//

///////////////////////////////////////////////////////////////
//  functor that defines delimiters needed for XML analysis

class xmlDelimiter {

  public:
    bool operator()(char ch);
};

//----< is this char a delimiter? >----------------------------

inline bool xmlDelimiter::operator()(char ch) {

  switch(ch) {
    case '=' : return 1;
    case '[' : return 1;
    case ']' : return 1;
    case '{' : return 1;
    case '}' : return 1;
    case '(' : return 1;
    case ')' : return 1;
    case '>' : return 1;
    case ';' : return 1;
    case ',' : return 1;
    default  : return 0;
  }
}

//
///////////////////////////////////////////////////////////////
//  tokenizer class

template <class T> class toker {

  public:
    toker(istream &input);
    bool getTok(string &buffer);
    unsigned int lines();
    unsigned int scopeLevel();
    void collectSingleQuotes(bool pred=true);
    void nestedComments(bool pred=true);
  private:
    T delim;
    istream &in;                // source of tokens
    unsigned int _lines;        // number of newlines
    unsigned int _scopeLevel;   // net number of open/closed braces
    enum charType { Char, NewLine, White, Punct };
    charType type(char ch);
    int  isComment(const string &buffer);
    int  isWhite(const string &buffer);
    int  eatWhite(string &buffer);
    int  eatComment(string &buffer);
    int  isQuote(const string &buffer);
    int  collectQuote(string &buffer);
    bool _collectSingleQuotes;
    bool _nestedComments;
};

template <class T>
inline unsigned int toker<T>::lines() { return _lines; }

template <class T>
inline unsigned int toker<T>::scopeLevel() { return _scopeLevel; }

template <class T>
inline void toker<T>::nestedComments(bool pred) { _nestedComments = pred; }

template <class T>
inline void toker<T>::collectSingleQuotes(bool pred) { 
  _collectSingleQuotes = pred;
}

//----< constructor >------------------------------------------

template <class T>
toker<T>::toker(istream &input) 
        : in(input), _lines(0), _scopeLevel(0),
          _collectSingleQuotes(true), _nestedComments(false) { }

//

//----< find char type >---------------------------------------

template <class T>
typename toker<T>::charType toker<T>::type(char ch) {

  if(isalnum(ch) || ch == '_') return Char;
  if(ch == '\n')  return NewLine;
  if(isspace(ch)) return White;
  return Punct;
}

//----< is this a comment? >-----------------------------------

template <class T>
int toker<T>::isComment(const string &buffer) {

  int next = buffer.length();
  if(next < 2) return 0;
  if(buffer[next-2] == '/') {
    if(buffer[next-1] == '/') return 1;
    if(buffer[next-1] == '*') return 1;
  }
  return 0;
}

//----< is buffer filled only with whitespace chars? >---------

template <class T>
int toker<T>::isWhite(const string &buffer) {

  int i;
  int next = buffer.length();
  if(next == 0) return 1;
  for(i=0; i<next; i++)
    if(!isspace(buffer[i])) return 0;
  return 1;
}

//

//----< consume whitespace >-----------------------------------

template <class T>
int toker<T>::eatWhite(string &buffer) {

  char ch;
  while(isspace(ch = in.get()) && ch != EOF) {
    if(ch == '\n') {
      ++_lines;
      buffer += '\n';
      return buffer.length();
    }
  }
  if(!isspace(ch))
    in.putback(ch);
  return buffer.length();
}

//
//----< consume comment characters >---------------------------

template <class T>
int toker<T>::eatComment(string &buffer) {

  char ch = ' ', chPrev = ' ';
  if(buffer[buffer.length()-1] == '/') {       // eat C++ comment
    while(ch != '\n' && ch != EOF) {
      ch = in.get();
      if(ch == '\n')
       in.putback(ch);
    }
    buffer.erase(buffer.length()-2);
    return buffer.length();
  }
  int level = 1;
  while(ch != EOF) {   // eat nested C comments
    ch = in.get();
    if(ch == '\n')
      ++_lines;
    if(chPrev == '/' && ch == '*')
      level++;
    else if(chPrev == '*' && ch == '/') {
      level--;
      if(!_nestedComments)  // added 21 Apr 2000
        level = 0;          //   "    "  "    "
    }
    chPrev = ch;
    if(level == 0) {
      buffer.erase(buffer.length()-2);
      return buffer.length();
    }
  }
  // next two lines added 21 Apr 2000

  cout << "\n  ---- end of file while in comment               ----\n"
       <<   "  ---- probably due to improperly nested comments ----\n";
  return buffer.length();   // should never get here
}

//
//----< is this start of a quote? >----------------------------

template <class T>
int toker<T>::isQuote(const string &buffer) {

  int next = buffer.length();
  if(buffer[next-1] == '\"') return 1;
  if(_collectSingleQuotes && buffer[next-1] == '\'') return 1;
  return 0;
}

//----< collect all chars in quote starting at buffer end >----

template <class T>
int toker<T>::collectQuote(string &buffer) {

  char ch, terminal = buffer[buffer.length()-1];
  while((ch = in.get()) != EOF) {
    buffer += ch;
    if(ch == '\n')
      ++_lines;
    if(ch == terminal)
      if(buffer[buffer.length()-2] != '\\' || 
         buffer[buffer.length()-3] == '\\')
        break;
  }
  return buffer.length();
}

//
//----< collect token from file >------------------------------

template <class T>
bool toker<T>::getTok(string &buffer) {

  char ch, chprev=' ';
  buffer.erase();
  if(eatWhite(buffer) > 0)
    return 1;
  while((ch = in.get()) != EOF) {
    switch(ch) {
      case '\n' : ++_lines;      break;
      case '{'  : ++_scopeLevel; break;
      case '}'  : --_scopeLevel; break;
    }
    if(buffer.length() == 0) {
      buffer += ch;
      if(delim(ch))
        break;
    }
    else if(type(ch) == type(chprev)) {
      if(!delim(ch)) {
        buffer += ch;
      }
      else {
        in.putback(ch);
        switch(ch) {
          case '\n' : --_lines;      break;
          case '{'  : --_scopeLevel; break;
          case '}'  : ++_scopeLevel; break;
        }
        break;
      }
    }
    else {    // either delimiter or change of charType 
              // so end this token collection           
      in.putback(ch);
      switch(ch) {
        case '\n' : --_lines;      break;
        case '{'  : --_scopeLevel; break;
        case '}'  : ++_scopeLevel; break;
      }
      break;
    }
    // don't collect C or C++ comments 

    if(isComment(buffer)) {
      eatComment(buffer);
      if(isWhite(buffer))
        eatWhite(buffer);
      if(buffer.length() > 0)
        break;
    }
    // collect quote as token

    if(isQuote(buffer)) {
      collectQuote(buffer);
      break;
    }
    chprev = ch;
  }
  // cout << "\n  debug in getTok: buffer = " << buffer.c_str();
  return (ch != EOF);
}

#endif