#ifndef TOKENIZER_H
#define TOKENIZER_H
/////////////////////////////////////////////////////////////////////
//  Tokenizer.h - Reads words from a file                          //
//  ver 1.3                                                        //
//                                                                 //
//  Language:      Visual C++ 2005                                 //
//  Platform:      Dell Dimension 9150, Windows XP SP2             //
//  Application:   Prototype for CSE687 Pr1, Sp06                  //
//  Author:        Jim Fawcett, CST 2-187, Syracuse University     //
//                 (315) 443-3948, jfawcett@twcny.rr.com           //
/////////////////////////////////////////////////////////////////////
/*
  Module Operations: 
  ==================
  This module defines a tokenizer class.  Its instances read words from 
  an attached file.  Word boundaries occur when a character sequence
  read from the file:
  - changes between any of the character types: alphanumeric, punctuator,
    or white space.
  - certain characters are treated as single character tokens, e.g., 
    "(", ")", "{", "}", "[". "]", ";", ".", and "\n".
  A tokenizer is an important part of a scanner, used to read and interpret
  source code.

  Public Interface:
  =================
  Toker t;                        // create tokenizer instance
  returnComments();               // request comments return as tokens
  if(t.attach(someFileName))      // select file for tokenizing
    string tok = t.getTok();      // extract first token
  int numLines = t.lines();       // return number of lines encountered
  t.lines() = 0;                  // reset line count

  Build Process:
  ==============
  Required files
    - tokenizer.h, tokenizer.cpp
  Build commands (either one)
    - devenv Project1HelpS06.sln
    - cl /EHsc /DTEST_TOKENIZER tokenizer.cpp /link setargv.obj

  Maintenance History:
  ====================
  ver 1.3 : 24 Feb 06
  - fixed bug in eat comment that hung on ending comment with no
    newline, by testing for stream state good.
  ver 1.2 : 06 Feb 06
  - added stream closing to destructor and attach memeber functions
  ver 1.1 : 01 Feb 06
  - added if test at end of getTok() to avoid returning space after 
    C comment as a token
  ver 1.0 : 12 Jan 06
  - first release

*/
//
#include <string>
#include <iostream>
#include <vector>

class Toker
{
public:
  Toker(const std::string& src = "", bool isFile = true);
  ~Toker();
  bool attach(const std::string& filename, bool isFile = true);
  std::string getTok();
  void returnComments(bool doReturn = true);
  int& lines();
  int  braceLevel();
  bool isFileEnd();

private:
  std::istream* pIn;
  char prevprevChar, prevChar, currChar, nextChar;
  std::vector<char> scTok;
  std::vector<char> putbacks;
  int numLines;
  int braceCount;
  bool doReturnComments;
  bool aCppComment;

  // private helper functions
  int  get();
  int  peek();
  void putback(int ch);
  bool getChar();
  bool isSingleCharTok(char ch);
  bool isTokEnd();
  void stripWhiteSpace();
  bool isIdentifierChar(char ch);
  bool isBeginComment();
  bool isEndComment();
  std::string eatComment();
  bool aSingleQuote;
  bool isBeginQuote();
  bool isEndQuote();
  std::string eatQuote();

  // prohibit copying and assignment
  Toker(const Toker &tkr);
  Toker& operator=(const Toker&);
};

inline void Toker::returnComments(bool doReturn)
{
  doReturnComments = doReturn;
}

inline bool Toker::isFileEnd() { return (nextChar == -1); }

inline int& Toker::lines() { return numLines; }

inline int Toker::braceLevel() { return braceCount; }

#endif