/////////////////////////////////////////////////////////////////////////
// Toker.cs  -  Tokenizer                                              //
//              Reads words and punctuation symbols from a file stream //
// ver 2.2                                                             //
// Language:    C#, Visual Studio 7.0, .Net Framework 1.0              //
// Platform:    Dell Dimension 8100, Windows 2000, SP 2                //
// Application: Pr#2 Help, CSE681, Fall 2003                           //
// Author:      Jim Fawcett, CST 2-187, Syracuse University            //
//              (315) 443-3948, jfawcett@twcny.rr.com                  //
/////////////////////////////////////////////////////////////////////////

/*
  * Module Operations
  * =================
  * Toker provides, via the class CToker, the facilities to tokenize ASCII
  * text files.  That is, it composes the file's stream of characters into
  * words and punctuation symbols.
  * 
  * CToker works with a private buffer of characters from an attached file.
  * When the buffer is emptied CToker silently fills it again, so tokens
  * are always available until the end of file is reached.  End of file is
  * reported by tok = getTok() returning an empty token, e.g., tok == "".  
  * 
  * Public Interface
  * ================
  * CToker toker = new CToker();       // constructs CToker object
  * if(toker.openFile(fileName)) ...   // attaches toker to specified file
  * if(toker.openString(str)) ...      // attaches toker to specified string
  * toker.close();                     // closes stream
  * string tok = toker.getTok();       // extracts next token from stream
  * string tok = toker.peekNextTok();  // peeks but does not extract
  * toker.pushBack(tok);               // puts token back on stream
  * 
  */
/*
  * Build Process
  * =============
  * Required Files:
  *   Toker.cs
  * 
  * Compiler Command:
  *   csc /target:exe /define:TEST_CTOKER CToker.cs
  * 
  * Maintenance History
  * ===================
  * ver 2.2 : 10 Jun 08
  *   - added IsGrammarPunctuation to make tokenizer treat underscore
  *     as an ASCII char rather than a punctuator and used that in
  *     fillBuffer and eatASCII
  * ver 2.1 : 14 Jun 05
  *   - fixed newline handling bug in buffer filling routines:
  *     readLine, getLine, fillbuffer
  *   - fixed newline handling bug in extractComment
  * ver 2.0 : 30 May 05
  *   - added extraction of comments and quotes as tokens
  *   - added openString(...) to attach tokenizer to string
  * ver 1.1 : 21 Sep 04
  *   - added toker.close() in test stub
  *   - added processing for all command line args
  * ver 1.0 : 31 Aug 03
  *   - first release
  * 
  */
//


using System;
using System.IO;
using System.Text;
using System.Collections;

namespace CStoker
{
  ///////////////////////////////////////////////////////////////////////
  // class CToker - tokenizer
  
  public class CToker
  {
    private TextReader ts = null;         // source of tokens
    private ArrayList tokBuffer = null;   // intermediate token store
    private string lineRemainder;         // unprocessed line fragment
    
    //----< constructor >------------------------------------------------
    
    public CToker()
    {
      tokBuffer = new ArrayList();
    }

    //----< opens file stream for tokenizing >---------------------------
    
    public bool openFile(string fileName)
    {
      lineRemainder = "";
      try
      {
        ts = new StreamReader(fileName);
      }
      catch(Exception)
      {
        return false;
      }
      return true;
    }

    //----< opens string for tokenizing >------------------------------
    
    public bool openString(string source)
    {
      lineRemainder = "";
      try
      {
        ts = new StringReader(source);
      }
      catch(Exception)
      {
        return false;
      }
      return true;
    }

    //----< closes filestream >------------------------------------------
    
    public void close()
    {
      ts.Close();
    }

    //----< read a single line, retaining newline character >------------
    
    public string readLine()
    {
      StringBuilder temp = new StringBuilder();
      while(true)
      {
        int i = ts.Read();
        if(i == -1)
        {
          return temp.ToString();
        }
        char ch = (char)i;
        temp.Append(ch);
        if(ch == '\n')
          break;
      }
      return temp.ToString();
    }

    //----< extracts line of text for tokenizing >-----------------------
    //
    //  Passes back a line to process for tokens with no comments
    //  as a side effect through the out string parameter.
    //  - if line has a leading comment or quote it is extracted and 
    //    saved in tokBuffer and remaining string is passed back
    //  - if line has a trailing comment or quote the line fragment
    //    at the front is passed back after saving the rest of the
    //    line for later processing
    //  - always passes back a line to process until end of file
    //  - returns true if end of file has not been reached
    //
    
    bool getLine(out string line)
    {
      do
      {
        if(lineRemainder == "")  // previously saved line fragment is empty
        {
          try
          {
            lineRemainder = readLine();

            if(lineRemainder == null || lineRemainder == "")
            {
              line = "";
              return false;     // end of file
            }
          }
          catch(Exception except)
          {
            line = except.Message.ToString();
            return false;       // error reading file
          }
        }
        line = extract(ref lineRemainder);
        
        // keep extracting until there is a line to tokenize
        // or tokBuffer has contents

      } while(line == "" && tokBuffer.Count == 0);
      return true;
    }

    //----< extract tokens and comments >------------------------------
    //
    //  Extract the first of:
    //    C++ comments, C comments, double quotes, single quotes
    //
    
    string extract(ref string lineRemainder)
    {
      char[] whiteChars = { ' ', '\t', '\r', '\f' };  // newlines are tokens
      lineRemainder = lineRemainder.TrimStart(whiteChars);

      int posCppComm = lineRemainder.IndexOf("//");
      int posCComm   = lineRemainder.IndexOf("/*");
      int posDQuote  = lineRemainder.IndexOf('\"');
      int posSQuote  = lineRemainder.IndexOf('\'');

      // find first of the above

      int[] positions = { posCppComm, posCComm, posDQuote, posSQuote };
      for(int i=0; i<positions.Length; ++i)
        if(positions[i] == -1)
          positions[i] = Int32.MaxValue;
      Array.Sort(positions);
      
      if(positions[0] == Int32.MaxValue)    // nothing to extract
      {
        string retStr = lineRemainder;
        lineRemainder = "";
        return retStr;
      }
      if(posCppComm == positions[0])
        return extractComment(ref lineRemainder);
      if(posCComm == positions[0])
        return extractComment(ref lineRemainder);
      if(posDQuote == positions[0])
        return extractDQuote(ref lineRemainder);
      if(posSQuote == positions[0])
        return extractSQuote(ref lineRemainder);
      throw new Exception("extract failed");
    }

    //----< extract double quote >-------------------------------------
    
    string extractDQuote(ref string lineRemainder)
    {
      string retStr;
      int pos = lineRemainder.IndexOf('\"');
      if(pos == 0)
      {
        StringBuilder quote = new StringBuilder();
        quote.Append('\"');
        for(int i=1; i<lineRemainder.Length; ++i)
        {
          quote.Append(lineRemainder[i]);
          if(lineRemainder[i] == '\"')
          {
            if(lineRemainder[i-1] != '\\' || lineRemainder[i-2] == '\\')
            {
              tokBuffer.Add(quote.ToString());
              lineRemainder = lineRemainder.Remove(0,i+1);
              return "";
            }
          }
        }
      }
      else
      {
        retStr = lineRemainder.Remove(pos,lineRemainder.Length-pos);
        lineRemainder = lineRemainder.Remove(0,pos);
        return retStr;
      }
      throw new Exception("extractDQuote failed");
    }

    //----< extract single quote >-------------------------------------
    
    string extractSQuote(ref string lineRemainder)
    {
      string retStr;
      int pos = lineRemainder.IndexOf('\'');
      if(pos == 0)
      {
        StringBuilder quote = new StringBuilder();
        quote.Append('\'');
        for(int i=1; i<lineRemainder.Length; ++i)
        {
          quote.Append(lineRemainder[i]);
          if(lineRemainder[i] == '\'')
          {
            if(lineRemainder[i-1] != '\\' || lineRemainder[i-2] == '\\')
            {
              tokBuffer.Add(quote.ToString());
              lineRemainder = lineRemainder.Remove(0,i+1);
              return "";
            }
          }
        }
      }
      else
      {
        retStr = lineRemainder.Remove(pos,lineRemainder.Length-pos);
        lineRemainder = lineRemainder.Remove(0,pos);
        return retStr;
      }
      throw new Exception("extractSQuote failed");
    }

    //----< extract comment >------------------------------------------
    
    string extractComment(ref string lineRemainder)
    {
      string line;
      int pos = lineRemainder.IndexOf("//");
      if(pos == 0)                          // whole line is C++ comment
      {
        if(lineRemainder[lineRemainder.Length-1] == '\n')
        {
          lineRemainder = lineRemainder.Remove(lineRemainder.Length-1,1);
          tokBuffer.Add(lineRemainder);
          lineRemainder = "";
          return "\n";
        }
        else
        {
          tokBuffer.Add(lineRemainder);
          lineRemainder = "";
        }
        return lineRemainder;
      }
      if(pos > -1)                          // end of line is C++ comment
      {
        line = lineRemainder.Remove(pos,lineRemainder.Length-pos);
        lineRemainder = lineRemainder.Remove(0,pos);
        return line;
      }
      pos = lineRemainder.IndexOf("/*");    // line contains C comment
      if(pos > -1)
      {
        if(pos == 0)
        {
          eatCComment();
          return "";
        }
        else
        {
          string retStr = lineRemainder.Remove(pos,lineRemainder.Length-pos);
          lineRemainder = lineRemainder.Remove(0,pos);
          return retStr;
        }
      }
      // if we get here there is no comment in line

      line = lineRemainder;
      lineRemainder = "";
      return line;
    }

    //----< eat C comment - may consume more lines >---------------------
    
    void eatCComment()
    {
      StringBuilder comment = new StringBuilder();
      while(true)
      {
        for(int i=0; i<lineRemainder.Length; ++i)
        {
          int pos = lineRemainder.IndexOf("*/");
          if(pos != i)
            comment.Append(lineRemainder[i]);
          else
          {
            comment.Append(lineRemainder[i]);
            comment.Append(lineRemainder[i+1]);
            tokBuffer.Add(comment.ToString());
            lineRemainder = lineRemainder.Remove(0,i+2);
            return;
          }
        }
        comment.Append('\n');
        lineRemainder = ts.ReadLine();
        if(lineRemainder == null)
        {
          throw new Exception("encountered eof while processing comment");
        }
      }
    }

    //----< treat underscore as ASCII >----------------------------------
    
    bool IsGrammarPunctuation(char ch)
    {
      if (ch == '_')
        return false;
      if (Char.IsPunctuation(ch))
        return true;
      return false;
    }

    //----< consumes ASCII characters from stream >----------------------
    
    string eatAscii(ref string tok)
    {
      string retStr = tok;
      for(int i=0; i<tok.Length; ++i)
      {
        if(IsGrammarPunctuation(tok[i]) || Char.IsSymbol(tok[i]))
        {
          retStr = tok.Remove(i,tok.Length-i);
          tok = tok.Remove(0,i);
          return retStr;
        }
      }
      tok = "";
      return retStr;
    }

    //----< consumes a single punctuator from stream >-------------------
    
    string eatPunctuationChar(ref string tok)
    {
      string retStr = tok.Remove(1,tok.Length-1);
      tok = tok.Remove(0,1);
      return retStr;
    }

    //----< fills internal buffer with tokens >--------------------------
    
    bool fillBuffer()
    {
      string line;
      if(!this.getLine(out line))
        return false;             // end of token source
      if(line == "")
        return (tokBuffer.Count > 0);
      char [] delim = { ' ', '\t', '\f', '\r' };
      string [] toks = line.Split(delim);
      foreach(string tok in toks)
      {
        string temp = tok;
        while(temp.Length > 0)
        {
          if(IsGrammarPunctuation(temp[0]) || Char.IsSymbol(temp[0]))
          {
            string punc = this.eatPunctuationChar(ref temp);
            tokBuffer.Add(punc);
          }
          else
          {
            string ascii = this.eatAscii(ref temp);
            tokBuffer.Add(ascii);
          }
        }
      }
      return true;
    }

    //----< extracts tokens from internal buffer, filling if needed >----
    
    public string getTok()
    {
      string tok = peekNextTok();
      if(tok != "")
        tokBuffer.RemoveAt(0);
      return tok;
    }

    //----< look at next token without extracting >----------------------
    
    public string peekNextTok()
    {
      if(tokBuffer.Count == 0)
        if(!fillBuffer())
          return "";
      string tok = (string)tokBuffer[0];
      return tok;
    }

    //----< put token back into tokBuffer >------------------------------
    
    public void pushBack(string tok)
    {
      tokBuffer.Insert(0,tok);
    }

    //----< test stub >--------------------------------------------------
    

#if(TEST_TOKER)

    [STAThread]
    static void Main(string[] args)
    {
      Console.Write("\n  Testing CToker - Tokenizer ");
      Console.Write("\n ============================\n");

      CToker toker = new CToker();

      if(args.Length == 0)
      {
        Console.Write("\n  Please enter name of file to tokenize\n\n");
        return;
      }
      foreach(string file in args)
      {
        string msg1;
        if(!toker.openFile(file))
        {
          msg1 = "Can't open file " + file;
          Console.Write("\n\n  {0}",msg1);
          Console.Write("\n  {0}",new string('-',msg1.Length));
        }
        else
        {
          msg1 = "Processing file " + file;
          Console.Write("\n\n  {0}",msg1);
          Console.Write("\n  {0}",new string('-',msg1.Length));
          string tok = "";
          while((tok = toker.getTok()) != "")
            if(tok != "\n")
              Console.Write("\n{0}",tok);
          toker.close();
        }
      }
      Console.Write("\n");
      //
      string[] msgs = new string[9];
      msgs[0] = "abc";
      msgs[1] = "string with double quotes \"first quote\""
                + " and \"second quote\" but no more";
      msgs[2] = "string with single quotes \'1\' and \'2\'";
      msgs[3] = "string with quotes \"first quote\" and \'2\'";
      msgs[4] = "string with C comments /* first */ and /*second*/ but no more";
      msgs[5] = "/* single C comment */";
      msgs[6] = " -- /* another single comment */ --";
      msgs[7] = "// a C++ comment\n";
      msgs[8] = "// another C++ comment\n";

      foreach(string msg in msgs)
      {
        if(!toker.openString(msg))
        {
          string msg2 = "Can't open string for reading";
          Console.Write("\n\n  {0}",msg2);
          Console.Write("\n  {0}",new string('-',msg2.Length));
        }
        else
        {
          string msg2 = "Processing \"" + msg + "\"";
          Console.Write("\n\n  {0}",msg2);
          Console.Write("\n  {0}",new string('-',msg2.Length));
          string tok = "";
          while((tok = toker.getTok()) != "")
          {
            if(tok != "\n")
              Console.Write("\n{0}",tok);
            else
              Console.Write("\nnewline");
          }
          toker.close();
        }
      }
      Console.Write("\n\n");
    }

#endif
  }
  
  public class Test
  {
  }
}

// a namespace for testing
namespace Test
{
  // a testing class
  private AnotherTest class
  {
  //----< a test function >----------------------------------------
    
    void someUninterestingFunction()
    {
      return;
    }
    bool anotherUninterestingFunction()
    {
      return false;
    }
  }
}