/////////////////////////////////////////////////////////////////////////
// Toker.cs  -  Tokenizer                                              //
//              Reads words and punctuation symbols from a file stream //
// ver 2.0                                                             //
// Language:    C#, Visual Studio 7.0, .Net Framework 1.0              //
// Platform:    Dell Dimension 8100, Windows 2000, SP 2                //
// Application: Pr#2 Help, CSE681, Fall 2003                           //
// Author:      Jim Fawcett, CST 2-187, Syracuse University            //
//              (315) 443-3948, jfawcett@twcny.rr.com                  //
//    ver2.0:   Noriyuki Takasaku, ntakasak@syr.edu                    //
/////////////////////////////////////////////////////////////////////////
/*
* Module Operations
* =================
* Toker provides, via the class CToker, the facilities to tokenize ASCII
* text files.  That is, it composes the file's stream of characters into
* words and punctuation symbols.
*
* CToker works with a private buffer of characters from an attached file.
* When the buffer is emptied CToker silently fills it again, so tokens
* are always available until the end of file is reached.  End of file is
* reported by tok = getTok() returning an empty token, e.g., tok == "".
*
* Public Interface
* ================
* CToker toker = new CToker();       // constructs CToker object
* if(toker.open(fileName)) ...       // attaches toker to specified file
* toker.close();                     // closes file stream
* string tok = toker.getTok();       // extracts next token from stream
* string tok = toker.peekNextTok();  // peeks but does not extract
* toker.pushBack(tok);               // puts token back on stream
*
*/
/*
* Build Process
* =============
* Required Files:
*   Toker.cs TestHarness.dll
*
* Compiler Command:
*   ver1.0 csc /target:exe /define:TEST_CTOKER CToker.cs
*   ver2.0 csc /reference:TestHarness.dll /define:TEST_CTOKER /target:library Toker.cs
*   *Note: ver2.0 generates a dll file.
*
* Maintenance History
* ===================
* ver 2.0 : 26 Sep 04
*   - added    toker.test() does the same thing as Main function.
*   - modified fillBuffer() so that it extracts C/C++ style comment
*   - added    toker.fillComment(string) which is helping function of fillBuffer
*   - added    toker.seekComment(bool) which is helping function of fillComment
*   - modified CToker inherits "Test" class of TestHarness system
* ver 1.1 : 21 Sep 04
*   - added toker.close() in test stub
*   - added processing for all command line args
* ver 1.0 : 31 Aug 03
*   - first release
*
*/
//
using
System
;
using
System
.
IO
;
using
System
.
Collections
;
using
TestHarness
;
namespace
CStoker
{
///////////////////////////////////////////////////////////////////////
// class CToker - tokenizer
class
CToker
:
Test
{
private
StreamReader
ts
=
null
;
/
/
source
of
tokens
private
ArrayList
tokBuffer
=
null
;
/
/
intermediate
token
store
//----< constructor >------------------------------------------------
public
CToker
(
)
{
tokBuffer
=
new
ArrayList
(
)
;
}
//----< opens file stream for tokenizing >---------------------------
public
bool
open
(
string
fileName
)
{
try
{
ts
=
new
StreamReader
(
fileName
)
;
}
catch
(
Exception
)
{
return
false
;
}
return
true
;
}
//----< closes filestream >------------------------------------------
public
void
close
(
)
{
ts
.
Close
(
)
;
}
//----< extracts line of text for tokenizing >-----------------------
bool
getLine
(
out
string
tok
)
{
try
{
tok
=
ts
.
ReadLine
(
)
;
return
(
tok
!
=
null
)
;
}
catch
(
Exception
except
)
{
tok
=
except
.
Message
.
ToString
(
)
;
return
false
;
}
}
//
//----< consumes ASCII characters from stream >----------------------
string
eatAscii
(
ref
string
tok
)
{
string
retStr
=
tok
;
for
(
int
i
=
0
;
i
<
tok
.
Length
;
+
+
i
)
{
if
(
Char
.
IsPunctuation
(
tok
[
i
]
)
|
|
Char
.
IsSymbol
(
tok
[
i
]
)
)
{
retStr
=
tok
.
Remove
(
i
,
tok
.
Length
-
i
)
;
tok
=
tok
.
Remove
(
0
,
i
)
;
return
retStr
;
}
}
tok
=
"
"
;
return
retStr
;
}
//----< consumes a single punctuator from stream >-------------------
string
eatPunctuationChar
(
ref
string
tok
)
{
string
retStr
=
tok
.
Remove
(
1
,
tok
.
Length
-
1
)
;
tok
=
tok
.
Remove
(
0
,
1
)
;
return
retStr
;
}
//----< fills internal buffer with tokens >--------------------------
bool
fillBuffer
(
)
{
string
line
=
"
"
;
string
aLine
=
"
"
;
if
(
!
this
.
getLine
(
out
aLine
)
)
return
false
;
line
=
fillComment
(
aLine
)
;
char
[
]
delim
=
{
'
'
,
'
\
t
'
,
'
\
f
'
,
'
\
r
'
}
;
string
[
]
toks
=
line
.
Split
(
delim
)
;
foreach
(
string
tok
in
toks
)
{
string
temp
=
tok
;
while
(
temp
.
Length
>
0
)
{
if
(
Char
.
IsPunctuation
(
temp
[
0
]
)
|
|
Char
.
IsSymbol
(
temp
[
0
]
)
)
{
string
punc
=
this
.
eatPunctuationChar
(
ref
temp
)
;
tokBuffer
.
Add
(
punc
)
;
}
else
{
string
ascii
=
this
.
eatAscii
(
ref
temp
)
;
tokBuffer
.
Add
(
ascii
)
;
}
}
}
tokBuffer
.
Add
(
"
\
n
"
)
;
/
/
restore
newline
stripped
by
getLine
(
)
return
true
;
}
//-----< seeks token until it finds the end comment "*/" >-----
private
string
seekComment
(
bool
findComment
)
{
string
line
=
"
"
;
string
trimedLine
;
int
j
;
while
(
findComment
)
{
bool
r
=
this
.
getLine
(
out
line
)
;
if
(
r
)
{
trimedLine
=
line
.
Trim
(
)
;
j
=
trimedLine
.
IndexOf
(
"
*
/
"
)
;
if
(
j
>
=
0
)
{
Console
.
WriteLine
(
trimedLine
)
;
tokBuffer
.
Add
(
trimedLine
.
Substring
(
j
,
j
+
2
)
)
;
if
(
j
+
2
<
trimedLine
.
Length
)
{
line
=
trimedLine
.
Substring
(
j
+
3
,
trimedLine
.
Length
-
1
)
;
}
findComment
=
false
;
}
else
{
tokBuffer
.
Add
(
trimedLine
+
"
\
n
"
)
;
}
}
}
return
line
;
}
//-----< fills internal buffers with C/C++ comment if it exists. >-----
private
string
fillComment
(
string
theLine
)
{
string
trimedLine
=
theLine
.
Trim
(
)
;
string
line
=
"
"
;
bool
findComment
=
false
;
if
(
trimedLine
.
Length
>
1
&
&
trimedLine
.
Substring
(
0
,
2
)
=
=
"
/
/
"
)
{
tokBuffer
.
Add
(
trimedLine
)
;
return
"
"
;
}
else
if
(
trimedLine
.
Length
>
1
&
&
trimedLine
.
Substring
(
0
,
2
)
=
=
"
/
*
"
)
{
int
i
=
trimedLine
.
IndexOf
(
"
*
/
"
)
;
if
(
i
>
0
)
{
tokBuffer
.
Add
(
trimedLine
.
Substring
(
0
,
i
+
2
)
)
;
if
(
i
+
2
<
trimedLine
.
Length
-
1
)
{
line
=
trimedLine
.
Substring
(
i
+
3
,
trimedLine
.
Length
-
1
)
;
}
}
else
{
int
j
=
trimedLine
.
IndexOf
(
"
*
/
"
)
;
if
(
j
>
0
)
{
tokBuffer
.
Add
(
trimedLine
.
Substring
(
0
,
j
)
)
;
line
=
trimedLine
.
Substring
(
j
+
1
,
trimedLine
.
Length
-
1
)
;
}
else
{
tokBuffer
.
Add
(
trimedLine
+
"
\
n
"
)
;
findComment
=
true
;
}
line
=
seekComment
(
findComment
)
;
}
}
else
{
line
=
theLine
;
}
return
line
;
}
//----< extracts tokens from internal buffer, filling if needed >----
public
string
getTok
(
)
{
string
tok
=
peekNextTok
(
)
;
if
(
tok
!
=
"
"
)
tokBuffer
.
RemoveAt
(
0
)
;
return
tok
;
}
//----< look at next token without extracting >----------------------
public
string
peekNextTok
(
)
{
if
(
tokBuffer
.
Count
=
=
0
)
if
(
!
fillBuffer
(
)
)
return
"
"
;
string
tok
=
(
string
)
tokBuffer
[
0
]
;
return
tok
;
}
//----< put token back into tokBuffer >------------------------------
public
void
pushBack
(
string
tok
)
{
tokBuffer
.
Insert
(
0
,
tok
)
;
}
public
override
bool
test
(
)
{
fileLogger
=
new
FileLogger
(
"
TokerTestOutput
"
)
;
consoleLogger
=
new
ConsoleLogger
(
)
;
consoleLogger
.
write
(
"
\
n
Testing
CToker
-
Tokenizer
"
)
;
consoleLogger
.
write
(
"
\
n
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
\
n
"
)
;
CToker
toker
=
new
CToker
(
)
;
string
file
=
"
shell
.
c
"
;
string
msg
;
if
(
!
toker
.
open
(
file
)
)
{
msg
=
"
Can
'
t
open
file
"
+
file
;
consoleLogger
.
write
(
msg
)
;
consoleLogger
.
write
(
new
string
(
'
-
'
,
msg
.
Length
)
)
;
}
else
{
msg
=
"
Processing
file
"
+
file
;
consoleLogger
.
write
(
msg
)
;
consoleLogger
.
write
(
new
string
(
'
-
'
,
msg
.
Length
)
)
;
string
tok
=
"
"
;
while
(
(
tok
=
toker
.
getTok
(
)
)
!
=
"
"
)
if
(
tok
!
=
"
\
n
"
)
{
consoleLogger
.
write
(
tok
)
;
fileLogger
.
write
(
tok
)
;
}
toker
.
close
(
)
;
}
return
true
;
}
//----< test stub >--------------------------------------------------
#
if
(
TEST
_
CTOKER
)
[
STAThread
]
static
void
Main
(
string
[
]
args
)
{
Console
.
Write
(
"
\
n
Testing
CToker
-
Tokenizer
"
)
;
Console
.
Write
(
"
\
n
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
=
\
n
"
)
;
CToker
toker
=
new
CToker
(
)
;
if
(
args
.
Length
=
=
0
)
{
Console
.
Write
(
"
\
n
Please
enter
name
of
file
to
tokenize
\
n
\
n
"
)
;
return
;
}
foreach
(
string
file
in
args
)
{
string
msg
;
if
(
!
toker
.
open
(
file
)
)
{
msg
=
"
Can
'
t
open
file
"
+
file
;
Console
.
Write
(
"
\
n
\
n
{
0
}
"
,
msg
)
;
Console
.
Write
(
"
\
n
{
0
}
"
,
new
string
(
'
-
'
,
msg
.
Length
)
)
;
}
else
{
msg
=
"
Processing
file
"
+
file
;
Console
.
Write
(
"
\
n
\
n
{
0
}
"
,
msg
)
;
Console
.
Write
(
"
\
n
{
0
}
"
,
new
string
(
'
-
'
,
msg
.
Length
)
)
;
string
tok
=
"
"
;
while
(
(
tok
=
toker
.
getTok
(
)
)
!
=
"
"
)
if
(
tok
!
=
"
\
n
"
)
Console
.
Write
(
"
\
n
{
0
}
"
,
tok
)
;
toker
.
close
(
)
;
}
}
Console
.
Write
(
"
\
n
\
n
"
)
;
//----< remove this >----------------------
}
#
endif
}
}
--< remove this >----------------------
}
#
endif
}
}
