#!/usr/local/bin/perl

# nearly the same as count1.pl, except for token def

# simple 1-gram counter to identify single word features
# this allows for frequency cutoffs
# output as regular expressions /word/ with attribute names
# 	to serve as input to xml2arff
#
# ted pedersen 22may01
# tpederse@d.umn.edu

sub usage {
    select(STDERR);
    print <<USAGE_END;
Usage: $0 frequency stoplist < input > output

words that occur < frequency times will not be counted
stoplist is a file, must be specified
input is from stdin 
output is to stdout 
this program filters out all non-word characters (\W)

USAGE_END

    exit 1;
}

# If there are no command line arguments then issue usage message
# this isn't foolproof since invalid arguments are not detected

($#ARGV >= 0) || &usage;

# get frequency cutoff value

$freq = shift @ARGV;

# get stoplist

$stoplist = shift @ARGV;
open (STOP, "$stoplist") || die "can't open $stoplist $!\n";

# perform word counts

while (<>) {
	@words = split(/\s+/,$_);
	foreach $word (@words) {
#	$word =~ 
#	(/(<head>[\w\x80-\xff]+\w+<\/head>)|[\w\x80-\xff]+|\d+/);     
		if ($word =~ /[\w\x80-\xff]+/) {
			$onegram{$&}++;
		}
	}

}

#
# load stop list
#

while(<STOP>) {
	chomp;
	/\/([\w\x80-\xff]+)\//;
	$stoplist{$1} = 1;
}

#
# produce output files
#

foreach (sort keys %onegram) {
	if (($onegram{$_}>=$freq) && ($stoplist{$_} != 1)) {
		print("/\\b$_\\b/ \@name=$_\n");
	}
}
	
