#!/usr/local/bin/perl

# this program could be replaced by count.pl in BSP v0.5
# however, since the Duluth systems in Senseval-2 were
# run using BSP v0.4, we continue to use that older method

#
# simple 1-gram counter to identify single word features
# this allows for frequency cutoffs
# output as regular expressions /word/ with attribute names
# 	to serve as input to xml2arff
#
# ted pedersen 22may01
# tpederse@d.umn.edu

sub usage {
    select(STDERR);
    print <<USAGE_END;
Usage: $0 frequency stoplist < input > output

words that occur < frequency times will not be counted
stoplist is a file, must be specified
input is from stdin 
output is to stdout 

USAGE_END

    exit 1;
}

# If there are no command line arguments then issue usage message
# this isn't foolproof since invalid arguments are not detected

($#ARGV >= 0) || &usage;

# get frequency cutoff value

$freq = shift @ARGV;

# get stoplist

$stoplist = shift @ARGV;
open (STOP, "$stoplist") || die "can't open $stoplist $!\n";

# perform word counts

while (<>) {
	@words = split(/\s+/,$_);
	foreach $word (@words) {
##	$word =~ 
##	(/(<head.*>\w+<lexelt=.*><\/head>)|(<sat.*>\w+<\/sat>)|
##		(\w+\.[nva])|((\w+)?(&(\w+);)+(\w+)?)|\w+/);

		if ($word =~ /[a-z]+/) {         
			$onegram{$&}++;
		}
	}

}

#
# load stop list
#

while(<STOP>) {
	chomp;
	/\/(\w+)\//;
	$stoplist{$1} = 1;
}

#
# produce output files
#

foreach (sort keys %onegram) {
	if (($onegram{$_}>=$freq) && ($stoplist{$_} != 1)) {
		print("/\\b$_\\b/ \@name=$_\n");
	}
}
	
