#!/usr/local/bin/perl

# find collocations that occur to left or right of ambiguous word
# output as regular expressions /word/ with attribute names
# 	to serve as input to xml2arff
#
# can do the same with gram2 by simply grepping on <head>
#
# ted pedersen 24may01
# tpederse@d.umn.edu

sub usage {
    select(STDERR);
    print <<USAGE_END;
Usage: $0 frequency < input > output

words that occur < frequency times will not be counted
input is from stdin 
output is to stdout 
this program filters out all non-word characters (\W)

USAGE_END

    exit 1;
}

# If there are no command line arguments then issue usage message
# this isn't foolproof since invalid arguments are not detected

($#ARGV >= 0) || &usage;

# get frequency cutoff value

$freq = shift @ARGV;

# get stoplist

$stoplist = shift @ARGV;

# perform word counts

while (<>) {
	/<head>\w+<\/head>\s+\w+/;     
	$coll{$&}++;
	/\w+\s+<\head>\w+<\/head>/;     
	$coll{$&}++;
}

#
# produce output files
#
# remove words from stoplist

foreach (sort keys %coll) {
	if ($coll{$_}>=$freq) {
		print("/$_/ \@name=$_\n");
	}
}
	
