#!/usr/local/bin/perl  -w

# create a stoplist from a list of input files
# the criteria are as follows - a word shall be included in the
# stop list if it occurs more than freq times, and occurs in
# all files submitted 
#
# ted pedersen 25may01
# tpederse@d.umn.edu

sub usage {
    select(STDERR);
    print <<USAGE_END;
Usage: $0 frequency list-of-files

create stop list from a list of files, where each word in the stop
list occurs in each file more than freq times

USAGE_END

    exit 1;
}

# If there are no command line arguments then issue usage message
# this isn't foolproof since invalid arguments are not detected

($#ARGV >= 0) || &usage;

# get command line arguments

$freq = shift @ARGV;

# perform word counts

$i=0;
foreach $file (@ARGV) {	
	%hash=();
	open (FILE , "$file") || die "can't open $file $!\n";
	while (<FILE>) {
		@words = split(/\s+/,$_);
		foreach $word (@words) {
			if ($word =~ /[a-z]+/) {
				$hash{$&}++;
				}
			}
	}
	$i++;

	# you are finished with this file, find all the words that
	# occur more than freq times. Only keep these.

	foreach (sort keys %hash) {
		if ($hash{$_} >= $freq) {
			$stoplist{$_}++;
		}
	}
}


## printf( "files %d\n",$i);

# check to see how many files the words that occur above freq
# times will occur in. 

foreach (sort keys %stoplist) {
		if ($stoplist{$_} == $i) {
			print "/$_/\n";
		}
}

