#!/usr/local/bin/perl -w

#### LAST UPDATED: July, 2003 ####
# ColinPrep.pl version 0.1

###########################################################################
#
# This program pre-processes data in the SOURCE file to  a format suitable 
# for parsing by Collins Parser. It takes as input part of speech tagged 
# sentences such that:
#
# a. Token and its part of speech are separated by a forward slash `/'.
# b. All sentences are on separate lines.
#
# Output is placed in the DESTINATION file. 
#
# USAGE :  ColinPrep.pl [OPTIONS] NUM DESTINATION SOURCE > FILTER
#
# The FILTER collects lines with greater than NUM words.
#
# Following is a list of its functions:
#
# a. Count number of tokens in sentence; place count at start of sentence
# b. Sentences which have more than NUM tokens are printed to standard 
#    output and may be captured in a file by redirection.
# c. Separate word and pos by a space; eliminate the separating slash
# d. Eliminate additional tags for word if specified - identified by `|'
# e. Covert Brill tags such as `(' and `)' which are not understood by 
#    Collins parser into most similar tags such as `SYM'.
#
###########################################################################

# Copyright (C) 2000-2002
# Ted Pedersen, University of Minnesota, Duluth
# tpederse@d.umn.edu
# Mohammad Saif, University of Minnesota, Duluth
# moha0149@d.umn.edu
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

# START OF PROGRAM
use Getopt::Long;
# CHANGE Dec, 03
# use utf8;

# Check if any options specified
GetOptions("version", "help");
if ( defined $opt_help )    { $help = $opt_help; }
else                        { $help = 0; }
# Print help if asked
if ( $help ) {
    &help();
    exit; }
# if version has been requested, show version
if ( defined $opt_version ) {
    $opt_version = 1;
    &showVersion();
    exit; }

# Check if command line options have been provided
# If not print usage information
if ( $#ARGV < 2 ) {
    print "Insufficient command line arguments.\n";
    &usage();
    exit;}

$NUM = shift();
# Check if NUM is a positive integer
if($NUM !~ /^\d+$/)     {
    print "NUM must be a positive integer.\n";
    &usage();
    exit;}

$opfl = shift();
$inpfl = shift();

if(-e $opfl) {  
    print "File/Directory with same name as DESTINATION already exists. Press `y' to overwrite\n";
    $resp = <STDIN>;
    chomp($resp);
    if($resp eq "y" || $resp eq "Y") {
	print "Overwriting the file.\n"; }
    else{
	exit; } }

# Check if SOURCE exists
if(!(-f $inpfl)) {
    print "SOURCE file does not exist.\n";
    &usage();
    exit; }

# Open the various files
open(INP,"<$inpfl");
open(OP,">$opfl");

$maxwd = 0;

# Read lines from input file
while($ln = <INP>) {
	# Ignore blank lines
	if($ln =~ /^\s*$/){
	    next;}

	# Eliminate white space pre-ceeding and at end of sentence
	$ln =~ s/^(\s*)//;
	$ln =~ s/(\s*)$//;

	# Capture and print number of tokens in sentence
	$tokens = 1;
	# CHANGE Jun1
	# while($ln =~ /[^\s]+\s/g) 	{ $tokens++; }
	while($ln =~ /\s+/g)     { $tokens++; }

	## TMP
	print "tokens: $tokens\n";
	######

	if($tokens > $NUM)		{ 
		$maxwd++;
		print "$ln\n";
		next; }
	print OP "$tokens ";	

	# Conversion of Brill pos tags not understood by Collin
	# Replace `//' (pretag symbol) with `/'
        $ln =~ s/([^\s])\/\/([^\s\/])/$1\/$2/g;
	# Replace ( and ) pos tags with SYM
	$ln =~ s/\/\((\s|$)/\/SYM$1/g;
	$ln =~ s/\/\)(\s|$)/\/SYM$1/g;

	# Replace quote symbol with ``
	$ln =~ s/"/``/g;

	# Replace / pos tag with CC
	$ln =~ s/\/\/(\s|$)/\/CC/g;

	# Replace `/' with ` '
	# Take care of //CC first: becomes / CC
	$ln =~ s/(^|\s)\/\//$1\/ /g;
	# Take care of general case - blah/BLH becomes blah BLH

	# CHANGE Dec, 03	
	# $ln =~ s/([^\s])\//$1 /g;
	$ln =~ s/\/([^\s])/ $1/g;

	# Remove the optional tag |
	$ln =~ s/\|[^\s]*//g;	
     	print OP "$ln\n"; }

print "\n\nMax wd: $maxwd\n\n";

# Subroutine to print help message
sub help{
    print "Usage: ColinPrep.pl [OPTIONS] NUM DESTINATION SOURCE > FILTER\n\n";
    print "This program pre-processes data in the SOURCE file to  a format suitable for parsing by Collins Parser. It takes as input part of speech tagged sentences in a format created by the Brill Part of Speech Tagger. Output is placed in the DESTINATION file. The FILTER collects lines with greater than NUM words.\n\n"; 
    print "OPTIONS:\n";
    print "  --version    Prints the version number.\n";
    print "  --help       Prints the help message.\n\n";}

sub usage{
    print "Usage: ColinPrep.pl [OPTIONS] NUM DESTINATION SOURCE > FILTER\n";
    AskHelp();}
# Message describing way to get help
sub AskHelp{
    print "Type ColinPrep.pl --help for help.\n";}
# function to output the version number
sub showVersion
{
    print STDERR "ColinPrep.pl      -        version 0.1\n";
    print STDERR "A component of the parSenseval 0.1\n";
    print STDERR "Copyright (C) 2000-2003, Saif Mohammad and Ted Pedersen\n";
}

