#!/usr/local/bin/perl -w

#### LAST UPDATED: Jul, 2003 ####
# xml2sval.pl version 0.1

###########################################################################
#
# This program takes as input two files. SOURCE1 and SOURCE2 have 
# the same tokens and in the same order. However, SOURCE1 and SOURCE2
# might have different XML tags. `xml2sval.pl' places a copy of  
# SOURCE2 in DESTINATION and adds the xml tokens from SOURCE1 at
# corresponding positions relative to the tokens. Each
# (non-xml) token in the SOURCE1 is compared with the corresponding token in 
# SOURCE2. Any mismatch causes the relevant lines of the two
# files to be printed in the file FILTER. This file is not created if
# there is perfect synchronization among the two files. The contextual
# alphabetic characters are converted to the lower case. 
# Other functions include:
#
# a. Non-contextual data within square brackets is put in angular braces
# b. Non-contextual data within curly braces (found in Senseval-1 data)
#    also placed in angular braces. An exception to thi sbeing typo
#    error information in curly braces; the information is replaced
#    by the correct spelling.
# c. Character references which were eliminated during pre-processing are
#    put in angular braces 
#
# USAGE : xml2sval.pl [OPTIONS] DESTINATION SOURCE1 SOURCE2 
#
###########################################################################

# Copyright (C) 2000-2002
# Ted Pedersen, University of Minnesota, Duluth
# tpederse@d.umn.edu
# Mohammad Saif, University of Minnesota, Duluth
# moha0149@d.umn.edu
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

# START OF PROGRAM
use Getopt::Long;

# Check if any options specified
GetOptions("version", "help");
if ( defined $opt_help )    { $help = $opt_help; }
else                        { $help = 0; }
# Print help if asked
if ( $help ) {
    &help();
    exit; }
# if version has been requested, show version
if ( defined $opt_version ) {
    $opt_version = 1;
    &showVersion();
    exit; }

# check if command line options have been provided
# If not print usage information
if ( $#ARGV < 2 ) {
    print "Insufficient command line arguments.\n";
    &usage();
    exit; }

# Store the files specified by the command line 
$opfl = shift @ARGV;
$xmlfl = shift @ARGV;
$tgdfl = shift @ARGV;

# Check if DESTINATION file specified already exist
if(-f $opfl) {
    print "Destination file already exists. Press `y' to overwrite\n";
    $resp = <STDIN>;
    chomp($resp);
    if($resp eq "y" || $resp eq "Y") {
        print "Overwriting the file.\n"; }
    else {
        exit; } }

# Check if Tagged file exists
if(!(-f $tgdfl)) {
    print "Tagged file does not exist.\n";
    &usage();
    exit; }
# Check if XML file exists
if(!(-f $xmlfl)) {
    print "XML file does not exist.\n";
    &usage();
    exit; }


# Open files involved
open(XML, "<$xmlfl") or die "could not open xml file";
open(TGD, "<$tgdfl") or die "could not open tagged file";
open(OUTPUT, ">$opfl") or die "could not open file";

# Read lines from the SOURCE file - line by line
$lnno=0;
$incontext = 0;
while(<XML>) {
    $xmlln = $_;
    
    # Increment line number of the line being processed in SOURCE1
    # displayed on stdout and FILTER in case of mismatch
    $lnno++;
    # No new line character should exist within angular braces
    if(/^\s?<!DOCTYPE corpus SYSTEM\s?\n/) {
	s/^(\s?<!DOCTYPE corpus SYSTEM\s?)\n/$1/;
	$exln = <XML>;
	$_ = $_.$exln; }

    # Jul16
    # Senseval-1 data has few instances with a lone square bracket
    # float-v.800152 has token `Horrors]' which is garbled and hence removed
    s/^Horrors\] / /;
    # promise-n.800121 and shirt-n.800409 have a lone ] which is removed
    s/(The people have won, Aylwin is President)\]/$1/;
    s/(^\`No, sir)\]/$1/;
    
    if(/^\s*<context>\s*$/) {
	print OUTPUT;
        $incontext = 1;
	next;
    }
    
    if(/^\s*<\/context>\s*$/) {
        $incontext = 0;
    }
    
    # Print whole line if not context
    if($incontext == 0){
	print OUTPUT;
	next;
    }
   
    # Jul16 
    # contextual data within square braces is placed between curved braces
    # s/\[(\s?[\w\.]+\s[\w\.]+[^\]]*)\]/\($1\)/g;
    s/\[(\s*[^\]\s]+\s+[^\]\s]+\s*[^\]]*)\]/\($1\)/g;
    # typo information as in {} braces is replaced by correct spelling
    s/\{typo bad=\"[^\"]*\",\s*good=\"([^\"]*)\"[^\}]*\}/ $1 /g;
    # Non-contextual data within square brackets is put in angular braces
    s/(\[[^\]]+\])/<$1>/g;

    # Jul17
    # s/\./ \. /g;
    # Jul17
    # s/(^|[^<]),/$1 , /g;

    # Eliminating data in {}
    # s/(\{[^\}]*\})/<$1>/g;

    # Jul17 Jul19 back
    # while(/(\{[^\}]*\})/g)
    # {
#	$curly = $1;
#	$oldcurly = $curly;
#	$curly =~ s/\s+/curlyspace/g;
#	$curly =~ s/\{/startcurly/;
#	$curly =~ s/\}/endcurly/;
#	s/$oldcurly/$curly/g;
#    }
#    s/startcurly\s*/<\{/g;
#    s/\s*endcurly/\}>/g;

    # Jul16
    s/(^|[^<])\)/$1 \) /g;
    s/(^|[^<])\(/$1 \( /g;
   
    # CHANGE Jul6 
    # -- is tokenised to  --
    s/--/ -- /g;

    # CHANGE Jul6
    # Tokenize non compound forming `-'
    # s/([^-])-([^-])/$1 - $2/g;
    # s/([\w])\s+-\s+([\w])/$1-$2/g;
    
    # Replace those character references which were not eliminated during pre-processing
    # by appropriate characters
    while(/&mdash;/g) {
	s/&mdash;/ -- /g; }
    while(/&ndash;/g) {
	s/&ndash;/ - /g; }
    while(/&([be]|[lr]d)?quo;/g) {
	s/&([be]|[lr]d)?quo;/ " /g; }
    while(/&amp;/g) {
        s/&amp;/ & /g; }
    while(/&lt;/g){
        s/&lt;/ < /g;}
    while(/&gt;/g){
        s/&gt;/ > /g;}
    while(/&apos;/g){
        s/&apos;/ ' /g;}
    while(/&lsquo;/g){
        s/&lsquo;/ ' /g;}
    while(/&rsquo;/g){
        s/&rsquo;/ ' /g;}
    # Jul16
    while(/(^|[^<])&lsqb;([^>]|$)/g){
        s/(^|[^<])&lsqb;([^>]|$)/$1 [ $2/g;}
    while(/[^<]&rsqb;[^>]/g){
        s/&rsqb;/ ] /g;}
    while(/&num;/g){
        s/&num;/ \# /g;}
    # Jul19 for interest
    while(/&lcub;/g){
        s/&lcub;/ \{ /g;}
    while(/&rcub;/g){
        s/&rcub;/ \} /g;}
    # Jul16
    while(/&equals\s*;/g){
        s/&equals\s*;/ = /g;}

    # Put the remaining character references in angular braces
    # CHANGE May
    # s/(^|[^<])(&[^&;\.\s]*[;\.])/$1 <$2> /g;
    # CHANGE May30
    s/(&[^&;\.\s]*[;\.])/ <$1> /g;
    
    # Ignore empty lines
    if(/^\s*$/) {
	next; }
   
    # tokenise punctuations
    s/</ </g;
    s/>/> /g;

    # Keep the curly tokens together
    # s/\s*<&curlyspace\;>\s*/<&curlyspace\;/g;
    #### CHANGE May30
    s/(^|\s)([^\s&<\;]+)\;/$1$2 \; /g;
    #### s/(^|[^<;]+);/$1 ; /g;

    s/(^|[^<]):/$1 : /g;
    # CHANGE
    # Jul17
     s/(^|[^<]),/$1 , /g;
    s/(^|[^<])\?/$1 \? /g;
    # Jul16
    # s/(^|[^<])\./$1 \. /g;

    # Jul17
     s/\./ \. /g;

    s/\"/ \" /g;
    # CHANGE FEB17
    while(s/(\{[^\s\}]*)\s+\"\s*/$1\"/g) {;}
    # CHANGE Jul16
    # while(/(^|[^<\s])\'/) {
    # 	s/(^|[^<\s])\'/$1 \'/;
    #}
    s/(^|[^<\'])\'/$1 \'/g;
    s/(^|[^<])(\`(\`)?)/$1 $2 /g;
    # s/(^|[^<])\'/$1 \' /g;
    ### s/(^|[^<])\`/$1 \` /g;
    ### s/' '/''/g;
    # Take care of ``
    ### s/` `/``/g;

    # Jul16
    # s/(^|[^<])\)/$1 \) /g;
    # s/(^|[^<])\(/$1 \( /g;

    s/([^<])\[/$1 \[/g;
    s/\]([^>])/\] $1/g;
    # CHANGE MAY30
    s/(^|[^<])&/$1 & /g;
    # CHANGE MAY30
    # s/& (\w+) ;/&$1;/g;
    s/&\s*(\w+)\s*\;/&$1\;/g;

    s/\// \/ /g;
    # s/([^-])-([^-])/$1 - $2/g;
    # s/([a-zA-Z])\s+-\s+([a-zA-Z])/$1-$2/g;
    s/!/ ! /g;
    s/\+/ \+ /g;

    s/(^|[^<])\$/$1 \$ /g;
    s/(^|[^<])%/$1 % /g;
    s/(^|[^<])\#/$1 \# /g;

    s/\226/ \226 /g;
    s/\227/ \227 /g;
    s/[^<]£[^<]/ £ /g;
    s/\*/ \* /g;
    s/\{/ \{ /g;
    s/\}/ \} /g;
    # CHANGE FEB17
    s/<\s*{\s*/<\{/g;
    s/\s*\}\s*>/\}>/g;
    # END CHANGE
    s/\\ \// \\\/ /g;
    s/ \/  \/ /\//;
    if (/\/ [^\s]+[\]>]/) {
	s/ \/ ([^\s]+[\]>])/\/$1/g; }

    # Jul16
    s/([a-zA-Z0-9])(-+)(\s|$)/$1 $2$3/g;
    s/(^|\s)(-+)([a-zA-Z0-9])/$1$2 $3/g;
    # Jul16
    s/(\'[^\s]+)-([^\s]+)/$1 - $2/g;

    # Jul17  Jul19-commented back for interest
     while(/(\{[^\}]*\})/g)
     {
       $curly = $1;
        $oldcurly = $curly;
        $curly =~ s/\s+/curlyspace/g;
        $curly =~ s/\{/startcurly/;
        $curly =~ s/\}/endcurly/;
        s/$oldcurly/$curly/g;
    }
    s/startcurly\s*/<\{/g;
    s/\s*endcurly/\}>/g;

    # Jul19 for interest
    # while(/&lcub;/g){
     #    s/&lcub;/ \{ /g;}
    # while(/&rcub;/g){
    #    s/&rcub;/ \} /g;}


    # PATCH
    if(/<\{curlyspacephelancurlyspace'scurlyspace\}>/) {; }
#	s/<\{curlyspacephelancurlyspace'scurlyspace\}>/\{ phelan  's  \}/; }
    # DBUG
    if (/phelan/) {
           print;
    }
   
    # Initialise word counter of XML line	
    $cntr1=0;
    $#xmlwd = -1;
    # CAPTURE TOKENS SEPARATED BY WHITE SPACE IN VARIABLE
    for $chunk1 (split) {
	if($chunk1 !~ /[^\s]/)	{ next; }
	$cntr1++;
	$xmlwd[$cntr1]=$chunk1; }
    
    # Initialise XML tag counter and empty line flag	
    $xmlcnt=0;
    $lnempty=1;
    $parcnt=1;
    # print OUTPUT "Total contextual Tokens:$cntr1 ";
    $tmphdtagfl = 0;
    # Loop till all tokens in XML line exhausted
    for($i=1;$i<($cntr1+1);$i++) {
	# If token is xml tag, print
	# Jul16
	# if($xmlwd[$i] =~ /<[^\s]+>|\226|\227|\\\//) {
	 if($xmlwd[$i] =~ /<[^\s]+|\226|\227|\\\//) {
	    # CHANGE FEB17
	    $xmlwd[$i] =~ s/curlyspace/ /g;

		#if (/^Among other/) {
                #       print "XML WDS: $xmlwd[$i]\n";
                #}
	    # If <head> tag dont print yet; store tag
	    if($xmlwd[$i] =~ /<head[^>]*>/) {
		$tmphdtag = $xmlwd[$i];
		$tmphdtagfl = 1;
		# DBUG
    		#if (/^Among other/) {
           	#	print "HDTAG: $tmphdtag\n";
    		#}
	    }
	    elsif ($xmlwd[$i] =~ /<head[^>]*$/) {
		$tmphdtag = $xmlwd[$i];
		$tmphdtagfl = 1;
		$i++;
		while($xmlwd[$i] =~ /[^>]$/) {
                 	$tmphdtag = $tmphdtag." ".$xmlwd[$i];	
                	# $xmlcnt++;
                	$i++ }
		$tmphdtag = $tmphdtag." ".$xmlwd[$i];
		# $xmlcnt++;
		#$i++;
		# DBUG
                # if (/^Among other/) {
            #            print "HDTAG: $tmphdtag\n";
                # }
	    }
	    else {
		print OUTPUT "$xmlwd[$i] ";
		if($xmlwd[$i] =~ /<[^\s>]+$/) { 
			$i++;
			while($xmlwd[$i] !~ />$/) {
				print OUTPUT "$xmlwd[$i] ";
				$i++;	
			}
			print OUTPUT "$xmlwd[$i] ";
		}
	    }
	    # Increment xml tag ctr
	    $xmlcnt++;
	    next; }
	# Print xml tags with multiple tokens within it	
	if($xmlwd[$i] =~ /<[^>]+$/) {
	    print OUTPUT "$xmlwd[$i] ";
	    $xmlcnt++;
	    $i++;
	    while($xmlwd[$i] =~ /[^>]$/) {
		print OUTPUT "$xmlwd[$i]";
		$xmlcnt++;
		$i++ }
	    print OUTPUT "$xmlwd[$i] ";
	    $xmlcnt++;
	    next; }
	
	# Read new line from TAGGED file only if XML line is devoid of any non-xml tokens
	if($lnempty==1) {
	    # Read new line from TAGGED file
	    $tgdln = <TGD>;
	    if(!(defined($tgdln)))	{ exit; }

	    # If tagged line is empty read another line	
	    while($tgdln =~ /^\s*$/) {
		$tgdln = <TGD>;
	    }

	    $tgdln =~ s/^\s*//;
	    $tgdln =~ s/\s*$//;
	    $cntr2=0;
	    @tgdwd="";
	    $#tgdwd = -1;
	    # CAPTURE TOKENS SEPARATED BY WHITE SPACE IN VARIABLE
	    for $chunk1 (split(/\s/,"$tgdln")) {
		$cntr2++;
		$tgdwd[$cntr2]=$chunk1; }
	    $tgdwdno = @tgdwd; }	
	    $lnempty=0;

	    # Print xml tags
	    while(($parcnt < ($tgdwdno-1)) && ($tgdwd[$parcnt] =~ /^</)) {
		# Print one xml tag
		while(1) {
		    print OUTPUT "$tgdwd[$parcnt] ";
		    $parcnt++;
		    if($tgdwd[$parcnt-1] =~ />$/) 	{ last; }
		}
	    }

	    # CHECK
	    if(!(defined $tgdwd[$parcnt])) {
		# Patch for line  &M.
		# if(/reverse-engi/) {
		#if(/Olivetti ,  Europe/) {
		#if(/already markets in Europe for/) {
		#if(/In a meeting on how to trim fat/) {
		print "Mismatch\n\n";
		print "Tagged Line: $tgdln\n";
		print "Orig: $_\n\n";
		exit;
	    }
	    if($tgdwd[$parcnt] =~ /^[a-z]+$/) {
		# check only if word is alphanumeric (" to '' conversion)
    if($xmlln !~ /($tgdwd[$parcnt])/) {
	print "Mismatch: $xmlln";
	print "WD: $tgdwd[$parcnt]\n";
	print "Tagged line: $tgdln\n";
	exit;
    }
}

# print head tag if present
if($tmphdtagfl) {
    print OUTPUT "$tmphdtag ";
    $tmphdtagfl = 0;
}

# Print the contextual token
$lcwd = lc($tgdwd[$parcnt]);
$parcnt++;
print OUTPUT "$lcwd "; 
} 
# Print remaining xml tags from parsed line
while(($parcnt < ($tgdwdno-1)) && ($tgdwd[$parcnt] =~ /^</)) {
    # Print one xml tag
    while(1) {
	print OUTPUT "$tgdwd[$parcnt] ";
	$parcnt++;
	if($tgdwd[$parcnt-1] =~ />$/)   { last; }
    }
}
print OUTPUT "\n"; }

# Subroutine to print help message
sub help {
    print "Usage: xml2sval.pl [OPTIONS] DESTINATION SOURCE1 SOURCE2\n\n"; 
    print "This program takes as input two files. SOURCE1 and SOURCE2 have the same tokens and in the same order. However, SOURCE1 and SOURCE2 might have different XML tags. `xml2sval.pl' places a copy of SOURCE2 in DESTINATION and adds the xml tokens from SOURCE1 at corresponding positions relative to the tokens. Each (non-xml) token in the SOURCE1 is compared with the corresponding token in SOURCE2. Any mismatch causes the relevant lines of the two files to be printed in the file FILTER. This file is not created if there is perfect synchronization among the two files. The contextual alphabetic characters are converted to the lower case.\n\n";
    print "OPTIONS:\n";
    print "  --version    Prints the version number.\n";
    print "  --help       Prints the help message.\n\n";}

sub usage {
    print "Usage: xml2sval.pl [OPTIONS] DESTINATION SOURCE1 SOURCE2\n"; 
    AskHelp(); }
# Message describing way to get help
sub AskHelp {
    print "Type xml2sval.pl --help for help.\n"; }
# function to output the version number
sub showVersion
{
    print STDERR "xml2sval.pl      -        version 0.1\n";
    print STDERR "A component of the parSenseval 0.1\n";
    print STDERR "Copyright (C) 2000-2003, Saif Mohammad and Ted Pedersen\n";
}
