#!/usr/bin/perl -w

#######################################
## Get a results file or exit
#######################################
if ($#ARGV != 1) {
	print STDERR "Usage: $0 -flags resultsfile\n";
	print STDERR "-n: non-restrictive (uses frame element boundaries, i.e., classification task)\n";
	print STDERR "-r: restrictive (without frame element boundaries)\n";
	print STDERR "-d: details for each test item\n";
	die "\n";
}
$flags = $ARGV[0];
$results_file = $ARGV[1];

#######################################
## Load the answer key
#######################################

open (KEY, "< SensSemRolesTestAns.txt") or die "\nUnable to open the answer key file \"wngloss.key\"\n";
open (P0, "< RoleAtPos0.txt") or die "\nUnable to open Role file at 0 position\n";

#######################################
## Store the answer key in a hash
#######################################
%answers = ();
while($line = <KEY>){
	chomp $line;
	$line =~ / /;
	$frameInst = $`;
	$frameElems = $';
	$answers{$frameInst} = $frameElems;
}
close KEY;

%pos0 = ();
while($line = <P0>){
	chomp $line;
	$line =~ / /;
	$frameInst = $`;
	$frameElems = $';
	$pos0{$frameInst} = $frameElems;
}
close P0;

#######################################
## Read in and store the system's
## answers
#######################################
open (RESULTS, "< $results_file") or die "\nUnable to open the results file: $results_file\n";
%results = ();
while($line = <RESULTS>){
	chomp $line;
	$line =~ / /;
	$frameInst = $`;
	$frameElems = $';
	$results{$frameInst} = $frameElems;
}
close RESULTS;

#######################################
## Score the system's answers
#######################################
%frItemCount = ();
%feItemMatch = ();
%feItemAttempted = ();
%feItemOverlap = ();
@ansarray = keys(%answers);
@ansarray = sort(@ansarray);
foreach $key (@ansarray){
	$frameSet = $answers{$key};
	$key =~ /\./;
	$frame = $`; # get the frame name
	@frAnsElems = split / /, $frameSet;
	if($flags =~ /d/){
		print "$key\n";
	}
	if(exists $results{$key} && $results{$key} ne ""){
		$sysResults = $results{$key};
		if($flags =~ /d/){
			print "\tSystem results: $sysResults\n";
		}
		@frResElems = split / /, $sysResults;
		for($i=0;$i<$#frAnsElems;$i+=2){
			$frElem = $frAnsElems[$i];
			$frExtent = $frAnsElems[$i+1];
			if($flags =~ /d/){
				print "\t$frElem $frExtent: ";
			}
			# count the number of non-null frames (for recall score)
			if($frExtent ne "(0,0)" || (exists $pos0{$key} && "$frElem $frExtent" eq $pos0{$key})){
				if(exists $frItemCount{$frame}){
					$frItemCount{$frame}++;
				} else {
					$frItemCount{$frame} = 1;
				}
			}
			$frFound = 0;
			$changed = 0;
			my $mark = 0;
			my $overlap = 0;
			for($j=0;$j<$#frResElems;$j+=2){
				if($frElem eq $frResElems[$j]){
					if($frResElems[$j+1] ne "(0,0)" || (exists $pos0{$key} && "$frElem $frExtent" eq $pos0{$key})){
						$frFound = 1;
						$frExtent =~ /\((.*?),(.*?)\)/;
						$fbeg = $1;
						$fend = $2;
						$flen = $fend - $fbeg + 1;
						$frResElems[$j+1] =~ /\((.*?),(.*?)\)/;
						$rbeg = $1;
						$rend = $2;
						$rlen = $rend - $rbeg + 1;
						# check to see if system's answer doesn't overlap
						# with correct answer
						if($rend < $fbeg || $rbeg > $fend){
							$frFound = 0 if $mark < 1;
							$changed = 1 if $mark < 1;
							$mark = 1 if $mark < 1;
							if($flags =~ /d/){
								#print "Score = 0.0 (Correct FE, but no overlap)\n";
							}
						}
						#determine the degree of overlap in the element boundaries
						#if($frFound && $flags =~ /r/){
						if($frFound ){
							$mark = 8 if $mark < 8;
							# score overlap between system result and official result
							if($frExtent eq $frResElems[$j+1]){
								$temp = 1.0;
								$mark = 9;
							} elsif($rbeg >= $fbeg && $rend <= $fend){
								$temp = $rlen/$flen;
							} elsif($rbeg >= $fbeg && $rend > $fend){
								$temp = ($rlen-($rend-$fend))/$flen;
							} elsif($rbeg < $fbeg && $rend <= $fend){
								$temp = ($rlen-($fbeg-$rbeg))/$flen;
							} elsif($rbeg < $fbeg && $rend > $fend){
								$temp = 1.0;
							}
							$overlap = $temp if $overlap < $temp;
							if($flags =~ /d/){
								#printf "Score = 1.0 (Overlap score = %4.2f)\n", $overlap;
							}
						} elsif($frFound && $flags =~ /d/){
							#print "Score = 1.0 (Correct assignment)\n";
						}
					} else {
						$changed = 1 if $mark < 3;
						$mark = 3 if $mark < 3;
						#print "Score = 0.0 (Null assignment is incorrect)\n";
					}
					last if $mark == 9;
				}
			}
			if ($mark == 9 || $mark == 8) {
				if ($flags =~ /r/) {
					if(exists $feItemOverlap{$frame}){
						$feItemOverlap{$frame} += $overlap;
					} else {
						$feItemOverlap{$frame} = $overlap;
					}
					if($flags =~ /d/){
						printf "Score = 1.0 (Overlap score = %4.2f)\n", $overlap;
					}
				} elsif ($flags =~ /d/) {
					print "Score = 1.0 (Correct assignment)\n";
				}
			} elsif ($mark == 1) {
				if($flags =~ /d/){
					print "Score = 0.0 (Correct FE, but no overlap)\n";
				}
			} elsif ($mark == 3) {
				if($flags =~ /d/){
					print "Score = 0.0 (Null assignment is incorrect)\n";
				}
			}	
			if($frFound){
				if(exists $feItemMatch{$frame}){
					$feItemMatch{$frame}++;
				} else {
					$feItemMatch{$frame} = 1;
				}
			}
			if((!$changed && !$frFound && ($frExtent ne "(0,0)" || (exists $pos0{$key} && "$frElem $frExtent" eq $pos0{$key}))) && $flags =~ /d/){
				print "Score = 0.0 (No result provided)\n";
			}
				
		}
		for($i=0;$i<$#frResElems;$i+=2){
			$frElem = $frResElems[$i];
			$frExtent = $frResElems[$i+1];
			if($frExtent ne "(0,0)" || (exists $pos0{$key} && "$frElem $frExtent" eq $pos0{$key})){
				if(exists $feItemAttempted{$frame}){
					$feItemAttempted{$frame}++;
				} else {
					$feItemAttempted{$frame} = 1;
				}
			}
		}
	} else {
		# system gave no answers, so just count items in answer key
		if($flags =~ /d/){
			print "\tSystem results: No system results for this item\n";
		}
		for($i=0;$i<$#frAnsElems;$i+=2){
			$frElem = $frAnsElems[$i];
			$frExtent = $frAnsElems[$i+1];
			# count the number of non-null frames (for recall score)
			if($frExtent ne "(0,0)" || (exists $pos0{$key} && "$frElem $frExtent" eq $pos0{$key})){
				if(exists $frItemCount{$frame}){
					$frItemCount{$frame}++;
				} else {
					$frItemCount{$frame} = 1;
				}
			}
		}
	}
}

#######################################
## Summarize the system's scores
## by frame and overall
#######################################
@cntarray = keys(%frItemCount);
@cntarray = sort(@cntarray);
$totItems = 0;
$totMatch = 0;
$totAttempted = 0;
$totOverlap = 0;
print "\n\nScores for $results_file\n";
print "\nFrame Scores:\n";
foreach $key (@cntarray){
	print "Frame: $key\n";
	$numitems = $frItemCount{$key};
	$totItems += $numitems;
	if(exists $feItemMatch{$key}){
		$numcorrect = $feItemMatch{$key};
	} else {
		$numcorrect = 0;
	}
	$totMatch += $numcorrect;
	if(exists $feItemAttempted{$key}){
		$numattempted = $feItemAttempted{$key};
	} else {
		$numattempted = 0;
	}
	$totAttempted += $numattempted;
	if($numattempted){
		$precision = $numcorrect/$numattempted;
	} else {
		$precision = 0.0;
	}
	printf " precision: %5.3f ($numcorrect correct of $numattempted attempted)\n", $precision;
	if($flags =~ /r/){
		if(exists $feItemOverlap{$key}){
			$overlap = $feItemOverlap{$key};
		} else {
			$overlap = 0;
		}
		$totOverlap += $overlap;
		if($numattempted){
			$overlap = $overlap/$numattempted;
		} else {
			$overlap = 0;
		}
		printf " overlap: %5.3f ($numcorrect correct of $numattempted attempted)\n", $overlap;
	}
	$recall = $numcorrect/$numitems;
	printf " recall: %5.3f ($numcorrect correct of $numitems in total)\n", $recall;
	$attempted = ($numattempted/$numitems) * 100.0;
	printf " attempted: %4.1f percent ($numattempted attempted of $numitems in total)\n", $attempted;
}

print "\nTotal Scores:\n";
$precision = $totMatch/$totAttempted;
printf " precision: %5.3f ($totMatch correct of $totAttempted attempted)\n", $precision;
if($flags =~ /r/){
	$overlap = $totOverlap/$totAttempted;
	printf " overlap: %5.3f ($totMatch correct of $totAttempted attempted)\n", $overlap;
}
$recall = $totMatch/$totItems;
printf " recall: %5.3f ($totMatch correct of $totItems in total)\n", $recall;
$attempted = ($totAttempted/$totItems) * 100.0;
printf " attempted: %4.1f percent ($totAttempted attempted of $totItems in total)\n", $attempted;