reamhinneacs

#!/usr/bin/perl

use strict;
use warnings;
use Lingua::GA::Stemmer;
use Lingua::GA::Caighdean;

if ($#ARGV != 0 or $ARGV[0] !~ m/^(?:ABAIRT|ABAIRT-CH|(?:NN|YY|YN)[YN])$/) {
	print "Usage: reamhinneacs (ABAIRT|ABAIRT-CH|YYY|YNY|YNN|YYN|NNY|NNN)\n";
	exit 1;
}
binmode STDOUT, ":encoding(iso-8859-1)";
binmode STDERR, ":encoding(iso-8859-1)";
binmode STDIN, ":bytes";

my $cineal = $ARGV[0];
my $obj;
if ($cineal eq 'ABAIRT') {
	$obj = new Lingua::GA::Gramadoir;   # get_sentences
}
elsif ($cineal eq 'ABAIRT-CH') {
	$obj = new Lingua::GA::Caighdean(fix_spelling => 0);
}
else {
	$obj = new Lingua::GA::Stemmer;
}

sub process_me
{
	(my $foinse, my $sprioc) = @_;
	(my $docno) = $foinse =~ m/([0-9]+)\.txt$/;
	open (SPRIOC, ">", $sprioc) or die "Could not open target file $sprioc: $!\n";
	open (FOINSE, "<", $foinse) or die "Could not open source file $foinse: $!\n";
	print SPRIOC "<DOC>\n<DOCNO>$docno</DOCNO>\n";
	if ($cineal =~ m/^ABAIRT/) {
		local $/;
		my $text = <FOINSE>;
		close FOINSE;
		if ($cineal eq 'ABAIRT') {
			foreach (@{$obj->get_sentences($text)}) {
				s/&quot;/"/g;
				s/&lt;/</g;
				s/&gt;/>/g;
				s/&amp;/&/g;
				# don't truncate here since these files
				# are used for indexing; do it in aimsigh.cgi
				print SPRIOC "$_\n";
			}
		}
		else {  # ABAIRT-CH
			print SPRIOC $obj->caighdean($text);
		}
	}
	else {
		my $dum = <FOINSE>;  # <DOC>
		$dum = <FOINSE>;     # <DOCNO>...</DOCNO>
		while (<FOINSE>) {  # now reading TREC file from ABAIRT*
			chomp;
			last if /^<\/DOC>$/;
			if ($cineal =~ m/^YY/) {
				my $s = $obj->stem($_);  # should be 1 line
		# strip mutation below is needed because the
		# stemmer is "too smart" - if a word is completely unknown
		# it won't strip mutations at all.  This is smart for
		# stuff like "chemical" but bad for new truly-Irish words
		# We choose then to strip chemical -> cemical in order
		# to make the equivalence relation defined by the 
		# aimsigh.com stemming option strictly coarser than the
		# "infhillte" option alone.
				while ($s =~ m/<[A-Z][^>]*>([^<]+)<\/[A-Z]>/g) {
					my $tok = $1;
					$tok =~ s/&quot;/"/g;  # in URLs only
					$tok =~ s/&lt;/</g;
					$tok =~ s/&gt;/>/g;
					$tok =~ s/&amp;/&/g;
					foreach (split / /,$tok) {  # "ar bith", etc.
						print SPRIOC $obj->strip_mutation($obj->tolower($_))." ";
					}
				}
			}
			else {  # $cineal eq 'YN.' or 'NN.'
				my $tokes = $obj->tokenize($_);
				foreach my $tok (@$tokes) {
					$tok = $obj->tolower($tok);
					$tok = $obj->strip_mutation($tok) if ($cineal =~ m/^Y/);
					print SPRIOC $tok." ";
				}
			}
			print SPRIOC "\n";
		}  # while (<FOINSE>)
		close FOINSE;
	}
	print SPRIOC "</DOC>\n";
	close SPRIOC;
}


my $base;
if ($cineal =~ m/^ABAIRT/) {
	$base='/usr/local/share/crubadan/ga/ciu';
}
elsif ($cineal =~ m/N$/) {
	$base="/snapshot/aimsigh/ABAIRT";
}
else {   # ..Y 
	$base="/snapshot/aimsigh/ABAIRT-CH";
}
my $range="/snapshot/aimsigh/$cineal";

sub do_from_queue
{
	open(ANCIU, "<", "/usr/local/share/crubadan/ga/ANCIU") or die "could not open ANCIU: $!\n";
	while (<ANCIU>) {
		chomp;
		my $doctxt = "$_.txt";
		print "Processing $doctxt (first time)...\n";
		process_me("$base/$doctxt", "$range/$doctxt");
	}
	close ANCIU;
}

sub do_from_timestamps
{
	opendir DIRH, $base or die "could not open $base: $!\n";
	foreach my $doctxt (readdir DIRH) {
		next if $doctxt !~ /\.txt$/;
	#	print "Processing document $doctxt for $cineal index...\n";
		my $sprioc = "$range/$doctxt";
		my $foinse = "$base/$doctxt";
		my $doit = 0;
		if (-e $sprioc) {
			my @stat1 = stat($foinse);
			my @stat2 = stat($sprioc);
			$doit = ($stat1[9] > $stat2[9]);
			print "Processing $doctxt (out of date)...\n" if $doit;
		}
		else {
			$doit=1;
			print "Processing $doctxt (first time)...\n";
		}
		process_me($foinse, $sprioc) if ($doit);
	}
	closedir DIRH;
}


# do_from_timestamps();
do_from_queue();
exit 0;