#!/usr/bin/perl -w
##-------------------------------------------------------------##
# $Author: oravecz $
# $Date: $
# $Header: $
#
# Desc: script to make xml version from celex tgd files
#
# Bugs:
# probably a lot
# and a lot more
# Todo:
#  - 
# Change log:
#
#_______________________________________________________________##
use strict;
if ($ENV{OSTYPE} =~ m/linux/i) {
require 5.004 or die "We need at least 5.004!";
use POSIX qw(locale_h);
} else {
  die "Alas! We need Linux...";
}
setlocale(LC_ALL, "hu_HU");
use locale;

use Getopt::Std;
my $debug = 0;
my %Opt;
my $usage = <<EUS ;

Script to make celex xml files from tgd format.

Usage: $0 [-h|d|s] [tgd file]
 s         use space as token separator (default: newline))
 d         print out debugging info
 h         print out usage info
EUS

getopts('hds', \%Opt) or do {print STDERR "$usage"; exit 1};
$debug = 1 if $Opt{'d'} ;
if ($Opt{'h'}) {print STDERR "$usage";exit 1};

my $sep = "\n";
$sep = " " if $Opt{'s'};

my $prolog = <<EOP;
<?xml version="1.0" encoding="iso-8859-2"?>
<!DOCTYPE celex SYSTEM "/home2/eurovoc/dtd/celex.dtd" []>
<celex>
<documents>
EOP

my $eop = <<EOE;
<descriptors>

<descriptor id="UniqueIdOfEurovocDescriptor">LabelOfEurovocDescriptor
</descriptor>

</descriptors>
<description>
EMPTY
</description>
</document>
</documents>
</celex>
EOE

print $prolog;

my $count = 0;
my $file = $ARGV[0];
my ($code) = $file =~ m%_([^-_]+)-HU%;
my $id = "hu." . $code;
my $sequence = "0.0";
my $tokens = '';
my $lemmas = '';
my (@tokens,@lemmas);
my ($ref,$type,$token,$bos,$ana);
my ($tokentitle,$lemmatitle,$lemma);

my $template = '(Az\s+Európai\s+|az\s+ellátási\s+ügynökség|A\s+TAGÁLLAMOK\s+|A\s+VÉGREHAJTÓ\s+BIZOTTSÁG)';
my $upper = 800;

print "<document id=\"$id\" sequence=\"$sequence\" celex=\"$code\">\n";
while (<>) {
  chomp;
  next if /CHUNK.*DIV/;
  # amp filter
  s/&(?!\w+;)/&amp;/g;
  if ($debug) {
    print STDERR "$.\r" if $. % 100 == 0;
  }
  if (/^\#.*<(\/?[PS])/) {
    $token = $lemma = "\L<$1>";
  } else {
    ($ref,$type,$token,$bos,$ana) = split(/\t/,$_);
    ($lemma) = $ana =~ m%(.*?)\\%;
  }
  push @tokens, $token;
  push @lemmas, $lemma;
}

$tokens = join("$sep",@tokens);
$lemmas = join("$sep",@lemmas);

# try to get end of title:
if ($tokens =~ m%^.{20,$upper}<p>\s*<s>\s*$template%is) {
  print STDERR "Possible title match in $file...\n";
  # probably up to this point it is the title
  $tokens =~ s%^(.{20,$upper}?)(<p>\s+<s>\s+$template)%$2%si;
  $tokentitle = $1;
  $lemmas =~ s%^(.{20,$upper}?)(<p>\s+<s>\s+$template)%$2%si;
  $lemmatitle = $1;
} else {
  print STDERR
    "No title detected in $file!!! Using first paragraph, must be checked!\n";
  $tokens =~ s%^(.*?</s>\s+</p>)%%s;
  $tokentitle = "$1\n";
  $lemmas =~ s%^(.*?</s>\s+</p>)%%s;
  $lemmatitle = "$1\n";
}

print "<title type=\"token\">\n$tokentitle</title>\n";
print "<title type=\"lemma\">\n$lemmatitle</title>\n";
$tokens =~ s/^\s+//;
$tokens =~ s%<p>\s+</p>%%g;
$lemmas =~ s/^\s+//;
$lemmas =~ s%<p>\s+</p>%%g;
print "<text>\n$tokens</text>\n";
print "<lemmatised>\n$lemmas</lemmatised>\n";
print $eop;

__END__

