#!/usr/bin/perl -w
##-------------------------------------------------------------##
# $Author: oravecz $
# $Date: $
# $Header: $
#
# Desc: script to make various freq and ngram lists from tgd files
#
# Bugs:
# probably a lot
# and a lot more
# Todo:
#  - 
# Change log:
#
#_______________________________________________________________##
use strict;
if ($ENV{OSTYPE} =~ m/linux/i) {
require 5.004 or die "We need at least 5.004!";
use POSIX qw(locale_h);
} else {
  die "Alas! We need Linux...";
}
setlocale(LC_ALL, "hu_HU");
use locale;

use Getopt::Std;
my $debug = 0;
my %Opt;
my $usage = <<EUS ;

Script to make wfls and ngrams from files in tgd format.

Usage: $0 [-h|d|s|l] [-n integer] [tgd file]
 s         normalize tokens to lowercase
 n num     produce ngram list (Num: n in ngrams)
 l         produce lemma frequency/ngram list (default: wordform)
 d         print out debugging info
 h         print out usage info
EUS

getopts('hdn:ls', \%Opt) or do {print STDERR "$usage"; exit 1};
$debug = 1 if $Opt{'d'} ;
if ($Opt{'h'}) {print STDERR "$usage";exit 1};

my %ngram;
my %words;
my $form;
my $count = 0;
my @string;

print STDERR "Processing input...\n";
while (<>) {
  next if /^\#/;
  print STDERR "$.\r" if $. % 10000 == 0;
  chomp;
  $count++;
  my ($ref,$type,$token,$bos,$ana) = split(/\t/,$_);
  my ($lemma) = $ana =~ m%(.*?)\\%;
  if ($Opt{'l'}) {
    $form = $lemma;
  } else {
    $form = $token;
  }
  if ($Opt{'s'}) {
    $form = "\L$form";
  }
  if ($Opt{'n'}) {
    #### ngram list ###
    push @string, $form;
    next if $count < $Opt{'n'};
    $ngram{join(" ", @string)}++;
    shift @string;
  } else {
    ####### wfl #######
    $words{$form}++;
  }
}


print STDERR "\nDone.\n";

if ($Opt{'n'}) {
  printout(\%ngram);
} else {
  printout(\%words);
}

sub printout {
  my $href = shift;
  map { printf "%8d\t%s\n", $href->{$_}, $_ }
    sort { $href->{$b} <=> $href->{$a} || $a cmp $b}  keys %$href;
}

__END__



if ($Opt{'a'}) {
  foreach my $word (sort keys %wordlist) {
    printf "%8d\t%s\n", $wordlist{$word}, $word;
  }
}


