#!/usr/bin/perl

use strict;
use warnings;

use HTML::Entities;
use Text::Unaccent;

my $guide = "Couldn't run the program: perl $0 directory";
die $guide unless @ARGV == 1;

my $directory = "$ARGV[0]";
$directory =~ s/[\/]$//;

my %ITEM_LIST = ();  # Inventaire des articles
my %TOPIC_LIST = (); # Inventaire des rubriques

# Parcours de l'arborescense des fichiers
&browse_local_folders($directory);

my @TOPICS = keys(%TOPIC_LIST);

# Fermeture des balises dans les fichiers XML de sortie
foreach my $topic (@TOPICS) {
	my $file_xml = "SORTIE_".$topic.".xml";
	open(my $OUT_XML, ">>:encoding(UTF-8)", $file_xml) or die "Couldn't open $file_xml: $!";
	print $OUT_XML "</FILTRAGE>\n";
	print $OUT_XML "</PARCOURS>";
	close($OUT_XML);
}

# Suppression des fichiers temporaires créés
unlink "temp_1.txt", "temp_2.txt", "tagged-output_1.txt", "tagged-output_2.txt", "tagged-output_1.txt.xml", "tagged-output_2.txt.xml";

exit;

# -----------------------------------------------------------------

sub browse_local_folders {
	my $path = shift(@_);
	opendir(my $DIR, $path) or die "Couldn't open $path: $!\n";
	my @FILES = readdir($DIR);
	closedir($DIR);

	foreach my $file (@FILES) {
		next if $file =~ /^\.\.?$/;
		$file = $path."/".$file;
		# Si $file est un répertoire, la fonction est alors récursive
		if(-d $file) {
			print "PARCOURS DE L'ARBORESCENCE [$file]\n";
			&browse_local_folders($file);
		}
		# Si $file est un fichier, celui-ci est alors traité
		if(-f $file) {
			if($file =~ /,0\.xml$/) {
				print "EN COURS DE TRAITEMENT : $file\n";

				open(my $INPUT, "<:encoding(UTF-8)", $file) or die "Couldn't open $file: $!\n";
				my $text = "";
				while(my $line = <$INPUT>) {
					chomp $line;
					$line =~ s/\r//g;
					$text .= $line;
				}
				$text =~ s/> +</></g;

				# Extraction du nom de la rubrique
				my $topic_name = "";
				if($text =~ /<channel>.+?<title>([^<]+)<\/title>/) {
					$topic_name = $1;
				}
				my $i = index($topic_name, ":");
				if($i == -1) {
					$topic_name = "Actualité à la Une";
				} else {
					$topic_name = substr($topic_name, 0, $i-1);
				}
				$topic_name = lc($topic_name);
				$topic_name =~ s/ /-/g;
				my $topic = unac_string("UTF-8", $topic_name);

				my $file_xml = "SORTIE_".$topic.".xml";
				my $file_txt = "SORTIE_".$topic.".txt";

				# Création des fichiers de sortie pour chaque rubrique s'ils n'existent pas déjà
				if(!(exists $TOPIC_LIST{$topic})) {
					open(my $OUT_XML, ">:encoding(UTF-8)", $file_xml) or die "Couldn't open $file_xml: $!";
					print $OUT_XML "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
					print $OUT_XML "<PARCOURS>\n";
					print $OUT_XML "<NOM>De Sousa</NOM>\n";
					print $OUT_XML "<FILTRAGE>\n";
					close($OUT_XML);

					open(my $OUT_TXT, ">:encoding(UTF-8)", $file_txt) or die "Couldn't open $file_txt: $!";
					close($OUT_TXT);

					$TOPIC_LIST{$topic} = 1;
				}

				# Récupération du contenu des articles
				while($text =~ /<item>.+?<title>(.+?)<\/title>.+?<description>(.+?)<\/description>/g) {
					my $title = $1;
					decode_entities($title);
					my $description = $2;
					$description =~ s/&lt;.+&gt;//g;
					$description =~ s/<[^>]+>//g;
					decode_entities($description);

					# Ajout et impression des articles s'ils n'ont pas déjà été enregistrés
					if(!(exists $ITEM_LIST{$title}) && ($description ne "")) {
						# Étiquetage en POS
						my($title_xml, $description_xml) = &pos_tagging($title, $description);

						open(my $OUT_XML, ">>:encoding(UTF-8)", $file_xml) or die "Couldn't open$file_xml: $!";
						print $OUT_XML "<item>\n<title>$title_xml</title>\n<description>$description_xml</description>\n</item>\n";
						close($OUT_XML);

						open(my $OUT_TXT, ">>:encoding(UTF-8)", $file_txt) or die "Couldn't open$file_txt: $!";
						print $OUT_TXT "Titre : $title\nDescription : $description\n";
						close($OUT_TXT);

						$ITEM_LIST{$title} = 1;
					}
				}

				close($INPUT);
			}
		}
	}
}

# -----------------------------------------------------------------

sub pos_tagging {
	my $title_1 = $_[0];
	my $description_2 = $_[1];

	# Ouverture d'un fichier temporaire pour l'étiquetage avec TreeTagger du titre
	open(my $TMP_1, ">:encoding(UTF8)", "temp_1.txt") or die "Couldn't open: $!";
	print $TMP_1 $title_1;
	system("perl tokenise-utf8.pl temp_1.txt | tree-tagger.exe -token -lemma -no-unknown french-oral-UTF-8.par > tagged-output_1.txt");
	system("perl treetagger2xml-utf8.pl tagged-output_1.txt utf8");
	close($TMP_1);

	open(my $FILE_1, "<:encoding(UTF8)", "tagged-output_1.txt.xml") or die "Couldn't open: $!";
	my $first_line = <$FILE_1>;
	my $title_xml = "";
	while(my $line = <$FILE_1>) {
		$title_xml .= $line;
	}
	close($FILE_1);

	# Ouverture d'un fichier temporaire pour l'étiquetage avec TreeTagger de la description
	open(my $TMP_2, ">:encoding(UTF8)", "temp_2.txt") or die "Couldn't open: $!";
	print $TMP_2 $description_2;
	system("perl tokenise-utf8.pl temp_2.txt | tree-tagger.exe -token -lemma -no-unknown french-oral-UTF-8.par > tagged-output_2.txt");
	system("perl treetagger2xml-utf8.pl tagged-output_2.txt utf8");
	close($TMP_2);

	open(my $FILE_2, "<:encoding(UTF8)", "tagged-output_2.txt.xml") or die "Couldn't open: $!";
	$first_line = <$FILE_2>;
	my $description_xml = "";
	while(my $line = <$FILE_2>) {
		$description_xml .= $line;
	}
	close($FILE_2);

	return($title_xml, $description_xml);
}