use DBI;
use strict;
use Lingua::Stem;

my $stemmer = Lingua::Stem->new(-locale=>'EN_UK');
my %stopwords = ();
my %tf = ();
my %all_tfs = ();
my $idf;
my %docs = ();
my %stem_map = ();
my $dir="/var/www/cron/mc/172_30_11_23/moneycontrol_trends";
load_stopwords();
my $db = "mmb";
#my $host = "172.29.5.1";
my $host = "172.29.5.253";
my $dbh = DBI->connect("DBI:mysql:database=$db;host=$host","mmbmcdb","M3m33db~181") or die("db not connected");
#my $query = "select msg_id,topic from msg_detail where MINUTE(TIMEDIFF(NOW(),ent_date))<5 order by ent_date";
my $query = "select msg_id,topic from msg_detail order by ent_date desc LIMIT 300";
my $mysql_handle = $dbh->prepare($query);
$mysql_handle->execute();
my $doc_set = $mysql_handle->rows;
while (my @row = $mysql_handle->fetchrow_array()){
	$query = "select message from msg_data where msg_id=?";
	my $result = $dbh->prepare($query);
	$result->execute($row[0]);
	my @msg_details = $result->fetchrow_array();
	parse_message($msg_details[0]." ".$row[1],$row[0]);
}

my $TF_LIMIT=2;
my %tfidfs = ();
foreach my $temp (keys %all_tfs)
{
	my $t = $all_tfs{$temp};
	my $sum = 0;
	foreach my $a (keys %$t)
	{
		$sum += $t->{$a};
	}	
	foreach my $k (keys %$t)
	{
		if ($t->{$k} >= $TF_LIMIT)
		{
			my $id = $idf->{$_};
			my $tf_idfs = ($t->{$k} / $sum) * log ($doc_set/(1+(scalar(keys %$id))));
			$tfidfs{$k} = $tf_idfs;
		}
	}
}

#open (HTML_TREND, "> /home/moneycontrol/moneycontrol_trends/trends.html") or die "File not found";
open (HTML_TREND, "> $dir/trends.html") or die "File trends.html not found";
print HTML_TREND "<html>\n";
print HTML_TREND "<head>\n";
print HTML_TREND "Moneycontrol Messageboard Trends\n";
print HTML_TREND "</head>\n";

print HTML_TREND "<body>\n";
print HTML_TREND "<table border=1>\n";
print HTML_TREND "<tr><th>Keyword</th><th>Weight</th></tr>";



#open (HTML_TREND1, "> /home/moneycontrol/moneycontrol_trends/trends_all.html") or die "File not found 1";
open (HTML_TREND1, "> $dir/trends_all.html") or die "File trends_all.html not found 1";
print HTML_TREND1 "<html>\n";
print HTML_TREND1 "<head>\n";
print HTML_TREND1 "Moneycontrol Messageboard Trends\n";
print HTML_TREND1 "</head>\n";

print HTML_TREND1 "<body>\n";
print HTML_TREND1 "<table border=1>\n";
print HTML_TREND1 "<tr><th>Keyword</th><th>Weight</th></tr>";





foreach my $temp (sort {$tfidfs{$b} <=> $tfidfs{$a}} keys %tfidfs)
{
	my @first_word = split (/~#~/,$stem_map{$temp});
	chomp(@first_word);
	if ($first_word[0] =~ m/^[A-Z].*?/)
	{
		print HTML_TREND "<tr>\n";
		print HTML_TREND "<td>" . $first_word[0] , "</td><td>", $tfidfs{$temp} , "</td>\n";
		print HTML_TREND "</tr>\n";
	}
	print HTML_TREND1 "<tr>\n";
	print HTML_TREND1 "<td>" . $first_word[0] , "</td><td>", $tfidfs{$temp} , "</td>\n";
	print HTML_TREND1 "</tr>\n";
}
print HTML_TREND "</table>\n";
print HTML_TREND "</body>\n";
print HTML_TREND "</html>\n";
print HTML_TREND1 "</table>\n";
print HTML_TREND1 "</body>\n";
print HTML_TREND1 "</html>\n";

close (HTML_TREND);
close (HTML_TREND1);
#foreach (sort{$tf{$b}<=>$tf{$a}}(keys %tf)){
#	print $_."\nTf:".$tf{$_}."\t";
#	my $temp = $idf->{$_};
#	my $idf = log($doc_set/(1+(scalar(keys %$temp))));
#	print "IDF:".$idf."\n";
#	print TRENDING_TOPICS $_."\t".($tf{$_}*$idf)."\n";
#}
sub parse_message{
	my ($message,$msg_id) = @_;
	my @words = split(/\b/,$message);
	my %tfs = ();
	foreach (@words){
		if(length($_)>1 and not $stopwords{lc($_)}){
			next if (lc($_) =~ m/\W+/);
			next if (lc($_) =~ m/\d+/);
			#print lc($_),"\n";
			my $stemmed_word = stem($_);
			$stem_map{$stemmed_word} .= $_ . "~#~";
			$tfs{$stemmed_word}++;	
			$idf->{$stemmed_word}->{msg_id}=1;
		}
	}
	$all_tfs{$msg_id}=\%tfs;
}

sub sanitize{
	my $str = shift;
	$str =~ s/[!('s)]//ig;
	if($str !~ /[a-zA-Z]/ims){
		$str = "";
	}
	$str =~ s/[_\/,\)\(.]/ /ig;
	my @temp = split(" ",lc($str));
	return @temp;
}


sub load_stopwords{

	open(FILE,"$dir/stopwords") or die "File Stopwords not found";
	while(<FILE>){
		my $temp = $_;
		chomp($temp);
		$stopwords{lc($temp)} = 1;
	}
	close(FILE);
	open(FILE,"$dir/stopwords1") or die "File Stopwords1 not found";
        while(<FILE>){
                my $temp = $_;
                chomp($temp);
                $stopwords{lc($temp)} = 1;
        }
        close(FILE);
}

sub stem{
	my $word = shift;
	my $temp = $stemmer->stem($word);
	#$stem_map{$word} = $temp->[0];
	return $temp->[0];
}
