use strict;
use DBI;
use LWP::Simple;
use CGI;
my $master_dsn = 'DBI:mysql:Distributed_ClusterDB:localhost';
#my $master_dsn = 'DBI:mysql:Distributed_ClusterDB:172.18.16.35';
my $master_db_user = 'root';
my $master_db_paswd = 'admin~414';

my $THRESHOLD_SIMILARITY = 0.90;
my $TITLE_THRESHOLD_SIMILARITY = 0.80;

my @signature_list = ();
$signature_list[0] =  "the";
$signature_list[1] = "is";
$signature_list[2] = "that";
$signature_list[3] = "a";
$signature_list[4] = "an";
$signature_list[5] = "in";
my %stopwords = ();
my $count = 0;

open (PROCESSCHECK, ("ps -ef | grep \"/usr/bin/perl /home/web18/entity/entity_duplicates.pl\" | wc -l |"));
my @lines = <PROCESSCHECK>;
chomp (@lines);
foreach my $temp (@lines)
{
        print $temp,"\n";
        if ($temp > 3)
        {
                print "There is a entity duplcicates process already running\n";
                #exit;
        }
}
close(PROCESSCHECK);


my $master_dbh = DBI->connect ($master_dsn, $master_db_user, $master_db_paswd, {mysql_auto_reconnect => 1}) or die( "db connection fail");

#my $master_sth = $master_dbh->prepare ("SELECT DATE_FORMAT(timestamp, '%Y%m%d%H%i%S') as timestamp from entity_fetch_time_duplicates");
#my $master_sth = $master_dbh->prepare ("SELECT DATE_FORMAT(timestamp - INTERVAL 1 HOUR, '%Y%m%d%H%i%S') as timestamp from entity_fetch_time_duplicates");
my $master_sth = $master_dbh->prepare ("SELECT DATE_FORMAT(timestamp - INTERVAL 300 MINUTE, '%Y%m%d%H%i%S') as timestamp from entity_fetch_time_duplicates");

$master_sth->execute ();

my $time = $master_sth->fetchrow_array();
#my $time = "20090907120000";
print $time , "\n";
#$time = "20120507000824";

my $master_sth_update = $master_dbh->prepare ("UPDATE entity_fetch_time_duplicates set timestamp=now()");
$master_sth_update->execute();
$master_sth_update->finish;

my $search_url = "http://api.in.com/active18/solr/rewamp/solr_search_read_inhouse.php?query=crawl_time:[$time+TO+*]&sortfield=crawl_time&sortorder=1";
#my $search_url = "http://api.in.com/active18/solr/solr_search_read_revamp.php?query=source:PTI+AND+crawl_time:[$time+TO+*]&sortfield=crawl_time&sortorder=1";
#my $search_url = "http://api.in.com/active18/solr/solr_search_read.php?query=id:eb44602598875ba01bb945afa2bd0aaecdd1368c&sortfield=crawl_time&sortorder=1";
print $search_url . "\n";

my $cont = '';
my $con = get $search_url;
my $count = 0;
if ($con =~ m/<length>(.*?)<\/length>/i)
{
    $count = $1;
}
print "Count: " .  $count, "\n";
#exit;

if($count eq 0)
{
        print "exiting because no records to process\n";
        exit;
}

#"172.30.254.189:3306","in-code-rw","EDhdfjh58ayjioG"
my $connect_dsn = 'DBI:mysql:inrevampdb:172.30.254.189:3306';
my $connect_db_user = 'in-code-rw';
my $connect_db_paswd = 'EDhdfjh58ayjioG';

my $connect_dbh = DBI->connect ($connect_dsn, $connect_db_user, $connect_db_paswd, {mysql_auto_reconnect => 1});

my $connect_sth = $connect_dbh->prepare ("SELECT entity_name, entity_id, cat_id from entity");

$connect_sth->execute ();
my $finalEntity = '';
my %entity_hash = ();
my %entity_cat_id = ();
my %entity_pr_id = ();
while (my ($entityname, $pr_id, $cat_id) = $connect_sth->fetchrow_array ())
{
	my $entitynameOriginal = $entityname;
        next if $entityname eq '';
        next if $entityname !~ m/^[A-Za-z0-9\. ]*?$/;
        $entityname =~ s/\(/\\(/;
        $entityname =~ s/\)/\\)/;
        $entityname = '\\b' . $entityname . '\\b';
        $entity_hash{$entityname} = $entitynameOriginal;
	$entity_cat_id{$entitynameOriginal} = $cat_id;
	$entity_pr_id{$entitynameOriginal} = $pr_id;
}
print "Added entities to the hash\n";

#open(STOPWORDS1, "/home/web18/entity/stopwords/1.stopwords") or die "Can't open file1";
#my $stopdir ="/webserver/home/news18.com/www/src/feeds/news/mc/hiten/stopwords";
my $stopdir ="/var/www/cron/mc/172_30_11_23";

open(STOPWORDS1, "$stopdir/1.stopwords") or die "Can't open file1";

my %stop_words_hash = ();
foreach my $stop (<STOPWORDS1>)
{
        $stop_words_hash{trim($stop)} = 1;
}
#open(STOPWORDS2, "/home/web18/entity/stopwords/2.stopwords") or die "Can't open file1";
open(STOPWORDS2, "$stopdir/2.stopwords") or die "Can't open file1";

foreach my $stop (<STOPWORDS2>)
{
        $stop_words_hash{trim($stop)} = 1;
}

print "Loaded stop words\n";

#exit;
my $index = 0;
my $limit = 200;
my $i = 0;
my $temp = 0;

my %final_list = ();
my %sources = ();
my %urls = ();
my %titles = ();
my %fullcontent = ();
my %duplicates = ();
my %originals = ();
my %ids = ();
my %signatures = ();
my %crawl_time = ();
#my $entList = '';
my %entities = ();
my %ids = ();
my %titleWords = ();
my %cat_ids = ();
my %pr_ids = ();

do
{
 my $query_url = "http://api.in.com/active18/solr/solr_search_read_revamp.php?query=crawl_time:[$time+TO+*]&sortfield=crawl_time&sortorder=0&start=" . $index . "&limit=" . $limit;
 #my $query_url = "http://api.in.com/active18/solr/solr_search_read.php?query=id:eb44602598875ba01bb945afa2bd0aaecdd1368c&sortfield=crawl_time&sortorder=0&start=" . $index . "&limit=" . $limit;

 $con = get $query_url;

	while ($con =~ s/<item>(.*?)<\/item>//)
    	{
		$i = $i + 1;
		print "Count:" . $i , "\n";
		my $text = $1;
		my $id = '';
		my $url = '';
		my $crawl_time = '';
		my $source = '';
		my $title = '';			
		my $content = '';
		my $dupTitle = '';
		my $dupContent = '';
		my $finalContent = '';
				
		if ($text =~ m/<id><!\[CDATA\[(.*?)\]\]><\/id>/)
		{
				$id = $1;
		}
		if ($text =~ m/<url><!\[CDATA\[(.*?)\]\]><\/url>/)
		{
				$url = CGI::escape($1);
		}
		if ($text =~ m/<crawl_time><!\[CDATA\[(.*?)\]\]><\/crawl_time>/)
		{
				$crawl_time = CGI::escape($1);
		}
		if ($text =~ m/<source><!\[CDATA\[(.*?)\]\]><\/source>/)
		{
				$source = CGI::escape($1);
		}
		if ($text =~ m/<title><!\[CDATA\[(.*?)\]\]><\/title>/)
		{
				$title = CGI::escape($1);
				$dupTitle = $1;
		}                                
		if ($text =~ m/<content><!\[CDATA\[(.*?)\]\]><\/content>/)
		{
				$content = CGI::escape($1);
				$dupContent = $1;
		}			
				
		
		my $compareText = $dupTitle . " " . $dupContent;
		my $ent = '';
		$ent = getEntities($compareText);
		my @entitySplit = split(/#/, $ent);

		
		print "\nNew Ent for id" . $id . " : " . $entitySplit[0];
		#exit;
		next if $entitySplit[0] eq '';
		$entities{$id} = $entitySplit[0];
		$crawl_time{$id} = $crawl_time;
		$sources{$id} = $source;
		$titles{$id} = $dupTitle;
		$fullcontent{$id} = $content;
		
		$urls{$id} = $url;
		$ids{$id} = $id;
		$pr_ids{$id} = $entitySplit[1];
		$cat_ids{$id} = $entitySplit[2];
		
		my @words = split ("[\\W0-9]+",$dupTitle);
				
		$titleWords{$id} = \@words;
		
		#my $temp_hash1 = get_signature (stripStopWords($dupTitle));
		my $temp_hash2 = get_signature (stripStopWords($dupContent));
		$crawl_time{$id} = $crawl_time;
		my %combine = ();
		#foreach my $temp (keys %$temp_hash1)
		#{
		#	print "\n Temp -->" . $temp ;
		#	$combine{$temp} = $temp_hash1->{$temp};
		#}
		foreach my $temp (keys %$temp_hash2)
		{
			#print "\n Temp -->" . $temp ;
			$combine{$temp} = $temp_hash2->{$temp};
		}
		$final_list{$id} = \%combine;		
		
		
		my $sig = '';
		foreach my $temp (keys %combine)
		{
			
			$sig = $temp . "#" . $sig;			
		}
		$signatures{$id} = $sig;		
		
	}

      $index = $index + $limit;
	  $i++;
}while ($index <= $count);	
	#exit;
	my %parent = ();
	print "\n#####################################################################\n";
	foreach my $temp1 (sort {$final_list{$a} cmp $final_list{$b}} keys %final_list)
	{
		# print "\n $temp---------------------------------------------------------\n";
		my $test_file1 = $final_list{$temp1};
		my %hell_file1 = %$test_file1;
		foreach my $temp2 (sort {$final_list{$a} cmp $final_list{$b}} keys %final_list)
		{
			
			next if $temp2 le $temp1;
			
			next if $entities{$temp2} ne $entities{$temp1};
				
			my $test_file2 = $final_list{$temp2};
			my %hell_file2 = %$test_file2;
			my $numerator = 0.0;
			my $denom1 = 0.0;
			my $denom2 = 0.0;
			
			my $cosine_sim = 0;
			my $cosine_sim = cosineDistance(\%hell_file1, \%hell_file2);
			
			my $cosine_sim_title = cosineDistanceTitle($temp1, $temp2);
			
			if ($cosine_sim_title > $TITLE_THRESHOLD_SIMILARITY || $cosine_sim > $THRESHOLD_SIMILARITY)
			{
				print "\nCosine Sim Title-->" . $cosine_sim_title;
				print "\nCosine Sim Content-->" . $cosine_sim;
				print "Duplicate detected\n";
				print "Check docs $temp1,  $temp2 \n ";
				my $orig = '';
				my $dup = '';
				if ($sources{$temp1} ne $sources{$temp2})
				{
					if ($crawl_time{$temp1} > $crawl_time{$temp2})
					{
						$duplicates{$temp1}=1;
						$originals{$temp2}=1;
						$parent{$temp1} = $temp2;
						delete $originals{$temp1};
					}
					else
					{
						$duplicates{$temp2}=1;	
						$originals{$temp1}=1;
						$parent{$temp2} = $temp1;
						delete $originals{$temp2};
					}
				}
				else
				{
					if ($crawl_time{$temp1} > $crawl_time{$temp2})
					{
						$duplicates{$temp2}=1;
						$originals{$temp1}=1;
						$parent{$temp2} = $temp1;
						delete $originals{$temp2};
					}
					else
					{
						$duplicates{$temp1}=1;
						$originals{$temp2}=1;
						$parent{$temp1} = $temp2;
						delete $originals{$temp1};
					}
				}
			}
		}
	}

	foreach my $temp (keys %parent)
        {
                my $val = $parent{$temp};
                if($parent{$val})
                {
                        $parent{$temp} = $parent{$val};
                }
        }

	print "Duplicates are \n";
	foreach my $temp (keys %duplicates)
	{
		print $temp . " :" ;
		print "Parent:" . $parent{$temp};
		print $entities{$temp}. "\n"
	}
	#exit;
	#print "\nHi\n";	
	my $sth = $master_dbh->prepare("SELECT docid, signature, source, title, entities from entity_document_duplicates where HOUR(TIMEDIFF(crawl_time,now())) < 96");
	$sth->execute ();
	
	my %db_signatures = ();
	my %db_sources = ();
	my %db_titles = ();
	my %db_entities = ();
	while (my ($docid, $signature, $source, $title, $entities) = $sth->fetchrow_array ())
	{
			#my temp2 = split(/\#/, $signature);						
			$db_signatures{$docid} = $signature;
			$db_sources{$docid} =  CGI::escape($source);
			$db_titles{$docid} = $title;
			$db_entities{$docid} = $entities;
	}
	#exit;	
	my %file1 = ();
	my %file2 = ();
	my %hell_file1 = ();
	my %hell_file2 = ();
	
	foreach my $temp2 (keys %ids)
	{
		#print "\nInside for\n";
		if(!$duplicates{$temp2})
		{
			my @temp1 = split(/\#/, $signatures{$temp2});
			foreach  my $temp1 (keys %db_signatures)
			{
				next if $temp2 eq $temp1;
				
				next if $entities{$temp2} ne $db_entities{$temp1};
				
				my @words = split ("[\\W0-9]+",$db_titles{$temp1});				
				$titleWords{$temp1} = \@words;	
				
				my @temp2 = split(/\#/, $db_signatures{$temp1});
				foreach my $t1 (@temp1) 
				{
					$hell_file1{$t1} = 1;				
				}
				foreach my $t2 (@temp2)
				{
					$hell_file2{$t2} = 1;
				}
				
				my $cosine_sim = 0;
				my $cosine_sim = cosineDistance(\%hell_file1, \%hell_file2);
				
				my $cosine_sim_title = cosineDistanceTitle($temp1, $temp2);
				#print "\nCosine Distance DB:" . $cosine_sim_title . " " . $temp1 . " : " . $temp2 ;	
				if ($cosine_sim_title > $TITLE_THRESHOLD_SIMILARITY || $cosine_sim > $THRESHOLD_SIMILARITY)
				{
					print "\nCosine Sim Title DB-->" . $cosine_sim_title;
					print "\nCosine Sim Content DB-->" . $cosine_sim;
					print "Duplicate detected\n";
					print "Check docs $temp2,  $temp1 \n ";
					if($sources{$temp2} eq $db_sources{$temp1})
					{
						my $sth = $master_dbh->prepare("DELETE from entity_document_duplicates where docid='" . $temp1 . "'");
						print "DB Delete:";
						print "DELETE from entity_document_duplicates where docid='" . $temp1 . "'";
						$sth->execute ();
						#	my $query_url = "http://api.in.com/active18/solr/solr_delete_read_entity_test.php?id=" . $temp1;
						#print "Solr Delete:" . "http://api.in.com/active18/solr/solr_delete_read_entity_test.php?id=" . $temp1;
						my $query_url = "http://api.in.com/active18/solr/solr_add_read_entity_test.php?url=" . $urls{$temp1} . "&crawl_time=" . $crawl_time{$temp1} . "&source=" . $sources{$temp1} . "&title=" . $titles{$temp1} . "&entity=" . CGI::escape($entities{$temp1}). "&isduplicate=1&parentid=" . $temp2 . "&content=" . $fullcontent{$temp1} . "&pr_id=" . $pr_ids{$temp1} . "&cat_id=" . $cat_ids{$temp1};
						#print "\n" . $query_url . "\n";
						print "\n".$temp . ":";
						my $con = get $query_url;
						print $con . "\n";	
						$parent{$temp1} = $temp2;
					}
					else
					{
						$duplicates{$temp2} = 1;
						$parent{$temp2} = $temp1;
					}
				}
			}			
		}	
	}
	
	print "\nnOriginals are \n";
	$master_dbh = DBI->connect ($master_dsn, $master_db_user, $master_db_paswd);
	foreach my $temp (keys %ids)
	{
		if(!$duplicates{$temp})
		{
			my $isduplicate = '';
			$isduplicate = 0;
			print $temp . " :";
			print $entities{$temp}. "\n";
			if(!$db_signatures{$temp})
			{
				my $tempTitle = $master_dbh->quote($titles{$temp});
				my $sth = $master_dbh->prepare("INSERT INTO entity_document_duplicates(docid, crawl_time, signature, insert_time, entities, source, title) values('" . $temp. "','" .  $crawl_time{$temp} . "','" . $signatures{$temp} . "', now() ,'" . $entities{$temp} . "','". $sources{$temp}. "'," . $tempTitle . ")");
				$sth->execute ();
			}
		}	
	}
	
	foreach my $temp (keys %ids)
	{
		my $isduplicate = '';
		if(!$duplicates{$temp})
		{
			$isduplicate = 0;
		}
		else
		{
			$isduplicate = 1;
		}
		
		my $query_url = "http://api.in.com/active18/solr/solr_add_read_entity_test.php?url=" . $urls{$temp} . "&crawl_time=" . $crawl_time{$temp} . "&source=" . $sources{$temp} . "&title=" . CGI::escape($titles{$temp}) . "&entity=" . CGI::escape($entities{$temp}). "&isduplicate=" . $isduplicate . "&parentid=" . $parent{$temp}  . "&content=" . $fullcontent{$temp} . "&pr_id=" . $pr_ids{$temp} . "&cat_id=" . $cat_ids{$temp};

		print "\n" . $query_url . "\n";
		print "\n".$temp . ":";
		my $con = get $query_url;
		print $con . "\n";	
		#exit;	
	}
	
	print "#####################################################################\n";

my $entitynotify = "http://www.in.com/active18/readnow/notify_entity.php?timefrom=" . $time;
get $entitynotify;
print $entitynotify;
	
sub cosineDistanceTitle
{
	#my (%hell_file1, %hell_file2) = @_;
	my $hell1 = $_[0];
	my $hell2 = $_[1];
	
	my $temp1 = $titleWords{$hell1};
	my @words1 = @$temp1;
	
	#print @words;
	my %keywords1 = ();
	
	foreach my $word (@words1)
	{
			if($keywords1{$word})
			{
				$keywords1{$word}++;
			}
			else
			{
				$keywords1{$word} = 1;
			}
	}
	
	my $temp2 = $titleWords{$hell2};
	my @words2 = @$temp2;
	
	#print @words;
	my %keywords2 = ();
	
	foreach my $word (@words2)
	{
			if($keywords2{$word})
			{
				$keywords2{$word}++;
			}
			else
			{
				$keywords2{$word} = 1;
			}
	}
	
	my $numerator = 0.0;
	my $denom1 = 0.0;
	my $denom2 = 0.0;
	foreach my $temp (keys %keywords1)
	{
		if (exists $keywords2{$temp})
		{
			$numerator += ($keywords1{$temp} * $keywords2{$temp});
		}
		$denom1 += ($keywords1{$temp} ** 2);
	} 
	foreach my $temp (keys %keywords2)
	{
		$denom2 += ($keywords2{$temp} ** 2);
	}
	my $cosine_sim = 0;
	if ($numerator > 0 && $denom1 > 0 && $denom2 > 0)
	{
		$cosine_sim = $numerator / (($denom1 ** 0.5) * ($denom2 ** 0.5));
	}
	
	return $cosine_sim;
}
	

sub cosineDistance
{
	#my (%hell_file1, %hell_file2) = @_;
	my $hell1 = $_[0];
	my $hell2 = $_[1];
	my %hell_file1 = %$hell1;
	my %hell_file2 = %$hell2;
	
	my $numerator = 0.0;
	my $denom1 = 0.0;
	my $denom2 = 0.0;
	foreach my $temp (keys %hell_file1)
	{
		if (exists $hell_file2{$temp})
		{
			$numerator += ($hell_file1{$temp} * $hell_file2{$temp});
		}
		$denom1 += ($hell_file1{$temp} ** 2);
	} 
	foreach my $temp (keys %hell_file2)
	{
		$denom2 += ($hell_file2{$temp} ** 2);
	}
	my $cosine_sim = 0;
	if ($numerator > 0 && $denom1 > 0 && $denom2 > 0)
	{
		$cosine_sim = $numerator / (($denom1 ** 0.5) * ($denom2 ** 0.5));
	}	
	return $cosine_sim;
}
		
sub get_signature
{
	my ($content) = @_;
	#print $content , "\n";
	my %global_list = ();
	foreach my $temp (@signature_list)
	{
		while ($content =~ s/([.,\s]+?)($temp) ([^\s]+?) ([^\s]+?) / /is)
		{
			my $final_str = lc($2) . ":" . lc($3) . ":" . lc($4);
			$final_str . "\n";
			if (exists $global_list{$final_str})
			{	
				$global_list{$final_str}+=1;	
			}
			else
			{
				$global_list{$final_str}=1;
			}
		}
	}
	return \%global_list;
}

sub loadStopwords
{
        open(FILE1, "/home/web18/entity/stopwords/1.stopwords");
        open(FILE2, "/home/web18/entity/stopwords/2.stopwords");

        my @stop_w = <FILE1>;
        chomp(@stop_w);
		my $signature_words = join ("|", @signature_list);
        foreach my $temp (@stop_w)
        {
                $temp =~ s/\n//;
		$temp =~ s/$signature_words//ig;
		$stopwords{$temp} = 1;
        }

        @stop_w = ();
        @stop_w = <FILE2>;
        chomp(@stop_w);
        foreach my $temp (@stop_w)
        {
		$temp =~ s/$signature_words//ig;
                $stopwords{$temp} = 1;

        }
        close (FILE1);
        close (FILE2);
}

sub stripStopWords
{
	my ($text) = @_;
	#print "Text: " . $text . "\n\n";
	my @words = split ("[\\W]+",$text);
	my $tempText = "";
	foreach my $temp (@words)
	{
		if(!exists($stopwords{$temp}))
	        {
		   $tempText = $tempText . $temp . " " ;	
		}	
		chomp($tempText);
	}
	#print "Stripped: " . $tempText , "\n";
	return $tempText;	
} 
  
sub trim($)
{
        my $string = shift;
        $string =~ s/^\s+//;
        $string =~ s/\s+$//;
        return $string;
}
  
sub getEntities
{
	my $compareText = $_[0];
	#print $compareText . "\n";
	
	$finalEntity = '';
	my $finalEntityCat = '';
	my $finalEntityProfile = '';
	my $count = 0;
	my %entityCount = ();
	foreach my $key (keys %entity_hash)
	{
		$count = 0;
                while($compareText =~m/$key/gs)
                {
                        $count++;
                 }
                if ($count > 0)
                {
                        $key =~ m/\\b(.*?)\\b/i;
                        #$entityCount{$1}=$count;
			my $lc_e = lc($1);
			next if $lc_e eq '..';
			if($lc_e =~ m/^[0-9]*$/)
                        {
                                #print "stopword digit " . $lc_e . "\n";
                                $entityCount{$lc_e}= 0;
                        }
                        if($stop_words_hash{$lc_e})
                        {
                                 $entityCount{$1}= 0;
                        }
                        else
                        {
                                $entityCount{$1}=$count;
                        }
                }
	}

	#chop($finalEntity);
	#$finalEntity = trim($finalEntity);
	foreach my $entity (sort {$entityCount{$b} <=> $entityCount{$a}} keys %entityCount)
        {
                print $entity , "-->", $entityCount{$entity} , "\n";
		next if $entityCount{$entity} == 0;
                $finalEntity .= $entity . ",";
				
				print $entity_cat_id{$entity} , "\n";
				$finalEntityCat .= $entity_cat_id{$entity} . ",";
				print $entity_pr_id{$entity} , "\n";
				$finalEntityProfile .= $entity_pr_id{$entity} . ",";				

        }
        $finalEntity =~ s/,$//g;
	$finalEntityProfile =~ s/,$//g;
        $finalEntityCat =~ s/,$//g;

	$finalEntity = $finalEntity . '#' . $finalEntityProfile . '#' . $finalEntityCat . "\n";		
	if($finalEntity ne '')
	{
		return $finalEntity;
	}
	else
	{
		return '';
	}
	
}
  
