use strict;
use DBI;
use LWP;
use LWP::Simple;
use CGI;
my $master_dsn = 'DBI:mysql:Distributed_ClusterDB:localhost';
my $master_db_user = 'root';
my $master_db_paswd = 'admin~414';

my %stopwords = ();
my $count = 0;

my @signature_list = ();
$signature_list[0] =  "the";
$signature_list[1] = "is";
$signature_list[2] = "that";
$signature_list[3] = "a";
$signature_list[4] = "an";
$signature_list[5] = "in";

open (PROCESSCHECK, ("ps -ef | grep \"/usr/bin/perl /home/web18/entity/entity_blog.pl\" | wc -l |"));
my @lines = <PROCESSCHECK>;
chomp (@lines);
foreach my $temp (@lines)
{
        print $temp,"\n";
        if ($temp > 3)
        {
                print "There is a entity blogs process already running\n";
                #exit;
        }
}
close(PROCESSCHECK);


my $master_dbh = DBI->connect ($master_dsn, $master_db_user, $master_db_paswd, {mysql_auto_reconnect => 1}) or die("db connection fail");

my $master_sth = $master_dbh->prepare ("SELECT DATE_FORMAT(timestamp, '%Y%m%d%H%i%S') as timestamp from entity_fetch_time_blog");

$master_sth->execute ();

my $time = $master_sth->fetchrow_array();
#my $time = "20090909000000";
print $time , "\n";

my $master_sth_update = $master_dbh->prepare ("UPDATE entity_fetch_time_blog set timestamp=now()");
$master_sth_update->execute();
$master_sth_update->finish;

my $search_url = "http://api.in.com/active18/solr/solr_search_blog.php?query=*:*+AND+insert_time:[$time+TO+*]&sortfield=crawl_time&sortorder=1";
print $search_url . "\n";

my $cont = '';
my $con = get $search_url;
my $count = 0;
if ($con =~ m/<length>(.*?)<\/length>/i)
{
    $count = $1;
}
print "Count: " .  $count, "\n";
#exit;

if($count eq 0)
{
        print "exiting because no records to process\n";
        exit;
}


my $connect_dsn = 'DBI:mysql:connect:172.29.44.253 ';
my $connect_db_user = 'irconnectdb';
my $connect_db_paswd = '13c077ect~131';

my $connect_dbh = DBI->connect ($connect_dsn, $connect_db_user, $connect_db_paswd, {mysql_auto_reconnect => 1});

my $connect_sth = $connect_dbh->prepare ("SELECT profile_name, pr_id, cat_id from connect_master");

$connect_sth->execute ();
my $finalEntity = '';
my %entity_hash = ();
my %entity_cat_id = ();
my %entity_pr_id = ();
while (my ($entityname, $pr_id, $cat_id) = $connect_sth->fetchrow_array ())
{
	my $entitynameOriginal = $entityname;
        next if $entityname eq '';
        next if $entityname !~ m/^[A-Za-z0-9\. ]*?$/;
        $entityname =~ s/\(/\\(/;
        $entityname =~ s/\)/\\)/;
        $entityname = '\\b' . $entityname . '\\b';
        $entity_hash{$entityname} = $entitynameOriginal;
	$entity_cat_id{$entitynameOriginal} = $cat_id;
	$entity_pr_id{$entitynameOriginal} = $pr_id;
}
print "Added entities to the hash\n";
my $stopdir ="/var/www/cron/mc/172_30_11_23";
open(STOPWORDS1, "$stopdir/1.stopwords") or die "Can't open file1";

my %stop_words_hash = ();
foreach my $stop (<STOPWORDS1>)
{
        $stop_words_hash{trim($stop)} = 1;
}
open(STOPWORDS2, "$stopdir/2.stopwords") or die "Can't open file1";

foreach my $stop (<STOPWORDS2>)
{
        $stop_words_hash{trim($stop)} = 1;
}

print "Loaded stop words\n";

#exit;
my $index = 0;
my $limit = 200;
my $i = 0;
my $temp = 0;

my %final_list = ();
my %sources = ();
my %urls = ();
my %titles = ();
my %fullcontent = ();
my %duplicates = ();
my %originals = ();
my %ids = ();
my %signatures = ();
my %crawl_time = ();
#my $entList = '';
my %entities = ();
my %ids = ();
my %titleWords = ();
my %cat_ids = ();
my %pr_ids = ();
my %category_ids = ();
my %author_ids = ();

do
{
 my $query_url = "http://api.in.com/active18/solr/solr_search_blog.php?query=*:*+AND+insert_time:[$time+TO+*]&sortfield=crawl_time&sortorder=0&start=" . $index . "&limit=" . $limit;
 #my $query_url = "http://api.in.com/active18/solr/solr_search_read.php?query=id:eb44602598875ba01bb945afa2bd0aaecdd1368c&sortfield=crawl_time&sortorder=0&start=" . $index . "&limit=" . $limit;

 $con = get $query_url;

	while ($con =~ s/<item>(.*?)<\/item>//)
    	{
		$i = $i + 1;
		print "Count:" . $i , "\n";
		my $text = $1;
		my $id = '';
		my $url = '';
		my $crawl_time = '';
		my $source = '';
		my $title = '';			
		my $content = '';
		my $dupTitle = '';
		my $dupContent = '';
		my $finalContent = '';
		my $category = '';
		my $author = '';
				
		if ($text =~ m/<id><!\[CDATA\[(.*?)\]\]><\/id>/)
		{
				$id = $1;
		}
		if ($text =~ m/<url><!\[CDATA\[(.*?)\]\]><\/url>/)
		{
				$url = CGI::escape($1);
		}
		if ($text =~ m/<crawl_time><!\[CDATA\[(.*?)\]\]><\/crawl_time>/)
		{
				$crawl_time = CGI::escape($1);
		}
		if ($text =~ m/<source><!\[CDATA\[(.*?)\]\]><\/source>/)
		{
				$source = CGI::escape($1);
		}
		if ($text =~ m/<title><!\[CDATA\[(.*?)\]\]><\/title>/)
		{
				$title = CGI::escape($1);
				$dupTitle = $1;
		}                                
		if ($text =~ m/<content><!\[CDATA\[(.*?)\]\]><\/content>/)
		{
				$content = CGI::escape($1);
				$dupContent = $1;
		}
		if ($text =~ m/<category><!\[CDATA\[(.*?)\]\]><\/category>/)
                {
                                $category = CGI::escape($1);
                }
			
		if ($text =~ m/<author><!\[CDATA\[(.*?)\]\]><\/author>/)
                {
                                $author = CGI::escape($1);
                }
		
		
		my $compareText = $dupTitle . " " . $dupContent;
		my $ent = '';
		$ent = getEntities($compareText);
		my @entitySplit = split(/#/, $ent);

                print $entitySplit[0] , "\n";
                #print $entitySplit[1] , "\n";
                #print $entitySplit[2] , "\n";
		
		print "\nNew Ent for id" . $id . " : " . $entitySplit[0];
		#exit;
		next if $entitySplit[0] eq '';
		$entities{$id} = $entitySplit[0];
		$crawl_time{$id} = $crawl_time;
		$sources{$id} = $source;
		$titles{$id} = $dupTitle;
		$fullcontent{$id} = $content;
		
		$urls{$id} = $url;
		$ids{$id} = $id;
		$pr_ids{$id} = $entitySplit[1];
		$cat_ids{$id} = $entitySplit[2];
		$category_ids{$id} = $category;
		$author_ids{$id} = $author;
		my @words = split ("[\\W0-9]+",$dupTitle);
				
		
		
	}

      $index = $index + $limit;
	  $i++;
}while ($index <= $count);	

	my $ua = LWP::UserAgent->new;			
	foreach my $temp (keys %ids)
	{
		#my $query_url = "http://api.in.com/active18/solr/solr_add_blog.php?url=" . $urls{$temp} . "&crawl_time=" . $crawl_time{$temp} . "&source=" . $sources{$temp} . "&title=" . CGI::escape($titles{$temp}) . "&entity=" . CGI::escape($entities{$temp}). "&content=" . $fullcontent{$temp} . "&pr_id=" . $pr_ids{$temp} . "&cat_id=" . $cat_ids{$temp} . "&category=" . $category_ids{$temp};
		my $query_url = "url=" . $urls{$temp} . "&crawl_time=" . $crawl_time{$temp} . "&source=" . $sources{$temp} . "&title=" . CGI::escape($titles{$temp}) . "&entity=" . CGI::escape($entities{$temp}). "&content=" . $fullcontent{$temp} . "&pr_id=" . $pr_ids{$temp} . "&cat_id=" . $cat_ids{$temp} . "&category=" . $category_ids{$temp} . "&author=" . $author_ids{$temp} ;

		my $request = HTTP::Request->new(POST => 'http://api.in.com/active18/solr/solr_add_blog_forced.php');
		$request->content_type( 'application/x-www-form-urlencoded' );
		$request->content($query_url);
		
		my $response = $ua->request( $request );
		print "\n" . $query_url . "\n";
		if ( $response->is_success )
		{
		  print $response->content."\n\n";
		}
		else
		{
		  print $response->error_as_HTML."\n\n";
		}
		#print "\n" . $query_url . "\n";
		print "\n".$temp . ":";
		#my $con = get $query_url;
		#print $con . "\n";	
		#exit;	
	}
	
	print "#####################################################################\n";
	
sub loadStopwords
{
        open(FILE1, "$stopdir/1.stopwords");
        open(FILE2, "$stopdir/2.stopwords");

        my @stop_w = <FILE1>;
        chomp(@stop_w);
		my $signature_words = join ("|", @signature_list);
        foreach my $temp (@stop_w)
        {
                $temp =~ s/\n//;
		$temp =~ s/$signature_words//ig;
		$stopwords{$temp} = 1;
        }

        @stop_w = ();
        @stop_w = <FILE2>;
        chomp(@stop_w);
        foreach my $temp (@stop_w)
        {
		$temp =~ s/$signature_words//ig;
                $stopwords{$temp} = 1;

        }
        close (FILE1);
        close (FILE2);
}

sub stripStopWords
{
	my ($text) = @_;
	#print "Text: " . $text . "\n\n";
	my @words = split ("[\\W]+",$text);
	my $tempText = "";
	foreach my $temp (@words)
	{
		if(!exists($stopwords{$temp}))
	        {
		   $tempText = $tempText . $temp . " " ;	
		}	
		chomp($tempText);
	}
	#print "Stripped: " . $tempText , "\n";
	return $tempText;	
} 
  
sub trim($)
{
        my $string = shift;
        $string =~ s/^\s+//;
        $string =~ s/\s+$//;
        return $string;
}
  
sub getEntities
{
	my $compareText = $_[0];
	#print $compareText . "\n";
	
	$finalEntity = '';
	my $finalEntityCat = '';
	my $finalEntityProfile = '';
	my $count = 0;
	my %entityCount = ();
	foreach my $key (keys %entity_hash)
	{
		$count = 0;
                while($compareText =~m/$key/gs)
                {
                        $count++;
                 }
                if ($count > 0)
                {
                        $key =~ m/\\b(.*?)\\b/i;
                        #$entityCount{$1}=$count;
			my $lc_e = lc($1);
			if($lc_e =~ m/^[0-9]*$/)
                        {
                                #print "stopword digit " . $lc_e . "\n";
                                $entityCount{$lc_e}= 0;
                        }
                        if($stop_words_hash{$lc_e})
                        {
                                 $entityCount{$1}= 0;
                        }
                        else
                        {
                                $entityCount{$1}=$count;
                        }
                }
	}

	#chop($finalEntity);
	#$finalEntity = trim($finalEntity);
		foreach my $entity (sort {$entityCount{$b} <=> $entityCount{$a}} keys %entityCount)
        {
                print $entity , "-->", $entityCount{$entity} , "\n";
		next if $entityCount{$entity} == 0;
                $finalEntity .= $entity . ",";
				
				print $entity_cat_id{$entity} , "\n";
				$finalEntityCat .= $entity_cat_id{$entity} . ",";
				print $entity_pr_id{$entity} , "\n";
				$finalEntityProfile .= $entity_pr_id{$entity} . ",";				

        }
        $finalEntity =~ s/,$//g;
	$finalEntityProfile =~ s/,$//g;
        $finalEntityCat =~ s/,$//g;

	$finalEntity = $finalEntity . '#' . $finalEntityProfile . '#' . $finalEntityCat . "\n";		
	if($finalEntity ne '')
	{
		return $finalEntity;
	}
	else
	{
		return '';
	}
	
}  
