use Data::Dumper;
use strict;
use Lingua::EN::Tagger;
use LWP::Simple;
use XML::Simple;
use URI::Escape;

my $p = new Lingua::EN::Tagger;
my %final_list_of_nouns = ();
my %stopwords = ("the" => 1,"and" => 1,"of" => 1);
my %all_stopwords = ();

loadStopWords();

my $text = $ARGV[0];
$text =~ s/\\//g;
my $replace_text = '';

my @sentences = extract_sentences($text);
my $xml_parser = new XML::Simple;
my $index = 0;
my %extracted_entities;
foreach my $sentence (@sentences)
{
	foreach my $sen (@$sentence)
	{
		my $all_nouns = get_nouns_from_text ($sen);
		my $tagged_text = $p->add_tags($sen);
		my $replace_sen = $sen;
		foreach my $noun (sort {length($b) <=> length($a)} keys %$all_nouns)
		{
			$final_list_of_nouns{$noun} += $all_nouns->{$noun};
			my $response = checkIfExtracted("\"".$noun."\"");
			if($response or exists($extracted_entities{$noun})){
				#print "#Found#";
				$extracted_entities{$noun} = 1;
				my $replace_noun = $noun;
                                print $replace_noun."#";
 #                               $replace_noun =~ s/\s/<sp>/g;
#                                $replace_sen =~ s/\b$noun\b/<span class="Entity">$replace_noun<\/span>/g;

			}
			else{
				#print "\nChecking Index for the entity:$noun\n";
				my $url = "http://172.30.40.81/entity_extraction/solr_search_blog.php?query=\"". $noun . "\"&sortfield=crawl_time&limit=-1";
				#print $url , "\n";
			
				my $checkNoun = get $url;
				my %checks = ();
				while ($checkNoun =~ s/(.*?)~#~(.*?)<br>//i)
				{
					$checks{$1}=$2;
				}
				my $max = -1;
				my $exactMax = -1;
				foreach my $temp (keys %checks)
				{
					if ($checks{$temp} > $max)
					{
						$max = $checks{$temp};
						if ($temp eq $noun)
						{
							$exactMax = $checks{$temp};
						}
					}
				}
				if ($exactMax >= $max)
				{
					$extracted_entities{$noun} = 1;
					my $replace_noun = $noun;
					print $replace_noun."#";
				#	$replace_noun =~ s/\s/<sp>/g;
				#	$replace_sen =~ s/\b$noun\b/<span class="Entity">$replace_noun<\/span>/g;
				}
			}
		}
		#$replace_text .= $replace_sen . "<br><br>";
	}
}

#$replace_text =~ s/<sp>/ /g;
#print "&#&\n",$replace_text , "\n";
#foreach my $word_phrase (sort {$final_list_of_nouns{$b} <=> $final_list_of_nouns{$a}} keys %final_list_of_nouns)
#{
#	print $word_phrase , "-->" , $final_list_of_nouns{$word_phrase} , "\n";
#}

sub checkIfExtracted{
	my %entities_found;
	my $noun = $_[0];
	my $url = "http://172.30.40.101:2525/entities/select/?q=entities:".$noun."&version=2.2&start=0&rows=10&indent=on&f=none";
	#print $url."\n";
        my $response = $xml_parser->XMLin(get $url);
        if($response->{result}->{numFound}){
		return 1;
	}
	else{
		return 0;
	}
}

sub extract_sentences{
	my @split_sentences = ();
	my $tagger = new Lingua::EN::Tagger;
	my $sentences = $tagger->get_sentences($text);
	foreach my $sentence (@$sentences){
        	#print "----------\n";
        	#print "ORIGINAL:".$sentence."\n";
        	my @temp = $sentence =~ /[\.]?(.*?\s\w{2,}[`'\)\s"]*)\.([\w{1,}]?)/imsgx;
        	my $i;
        	#print "---------\n";
        	for ($i =0;$i<scalar(@temp);){
        	        if ($i > 1){
        	                #print $temp[$i-1].$temp[$i]."\n";
				@split_sentences = (@split_sentences,$temp[$i-1].$temp[$i]);
        	        }
        	        else{
        	#                print $temp[$i]."\n";
				@split_sentences = (@split_sentences,$temp[$i]);
        	        }
               		$i+=2;
        	}
		@split_sentences = (@split_sentences,$temp[scalar(@temp)-1]);
        	#print $temp[scalar(@temp)-1]."\n";
	}
	return \@split_sentences;
}


sub get_nouns_from_text
{
	my ($text) = @_;
	#print $text , "\n";
	my $tagged_text = $p->add_tags($text);
	my @words = split (/\s+/sg, $tagged_text);
	chomp(@words);
	my $noun_phrase = '';
	my %nouns = ();
	my $is_ending_with_stopword = 0;
	foreach my $word (@words)
	{
		if ($word =~ m/<(.*?)>(.*?)<(.*?)>/)
		{
			my $w = $2;
			if ($w =~ m/^\W+/)
			{
				if ($noun_phrase ne '')
				{
					assignEntity($is_ending_with_stopword, $noun_phrase,\%nouns);
					$is_ending_with_stopword = 0;
					$noun_phrase = '';	
				}
			}
			else
			{
				$w =~ s/^\W+|\W+$//g;
				if ($w =~ m/^[A-Z].*?$/)
				{
					$noun_phrase = $noun_phrase . $w . " ";
				}
				elsif (exists $stopwords{lc($w)})
				{
					$noun_phrase = $noun_phrase . $w . " ";
				}
				else
				{
					assignEntity($is_ending_with_stopword, $noun_phrase,\%nouns);
					$is_ending_with_stopword = 0;
					$noun_phrase = '';	
				}
			}	
		}
	}	
	if ($noun_phrase ne '')
	{
		assignEntity($is_ending_with_stopword, $noun_phrase,\%nouns);
		$is_ending_with_stopword = 0;
		$noun_phrase = '';	
	}
	return \%nouns;
}

sub assignEntity
{
	my ($is_ending_with_stopword, $noun_phrase,$nouns) = @_;
	if ($noun_phrase ne '')
	{
		$noun_phrase =~ s/\s$//i;
		my $loop_flag = 1;
		
		while ($loop_flag == 1)
		{
			my @words = split (/\s/,$noun_phrase);
			chomp(@words);
			my $phrase_length = @words;
			my $begin_is_stopword = -1;
			my $end_is_stopword = -1;
			my $is_stopword = -1;
			my $flag = 0;
			#print $noun_phrase , "---<br>";
			#print $phrase_length , "<br>";		
			for (my $i = $phrase_length - 1; $i >= 0; $i--)
			{
				if (exists ($all_stopwords{lc($words[$i])}))
				{
					#print $noun_phrase , "--", $phrase_length , "<br>";
					if ($phrase_length == 1)
					{
						if ($i == $phrase_length - 1)
						{
							$end_is_stopword = 1;
							$flag = 1;
						}
						if ($i == 0)
						{
							$begin_is_stopword = 1;
							$flag = 1;
						}
					}
				}
				elsif (exists ($stopwords{lc($words[$i])}))
				{
					#print $noun_phrase , "<br>";
					if ($words[$i] !~ m/^[A-Z]+$/)
					{
						if ($i == $phrase_length - 1)
						{
							$end_is_stopword = 1;
							$flag = 1;
						}
						if ($i == 0)
						{
							#print "--->came here#" , $words[$i] , "#<br>";
							if ($words[$i] !~ m/^[A-Z].*?$/)
							{
							
								$begin_is_stopword = 1;
								$flag = 1;
							}
						}
					}
				}
			}
			if ($flag == 1 && $noun_phrase ne '')
			{
				#print $noun_phrase , "#####", $end_is_stopword , "#####", $begin_is_stopword , "#####<br>";
				if ($phrase_length == 1)
				{
					if ($end_is_stopword == 1 || $begin_is_stopword == 1)
					{
						$noun_phrase = '';
					}
				}
				else
				{	
					if ($end_is_stopword == 1)
					{
						$noun_phrase =~ s/\s([^ ]*?)$//;
					}
					if ($begin_is_stopword == 1)
					{
						$noun_phrase =~ s/^(.*?)\s//;
					}
					#print $noun_phrase , "--><br>";
				}
				$flag = 0;
				$end_is_stopword = 0;
				$begin_is_stopword = 0;
				#print $noun_phrase , "<br>";
			}

			else
			{
				if ($noun_phrase ne '')
				{
					if ($end_is_stopword == 1)
                                        {
                                                $noun_phrase =~ s/\s([^ ]*?)$//;
                                        }
                                        if ($begin_is_stopword == 1)
                                        {
                                                $noun_phrase =~ s/^(.*?)\s//;
                                        }
					$nouns->{$noun_phrase}++;
					#print $noun_phrase , "---244\n";
				}
				$loop_flag = 0;
			}
			#print $noun_phrase , "-->\n";
		}
	}
}

sub loadStopWords
{
	open (FILENAME, "/home/moneycontrol/entity_extraction/stopwords");
	my @all_words = <FILENAME>;
	chomp(@all_words);
	foreach my $word (@all_words)
	{	
		$all_stopwords{$word} = 1;
	}
	close(FILENAME);
	foreach my $word (keys %stopwords)
	{
		if (exists $all_stopwords{$word})
		{
			delete $all_stopwords{$word};
		}
	}
}


