use strict;
use LWP::Simple;
use LWP::UserAgent;
use CGI;

open (PROCESSCHECK, ("ps -ef | grep \"/usr/bin/perl /home/web18/universal_search/news/index_search.pl\" | wc -l |"));
my @lines = <PROCESSCHECK>;
chomp (@lines);
foreach my $temp (@lines)
{
        print $temp,"\n";
        if ($temp > 3)
        {
                print "There is a entity process already running\n";
                exit;
        }
}
close(PROCESSCHECK);


#nprint " in here\n";

######## INPUT URL AND SOURCE NAME

my $mainurl = "http://cms.moneycontrol.com/mccode/news/mc_news.php";
my $url = $mainurl."?&start=0&newscnt=1"; 
#my $sourcename = 'Moneycontrol';
my $content = get $url or die $!;
my $max_count = 0;
#print $content;
#exit;
print "in here\n";
while($content =~ s/<totalnews>(.*?)<\/totalnews>//)
{
	 $max_count = $1;
}
$max_count =100;
print $max_count . "\n";
my $start =0;
#my $start = 215200;
my $count = 50;

do
{
        my $query_url = $mainurl."?&start=".$start."&newscnt=".$count;
	print "round = $start\n";
	#print $query_url."----------------------------\n\n";
        my $con = get $query_url;
#	print $con."\n";
#	exit;
#	print "here\n";


my $i=0;


	while($con =~ s/<article>(.*?)<\/article>//ism)
 	{
                #print "Count:" . $i . "\n";
                $i++;
                my $text = $1;
                my $source_article_id = 0;
		my $content_type ='';
		my $category = '';
		my $subcategory = '';
		my $main_headline = '';
		my $secondary_headline = '';
		my $summary = '';
		my $thumbnail_100x75 = '';
		my $thumbnail_50x38 = '';
		my $thumbnail_240x180 = '';
		my $first_author = '';
		my $second_author = '';
		my $article_url = '';
		my $video_title = '';
		my $videourl = '';
		my $audiourl = '';
		my $keywords = '';
		my $description = '';
                my $crawl_time = '';
		my $sourcename = '';
		my $viewcount = 0;
		my $publishdate = '';
		my $prod_category = '';
		my $download_url = '';
		my $attachment_url = '';
		my $person_name = '';
		my $person_organization = '';
		my $person_designation = '';
		my $mf_code = '';
		my $stock_code = '';
		my $price_chart_scdid = '';
		my $industry_code = '';
		my $entity_tags = '';
		my $quotes = '';
		my $mf_name = '';
		my $stock_name = '';
		my $industry_name = '';
		my $bse_nse_codes = '';
		my $mf_url = '';
		my $stock_url = '';
		my $parent_category = '';
		my $large_img_path ='';

                if ($text =~ m/<article_id>(.*?)<\/article_id>/)
                {
                        $source_article_id = $1;
			
                }
                if ($text =~ m/<contenttype>(.*?)<\/contenttype>/)
                {
                        $content_type = CGI::escape($1);
                }
                if ($text =~ m/<category>(.*?)<\/category>/)
                {
                        $category = CGI::escape($1);
                }
                if ($text =~ m/<subcategory>(.*?)<\/subcategory>/)
                {
                        $subcategory = CGI::escape($1);
                }
                if ($text =~ m/<summary>(.*?)<\/summary>/)
                {
                        $summary= CGI::escape($1);
                }

                if ($text =~ m/<main_headline>(.*?)<\/main_headline>/)
                {
                        $main_headline = CGI::escape($1);
                }
                if ($text =~ m/<secondary_headline>(.*?)<\/secondary_headline>/)
                {
                        $secondary_headline = CGI::escape($1);
                }
                if ($text =~ m/<thumbnail_img>(.*?)<\/thumbnail_img>/) # 100X75 is stored like this
                {
                                $thumbnail_100x75 = $1;
                }
                if ($text =~ m/<thumbnail_50x38>(.*?)<\/thumbnail_50x38>/)
                {
                        $thumbnail_50x38 = CGI::escape($1);
                }
                if ($text =~ m/<thumbnail_240x180>(.*?)<\/thumbnail_240x180>/)
                {
                        $thumbnail_240x180 = CGI::escape($1);
                }
                if ($text =~ m/<first_author>(.*?)<\/first_author>/)
                {
                        $first_author = CGI::escape($1);
                }
                if ($text =~ m/<second_author>(.*?)<\/second_author>/)
                {
                        $second_author = CGI::escape($1);
                }
                if ($text =~ m/<url>(.*?)<\/url>/)
                {
                        $article_url = CGI::escape($1);
                }
                if ($text =~ m/<video_title>(.*?)<\/video_title>/)
                {
                        $video_title = CGI::escape($1);
                }
                if ($text =~ m/<videourl>(.*?)<\/videourl>/)
                {
                        $videourl = CGI::escape($1);
                }
                if ($text =~ m/<audiourl>(.*?)<\/audiourl>/)
                {
                        $audiourl = CGI::escape($1);
                }
                if ($text =~ m/<keywords>(.*?)<\/keywords>/)
                {
                        $keywords = CGI::escape($1);
                }
                if ($text =~ m/<description>(.*?)<\/description>/)
                {
                        $description = CGI::escape($1);
                }
                if ($text =~ m/<sourcename>(.*?)<\/sourcename>/)
                {
                        $sourcename = CGI::escape($1);
                }
                if ($text =~ m/<viewcount>(.*?)<\/viewcount>/)
                {
                        $viewcount = CGI::escape($1);
			 if($viewcount eq '')
                        {
                                $viewcount = 0;
                        }

                }
                if ($text =~ m/<publishdate>(.*?)<\/publishdate>/)
                {
                        $publishdate = $1;

			#INPUT FORMAT 2009-04-09 13:46:54
			if($publishdate =~ s/(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)//)	
			{
				$publishdate= $1. $2. $3. $4. $5. $6;	
			}
			$publishdate = CGI::escape($publishdate); 
			#exit;
                }
                if ($text =~ m/<prod_category>(.*?)<\/prod_category>/)
                {
                        $prod_category = CGI::escape($1);
                }
                if ($text =~ m/<download_url>(.*?)<\/download_url>/)
                {
                        $download_url = CGI::escape($1);
                }
                if ($text =~ m/<attachment>(.*?)<\/attachment>/)
                {
                        $attachment_url = CGI::escape($1);
                }
                if ($text =~ m/<person_name>(.*?)<\/person_name>/)
                {
                        $person_name = CGI::escape($1);
                }
                if ($text =~ m/<person_organization>(.*?)<\/person_organization>/)
                {
                        $person_organization = CGI::escape($1);
                }
                if ($text =~ m/<person_designation>(.*?)<\/person_designation>/)
                {
                        $person_designation = CGI::escape($1);
                }
                if ($text =~ m/<mf_code>(.*?)<\/mf_code>/)
                {
                        $mf_code = CGI::escape($1);
                }
                if ($text =~ m/<stock_code>(.*?)<\/stock_code>/)
                {
                        $stock_code = CGI::escape($1);
                }
		if ($text =~ m/<price_chart_scdid>(.*?)<\/price_chart_scdid>/)
                {
                        $price_chart_scdid = CGI::escape($1);
                }
                if ($text =~ m/<industry_code>(.*?)<\/industry_code>/)
                {
                        $industry_code = CGI::escape($1);
                }
                if ($text =~ m/<entity_tags>(.*?)<\/entity_tags>/)
                {
                        $entity_tags = CGI::escape($1);
                }
                if ($text =~ m/<quotes>(.*?)<\/quotes>/)
                {
                        $quotes = CGI::escape($1);
                }
                if ($text =~ m/<mf_name>(.*?)<\/mf_name>/)
                {
                        $mf_name = CGI::escape($1);
                }
                if ($text =~ m/<mf_url>(.*?)<\/mf_url>/)
                {
                        $mf_url = CGI::escape($1);
                }
                if ($text =~ m/<stock_name>(.*?)<\/stock_name>/)
                {
                        $stock_name = CGI::escape($1);
                }
                if ($text =~ m/<stock_url>(.*?)<\/stock_url>/)
                {
                        $stock_url = CGI::escape($1);
                }
                if ($text =~ m/<industry_name>(.*?)<\/industry_name>/)
                {
                        $industry_name = CGI::escape($1);
                }
                if ($text =~ m/<bse_nse_codes>(.*?)<\/bse_nse_codes>/)
                {
                        $bse_nse_codes = CGI::escape($1);
                }

		if ($text =~ m/<main_category>(.*?)<\/main_category>/)
                {
                        $parent_category = CGI::escape($1);
                }
		if($text=~m/<large_img_path>(.*?)<\/large_img_path>/)
		{
			$large_img_path	= CGI::escape($1);
		}
#large_img_path
		#print "Main Category:" . $main_category;
		#exit;
		my $authors = '';
		if($second_author ne '')
		{
			$authors = $first_author.",".$second_author;
		}
		else
		{
			$authors = $first_author;
		}
	#	exit;	
#		my $posturl = "http://api2.in.com/active18/solr/solr_add_moneycontrol_news.php";
		#my $posturl = "http://api.moneycontrol.com/solr/solr_add_moneycontrol_news_test.php";
		my $posturl = "http://solrplatform.in.com/mc/solr_add_moneycontrol_news.php";
		my %formdata = ();
		print $source_article_id."=id,$publishdate=date\n";
		#print $source_article_id . " again\n";
		#print $stock_url . " \n";
		#print "Contenttype=".$content_type;
		#exit;
		$formdata{'large_img_path'}=$large_img_path;
		$formdata{'source_article_id'}=$source_article_id;
$formdata{'category'}=$category;
$formdata{'sub_category'}=$subcategory;
$formdata{'content_type'}=$content_type;
$formdata{'main_headline'}=$main_headline;
$formdata{'secondary_headline'}=$secondary_headline;
$formdata{'summary'}=$summary;
$formdata{'description'}=$description;
$formdata{'article_url'}=$article_url;
$formdata{'keywords'}=$keywords;
$formdata{'authors'}=$authors;
$formdata{'source'}=$sourcename;
$formdata{'thumbnail_100x75'}=$thumbnail_100x75;
$formdata{'thumbnail_50x38'}=$thumbnail_50x38;
$formdata{'thumbnail_240x180'}=$thumbnail_240x180;
$formdata{'audio_url'}=$audiourl;
$formdata{'video_title'}=$video_title;
$formdata{'video_url'}=$videourl;
$formdata{'publish_date'}=$publishdate;
$formdata{'viewcount'}=$viewcount;
$formdata{'product_category'}=$prod_category;
$formdata{'attachment_url'}=$attachment_url;
$formdata{'person_name'}=$person_name;
$formdata{'person_organization'}=$person_organization;
$formdata{'person_designation'}=$person_designation;
$formdata{'mf_code'}=$mf_code;
$formdata{'stock_code'}=$stock_code;
$formdata{'price_chart_scdid'} = $price_chart_scdid;
$formdata{'industry_code'}=$industry_code;
$formdata{'mf_name'}=$mf_name;
$formdata{'stock_name'}=$stock_name;
$formdata{'industry_name'}=$industry_name;
$formdata{'mf_url'}=$mf_code;
$formdata{'stock_url'}=$stock_url;
$formdata{'bse_nse_codes'}=$bse_nse_codes;
$formdata{'parent_category'}=$parent_category;

	foreach my $key (keys %formdata)
	{
		if ($formdata{$key} eq 'NULL')
		{
			$formdata{$key} = '';
			
		}
	}
		my $url="";
		foreach my $key (keys %formdata)
        	{
			$url.=$key."=".$formdata{$key}."&";
		}
#print $url;
#exit;
		my $ua = new LWP::UserAgent;
		my $response  = $ua->post( $posturl, \%formdata );
		my $content = $response->content; 

		#print $query."\n";
		print $content."  record $i \n";
		#exit;

        }
	#exit;
                $start = $start + $count;
#        $i++;
}while ($start <= $max_count);

#my $url = "http://api2.in.com/active18/solr/solr_commit_moneycontrol_news.php";
#my $res = get $url;
print "ran commit successfully\n";
