unable to scrape images and full text , only only title and featured image is inserted.how can i correct this error . save this code as grab.php and run
check it on my server http://ift.tt/1JTGnSMhttp://ift.tt/1AKvnPp
and you can see the results here
http://ift.tt/1JTGoWT
my scrapper uploads the featurued image of each post on my server but if the images that are includedd inside posts doesn't upload them on my site to wp uploads folder
<head profile="http://gmpg.org/xfn/11">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta charset="UTF-8">
<?php
header('Content-Type: text/html; charset=utf-8');
//RUN IT AS grab.php? c=215&p=1&site=diaforetiko.gr&s=http://ift.tt/1AKvo5E
//Where c= Your cat id
//Where p= number of posts to grab
//Where site= Source site
//Where s= Source Sites full category path
$source = $_GET['s'];
$n = $_GET['p'];
$site = $_GET['site'];
$mcat = $_GET['c'];
$cat = $_GET['c'];
require( dirname(__FILE__) . '/../wp-load.php' );
//OTHERSIDE SCRAPPER
if ($site == "otherside.gr")
{
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $source);
curl_setopt($curl, CURLOPT_HEADER, 0);
curl_setopt($curl1, CURLOPT_POST, true);
curl_setopt($curl1, CURLOPT_POSTFIELDS, $post);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION,true);
$result = curl_exec($curl); curl_close($curl);
preg_match_all('/<h2 class="title front-view-title">(.*?)<\/h2>/s', $result, $matches, PREG_OFFSET_CAPTURE);
$arr = array();
$doc = new DOMDocument();
//extract the single block post URLs
for($ik=0;$ik<=$n;$ik++)
{
$doc->loadHTML($matches[1][$ik][0]);
$imageTags = $doc->getElementsByTagName('a');
foreach($imageTags as $tag) {
$arr[]=$tag->getAttribute('href');
}
}
//get first 6 post result
for($i=0;$i<=$n;$i++)
{
$curl1 = curl_init();
curl_setopt($curl1, CURLOPT_URL, $arr[$i]);
curl_setopt($curl1, CURLOPT_HEADER, 0);
curl_setopt($curl1, CURLOPT_POST, true);
curl_setopt($curl1, CURLOPT_POSTFIELDS, $post);
curl_setopt($curl1, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl1, CURLOPT_FOLLOWLOCATION,true);
$result1 = curl_exec($curl1); curl_close($curl1);
## get the title
preg_match_all('/<h1 class="title single-title">(.*?)<\/h1>/s', $result1, $matches1, PREG_OFFSET_CAPTURE);
$my_title =$matches1[1][0][0];
require_once('url_to_absolute/simple_html_dom.php');
require_once('url_to_absolute/url_to_absolute.php');
$arr4 = array();
$arr41 = array();
$arr44 = array();
$html = file_get_html($arr[$i]);
## get the content
foreach($html->find('div[class=post-single-content box mark-links]') as $table)
{
$arr44[]= $table->innertext ;
}
## get the image
foreach($html->find('div[class=post-single-content box mark-links] img') as $table1)
{
$size = getimagesize(url_to_absolute($url, $table1->src));
if($size['mime']=="image/jpeg" || $size['mime']=="image/JPEG" || $size['mime']=="image/jpg" || $size['mime']=="image/JPG" || $size['mime']=="image/png" )
{
$arr4[] = url_to_absolute($url, $table1->src);
}
}
$r_image=$arr4[0];
// check if the content was already updated in our database
$querystr = "SELECT * FROM $wpdb->posts
LEFT JOIN $wpdb->term_relationships ON($wpdb->posts.ID = $wpdb->term_relationships.object_id)
LEFT JOIN $wpdb->term_taxonomy ON($wpdb->term_relationships.term_taxonomy_id = $wpdb->term_taxonomy.term_taxonomy_id)
WHERE ($wpdb->term_taxonomy.term_id = ".$_GET['c']."
AND $wpdb->term_taxonomy.taxonomy = 'category'
AND $wpdb->posts.post_status = 'publish'
AND $wpdb->posts.post_title like '%".$my_title."%')";
$pageposts = $wpdb->get_results($querystr);
if(empty($pageposts[0]->ID))
{
$mcat = $_GET['ca'];
//Database insert query
$my_post = array(
'post_title' => trim($my_title),
'post_status' => 'publish',
'post_author' => 1,
'post_category' => array($_GET['c']),
'post_date' => date('Y-m-d H:i:s'),
'post_date_gmt' => date('Y-m-d H:i:s'),
'post_type' => 'post'
);
$post_id=wp_insert_post( $my_post );
//update with new image and the new content
$image_url = $r_image;
$upload_dir = wp_upload_dir();
$image_data = file_get_contents($image_url);
$filename = basename($image_url);
if(wp_mkdir_p($upload_dir['path']))
$file = $upload_dir['path'] . '/' . $filename;
else
$file = $upload_dir['basedir'] . '/' . $filename;
file_put_contents($file, $image_data);
$wp_filetype = wp_check_filetype($filename, null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => sanitize_file_name($filename),
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file, $post_id );
require_once(ABSPATH . 'wp-admin/includes/image.php');
$attach_data = wp_generate_attachment_metadata( $attach_id, $file );
wp_update_attachment_metadata( $attach_id, $attach_data );
set_post_thumbnail( $post_id , $attach_id );
$Img_Uploaded_Url=get_site_url()."/wp-content/uploads/".$attach_data['file'];
// - concatenate both image and content
## I need to wrap the text
//$remove_html_first= strip_tags(implode("</p><p>", $arr44));
//$mylink = " <a href=http://ift.tt/1JTGoWV".$arr[$i]."> ...συνέχεια ΕΔΩ!</a>";
//$then_truncate_the_value=mb_substr($remove_html_first , 0, 300);
//$finallink = $then_truncate_the_value.$mylink;
//$res_final= "<p><img src=".$Img_Uploaded_Url."></p>"."<p>".$finallink."</p>";
## I need full text
$linkz='<p><a href="http://otherside.gr" target="_blank">Πηγή</a></p>';
$res_final= implode("</p><p>", $arr44).$linkz;
$d = new DOMDocument();
$d->loadHTML('<meta http-equiv="content-type" content="text/html; charset=utf-8">'.$res_final);
$s = new DOMXPath($d);
foreach($s->query('//div[contains(attribute::class, "sharebar-wrap")]') as $t )
$t->parentNode->removeChild($t);
foreach($s->query('//*[contains(@class, "sharebar-wrap")]') as $t )
$t->parentNode->removeChild($t);
$res_finals = $d->saveHTML();
$wpdb->query("UPDATE $wpdb->posts SET post_content = '".str_replace("'", "", $res_finals)."' WHERE ID = '".$post_id."'");
}
}
echo '<div style="margin-right:10%; margin-top:12%; margin-left:39%; margin-bottom:16%; color:#339933; font-size:16px;"> Successfully scrapped the contents</div>';
$html->clear();
unset($html);
}//END OTHERSIDE
//TILESTWRA.COM SCRAPPER
if ($site == "tilestwra.com")
{
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $source);
curl_setopt($curl, CURLOPT_HEADER, 0);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION,true);
$result = curl_exec($curl); curl_close($curl);
preg_match_all('/<h3>(.*?)<\/h3>/s', $result, $matches, PREG_OFFSET_CAPTURE);
$arr = array();
$doc = new DOMDocument();
//extract the single block post URLs
for($ik=0;$ik<=$n;$ik++)
{
$doc->loadHTML($matches[1][$ik][0]);
$imageTags = $doc->getElementsByTagName('a');
foreach($imageTags as $tag) {
$arr[]=$tag->getAttribute('href');
}
}
//get first 6 post result
for($i=0;$i<=$n;$i++)
{
$curl1 = curl_init();
curl_setopt($curl1, CURLOPT_URL, $arr[$i]);
curl_setopt($curl1, CURLOPT_HEADER, 0);
curl_setopt($curl1, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl1, CURLOPT_FOLLOWLOCATION,true);
$result1 = curl_exec($curl1); curl_close($curl1);
## get the title
preg_match_all('/<h1 class="light-title">(.*?)<\/h1>/s', $result1, $matches1, PREG_OFFSET_CAPTURE);
$my_title = $matches1[1][0][0];
require_once('url_to_absolute/simple_html_dom.php');
require_once('url_to_absolute/url_to_absolute.php');
$arr4 = array();
$arr41 = array();
$arr44 = array();
$html = file_get_html($arr[$i]);
## get the content
foreach($html->find('div[class=item-content]') as $table)
{
$arr44[]= $table->innertext;
}
## get the image
foreach($html->find('div[class=single-inbox] img') as $table1)
{
$size = getimagesize(url_to_absolute($url, $table1->src));
if($size['mime']=="image/jpeg" || $size['mime']=="image/JPEG" || $size['mime']=="image/jpg" || $size['mime']=="image/JPG" || $size['mime']=="image/png" )
{
$arr4[] = url_to_absolute($url, $table1->src);
}
}
$r_image=$arr4[0];
// check if the content was already updated in our database
$querystr = "SELECT * FROM $wpdb->posts
LEFT JOIN $wpdb->term_relationships ON($wpdb->posts.ID = $wpdb->term_relationships.object_id)
LEFT JOIN $wpdb->term_taxonomy ON($wpdb->term_relationships.term_taxonomy_id = $wpdb->term_taxonomy.term_taxonomy_id)
WHERE ($wpdb->term_taxonomy.term_id = ".$_GET['c']."
AND $wpdb->term_taxonomy.taxonomy = 'category'
AND $wpdb->posts.post_status = 'publish'
AND $wpdb->posts.post_title like '%".trim($my_title)."%')";
$pageposts = $wpdb->get_results($querystr);
if(empty($pageposts[0]->ID))
{
//Database insert query
$my_post = array(
'post_title' => trim($my_title),
'post_status' => 'publish',
'post_author' => 1,
'post_category' => array($_GET['c']),
'post_date' => date('Y-m-d H:i:s'),
'post_date_gmt' => date('Y-m-d H:i:s'),
'post_type' => 'post'
);
$post_id=wp_insert_post( $my_post );
//update with new image and the new content
$image_url = $r_image;
$upload_dir = wp_upload_dir();
$image_data = file_get_contents($image_url);
$filename = basename($image_url);
if(wp_mkdir_p($upload_dir['path']))
$file = $upload_dir['path'] . '/' . $filename;
else
$file = $upload_dir['basedir'] . '/' . $filename;
file_put_contents($file, $image_data);
$wp_filetype = wp_check_filetype($filename, null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => sanitize_file_name($filename),
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file, $post_id );
require_once(ABSPATH . 'wp-admin/includes/image.php');
$attach_data = wp_generate_attachment_metadata( $attach_id, $file );
wp_update_attachment_metadata( $attach_id, $attach_data );
set_post_thumbnail( $post_id , $attach_id );
$Img_Uploaded_Url=get_site_url()."/wp-content/uploads/".$attach_data['file'];
// - concatenate both image and content
## I need full text AND REMOVE SCRIPTS AND DIVS
$linkz='<p><a href="http://www.tilestwra.gr" target="_blank">Πηγή</a></p>';
$arr44c = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', "", $arr44);
$arr44d = preg_replace('#<div id="facebook-comments">(.*?)</div>#', ' ', $arr44c);
$arr44e = preg_replace('#<div class="mobileno adbox(.*?)</div>#', ' ', $arr44d);
$arr44b = preg_replace('#<div class="mobileno adbsox(.*?)</div>#', ' ', $arr44e);
$res_final= '<div style="font-size:16px; color:#000;">' . implode("</p><p>", $arr44b).$linkz . '</div>';
$d = new DOMDocument();
$d->loadHTML('<meta http-equiv="content-type" content="text/html; charset=utf-8">'.$res_final);
$s = new DOMXPath($d);
//REMOVE DIVS
foreach($s->query('//div[contains(attribute::class, "fbwithcount")]') as $t )
$t->parentNode->removeChild($t);
foreach($s->query('//*[contains(@class, "fbwithcount")]') as $t )
$t->parentNode->removeChild($t);
$res_finals = $d->saveHTML();
$wpdb->query("UPDATE $wpdb->posts SET post_content = '".str_replace("'", "", $res_finals)."' WHERE ID = '".$post_id."'");
}
}
echo '<div style="margin-right:10%; margin-top:12%; margin-left:39%; margin-bottom:16%; color:#339933; font-size:16px;"> Successfully scrapped the contents</div>';
$html->clear();
unset($html);
}//END TILESTWRA.COM
//DIAFORETIKO SCRAPPER
if ($site == "diaforetiko.gr")
{
$website_url='www.diaforetiko.gr';
$curl = curl_init($website_url);
curl_setopt($curl, CURLOPT_URL, $source);
curl_setopt($curl, CURLOPT_HEADER, 0);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION,true);
$result = curl_exec($curl); curl_close($curl);
preg_match_all('/<h2>(.*?)<\/h2>/s', $result, $matches, PREG_OFFSET_CAPTURE);
$arr = array();
$doc = new DOMDocument();
//extract the single block post URLs
for($ik=0;$ik<=$n;$ik++)
{
$doc->loadHTML($matches[1][$ik][0]);
$imageTags = $doc->getElementsByTagName('a');
foreach($imageTags as $tag) {
$arr[]=$tag->getAttribute('href');
}
}
//get first 6 post result
for($i=0;$i<=$n;$i++)
{
$curl1 = curl_init();
curl_setopt($curl1, CURLOPT_URL, $arr[$i]);
curl_setopt($curl1, CURLOPT_HEADER, 0);
curl_setopt($curl1, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl1, CURLOPT_FOLLOWLOCATION,true);
$result1 = curl_exec($curl1); curl_close($curl1);
## get the title
preg_match_all('/<h1>(.*?)<\/h1>/s', $result1, $matches1, PREG_OFFSET_CAPTURE);
$my_title =$matches1[1][0][0];
require_once('url_to_absolute/simple_html_dom.php');
require_once('url_to_absolute/url_to_absolute.php');
$arr4 = array();
$arr41 = array();
$arr44 = array();
$html = file_get_html($arr[$i]);
## get the content
foreach($html->find('div[class=post-content]') as $table)
{
$arr44[]= $table->innertext ;
}
## get the image
foreach($html->find('div[class=post-content] img') as $table1)
{
$size = getimagesize(url_to_absolute($url, $table1->src));
if($size['mime']=="image/jpeg" || $size['mime']=="image/JPEG" || $size['mime']=="image/jpg" || $size['mime']=="image/JPG" || $size['mime']=="image/png" )
{
$arr4[] = url_to_absolute($url, $table1->src);
}
}
$r_image=$arr4[0];
// check if the content was already updated in our database
$querystr = "SELECT * FROM $wpdb->posts
LEFT JOIN $wpdb->term_relationships ON($wpdb->posts.ID = $wpdb->term_relationships.object_id)
LEFT JOIN $wpdb->term_taxonomy ON($wpdb->term_relationships.term_taxonomy_id = $wpdb->term_taxonomy.term_taxonomy_id)
WHERE ($wpdb->term_taxonomy.term_id = ".$_GET['c']."
AND $wpdb->term_taxonomy.taxonomy = 'category'
AND $wpdb->posts.post_status = 'publish'
AND $wpdb->posts.post_title like '%".trim($my_title)."%')";
$pageposts = $wpdb->get_results($querystr);
if(empty($pageposts[0]->ID))
{
$mcat = $_GET['ca'];
//Database insert query
$my_post = array(
'post_title' => trim($my_title),
'post_status' => 'publish',
'post_author' => 1,
'post_category' => array($_GET['c']),
'post_date' => date('Y-m-d H:i:s'),
'post_date_gmt' => date('Y-m-d H:i:s'),
'post_type' => 'post'
);
$post_id=wp_insert_post( $my_post );
//update with new image and the new content
$image_url = $r_image;
$upload_dir = wp_upload_dir();
$image_data = file_get_contents($image_url);
$filename = basename($image_url);
if(wp_mkdir_p($upload_dir['path']))
$file = $upload_dir['path'] . '/' . $filename;
else
$file = $upload_dir['basedir'] . '/' . $filename;
file_put_contents($file, $image_data);
$wp_filetype = wp_check_filetype($filename, null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => sanitize_file_name($filename),
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file, $post_id );
require_once(ABSPATH . 'wp-admin/includes/image.php');
$attach_data = wp_generate_attachment_metadata( $attach_id, $file );
wp_update_attachment_metadata( $attach_id, $attach_data );
set_post_thumbnail( $post_id , $attach_id );
$Img_Uploaded_Url=get_site_url()."/wp-content/uploads/".$attach_data['file'];
// - concatenate both image and content
## I need to wrap the text
//$remove_html_first= strip_tags(implode("</p><p>", $arr44));
//$mylink = " <a href=http://ift.tt/1JTGoWV".$arr[$i]."> ...συνέχεια ΕΔΩ!</a>";
//$then_truncate_the_value=mb_substr($remove_html_first , 0, 300);
//$finallink = $then_truncate_the_value.$mylink;
//$res_final= "<p><img src=".$Img_Uploaded_Url."></p>"."<p>".$finallink."</p>";
## I need full text
$linkz='Μην ξεχάσετε να κάνετε κοινοποίηση';
$arr44d = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', "", $arr44);
$arr44e = preg_replace('#<div id="minions">(.*?)</div>#', ' ', $arr44d);
$arr44f = preg_replace('#<div align="left">(.*?)</div>#', ' ', $arr44e);
$arr44g = preg_replace('#<h3>(.*?)</h3>#', ' ', $arr44f);
$arr44c = preg_replace('#<div class="fb-comments fb_iframe_widget">(.*?)</div>#', ' ', $arr44g);
$res_final= implode("</p><p>", $arr44c).$linkz;
$d = new DOMDocument();
//libxml_use_internal_errors(true);
$d->loadHTML('<meta http-equiv="Content-Type" content="text/html"; charset="utf-8">'.$res_finals);
$s = new DOMXPath($d);
foreach($s->query('//div[contains(attribute::class, "sharebar-wrap")]') as $t )
$t->parentNode->removeChild($t);
foreach($s->query('//*[contains(@class, "sharebar-wrap")]') as $t )
$t->parentNode->removeChild($t);
foreach($s->query('//div[contains(attribute::class, "fb-comments fb_iframe_widget")]') as $t )
$t->parentNode->removeChild($t);
foreach($s->query('//*[contains(@class, "fb-comments fb_iframe_widget")]') as $t )
$t->parentNode->removeChild($t);
$res_finals = $d->saveHTML();
$wpdb->query("UPDATE $wpdb->posts SET post_content = '".str_replace("'", "", $res_finals)."' WHERE ID = '".$post_id."'");
}
}
echo '<div style="margin-right:10%; margin-top:12%; margin-left:39%; margin-bottom:16%; color:#339933; font-size:16px;"> Successfully scrapped the contents</div>';
$html->clear();
unset($html);
}//END DIAFORETIKO
?>
