shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update AvasScraper.php

Browse Source
This commit is contained in:
Mohamed Jinas
2020-10-07 21:17:40 +05:00
parent 32888bd5b2
commit 38ef1570f3
1 changed files with 50 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
app/Services/Scrapers/AvasScraper.php
View File

@@ -1,8 +1,10 @@
<?php <?php
namespace App\Services\Scrapers; namespace App\Services\Scrapers;
use Goutte\Client; use Goutte\Client;
use Illuminate\Support\Carbon; use Illuminate\Support\Carbon;
use Exception;
class AvasScraper class AvasScraper
{ {
@@ -12,7 +14,7 @@ class AvasScraper
protected $author; protected $author;
protected $image = "images/noimg.jpg"; protected $image = "images/noimg.jpg";
protected $topics = []; protected $topics = [];
public function __construct() public function __construct()
{ {
$this->client = new Client; $this->client = new Client;
@@ -20,59 +22,58 @@ class AvasScraper
public function extract($url, $date) public function extract($url, $date)
{ {
try {
$crawler = $this->client->request('GET', $url);
$crawler = $this->client->request('GET', $url); $title = $crawler->filter('h1')->first()->text();
$title = $crawler->filter('h1')->first()->text(); if ($crawler->filter('figure img')->count() > 0) {
$this->image = $crawler->filter('figure img')->first()->attr('src');
}
if($crawler->filter('figure img')->count() > 0) if ($crawler->filter('.post_content p')->count() == 0) {
{
$this->image = $crawler->filter('figure img')->first()->attr('src');
}
if($crawler->filter('.post_content p')->count() == 0)
{
return;
}
$crawler->filter('.post_content p')->each(function ($node) {
$this->content[] = preg_replace("/[a-zA-Z]/","",$node->text());
});
$crawler->filter('div[class*="border-t border-grey-light border-dotted mt-7 py-3"] a')->each(function ($node) {
//Removing the show more tags button
if($node->text() == "+")
{
return; return;
} }
$this->topics[] = [
"name" => $node->text(), $crawler->filter('.post_content p')->each(function ($node) {
"slug" => str_replace("/", "", $node->attr('href')) $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());
});
$crawler->filter('div[class*="border-t border-grey-light border-dotted mt-7 py-3"] a')->each(function ($node) {
//Removing the show more tags button
if ($node->text() == "+") {
return;
}
$this->topics[] = [
"name" => $node->text(),
"slug" => str_replace("/", "", $node->attr('href'))
];
});
if ($crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->count() == 1) {
$this->author = $crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->first()->text();
}
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
return [
'source' => 'Avas',
'title' => $title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
'image' => $this->image,
'content' => $this->content,
'url' => $url,
'date' => $date,
'guid' => str_replace("https://avas.mv/", "", $url),
'author' => $this->author,
'topics' => $this->topics
]; ];
}); } catch (\Exception $e) {
if($crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->count() == 1)
{
$this->author = $crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->first()->text();
} }
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
return [
'source' => 'Avas',
'title' => $title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
'image' => $this->image,
'content' => $this->content,
'url' => $url,
'date' => $date,
'guid' => str_replace("https://avas.mv/","",$url),
'author' => $this->author,
'topics' => $this->topics
];
} }
} }