shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Sun Scraper and service fix

Browse Source
This commit is contained in:
Mohamed Jinas
2020-08-11 04:25:56 +05:00
parent 56fca8e8ba
commit 0687aa1687
2 changed files with 38 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
app/Services/Scrapers/SunScraper.php
View File

@@ -11,7 +11,7 @@ class SunScraper
protected $title; protected $title;
protected $content; protected $content;
protected $topics = []; protected $topics = [];
protected $author; protected $author = "unknown";
public function __construct() public function __construct()
{ {
@@ -23,25 +23,29 @@ class SunScraper
$crawler = $this->client->request('GET', $url); $crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) { $crawler->filter('.component-article-title h1')->each(function ($node) {
$this->title = $node->text(); $this->title = $node->text();
}); });
$crawler->filter('.component-article-content p')->each(function ($node) { $crawler->filter('.component-article-content p')->each(function ($node) {
$this->content[] = preg_replace("/[a-zA-Z]/","",$node->text()); $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());
}); });
$crawler->filter('.component-article-tag li a')->each(function ($node) { $crawler->filter('.component-article-tag li a')->each(function ($node) {
$this->topics[] = [ $this->topics[] = [
"name" => $node->first()->text(), "name" => $node->first()->text(),
"slug" => str_replace("https://sun.mv/", "", $node->first()->attr('href')) "slug" => str_replace("https://sun.mv/", "", $node->first()->attr('href'))
]; ];
}); });
if($crawler->filter(".author .name")->count() == 1)
{
$this->author = $crawler->filter(".author .name")->first()->text();
}
//Remove all the alphabets from string //Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string); //preg_replace("/[a-zA-Z]/", "",$string);
return [ return [
'source' => 'Sun', 'source' => 'Sun',
'title' => $this->title, 'title' => $this->title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
@@ -49,8 +53,8 @@ class SunScraper
'content' => $this->content, 'content' => $this->content,
'url' => $url, 'url' => $url,
'date' => $crawler->filter(".author .time")->first()->text(), 'date' => $crawler->filter(".author .time")->first()->text(),
'guid' => str_replace("https://sun.mv/","",$url), 'guid' => str_replace("https://sun.mv/", "", $url),
'author' => $crawler->filter(".author .name")->first()->text(), 'author' => $this->author,
'topics' => collect($this->topics)->unique()->values()->toArray() 'topics' => collect($this->topics)->unique()->values()->toArray()
]; ];
} }

27
app/Services/SunService.php
View File

@@ -0,0 +1,27 @@
<?php
namespace App\Services;
use App\Services\Feeds\SunFeed;
use App\Services\Scrapers\SunScraper;
class SunService
{
/**
* Scrap all the rss articles from Sun
*
* @return array
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = (new SunFeed)->get();
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
$articlesitems[] = (new SunScraper)->extract("https://sun.mv/".$article["id"]);
}
return $articlesitems;
}
}