Sun Scraper and service fix

This commit is contained in:
2020-08-11 04:25:56 +05:00
parent 56fca8e8ba
commit 0687aa1687
2 changed files with 38 additions and 7 deletions

View File

@@ -11,7 +11,7 @@ class SunScraper
protected $title;
protected $content;
protected $topics = [];
protected $author;
protected $author = "unknown";
public function __construct()
{
@@ -23,7 +23,7 @@ class SunScraper
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$crawler->filter('.component-article-title h1')->each(function ($node) {
$this->title = $node->text();
});
@@ -39,6 +39,10 @@ class SunScraper
];
});
if($crawler->filter(".author .name")->count() == 1)
{
$this->author = $crawler->filter(".author .name")->first()->text();
}
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
return [
@@ -50,7 +54,7 @@ class SunScraper
'url' => $url,
'date' => $crawler->filter(".author .time")->first()->text(),
'guid' => str_replace("https://sun.mv/", "", $url),
'author' => $crawler->filter(".author .name")->first()->text(),
'author' => $this->author,
'topics' => collect($this->topics)->unique()->values()->toArray()
];
}

View File

@@ -0,0 +1,27 @@
<?php
namespace App\Services;
use App\Services\Feeds\SunFeed;
use App\Services\Scrapers\SunScraper;
class SunService
{
/**
* Scrap all the rss articles from Sun
*
* @return array
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = (new SunFeed)->get();
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
$articlesitems[] = (new SunScraper)->extract("https://sun.mv/".$article["id"]);
}
return $articlesitems;
}
}