Thiladhun switch to virtual feed

This commit is contained in:
2021-02-05 03:58:14 +05:00
parent d33c727b68
commit 8ea594723f
3 changed files with 45 additions and 8 deletions

View File

@@ -0,0 +1,39 @@
<?php
namespace App\Services\Feeds;
use Goutte\Client;
class ThiladhunFeed implements Feed
{
protected $client;
public function __construct()
{
$this->client = new Client();
}
/**
* Get all the latest news
*
* @return array
*/
public function get() : array
{
$crawler = $this->client->request('GET', "https://www.thiladhun.com");
$feeds = [];
$crawler->filter('div[class*="posts-listing posts-list"] article')->each(function ($node) use (&$feeds) {
$feeds[] = [
"title" => $node->filter('.post__title a')->text(),
"link" => $node->filter('.post__title a')->attr('href'),
"date" => $node->filter('time')->first()->attr('datetime')
];
});
return $feeds;
}
}

View File

@@ -10,7 +10,6 @@ class ThiladhunScraper
protected $title; protected $title;
protected $content; protected $content;
protected $guid;
protected $image; protected $image;
protected $author; protected $author;
protected $topics = []; protected $topics = [];
@@ -34,9 +33,8 @@ class ThiladhunScraper
* *
* @return array * @return array
*/ */
public function extract($url) public function extract($url, $date)
{ {
$this->guid = str_replace('https://thiladhun.com/', '', $url);
$crawler = $this->client->request('GET', $url); $crawler = $this->client->request('GET', $url);
@@ -75,10 +73,10 @@ class ThiladhunScraper
'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')), 'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
'image' => $this->image, 'image' => $this->image,
'content' => $this->content, 'content' => $this->content,
'date' => $crawler->filter('.entry-meta time')->attr('datetime'), 'date' => $date,
'url' => $url, 'url' => $url,
'author' => $this->author, 'author' => $this->author,
'guid' => $this->guid, 'guid' => basename($url),
'topics' => $this->topics ? : [ 'topics' => $this->topics ? : [
[ [
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",

View File

@@ -2,6 +2,7 @@
namespace App\Services; namespace App\Services;
use App\Services\Feeds\ThiladhunFeed;
use App\Services\Scrapers\ThiladhunScraper; use App\Services\Scrapers\ThiladhunScraper;
class ThiladhunService extends Client class ThiladhunService extends Client
@@ -14,13 +15,12 @@ class ThiladhunService extends Client
public function scrape(): array public function scrape(): array
{ {
//Return only the rss that contains "news" keyboard in its url //Return only the rss that contains "news" keyboard in its url
$articles = $this->get("https://thiladhun.com/feed")["channel"]["item"]; $articles = (new ThiladhunFeed)->get();
$articlesitems = []; $articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper. //Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) { foreach ($articles as $article) {
$link = $article['link']; $articlesitems[] = (new ThiladhunScraper)->extract($article["link"], $article["date"]);
$articlesitems[] = (new ThiladhunScraper)->extract($link);
} }
return $articlesitems; return $articlesitems;