shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix dhiyares

Browse Source
This commit is contained in:
Mohamed jinas
2024-01-06 15:58:59 +05:00
parent 1b56bcb510
commit b7bda9b09e
3 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/Console/Commands/ScrapeDhiyaresCommand.php
View File

@@ -45,6 +45,7 @@ class ScrapeDhiyaresCommand extends Command
$articles = (new DhiyaresService)->scrape(); $articles = (new DhiyaresService)->scrape();
foreach ($articles as $article) { foreach ($articles as $article) {
// Attach the relationship between source and article and return the curren article instance // Attach the relationship between source and article and return the curren article instance

13
app/Services/DhiyaresService.php
View File

@@ -14,15 +14,16 @@ class DhiyaresService extends Client
*/ */
public function scrape(): array public function scrape(): array
{ {
//Return only the rss that contains "news" keyboard in its url
$articles = (new DhiyaresFeed)->get(); $articles = (new DhiyaresFeed)->get();
$articlesitems = []; $articleItems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) { foreach ($articles as $article) {
$articlesitems[] = (new DhiyaresScraper)->extract($article["link"], $article["date"]); $scrapedData = (new DhiyaresScraper)->extract($article["link"], $article["date"]);
if ($scrapedData !== null) {
$articlesitems[] = $scrapedData;
}
} }
return $articlesitems; return $articleItems;
} }
} }

37
app/Services/Scrapers/DhiyaresScraper.php
View File

@@ -13,6 +13,7 @@ class DhiyaresScraper
protected $content; protected $content;
protected $topics = []; protected $topics = [];
protected $author = "unknown"; protected $author = "unknown";
protected $image;
public function __construct() public function __construct()
{ {
@@ -24,38 +25,50 @@ class DhiyaresScraper
$crawler = $this->client->request('GET', $url); $crawler = $this->client->request('GET', $url);
$crawler->filter('.content p')->each(function ($node) { $ogTitle = $crawler->filter('title')->first()->text();
if (strpos($ogTitle, 'Gallery') === 0) {
return null;
}
$crawler->filter('.content .block-text')->each(function ($node) {
$this->content[] = $node->text(); $this->content[] = $node->text();
}); });
$crawler->filter('a[class*="bg-blue-50 border border-blue-50 font-normal text-blue-400 hover:text-white hover:bg-base p-1 px-4 mx-2 mb-4 rounded-full font-mv-bold text-base"]')->each(function ($node) { $crawler->filter('a[class*="bg-blue-50 border border-blue-50 font-normal text-blue-400 hover:text-white hover:bg-base p-1 px-4 mx-2 mb-4 rounded-full font-mv-bold text-base"]')->each(function ($node) {
if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) if (!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) {
{ $this->topics[] = [
$this->topics[] = [ "name" => $node->text(),
"name" => $node->text(), "slug" => basename($node->attr('href'))
"slug" => basename($node->attr('href')) ];
]; }
}
}); });
if ($crawler->filter("figure img")->count() > 0) {
$this->image = $crawler->filter("figure img")->first()->attr('src');
} else {
$this->image = ''; // or a default image path
}
if ($crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->count() == 1) { if ($crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->count() == 1) {
$this->author = $crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->first()->text(); $this->author = $crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->first()->text();
} }
//Remove all the alphabets from string //Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string); //preg_replace("/[a-zA-Z]/", "",$string);
return [ return [
'source' => 'Dhiyares', 'source' => 'Dhiyares',
'title' => $crawler->filter('h1')->first()->text(), 'title' => $crawler->filter('h1')->first()->text(),
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
'image' => $crawler->filter("figure img")->first()->attr('src'), 'image' => $this->image,
'content' => $this->content, 'content' => $this->content,
'url' => $url, 'url' => $url,
'date' => Carbon::parse($date)->format("Y-m-d H:i:s"), 'date' => Carbon::parse($date)->format("Y-m-d H:i:s"),
'guid' =>basename($url), 'guid' => basename($url),
'author' => $this->author, 'author' => $this->author,
'topics' => $this->topics ? : [ 'topics' => $this->topics ?: [
[ [
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "no-specific-topic" "slug" => "no-specific-topic"