shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix dhiyares

Browse Source
This commit is contained in:
Mohamed jinas
2024-01-06 15:58:59 +05:00
parent 1b56bcb510
commit b7bda9b09e
3 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/Console/Commands/ScrapeDhiyaresCommand.php
View File

@@ -45,6 +45,7 @@ class ScrapeDhiyaresCommand extends Command
$articles = (new DhiyaresService)->scrape();
foreach ($articles as $article) {
// Attach the relationship between source and article and return the curren article instance

11
app/Services/DhiyaresService.php
View File

@@ -14,15 +14,16 @@ class DhiyaresService extends Client
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = (new DhiyaresFeed)->get();
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
$articleItems = [];
foreach ($articles as $article) {
$articlesitems[] = (new DhiyaresScraper)->extract($article["link"], $article["date"]);
$scrapedData = (new DhiyaresScraper)->extract($article["link"], $article["date"]);
if ($scrapedData !== null) {
$articlesitems[] = $scrapedData;
}
}
return $articlesitems;
return $articleItems;
}
}

37
app/Services/Scrapers/DhiyaresScraper.php
View File

@@ -13,6 +13,7 @@ class DhiyaresScraper
protected $content;
protected $topics = [];
protected $author = "unknown";
protected $image;
public function __construct()
{
@@ -24,38 +25,50 @@ class DhiyaresScraper
$crawler = $this->client->request('GET', $url);
$crawler->filter('.content p')->each(function ($node) {
$ogTitle = $crawler->filter('title')->first()->text();
if (strpos($ogTitle, 'Gallery') === 0) {
return null;
}
$crawler->filter('.content .block-text')->each(function ($node) {
$this->content[] = $node->text();
});
$crawler->filter('a[class*="bg-blue-50 border border-blue-50 font-normal text-blue-400 hover:text-white hover:bg-base p-1 px-4 mx-2 mb-4 rounded-full font-mv-bold text-base"]')->each(function ($node) {
if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href'))))
{
$this->topics[] = [
"name" => $node->text(),
"slug" => basename($node->attr('href'))
];
}
$crawler->filter('a[class*="bg-blue-50 border border-blue-50 font-normal text-blue-400 hover:text-white hover:bg-base p-1 px-4 mx-2 mb-4 rounded-full font-mv-bold text-base"]')->each(function ($node) {
if (!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) {
$this->topics[] = [
"name" => $node->text(),
"slug" => basename($node->attr('href'))
];
}
});
if ($crawler->filter("figure img")->count() > 0) {
$this->image = $crawler->filter("figure img")->first()->attr('src');
} else {
$this->image = ''; // or a default image path
}
if ($crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->count() == 1) {
$this->author = $crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->first()->text();
}
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
return [
'source' => 'Dhiyares',
'title' => $crawler->filter('h1')->first()->text(),
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
'image' => $crawler->filter("figure img")->first()->attr('src'),
'image' => $this->image,
'content' => $this->content,
'url' => $url,
'date' => Carbon::parse($date)->format("Y-m-d H:i:s"),
'guid' =>basename($url),
'guid' => basename($url),
'author' => $this->author,
'topics' => $this->topics ? : [
'topics' => $this->topics ?: [
[
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "no-specific-topic"