shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix thiladhun

Browse Source
This commit is contained in:
Mohamed jinas
2024-01-07 21:42:31 +05:00
parent c8329962b7
commit 0589b19843
3 changed files with 29 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/Services/Feeds/ThiladhunFeed.php
View File

@@ -18,22 +18,23 @@ class ThiladhunFeed implements Feed
*/ */
public function get() : array public function get() : array
{ {
$crawler = $this->client->request('GET', "https://www.thiladhun.com"); $crawler = $this->client->request('GET', "https://www.thiladhun.com");
$feeds = []; $feeds = [];
$crawler->filter('div[class*="posts-listing posts-list"] article')->each(function ($node) use (&$feeds) { // Assuming 'ފަހުގެ' is a unique identifier for the section of interest
$crawler->filter('.elementor-element:contains("ފަހުގެ")')->nextAll('.elementor-post')->each(function ($node) use (&$feeds) {
$title = $node->filter('.elementor-heading-title a')->count() ? $node->filter('.elementor-heading-title a')->text() : null;
$link = $node->filter('.elementor-heading-title a')->count() ? $node->filter('.elementor-heading-title a')->attr('href') : null;
if ($title && $link) {
$feeds[] = [ $feeds[] = [
"title" => $node->filter('.post__title a')->text(), "title" => $title,
"link" => $node->filter('.post__title a')->attr('href'), "link" => $link,
"date" => $node->filter('time')->first()->attr('datetime')
]; ];
}
}); });
return $feeds; return $feeds;
} }
} }

19
app/Services/Scrapers/ThiladhunScraper.php
View File

@@ -34,7 +34,7 @@ class ThiladhunScraper
* *
* @return array * @return array
*/ */
public function extract($url, $date) public function extract($url)
{ {
$crawler = $this->client->request('GET', $url); $crawler = $this->client->request('GET', $url);
@@ -51,20 +51,23 @@ class ThiladhunScraper
$this->image = $crawler->filter('div[data-widget_type*="theme-post-featured-image.default"] figure img')->first()->attr('src'); $this->image = $crawler->filter('div[data-widget_type*="theme-post-featured-image.default"] figure img')->first()->attr('src');
$crawler->filter('.entry-tags a')->each(function ($node) { $crawler->filter('.entry-tags a')->each(function ($node) {
if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) if (!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) {
{
$this->topics[] = [ $this->topics[] = [
"name" => $node->text(), "name" => $node->text(),
"slug" => basename($node->attr('href')) "slug" => basename($node->attr('href'))
]; ];
} }
}); });
$crawler->filter('a > .elementor-post-info__item--type-author')->each(function ($node) { $crawler->filter('a > .elementor-post-info__item--type-author')->each(function ($node) {
$this->author = $node->text(); $this->author = $node->text();
}); });
$urlComponents = parse_url($url);
$cleanUrl = $urlComponents['scheme'] . '://' . $urlComponents['host'] . $urlComponents['path'];
// Extract and modify the guid
$guid = basename($urlComponents['path']);
return [ return [
'source' => 'Thiladhun News', 'source' => 'Thiladhun News',
@@ -72,11 +75,11 @@ class ThiladhunScraper
'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')), 'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
'image' => $this->image, 'image' => $this->image,
'content' => $this->content, 'content' => $this->content,
'date' => Carbon::parse($date)->format("Y-m-d H:i:s"), 'date' => Carbon::now(),
'url' => $url, 'url' => $cleanUrl,
'author' => $this->author, 'author' => $this->author,
'guid' => basename($url), 'guid' => $guid,
'topics' => $this->topics ? : [ 'topics' => $this->topics ?: [
[ [
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "no-specific-topic" "slug" => "no-specific-topic"

2
app/Services/ThiladhunService.php
View File

@@ -25,7 +25,7 @@ class ThiladhunService extends Client
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper. //Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) { foreach ($articles as $article) {
//Remove query strings //Remove query strings
$articlesitems[] = (new ThiladhunScraper)->extract(Str::before($article["link"], '?'), $article["pubDate"]); $articlesitems[] = (new ThiladhunScraper)->extract($article["link"]);
} }
return $articlesitems; return $articlesitems;