fix thiladhun

This commit is contained in:
Mohamed jinas
2024-01-07 21:42:31 +05:00
parent c8329962b7
commit 0589b19843
3 changed files with 29 additions and 25 deletions

View File

@@ -18,22 +18,23 @@ class ThiladhunFeed implements Feed
*/
public function get() : array
{
$crawler = $this->client->request('GET', "https://www.thiladhun.com");
$feeds = [];
$crawler->filter('div[class*="posts-listing posts-list"] article')->each(function ($node) use (&$feeds) {
// Assuming 'ފަހުގެ' is a unique identifier for the section of interest
$crawler->filter('.elementor-element:contains("ފަހުގެ")')->nextAll('.elementor-post')->each(function ($node) use (&$feeds) {
$title = $node->filter('.elementor-heading-title a')->count() ? $node->filter('.elementor-heading-title a')->text() : null;
$link = $node->filter('.elementor-heading-title a')->count() ? $node->filter('.elementor-heading-title a')->attr('href') : null;
if ($title && $link) {
$feeds[] = [
"title" => $node->filter('.post__title a')->text(),
"link" => $node->filter('.post__title a')->attr('href'),
"date" => $node->filter('time')->first()->attr('datetime')
"title" => $title,
"link" => $link,
];
}
});
return $feeds;
}
}

View File

@@ -34,7 +34,7 @@ class ThiladhunScraper
*
* @return array
*/
public function extract($url, $date)
public function extract($url)
{
$crawler = $this->client->request('GET', $url);
@@ -51,20 +51,23 @@ class ThiladhunScraper
$this->image = $crawler->filter('div[data-widget_type*="theme-post-featured-image.default"] figure img')->first()->attr('src');
$crawler->filter('.entry-tags a')->each(function ($node) {
if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href'))))
{
if (!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) {
$this->topics[] = [
"name" => $node->text(),
"slug" => basename($node->attr('href'))
];
}
});
$crawler->filter('a > .elementor-post-info__item--type-author')->each(function ($node) {
$this->author = $node->text();
});
$urlComponents = parse_url($url);
$cleanUrl = $urlComponents['scheme'] . '://' . $urlComponents['host'] . $urlComponents['path'];
// Extract and modify the guid
$guid = basename($urlComponents['path']);
return [
'source' => 'Thiladhun News',
@@ -72,11 +75,11 @@ class ThiladhunScraper
'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
'image' => $this->image,
'content' => $this->content,
'date' => Carbon::parse($date)->format("Y-m-d H:i:s"),
'url' => $url,
'date' => Carbon::now(),
'url' => $cleanUrl,
'author' => $this->author,
'guid' => basename($url),
'topics' => $this->topics ? : [
'guid' => $guid,
'topics' => $this->topics ?: [
[
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "no-specific-topic"

View File

@@ -25,7 +25,7 @@ class ThiladhunService extends Client
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
//Remove query strings
$articlesitems[] = (new ThiladhunScraper)->extract(Str::before($article["link"], '?'), $article["pubDate"]);
$articlesitems[] = (new ThiladhunScraper)->extract($article["link"]);
}
return $articlesitems;