diff --git a/app/Services/DhuvasService.php b/app/Services/DhuvasService.php new file mode 100644 index 0000000..f3d3945 --- /dev/null +++ b/app/Services/DhuvasService.php @@ -0,0 +1,29 @@ +get("https://dhuvas.mv/feed")["channel"]["item"]; + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $link = $article['link']; + $date = $article['pubDate']; + $articlesitems[] = (new DhuvasScraper)->extract($link,$date); + } + + return $articlesitems; + } +} diff --git a/app/Services/Scrapers/DhenScraper.php b/app/Services/Scrapers/DhenScraper.php index 36ca2e6..10d558a 100644 --- a/app/Services/Scrapers/DhenScraper.php +++ b/app/Services/Scrapers/DhenScraper.php @@ -50,10 +50,10 @@ class DhenScraper 'date' => Carbon::parse(str_replace("- ", "", $crawler->filter('.story-item--meta time')->first()->text()))->format("Y-m-d H:i:s"), 'guid' => str_replace("https://dhen.mv/","",$url), 'author' => $this->author, - 'topics' => [ + 'topics' => [ [ "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", - "slug" => "uncategorized" + "slug" => "no-specific-topic" ] ] ]; diff --git a/app/Services/Scrapers/DhuvasScraper.php b/app/Services/Scrapers/DhuvasScraper.php new file mode 100644 index 0000000..2428a76 --- /dev/null +++ b/app/Services/Scrapers/DhuvasScraper.php @@ -0,0 +1,62 @@ +client = new Client; + } + + public function extract($url, $date = null) + { + + $crawler = $this->client->request('GET', $url); + + $crawler->filter('h1')->each(function ($node) { + $this->title = $node->text(); + }); + + $crawler->filter('.elementor-widget-container > p')->each(function ($node) { + $this->content[] = preg_replace("/[a-zA-Z]/","",$node->text()); + }); + + if($crawler->filter('span[class*="elementor-icon-list-text elementor-post-info__item elementor-post-info__item--type-author"]')->count() == 1) + { + $this->author = $crawler->filter('span[class*="elementor-icon-list-text elementor-post-info__item elementor-post-info__item--type-author"]')->first()->text(); + } + + + + + //Remove all the alphabets from string + //preg_replace("/[a-zA-Z]/", "",$string); + return [ + 'source' => 'Dhuvas', + 'title' => $this->title, + 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), + 'image' => $crawler->filter('.elementor-widget-theme-post-featured-image .elementor-image a img')->first()->attr('src'), + 'content' => $this->content, + 'url' => $url, + 'date' => Carbon::parse($date)->format("Y-m-d H:i:s"), + 'guid' => str_replace("https://dhuvas.mv/","",$url), + 'author' => $this->author, + 'topics' => [ + [ + "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", + "slug" => "no-specific-topic" + ] + ] + ]; + } +}