diff --git a/app/Console/Commands/ScrapeDhiyaresCommand.php b/app/Console/Commands/ScrapeDhiyaresCommand.php new file mode 100644 index 0000000..6f597f4 --- /dev/null +++ b/app/Console/Commands/ScrapeDhiyaresCommand.php @@ -0,0 +1,74 @@ +first(); + + $articles = (new DhiyaresService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->updateOrCreate( + ["guid" => $article["guid"]], + [ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "published_date" => $article["date"], + "meta" => [ + "title" => $article["og_title"] + ] + + ] + ); + + collect($article["topics"])->each(function ($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]], ["name" => $topic["name"]]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 64d4165..3bf1f3e 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -100,6 +100,10 @@ class Kernel extends ConsoleKernel $schedule->command('scrape:jazeera')->everyFiveMinutes() ->runInBackground() ->pingOnSuccess(env('APP_URL') . "/api/ping/jazeera"); + + $schedule->command('scrape:dhiyares')->everyFiveMinutes() + ->runInBackground() + ->pingOnSuccess(env('APP_URL') . "/api/ping/dhiyares"); } /** diff --git a/app/Services/DhiyaresService.php b/app/Services/DhiyaresService.php new file mode 100644 index 0000000..355a5cf --- /dev/null +++ b/app/Services/DhiyaresService.php @@ -0,0 +1,28 @@ +get(); + + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $articlesitems[] = (new DhiyaresScraper)->extract($article["link"], $article["date"]); + } + + return $articlesitems; + } +} diff --git a/app/Services/Feeds/DhiyaresFeed.php b/app/Services/Feeds/DhiyaresFeed.php new file mode 100644 index 0000000..de8c12c --- /dev/null +++ b/app/Services/Feeds/DhiyaresFeed.php @@ -0,0 +1,39 @@ +client = new Client(); + } + /** + * Get all the latest news + * + * @return array + */ + public function get() : array + { + + $crawler = $this->client->request('GET', "https://dhiyares.com"); + + $feeds = []; + $crawler->filter('.recent-list')->each(function ($node) use (&$feeds) { + + + $feeds[] = [ + "title" => $node->filter('h3')->text(), + "link" => $node->filter('a')->attr('href'), + "date" => $node->filter('time-comment')->first()->attr('datetime') + ]; + + }); + + return $feeds; + + } +} \ No newline at end of file diff --git a/app/Services/Scrapers/DhiyaresScraper.php b/app/Services/Scrapers/DhiyaresScraper.php new file mode 100644 index 0000000..03ad971 --- /dev/null +++ b/app/Services/Scrapers/DhiyaresScraper.php @@ -0,0 +1,66 @@ +client = new Client; + } + + public function extract($url, $date) + { + + $crawler = $this->client->request('GET', $url); + + $crawler->filter('.content p')->each(function ($node) { + $this->content[] = $node->text(); + }); + + $crawler->filter('a[class*="bg-blue-50 border border-blue-50 font-normal text-blue-400 hover:text-white hover:bg-base p-1 px-4 mx-2 mb-4 rounded-full font-mv-bold text-base"]')->each(function ($node) { + if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href')))) + { + $this->topics[] = [ + "name" => $node->text(), + "slug" => basename($node->attr('href')) + ]; + } + + }); + + + if ($crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->count() == 1) { + $this->author = $crawler->filter('div[class*="text-base font-mv-bold"] a[class*="py-4"]')->first()->text(); + } + //Remove all the alphabets from string + //preg_replace("/[a-zA-Z]/", "",$string); + return [ + 'source' => 'Dhiyares', + 'title' => $crawler->filter('h1')->first()->text(), + 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), + 'image' => $crawler->filter("figure img")->first()->attr('src'), + 'content' => $this->content, + 'url' => $url, + 'date' => Carbon::parse($date)->format("Y-m-d H:i:s"), + 'guid' =>basename($url), + 'author' => $this->author, + 'topics' => $this->topics ? : [ + [ + "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", + "slug" => "no-specific-topic" + ] + ] + ]; + } +}