diff --git a/app/Console/Commands/ScrapePsmCommand.php b/app/Console/Commands/ScrapePsmCommand.php new file mode 100644 index 0000000..e2234ba --- /dev/null +++ b/app/Console/Commands/ScrapePsmCommand.php @@ -0,0 +1,73 @@ +first(); + + $articles = (new PsmService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->updateOrCreate(["guid" => $article["guid"]], + [ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "published_date" => Carbon::parse($article["date"])->format("Y-m-d H:i:s"), + "meta" => [ + "title" => $article["og_title"] + ] + + ]); + + collect($article["topics"])->each(function($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]],["name" => $topic["name"]]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index aed0447..491efb7 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -51,6 +51,9 @@ class Kernel extends ConsoleKernel $schedule->command('scrape:dhen')->everyFiveMinutes() ->pingOnSuccess(env('APP_URL')."/api/ping/dhen"); + + $schedule->command('scrape:psm')->everyFiveMinutes() + ->pingOnSuccess(env('APP_URL')."/api/ping/psm"); } /** diff --git a/app/Services/PsmService.php b/app/Services/PsmService.php new file mode 100644 index 0000000..f432f9a --- /dev/null +++ b/app/Services/PsmService.php @@ -0,0 +1,29 @@ +get("https://www.psmnews.mv/feed")["entry"]; + + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $link = $article['id']; + $date = $article['updated']; + $articlesitems[] = (new PsmScraper)->extract($link, $date); + } + + return $articlesitems; + } +} diff --git a/app/Services/Scrapers/PsmScraper.php b/app/Services/Scrapers/PsmScraper.php new file mode 100644 index 0000000..24657da --- /dev/null +++ b/app/Services/Scrapers/PsmScraper.php @@ -0,0 +1,59 @@ +client = new Client; + } + + public function extract($url, $date) + { + + $crawler = $this->client->request('GET', $url); + + $crawler->filter('h1')->each(function ($node) { + $this->title = $node->text(); + }); + + $crawler->filter('.content > p')->each(function ($node) { + $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text()); + }); + + + if($crawler->filter('div[class*="flex items-center border-grey-light border-solid"] p a')->count() == 1) + { + $this->author = $crawler->filter('div[class*="flex items-center border-grey-light border-solid"] p a')->first()->text(); + } + //Remove all the alphabets from string + //preg_replace("/[a-zA-Z]/", "",$string); + return [ + 'source' => 'PSM', + 'title' => $this->title, + 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), + 'image' => $crawler->filter("figure img")->first()->attr('src'), + 'content' => $this->content, + 'url' => $url, + 'date' => $date, + 'guid' => str_replace("https://www.psmnews.mv/", "", $url), + 'author' => $this->author, + 'topics' => [ + [ + "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", + "slug" => "no-specific-topic" + ] + ] + ]; + } +}