diff --git a/app/Console/Commands/ScrapeMinoosCommand.php b/app/Console/Commands/ScrapeMinoosCommand.php new file mode 100644 index 0000000..353221a --- /dev/null +++ b/app/Console/Commands/ScrapeMinoosCommand.php @@ -0,0 +1,73 @@ +first(); + + $articles = (new MinoosService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->updateOrCreate(["guid" => $article["guid"]], + [ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "published_date" => $article["date"], + "meta" => [ + "title" => $article["title"], + "highlights" => $article["highlights"] + ] + + ]); + + collect($article["topics"])->each(function($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]],["name" => $topic["name"]]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index b1b87ca..8a32f19 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -105,6 +105,10 @@ class Kernel extends ConsoleKernel $schedule->command('scrape:adhadhu')->everyFiveMinutes() ->runInBackground() ->pingOnSuccess(config('app.url') . "/api/ping/adhadhu"); + + $schedule->command('scrape:minoos')->everyFiveMinutes() + ->runInBackground() + ->pingOnSuccess(config('app.url') . "/api/ping/minoos"); } diff --git a/app/Services/Feeds/MinoosFeed.php b/app/Services/Feeds/MinoosFeed.php new file mode 100644 index 0000000..816df03 --- /dev/null +++ b/app/Services/Feeds/MinoosFeed.php @@ -0,0 +1,38 @@ +client = new Client(); + } + + /** + * Get all the latest news + * + * @return array + */ + public function get() : array + { + $response = $this->client->request('GET', "https://fili.minoos.mv/api/category/news/posts"); + $data = json_decode($response->getBody(), true); + + $feeds = []; + foreach ($data['data'] as $item) { + $feeds[] = [ + "title" => $item['heading'], + "link" => "https://minoos.mv/" . $item['id'], + "date" => $item['published_at'], + "highlights" => $item['highlights'], + ]; + } + + return $feeds; + + } +} diff --git a/app/Services/MinoosService.php b/app/Services/MinoosService.php new file mode 100644 index 0000000..d426f20 --- /dev/null +++ b/app/Services/MinoosService.php @@ -0,0 +1,31 @@ +get(); + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $scraped_article = (new MinoosScraper)->extract($article["link"], $article["date"], $article["highlights"]); + if (!is_null($scraped_article)) { + $articlesitems[] = $scraped_article; + } + } + + return $articlesitems; + } +} diff --git a/app/Services/Scrapers/MinoosScraper.php b/app/Services/Scrapers/MinoosScraper.php new file mode 100644 index 0000000..a3bf659 --- /dev/null +++ b/app/Services/Scrapers/MinoosScraper.php @@ -0,0 +1,48 @@ +client = new Client(); + } + + public function extract($url, $date = null, $highlights = null) + { + $crawler = $this->client->request('GET', $url); + + $title = $crawler->filter('h1')->first()->text(); + $content = $crawler->filter('.doc-text')->each(function (Crawler $node) { + return $node->text(); + }); + $image = $crawler->filter('meta[property="og:image"]')->attr('content'); + + $topics = $crawler->filter('a[href^="/tags/"]')->each(function (Crawler $node) { + return [ + 'name' => $node->text(), + 'slug' => str_replace("/tags/", "", $node->attr('href')) + ]; + }); + + return [ + 'source' => 'minoos', + 'title' => $title, + 'guid' => basename($url), + 'content' => $content, + 'author' => 'unknown', + 'image' => $image, + 'highlights' => $highlights, + 'url' => $url, + 'date' => Carbon::parse($date)->format("Y-m-d H:i:s"), + 'topics' => $topics + ]; + } +}