diff --git a/app/Console/Commands/ScrapeFunadhooTimesCommand.php b/app/Console/Commands/ScrapeFunadhooTimesCommand.php new file mode 100644 index 0000000..9757b3e --- /dev/null +++ b/app/Console/Commands/ScrapeFunadhooTimesCommand.php @@ -0,0 +1,74 @@ +first(); + + $articles = (new FunadhooTimesService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->updateOrCreate( + ["guid" => $article["guid"]], + [ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "published_date" => $article["date"], + "meta" => [ + "title" => $article["og_title"] + ] + + ] + ); + + collect($article["topics"])->each(function ($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]], ["name" => $topic["name"]]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + } + } +} diff --git a/app/Services/FunadhooTimesService.php b/app/Services/FunadhooTimesService.php new file mode 100644 index 0000000..68bbb32 --- /dev/null +++ b/app/Services/FunadhooTimesService.php @@ -0,0 +1,30 @@ +get("https://funadhootimes.com/rss.xml")["channel"]["article"]; + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $link = $article['link']; + $title = $article['title']; + $articlesitems[] = (new FunadhooTimesScraper)->extract($link, $title); + } + + return $articlesitems; + } +} diff --git a/app/Services/Scrapers/FunadhooTimesScraper.php b/app/Services/Scrapers/FunadhooTimesScraper.php new file mode 100644 index 0000000..6429476 --- /dev/null +++ b/app/Services/Scrapers/FunadhooTimesScraper.php @@ -0,0 +1,60 @@ +client = new Client; + } + + public function extract($url,$title) + { + $crawler = $this->client->request('GET', $url); + + $crawler->filter('.card-body > p')->each(function ($node) { + $this->content[] = $node->text(); + }); + + + $crawler->filter('.article-tags a')->each(function ($node) { + $this->topics[] = [ + "name" => $node->text(), + "slug" => str_replace("https://hama.mv/", "", $node->attr('href')) + ]; + }); + + + //Remove all the alphabets from string + //preg_replace("/[a-zA-Z]/", "",$string); + return [ + 'source' => 'Funadhoo times', + 'title' => $title, + 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), + 'image' => "https://funadhootimes.com/".$crawler->filter(".card img")->first()->attr('src'), + 'content' => $this->content, + 'url' => $url, + 'date' => Carbon::parse(explode('|',$crawler->filter('div[class*="d-flex flex-row-reverse justify-content-between"] small')->first()->text())[0])->format("Y-m-d H:i:s"), + 'guid' => str_replace("https://www.funadhootimes.com/p.php?id=","",$url), + 'author' => explode('|',$crawler->filter('div[class*="d-flex flex-row-reverse justify-content-between"] small')->first()->text())[1], + 'topics' => [ + [ + "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", + "slug" => "no-specific-topic" + ] + ] + ]; + + } +} diff --git a/public/images/source/funadhoo-times.jpg b/public/images/source/funadhoo-times.jpg new file mode 100644 index 0000000..9c81e73 Binary files /dev/null and b/public/images/source/funadhoo-times.jpg differ