diff --git a/app/Console/Commands/ScrapeAvasCommand.php b/app/Console/Commands/ScrapeAvasCommand.php new file mode 100644 index 0000000..8accc35 --- /dev/null +++ b/app/Console/Commands/ScrapeAvasCommand.php @@ -0,0 +1,72 @@ +first(); + + $articles = (new AvasService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->firstOrCreate(["guid" => $article["guid"]], + [ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "published_date" => $article["date"], + "meta" => [ + "title" => $article["og_title"] + ] + + ]); + + collect($article["topics"])->each(function($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]],["name" => $topic["name"]]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + + } + } +} diff --git a/app/Services/AvasService.php b/app/Services/AvasService.php new file mode 100644 index 0000000..a082339 --- /dev/null +++ b/app/Services/AvasService.php @@ -0,0 +1,26 @@ +get(); + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $articlesitems[] = (new AvasScraper)->extract($article["link"]); + } + + return $articlesitems; + } +} \ No newline at end of file diff --git a/app/Services/Feeds/AvasFeed.php b/app/Services/Feeds/AvasFeed.php new file mode 100644 index 0000000..70a1599 --- /dev/null +++ b/app/Services/Feeds/AvasFeed.php @@ -0,0 +1,37 @@ +client = new Client(); + } + /** + * Return the latest articles from avas + * + * @return array + */ + public function get() : array + { + + $crawler = $this->client->request('GET', "https://avas.mv/"); + + $feeds = []; + + $crawler->filter('div[class*="flex rtl -mx-4 flex-wrap md:px-0"] div[class*="w-full md:w-1/3 px-4 mb-7"] div a')->each(function($node) use (&$feeds){ + $feeds[] = [ + "title" => trim($node->text()), + "link" => "https://avas.mv".$node->attr('href') + ]; + }); + + + return array_slice($feeds,0,18); + + } +} \ No newline at end of file diff --git a/app/Services/Scrapers/AvasScraper.php b/app/Services/Scrapers/AvasScraper.php new file mode 100644 index 0000000..054b5ee --- /dev/null +++ b/app/Services/Scrapers/AvasScraper.php @@ -0,0 +1,71 @@ +client = new Client; + } + + public function extract($url) + { + + $crawler = $this->client->request('GET', $url); + + $title = $crawler->filter('h1')->first()->text(); + + + + $image = $crawler->filter('figure img')->first()->attr('src'); + + $crawler->filter('.post_content p')->each(function ($node) { + $this->content[] = preg_replace("/[a-zA-Z]/","",$node->text()); + }); + + $crawler->filter('div[class*="border-t border-grey-light border-dotted mt-7 py-3"] a')->each(function ($node) { + + //Removing the show more tags button + if($node->text() == "+") + { + return; + } + $this->topics[] = [ + "name" => $node->text(), + "slug" => str_replace("/", "", $node->attr('href')) + ]; + }); + + + + if($crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->count() == 1) + { + $this->author = $crawler->filter('div[class*="font-waheed text-grey ml-3 pl-3 text-lg border-l border-grey border-dotted"] a')->first()->text(); + } + + + //Remove all the alphabets from string + //preg_replace("/[a-zA-Z]/", "",$string); + return [ + 'source' => 'Avas', + 'title' => $title, + 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), + 'image' => $image, + 'content' => $this->content, + 'url' => $url, + 'date' => Carbon::parse($crawler->filter('timeago')->first()->attr('datetime'))->format("Y-m-d H:i:s"), + 'guid' => str_replace("https://avas.mv/","",$url), + 'author' => $this->author, + 'topics' => $this->topics + ]; + } +} \ No newline at end of file diff --git a/database/migrations/2020_08_20_153956_change_articles_table_column.php b/database/migrations/2020_08_20_153956_change_articles_table_column.php new file mode 100644 index 0000000..f3dd743 --- /dev/null +++ b/database/migrations/2020_08_20_153956_change_articles_table_column.php @@ -0,0 +1,30 @@ +longText('body')->change(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + // + } +}