Bug Fixes and thiladhun scraper intergration

app/Console/Commands/ScrapeThiladhunCommand.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Source;
+use App\Services\ThiladhunService;
+use App\Topic;
+use Illuminate\Support\Carbon;
+
+class ScrapeThiladhunCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'scrape:thiladhun';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Scrape Thiladhun';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        $source = Source::where('slug', 'thiladhun')->first();
+
+        $articles = (new ThiladhunService)->scrape();
+
+        foreach ($articles as $article) {
+
+            // Attach the relationship between source and article and return the curren article instance
+            $articleModel = $source->articles()->firstOrCreate([
+                "title" => $article["title"],
+                "url" => $article["url"],
+                "author" => $article["author"],
+                "featured_image" => $article["image"],
+                "body" => $article["content"],
+                "guid" => $article["guid"],
+                "published_date" => Carbon::parse($article["date"])->format("Y-m-d H:i:s"),
+                "meta" => [
+                    "title" => $article["og_title"]
+                ]
+
+            ]);
+
+            collect($article["topics"])->each(function ($topic) use ($articleModel) {
+                $topicModel = Topic::firstOrCreate([
+                    "name" => $topic["name"],
+                    "slug" => $topic["slug"],
+                ]);
+
+                $topicModel->articles()->syncWithoutDetaching($articleModel);
+            });
+        }
+    }
+}

app/Http/Controllers/RecentArticles.php

@@ -15,9 +15,8 @@ final class RecentArticles extends Controller
      */
     public function __invoke()
     {
-        return ArticleResource::collection(Article::with('source', 'topics')
+        return Article::with('source', 'topics')
                 ->latest("published_date")
-                ->paginate(8)
+                ->paginate(8);
-        );
     }
 }

app/Http/Controllers/TodaysPick.php

@@ -18,11 +18,11 @@ class TodaysPick extends Controller
      */
     public function __invoke()
     {
-        return ArticleResource::collection(Article::with('topics', 'source')
+       return Article::with('topics', 'source')
                         ->whereDate('published_date', Carbon::today())
                         ->inRandomOrder()
                         ->take(8)
                         ->get()
-                        ->unique('source.name'));
+                        ->unique('source.name')->values()->toArray();
     }
 }

app/Services/Scrapers/MihaaruScraper.php

@@ -11,7 +11,7 @@ class MihaaruScraper
     protected $title;
     protected $content;
     protected $image;
-    protected $tags = [];
+    protected $topics = [];
     protected $author;
 
     public function __construct()
@@ -25,13 +25,11 @@ class MihaaruScraper
         $crawler = $this->client->request('GET', $url);
 
         $crawler->filter('h1')->each(function ($node) {
-            $title = $node->text();
+            $this->title = $node->text();
-            $this->title = $title;
         });
 
         $crawler->filter('.container  img')->eq(3)->each(function ($node) {
-            $image = $node->attr('src');
+            $this->image = $node->attr('src');
-            $this->image = $image;
         });
 
         $crawler->filter('.by-line address')->each(function ($node) {
@@ -49,7 +47,7 @@ class MihaaruScraper
 
         $crawler->filter('.article-tags')->each(function ($node) {
            
-            $this->tags[] = [
+            $this->topics[] = [
                 "name" => $node->text(),
                 "slug" => str_replace("https://mihaaru.com/", "", $node->attr('href'))
             ];
@@ -57,7 +55,7 @@ class MihaaruScraper
 
         //Remove all the alphabets from string
         //preg_replace("/[a-zA-Z]/", "",$string);
-        $data = [
+       return [
             'source'    => 'Mihaaru',
             'title'      => $this->title,
             'og_title'   => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
@@ -67,9 +65,7 @@ class MihaaruScraper
             'date'       => $date,
             'guid'       => $guid,
             'author'     => $this->author,
-            'topics'       => $this->tags,
+            'topics'       => $this->topics
         ];
-
-        return $data;
     }
 }

app/Services/Scrapers/ThiladhunScraper.php

@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Services\Scrapers;
+
+use Goutte\Client;
+
+class ThiladhunScraper
+{
+    protected $client;
+
+    protected $title;
+    protected $content;
+    protected $guid;
+    protected $image;
+    protected $author;
+
+    /**
+     * __construct.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        $this->client = new Client();
+    }
+
+    /**
+     * extract.
+     *
+     * @param mixed $url
+     * @param mixed $date
+     * @param mixed $guid
+     *
+     * @return array
+     */
+    public function extract($url, $date = null, $guid = null)
+    {
+        $this->guid = str_replace('https://thiladhun.com/', '', $url);
+
+        $crawler = $this->client->request('GET', $url);
+
+        $crawler->filter('h1')->each(function ($node) {
+            $this->title = $node->text();
+        });
+
+
+        $crawler->filter('div.single-body.entry-content.typography-copy p')->each(function ($node) {
+            $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());;
+        });
+
+        $crawler->filter('div[class*="entry-thumb single-entry-thumb"] img')->each(function ($node) {
+            $this->image = $node->attr('src');
+        });
+
+        $crawler->filter('a[class*="entry-author__name"]')->each(function ($node) {
+            $this->author = $node->text();
+        });
+
+
+        return [
+            'service'    => 'Thiladhun News',
+            'title'      => $this->title,
+            'og_title'   => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
+            'image'      => $this->image,
+            'content'    => $this->content,
+            'date'       => $date,
+            'url'        => $url,
+            'author'     => $this->author,
+            'guid'       => $this->guid,
+            'topics'       =>  [
+                [
+                    "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
+                    "slug" => "uncategorized"
+                ]
+            ]
+        ];
+    }
+}

app/Services/ThiladhunService.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Services;
+
+use App\Services\Scrapers\ThiladhunScraper;
+
+class ThiladhunService extends Client
+{
+    /**
+     * Scrap all the rss articles from mihaaru
+     *
+     * @return array
+     */
+    public function scrape(): array
+    {
+        //Return only the rss that contains "news" keyboard in its url
+        $articles = $this->get("https://thiladhun.com/feed")["channel"]["item"];
+
+        $articlesitems = [];
+        //Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
+        foreach ($articles as $article) {
+            $link = $article['link'];
+            $date = $article['pubDate'];
+            $guid = $article['guid'];
+            $articlesitems[] = (new ThiladhunScraper)->extract($link, $date, $guid);
+        }
+
+        return $articlesitems;
+    }
+}

package-lock.json

@@ -9411,6 +9411,21 @@
                 "vue-style-loader": "^4.1.0"
             }
         },
+        "vue-meta": {
+            "version": "2.4.0",
+            "resolved": "https://registry.npmjs.org/vue-meta/-/vue-meta-2.4.0.tgz",
+            "integrity": "sha512-XEeZUmlVeODclAjCNpWDnjgw+t3WA6gdzs6ENoIAgwO1J1d5p1tezDhtteLUFwcaQaTtayRrsx7GL6oXp/m2Jw==",
+            "requires": {
+                "deepmerge": "^4.2.2"
+            },
+            "dependencies": {
+                "deepmerge": {
+                    "version": "4.2.2",
+                    "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
+                    "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg=="
+                }
+            }
+        },
         "vue-moment": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/vue-moment/-/vue-moment-4.1.0.tgz",

package.json

@@ -25,6 +25,7 @@
     "dependencies": {
         "tailwindcss": "^1.6.2",
         "vue": "^2.6.11",
+        "vue-meta": "^2.4.0",
         "vue-moment": "^4.1.0",
         "vue-router": "^3.4.2"
     }

resources/js/app.js

@@ -4,7 +4,6 @@ import Vue from 'vue';
 import VueRouter from 'vue-router';
 import routes from './routes';
 
-
 Vue.use(require('vue-moment'));
 Vue.use(VueRouter);
 

resources/js/components/TodaysPick.vue

@@ -30,12 +30,12 @@
                              <div>
                                 <p
                                     class="font-semibold text-gray-700 text-sm capitalize MvTyper"
-                                    v-text="article.source.name"
+                                    v-text="subarticle.source.name"
                                 ></p>
                             </div>
 
                             <img
-                                :src="article.source.logo"
+                                :src="subarticle.source.logo"
                                 class="h-10 w-10 rounded-full ml-1 object-cover"
                             />
 
@@ -59,7 +59,7 @@
                                 />
                             </svg>
 
-                            <span class="ml-1" v-text="article.readtime"></span>
+                            <span class="ml-1" v-text="subarticle.readtime"></span>
                         </div>
 
                         <p
@@ -153,8 +153,8 @@ export default {
         axios
             .get("api/today")
             .then(response => {
-                this.article = response.data.data[0];
+                this.article = response.data[0];
-                this.subarticles = response.data.data.slice(1, 5);
+                this.subarticles = response.data.slice(1, 5);
             })
             .catch(error => {
                 console.log(error);

resources/js/pages/Home.vue

@@ -6,15 +6,14 @@
     </div>
 </template>
 <script>
-import TodaysPick from '../components/TodaysPick';
+import TodaysPick from "../components/TodaysPick";
-import DiscoverTopics from '../components/DiscoverTopics';
+import DiscoverTopics from "../components/DiscoverTopics";
-import RecentStories from '../components/RecentStories';
+import RecentStories from "../components/RecentStories";
 export default {
     components: {
         TodaysPick,
         DiscoverTopics,
         RecentStories
-
     }
-}
+};
-</script>
+</script>

resources/js/routes.js

@@ -5,8 +5,7 @@ export default {
     mode: 'history',
 
 
-    routes: [
+    routes: [{
-        {
             path: '/',
             component: Home
         },
@@ -16,4 +15,4 @@ export default {
             name: 'article.show'
         }
     ]
-};
+};

resources/views/app.blade.php

@@ -5,8 +5,6 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-    <title>Baraveli News</title>
-
     <link rel="stylesheet" href="/css/app.css">
     <style>
         @font-face {