Bug Fixes and thiladhun scraper intergration

2020-08-11 03:39:08 +05:00
parent 0cede5b708
commit 5df4011f13
14 changed files with 4478 additions and 137 deletions
--- a/app/Services/Scrapers/ThiladhunScraper.php
+++ b/app/Services/Scrapers/ThiladhunScraper.php
@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Services\Scrapers;
+
+use Goutte\Client;
+
+class ThiladhunScraper
+{
+    protected $client;
+
+    protected $title;
+    protected $content;
+    protected $guid;
+    protected $image;
+    protected $author;
+
+    /**
+     * __construct.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        $this->client = new Client();
+    }
+
+    /**
+     * extract.
+     *
+     * @param mixed $url
+     * @param mixed $date
+     * @param mixed $guid
+     *
+     * @return array
+     */
+    public function extract($url, $date = null, $guid = null)
+    {
+        $this->guid = str_replace('https://thiladhun.com/', '', $url);
+
+        $crawler = $this->client->request('GET', $url);
+
+        $crawler->filter('h1')->each(function ($node) {
+            $this->title = $node->text();
+        });
+
+
+        $crawler->filter('div.single-body.entry-content.typography-copy p')->each(function ($node) {
+            $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());;
+        });
+
+        $crawler->filter('div[class*="entry-thumb single-entry-thumb"] img')->each(function ($node) {
+            $this->image = $node->attr('src');
+        });
+
+        $crawler->filter('a[class*="entry-author__name"]')->each(function ($node) {
+            $this->author = $node->text();
+        });
+
+
+        return [
+            'service'    => 'Thiladhun News',
+            'title'      => $this->title,
+            'og_title'   => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
+            'image'      => $this->image,
+            'content'    => $this->content,
+            'date'       => $date,
+            'url'        => $url,
+            'author'     => $this->author,
+            'guid'       => $this->guid,
+            'topics'       =>  [
+                [
+                    "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
+                    "slug" => "uncategorized"
+                ]
+            ]
+        ];
+    }
+}