From 963725463b9b187a7cf72b6d384ddf00716914fe Mon Sep 17 00:00:00 2001
From: Mohamed Jinas <j@live.mv>
Date: Fri, 5 Feb 2021 03:27:37 +0500
Subject: [PATCH] Jazeera intergration

---
 app/Console/Commands/ScrapeJazeeraCommand.php | 74 +++++++++++++++++++
 app/Console/Kernel.php                        |  4 +
 app/Services/JazeeraService.php               | 30 ++++++++
 app/Services/Scrapers/JazeeraScraper.php      | 66 +++++++++++++++++
 4 files changed, 174 insertions(+)
 create mode 100644 app/Console/Commands/ScrapeJazeeraCommand.php
 create mode 100644 app/Services/JazeeraService.php
 create mode 100644 app/Services/Scrapers/JazeeraScraper.php

diff --git a/app/Console/Commands/ScrapeJazeeraCommand.php b/app/Console/Commands/ScrapeJazeeraCommand.php
new file mode 100644
index 0000000..d6868b7
--- /dev/null
+++ b/app/Console/Commands/ScrapeJazeeraCommand.php
@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Source;
+use App\Services\JazeeraService;
+use App\Topic;
+use Illuminate\Support\Carbon;
+
+class ScrapeJazeeraCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'scrape:jazeera';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Scrape Jazeera';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        $source = Source::where('slug', 'jazeera')->first();
+
+        $articles = (new JazeeraService)->scrape();
+
+        foreach ($articles as $article) {
+
+            // Attach the relationship between source and article and return the curren article instance
+            $articleModel = $source->articles()->updateOrCreate(
+                ["guid" => $article["guid"]],
+                [
+                    "title" => $article["title"],
+                    "url" => $article["url"],
+                    "author" => $article["author"],
+                    "featured_image" => $article["image"],
+                    "body" => $article["content"],
+                    "published_date" => $article["date"],
+                    "meta" => [
+                        "title" => $article["og_title"]
+                    ]
+
+                ]
+            );
+
+            collect($article["topics"])->each(function ($topic) use ($articleModel) {
+                $topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]], ["name" => $topic["name"]]);
+
+                $topicModel->articles()->syncWithoutDetaching($articleModel);
+            });
+        }
+    }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 77f4dce..64d4165 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -96,6 +96,10 @@ class Kernel extends ConsoleKernel
         $schedule->command('scrape:zaviyani')->everyFiveMinutes()
             ->runInBackground()
             ->pingOnSuccess(env('APP_URL') . "/api/ping/zaviyani");
+
+        $schedule->command('scrape:jazeera')->everyFiveMinutes()
+            ->runInBackground()
+            ->pingOnSuccess(env('APP_URL') . "/api/ping/jazeera");
     }
 
     /**
diff --git a/app/Services/JazeeraService.php b/app/Services/JazeeraService.php
new file mode 100644
index 0000000..96dcbad
--- /dev/null
+++ b/app/Services/JazeeraService.php
@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Services;
+
+use App\Services\Scrapers\JazeeraScraper;
+use Illuminate\Support\Facades\Http;
+
+class JazeeraService extends Client
+{
+    /**
+     * Scrap all the rss articles from Thiladhun
+     *
+     * @return array
+     */
+    public function scrape(): array
+    {
+        //Return only the rss that contains "news" keyboard in its url
+        $articles = json_decode(json_encode(simplexml_load_string(Http::get("https://jazeera.mv/feed/")->body())),true)["channel"]["item"];
+        
+        $articlesitems = [];
+        //Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
+        foreach ($articles as $article) {
+            $link = $article['link'];
+            $date = $article['pubDate'];
+            $articlesitems[] = (new JazeeraScraper)->extract($link,$date);
+        }
+
+        return $articlesitems;
+    }
+}
diff --git a/app/Services/Scrapers/JazeeraScraper.php b/app/Services/Scrapers/JazeeraScraper.php
new file mode 100644
index 0000000..4d3f630
--- /dev/null
+++ b/app/Services/Scrapers/JazeeraScraper.php
@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services\Scrapers;
+
+use Goutte\Client;
+use Illuminate\Support\Carbon;
+
+class JazeeraScraper
+{
+    protected $client;
+
+    protected $title;
+    protected $content;
+    protected $topics = [];
+    protected $author = "unknown";
+
+    public function __construct()
+    {
+        $this->client = new Client;
+    }
+
+    public function extract($url, $date)
+    {
+
+        $crawler = $this->client->request('GET', $url);
+
+        $crawler->filter('.single-body--content p')->each(function ($node) {
+            $this->content[] = $node->text();
+        });
+
+        $crawler->filter('.entry-tags a')->each(function ($node) {
+           if(!preg_match('/[^A-Za-z0-9-]/', basename($node->attr('href'))))
+           {
+            $this->topics[] = [
+                "name" => $node->text(),
+                "slug" => basename($node->attr('href'))
+            ];
+           }
+            
+        });
+
+
+        if ($crawler->filter('a[class*="entry-author__name"]')->count() == 1) {
+            $this->author = $crawler->filter('a[class*="entry-author__name"]')->first()->text();
+        }
+        //Remove all the alphabets from string
+        //preg_replace("/[a-zA-Z]/", "",$string);
+        return [
+            'source'    => 'Jazeera',
+            'title'      => $crawler->filter('h1')->first()->text(),
+            'og_title'   => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
+            'image'      => $crawler->filter(".single-entry-thumb img")->first()->attr('data-src'),
+            'content'    => $this->content,
+            'url'        => strtok($url,'?'),
+            'date'       => Carbon::parse($date)->format("Y-m-d H:i:s"),
+            'guid'       =>basename(strtok($url,'?')),
+            'author'     => $this->author,
+            'topics'       => $this->topics ? : [
+                [
+                    "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
+                    "slug" => "no-specific-topic"
+                ]
+            ]
+        ];
+    }
+}