shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OneOnline Support

Browse Source
This commit is contained in:
Mohamed Jinas
2020-09-27 23:44:54 +05:00
parent 94fda48b90
commit 179bedead9
5 changed files with 216 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
app/Console/Commands/ScrapeOneOnlineCommand.php Normal file
View File

@@ -0,0 +1,72 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Services\OneOnlineService;
use App\Topic;
use App\Source;
class ScrapeOneOnlineCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'scrape:oneonline';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Scrape OneOnline';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$source = Source::where('slug', 'oneonline')->first();
$articles = (new OneOnlineService)->scrape();
foreach ($articles as $article) {
// Attach the relationship between source and article and return the curren article instance
$articleModel = $source->articles()->updateOrCreate(["guid" => $article["guid"]],
[
"title" => $article["title"],
"url" => $article["url"],
"author" => $article["author"],
"featured_image" => $article["image"],
"body" => $article["content"],
"published_date" => $article["date"],
"meta" => [
"title" => $article["og_title"]
]
]);
collect($article["topics"])->each(function($topic) use ($articleModel) {
$topicModel = Topic::firstOrCreate(["slug" => $topic["slug"]],["name" => $topic["name"]]);
$topicModel->articles()->syncWithoutDetaching($articleModel);
});
}
}
}

27
app/Console/Kernel.php
View File

@@ -26,34 +26,37 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule)
{
$schedule->command('scrape:mihaaru')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/mihaaru");
->pingOnSuccess(env('APP_URL') . "/api/ping/mihaaru");
$schedule->command('scrape:sun')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/sun");
->pingOnSuccess(env('APP_URL') . "/api/ping/sun");
$schedule->command('scrape:avas')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/avas");
->pingOnSuccess(env('APP_URL') . "/api/ping/avas");
$schedule->command('scrape:dhuvas')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/dhuvas");
->pingOnSuccess(env('APP_URL') . "/api/ping/dhuvas");
$schedule->command('scrape:thiladhun')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/thiladhun");
->pingOnSuccess(env('APP_URL') . "/api/ping/thiladhun");
$schedule->command('scrape:thepress')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/thepress");
->pingOnSuccess(env('APP_URL') . "/api/ping/thepress");
$schedule->command('scrape:addulive')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/addulive");
->pingOnSuccess(env('APP_URL') . "/api/ping/addulive");
$schedule->command('scrape:voice')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/voice");
->pingOnSuccess(env('APP_URL') . "/api/ping/voice");
$schedule->command('scrape:dhen')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/dhen");
->pingOnSuccess(env('APP_URL') . "/api/ping/dhen");
$schedule->command('scrape:psm')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL')."/api/ping/psm");
->pingOnSuccess(env('APP_URL') . "/api/ping/psm");
$schedule->command('scrape:oneonline')->everyFiveMinutes()
->pingOnSuccess(env('APP_URL') . "/api/ping/oneonline");
}
/**
@@ -63,7 +66,7 @@ class Kernel extends ConsoleKernel
*/
protected function commands()
{
$this->load(__DIR__.'/Commands');
$this->load(__DIR__ . '/Commands');
require base_path('routes/console.php');
}

37
app/Services/Feeds/OneOnlineFeed.php Normal file
View File

@@ -0,0 +1,37 @@
<?php
namespace App\Services\Feeds;
use Goutte\Client;
class OneOnlineFeed
{
protected $client;
public function __construct()
{
$this->client = new Client();
}
/**
* Return the latest articles from avas
*
* @return array
*/
public function get() : array
{
$crawler = $this->client->request('GET', "https://oneonline.mv/");
$feeds = [];
$crawler->filter('.latest-listing a')->each(function($node) use (&$feeds){
$feeds[] = [
"title" => $node->text(),
"link" => $node->attr('href')
];
});
return $feeds;
}
}

26
app/Services/OneOnlineService.php Normal file
View File

@@ -0,0 +1,26 @@
<?php
namespace App\Services;
use App\Services\Feeds\OneOnlineFeed;
use App\Services\Scrapers\OneOnlineScraper;
class OneOnlineService
{
/**
* Scrap all the rss articles from Sun
*
* @return array
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = (new OneOnlineFeed)->get();
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
$articlesitems[] = (new OneOnlineScraper)->extract($article["link"]);
}
return $articlesitems;
}
}

66
app/Services/Scrapers/OneOnlineScraper.php Normal file
View File

@@ -0,0 +1,66 @@
<?php
namespace App\Services\Scrapers;
use Goutte\Client;
use Illuminate\Support\Carbon;
class OneOnlineScraper
{
protected $client;
protected $title;
protected $content;
protected $topics = [];
protected $author = "unknown";
public function __construct()
{
$this->client = new Client;
}
public function extract($url)
{
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$this->title = $node->text();
});
$crawler->filter('.content p')->each(function ($node) {
$this->content[] = $node->text();
});
$crawler->filter('.tags a')->each(function ($node) {
//Removing the show more tags button
if ($node->text() == "ގުޅޭ ޓެގު") {
return;
}
$this->topics[] = [
"name" => $node->text(),
"slug" => str_replace("https://oneonline.mv/", "", $node->attr('href'))
];
});
if ($crawler->filter('div[class*="text-grey-dark font-waheed flex flex-row items-center"] span')->count() == 1) {
$this->author = $crawler->filter('div[class*="text-grey-dark font-waheed flex flex-row items-center"] span')->first()->text();
}
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
return [
'source' => 'OneOnline',
'title' => $this->title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
'image' => $crawler->filter("figure img")->first()->attr('src'),
'content' => $this->content,
'url' => $url,
'date' => Carbon::parse($crawler->filter('time')->first()->attr('datetime'))->format("Y-m-d H:i:s"),
'guid' => str_replace("https://oneonline.mv/", "", $url),
'author' => $this->author,
'topics' => $this->topics
];
}
}