shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug Fixes and thiladhun scraper intergration

Browse Source
This commit is contained in:
Mohamed Jinas
2020-08-11 03:39:08 +05:00
parent 0cede5b708
commit 5df4011f13
14 changed files with 4478 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/Services/Scrapers/MihaaruScraper.php
View File

@@ -11,7 +11,7 @@ class MihaaruScraper
protected $title;
protected $content;
protected $image;
protected $tags = [];
protected $topics = [];
protected $author;
public function __construct()
@@ -25,13 +25,11 @@ class MihaaruScraper
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$title = $node->text();
$this->title = $title;
$this->title = $node->text();
});
$crawler->filter('.container img')->eq(3)->each(function ($node) {
$image = $node->attr('src');
$this->image = $image;
$this->image = $node->attr('src');
});
$crawler->filter('.by-line address')->each(function ($node) {
@@ -49,7 +47,7 @@ class MihaaruScraper
$crawler->filter('.article-tags')->each(function ($node) {
$this->tags[] = [
$this->topics[] = [
"name" => $node->text(),
"slug" => str_replace("https://mihaaru.com/", "", $node->attr('href'))
];
@@ -57,7 +55,7 @@ class MihaaruScraper
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
$data = [
return [
'source' => 'Mihaaru',
'title' => $this->title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
@@ -67,9 +65,7 @@ class MihaaruScraper
'date' => $date,
'guid' => $guid,
'author' => $this->author,
'topics' => $this->tags,
'topics' => $this->topics
];
return $data;
}
}

78
app/Services/Scrapers/ThiladhunScraper.php Normal file
View File

@@ -0,0 +1,78 @@
<?php
namespace App\Services\Scrapers;
use Goutte\Client;
class ThiladhunScraper
{
protected $client;
protected $title;
protected $content;
protected $guid;
protected $image;
protected $author;
/**
* __construct.
*
* @return void
*/
public function __construct()
{
$this->client = new Client();
}
/**
* extract.
*
* @param mixed $url
* @param mixed $date
* @param mixed $guid
*
* @return array
*/
public function extract($url, $date = null, $guid = null)
{
$this->guid = str_replace('https://thiladhun.com/', '', $url);
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$this->title = $node->text();
});
$crawler->filter('div.single-body.entry-content.typography-copy p')->each(function ($node) {
$this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());;
});
$crawler->filter('div[class*="entry-thumb single-entry-thumb"] img')->each(function ($node) {
$this->image = $node->attr('src');
});
$crawler->filter('a[class*="entry-author__name"]')->each(function ($node) {
$this->author = $node->text();
});
return [
'service' => 'Thiladhun News',
'title' => $this->title,
'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
'image' => $this->image,
'content' => $this->content,
'date' => $date,
'url' => $url,
'author' => $this->author,
'guid' => $this->guid,
'topics' => [
[
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "uncategorized"
]
]
];
}
}

30
app/Services/ThiladhunService.php Normal file
View File

@@ -0,0 +1,30 @@
<?php
namespace App\Services;
use App\Services\Scrapers\ThiladhunScraper;
class ThiladhunService extends Client
{
/**
* Scrap all the rss articles from mihaaru
*
* @return array
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = $this->get("https://thiladhun.com/feed")["channel"]["item"];
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
$link = $article['link'];
$date = $article['pubDate'];
$guid = $article['guid'];
$articlesitems[] = (new ThiladhunScraper)->extract($link, $date, $guid);
}
return $articlesitems;
}
}