shihaam/karudhaas
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug Fixes and thiladhun scraper intergration

Browse Source
This commit is contained in:
Mohamed Jinas
2020-08-11 03:39:08 +05:00
parent 0cede5b708
commit 5df4011f13
14 changed files with 4478 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
app/Console/Commands/ScrapeThiladhunCommand.php Normal file
View File

@@ -0,0 +1,75 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Source;
use App\Services\ThiladhunService;
use App\Topic;
use Illuminate\Support\Carbon;
class ScrapeThiladhunCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'scrape:thiladhun';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Scrape Thiladhun';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$source = Source::where('slug', 'thiladhun')->first();
$articles = (new ThiladhunService)->scrape();
foreach ($articles as $article) {
// Attach the relationship between source and article and return the curren article instance
$articleModel = $source->articles()->firstOrCreate([
"title" => $article["title"],
"url" => $article["url"],
"author" => $article["author"],
"featured_image" => $article["image"],
"body" => $article["content"],
"guid" => $article["guid"],
"published_date" => Carbon::parse($article["date"])->format("Y-m-d H:i:s"),
"meta" => [
"title" => $article["og_title"]
]
]);
collect($article["topics"])->each(function ($topic) use ($articleModel) {
$topicModel = Topic::firstOrCreate([
"name" => $topic["name"],
"slug" => $topic["slug"],
]);
$topicModel->articles()->syncWithoutDetaching($articleModel);
});
}
}
}

5
app/Http/Controllers/RecentArticles.php
View File

@@ -15,9 +15,8 @@ final class RecentArticles extends Controller
*/
public function __invoke()
{
return ArticleResource::collection(Article::with('source', 'topics')
return Article::with('source', 'topics')
->latest("published_date")
->paginate(8)
);
->paginate(8);
}
}

4
app/Http/Controllers/TodaysPick.php
View File

@@ -18,11 +18,11 @@ class TodaysPick extends Controller
*/
public function __invoke()
{
return ArticleResource::collection(Article::with('topics', 'source')
return Article::with('topics', 'source')
->whereDate('published_date', Carbon::today())
->inRandomOrder()
->take(8)
->get()
->unique('source.name'));
->unique('source.name')->values()->toArray();
}
}

16
app/Services/Scrapers/MihaaruScraper.php
View File

@@ -11,7 +11,7 @@ class MihaaruScraper
protected $title;
protected $content;
protected $image;
protected $tags = [];
protected $topics = [];
protected $author;
public function __construct()
@@ -25,13 +25,11 @@ class MihaaruScraper
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$title = $node->text();
$this->title = $title;
$this->title = $node->text();
});
$crawler->filter('.container img')->eq(3)->each(function ($node) {
$image = $node->attr('src');
$this->image = $image;
$this->image = $node->attr('src');
});
$crawler->filter('.by-line address')->each(function ($node) {
@@ -49,7 +47,7 @@ class MihaaruScraper
$crawler->filter('.article-tags')->each(function ($node) {
$this->tags[] = [
$this->topics[] = [
"name" => $node->text(),
"slug" => str_replace("https://mihaaru.com/", "", $node->attr('href'))
];
@@ -57,7 +55,7 @@ class MihaaruScraper
//Remove all the alphabets from string
//preg_replace("/[a-zA-Z]/", "",$string);
$data = [
return [
'source' => 'Mihaaru',
'title' => $this->title,
'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'),
@@ -67,9 +65,7 @@ class MihaaruScraper
'date' => $date,
'guid' => $guid,
'author' => $this->author,
'topics' => $this->tags,
'topics' => $this->topics
];
return $data;
}
}

78
app/Services/Scrapers/ThiladhunScraper.php Normal file
View File

@@ -0,0 +1,78 @@
<?php
namespace App\Services\Scrapers;
use Goutte\Client;
class ThiladhunScraper
{
protected $client;
protected $title;
protected $content;
protected $guid;
protected $image;
protected $author;
/**
* __construct.
*
* @return void
*/
public function __construct()
{
$this->client = new Client();
}
/**
* extract.
*
* @param mixed $url
* @param mixed $date
* @param mixed $guid
*
* @return array
*/
public function extract($url, $date = null, $guid = null)
{
$this->guid = str_replace('https://thiladhun.com/', '', $url);
$crawler = $this->client->request('GET', $url);
$crawler->filter('h1')->each(function ($node) {
$this->title = $node->text();
});
$crawler->filter('div.single-body.entry-content.typography-copy p')->each(function ($node) {
$this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());;
});
$crawler->filter('div[class*="entry-thumb single-entry-thumb"] img')->each(function ($node) {
$this->image = $node->attr('src');
});
$crawler->filter('a[class*="entry-author__name"]')->each(function ($node) {
$this->author = $node->text();
});
return [
'service' => 'Thiladhun News',
'title' => $this->title,
'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')),
'image' => $this->image,
'content' => $this->content,
'date' => $date,
'url' => $url,
'author' => $this->author,
'guid' => $this->guid,
'topics' => [
[
"name" => "ވަކި މަޢުލޫއެއް ނޭންގެ",
"slug" => "uncategorized"
]
]
];
}
}

30
app/Services/ThiladhunService.php Normal file
View File

@@ -0,0 +1,30 @@
<?php
namespace App\Services;
use App\Services\Scrapers\ThiladhunScraper;
class ThiladhunService extends Client
{
/**
* Scrap all the rss articles from mihaaru
*
* @return array
*/
public function scrape(): array
{
//Return only the rss that contains "news" keyboard in its url
$articles = $this->get("https://thiladhun.com/feed")["channel"]["item"];
$articlesitems = [];
//Looping through the articles and scraping and while scraping it creates a new instance of the scraper.
foreach ($articles as $article) {
$link = $article['link'];
$date = $article['pubDate'];
$guid = $article['guid'];
$articlesitems[] = (new ThiladhunScraper)->extract($link, $date, $guid);
}
return $articlesitems;
}
}

15
package-lock.json generated
View File

@@ -9411,6 +9411,21 @@
"vue-style-loader": "^4.1.0"
}
},
"vue-meta": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/vue-meta/-/vue-meta-2.4.0.tgz",
"integrity": "sha512-XEeZUmlVeODclAjCNpWDnjgw+t3WA6gdzs6ENoIAgwO1J1d5p1tezDhtteLUFwcaQaTtayRrsx7GL6oXp/m2Jw==",
"requires": {
"deepmerge": "^4.2.2"
},
"dependencies": {
"deepmerge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
"integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg=="
}
}
},
"vue-moment": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/vue-moment/-/vue-moment-4.1.0.tgz",

1
package.json
View File

@@ -25,6 +25,7 @@
"dependencies": {
"tailwindcss": "^1.6.2",
"vue": "^2.6.11",
"vue-meta": "^2.4.0",
"vue-moment": "^4.1.0",
"vue-router": "^3.4.2"
}

4250
public/js/app.js vendored
View File

File diff suppressed because it is too large Load Diff

1
resources/js/app.js vendored
View File

@@ -4,7 +4,6 @@ import Vue from 'vue';
import VueRouter from 'vue-router';
import routes from './routes';
Vue.use(require('vue-moment'));
Vue.use(VueRouter);

10
resources/js/components/TodaysPick.vue
View File

@@ -30,12 +30,12 @@
<div>
<p
class="font-semibold text-gray-700 text-sm capitalize MvTyper"
v-text="article.source.name"
v-text="subarticle.source.name"
></p>
</div>
<img
:src="article.source.logo"
:src="subarticle.source.logo"
class="h-10 w-10 rounded-full ml-1 object-cover"
/>
@@ -59,7 +59,7 @@
/>
</svg>
<span class="ml-1" v-text="article.readtime"></span>
<span class="ml-1" v-text="subarticle.readtime"></span>
</div>
<p
@@ -153,8 +153,8 @@ export default {
axios
.get("api/today")
.then(response => {
this.article = response.data.data[0];
this.subarticles = response.data.data.slice(1, 5);
this.article = response.data[0];
this.subarticles = response.data.slice(1, 5);
})
.catch(error => {
console.log(error);

9
resources/js/pages/Home.vue
View File

@@ -6,15 +6,14 @@
</div>
</template>
<script>
import TodaysPick from '../components/TodaysPick';
import DiscoverTopics from '../components/DiscoverTopics';
import RecentStories from '../components/RecentStories';
import TodaysPick from "../components/TodaysPick";
import DiscoverTopics from "../components/DiscoverTopics";
import RecentStories from "../components/RecentStories";
export default {
components: {
TodaysPick,
DiscoverTopics,
RecentStories
}
}
};
</script>

3
resources/js/routes.js vendored
View File

@@ -5,8 +5,7 @@ export default {
mode: 'history',
routes: [
{
routes: [{
path: '/',
component: Home
},

2
resources/views/app.blade.php
View File

@@ -5,8 +5,6 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Baraveli News</title>
<link rel="stylesheet" href="/css/app.css">
<style>
@font-face {