diff --git a/app/Console/Commands/ScrapeThiladhunCommand.php b/app/Console/Commands/ScrapeThiladhunCommand.php new file mode 100644 index 0000000..43f60f4 --- /dev/null +++ b/app/Console/Commands/ScrapeThiladhunCommand.php @@ -0,0 +1,75 @@ +first(); + + $articles = (new ThiladhunService)->scrape(); + + foreach ($articles as $article) { + + // Attach the relationship between source and article and return the curren article instance + $articleModel = $source->articles()->firstOrCreate([ + "title" => $article["title"], + "url" => $article["url"], + "author" => $article["author"], + "featured_image" => $article["image"], + "body" => $article["content"], + "guid" => $article["guid"], + "published_date" => Carbon::parse($article["date"])->format("Y-m-d H:i:s"), + "meta" => [ + "title" => $article["og_title"] + ] + + ]); + + collect($article["topics"])->each(function ($topic) use ($articleModel) { + $topicModel = Topic::firstOrCreate([ + "name" => $topic["name"], + "slug" => $topic["slug"], + ]); + + $topicModel->articles()->syncWithoutDetaching($articleModel); + }); + } + } +} diff --git a/app/Http/Controllers/RecentArticles.php b/app/Http/Controllers/RecentArticles.php index 918be12..3aa8c01 100644 --- a/app/Http/Controllers/RecentArticles.php +++ b/app/Http/Controllers/RecentArticles.php @@ -15,9 +15,8 @@ final class RecentArticles extends Controller */ public function __invoke() { - return ArticleResource::collection(Article::with('source', 'topics') + return Article::with('source', 'topics') ->latest("published_date") - ->paginate(8) - ); + ->paginate(8); } } diff --git a/app/Http/Controllers/TodaysPick.php b/app/Http/Controllers/TodaysPick.php index 75abdf9..589de4e 100644 --- a/app/Http/Controllers/TodaysPick.php +++ b/app/Http/Controllers/TodaysPick.php @@ -18,11 +18,11 @@ class TodaysPick extends Controller */ public function __invoke() { - return ArticleResource::collection(Article::with('topics', 'source') + return Article::with('topics', 'source') ->whereDate('published_date', Carbon::today()) ->inRandomOrder() ->take(8) ->get() - ->unique('source.name')); + ->unique('source.name')->values()->toArray(); } } diff --git a/app/Services/Scrapers/MihaaruScraper.php b/app/Services/Scrapers/MihaaruScraper.php index 72554d6..c28890a 100644 --- a/app/Services/Scrapers/MihaaruScraper.php +++ b/app/Services/Scrapers/MihaaruScraper.php @@ -11,7 +11,7 @@ class MihaaruScraper protected $title; protected $content; protected $image; - protected $tags = []; + protected $topics = []; protected $author; public function __construct() @@ -25,13 +25,11 @@ class MihaaruScraper $crawler = $this->client->request('GET', $url); $crawler->filter('h1')->each(function ($node) { - $title = $node->text(); - $this->title = $title; + $this->title = $node->text(); }); $crawler->filter('.container img')->eq(3)->each(function ($node) { - $image = $node->attr('src'); - $this->image = $image; + $this->image = $node->attr('src'); }); $crawler->filter('.by-line address')->each(function ($node) { @@ -49,7 +47,7 @@ class MihaaruScraper $crawler->filter('.article-tags')->each(function ($node) { - $this->tags[] = [ + $this->topics[] = [ "name" => $node->text(), "slug" => str_replace("https://mihaaru.com/", "", $node->attr('href')) ]; @@ -57,7 +55,7 @@ class MihaaruScraper //Remove all the alphabets from string //preg_replace("/[a-zA-Z]/", "",$string); - $data = [ + return [ 'source' => 'Mihaaru', 'title' => $this->title, 'og_title' => $crawler->filter('meta[property*="og:title"]')->first()->attr('content'), @@ -67,9 +65,7 @@ class MihaaruScraper 'date' => $date, 'guid' => $guid, 'author' => $this->author, - 'topics' => $this->tags, + 'topics' => $this->topics ]; - - return $data; } } diff --git a/app/Services/Scrapers/ThiladhunScraper.php b/app/Services/Scrapers/ThiladhunScraper.php new file mode 100644 index 0000000..1c340c6 --- /dev/null +++ b/app/Services/Scrapers/ThiladhunScraper.php @@ -0,0 +1,78 @@ +client = new Client(); + } + + /** + * extract. + * + * @param mixed $url + * @param mixed $date + * @param mixed $guid + * + * @return array + */ + public function extract($url, $date = null, $guid = null) + { + $this->guid = str_replace('https://thiladhun.com/', '', $url); + + $crawler = $this->client->request('GET', $url); + + $crawler->filter('h1')->each(function ($node) { + $this->title = $node->text(); + }); + + + $crawler->filter('div.single-body.entry-content.typography-copy p')->each(function ($node) { + $this->content[] = preg_replace("/[a-zA-Z]/", "", $node->text());; + }); + + $crawler->filter('div[class*="entry-thumb single-entry-thumb"] img')->each(function ($node) { + $this->image = $node->attr('src'); + }); + + $crawler->filter('a[class*="entry-author__name"]')->each(function ($node) { + $this->author = $node->text(); + }); + + + return [ + 'service' => 'Thiladhun News', + 'title' => $this->title, + 'og_title' => str_replace(" | Thiladhun", "", $crawler->filter('title')->first()->text('content')), + 'image' => $this->image, + 'content' => $this->content, + 'date' => $date, + 'url' => $url, + 'author' => $this->author, + 'guid' => $this->guid, + 'topics' => [ + [ + "name" => "ވަކި މަޢުލޫއެއް ނޭންގެ", + "slug" => "uncategorized" + ] + ] + ]; + } +} diff --git a/app/Services/ThiladhunService.php b/app/Services/ThiladhunService.php new file mode 100644 index 0000000..1d5a538 --- /dev/null +++ b/app/Services/ThiladhunService.php @@ -0,0 +1,30 @@ +get("https://thiladhun.com/feed")["channel"]["item"]; + + $articlesitems = []; + //Looping through the articles and scraping and while scraping it creates a new instance of the scraper. + foreach ($articles as $article) { + $link = $article['link']; + $date = $article['pubDate']; + $guid = $article['guid']; + $articlesitems[] = (new ThiladhunScraper)->extract($link, $date, $guid); + } + + return $articlesitems; + } +} diff --git a/package-lock.json b/package-lock.json index 895ecec..c4dd36a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9411,6 +9411,21 @@ "vue-style-loader": "^4.1.0" } }, + "vue-meta": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/vue-meta/-/vue-meta-2.4.0.tgz", + "integrity": "sha512-XEeZUmlVeODclAjCNpWDnjgw+t3WA6gdzs6ENoIAgwO1J1d5p1tezDhtteLUFwcaQaTtayRrsx7GL6oXp/m2Jw==", + "requires": { + "deepmerge": "^4.2.2" + }, + "dependencies": { + "deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==" + } + } + }, "vue-moment": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/vue-moment/-/vue-moment-4.1.0.tgz", diff --git a/package.json b/package.json index 250134d..f3c2efa 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "dependencies": { "tailwindcss": "^1.6.2", "vue": "^2.6.11", + "vue-meta": "^2.4.0", "vue-moment": "^4.1.0", "vue-router": "^3.4.2" } diff --git a/public/js/app.js b/public/js/app.js index c52d0a5..6708a3a 100644 --- a/public/js/app.js +++ b/public/js/app.js @@ -2015,6 +2015,17 @@ __webpack_require__.r(__webpack_exports__); // // // +// +// +// +// +// +// +// +// +// +// +// /* harmony default export */ __webpack_exports__["default"] = ({ name: "recent-stories", data: function data() { @@ -2176,6 +2187,15 @@ __webpack_require__.r(__webpack_exports__); // // // +// +// +// +// +// +// +// +// +// /* harmony default export */ __webpack_exports__["default"] = ({ name: "todays-pick", data: function data() { @@ -2188,14 +2208,208 @@ __webpack_require__.r(__webpack_exports__); var _this = this; axios.get("api/today").then(function (response) { - _this.article = response.data.data[0]; - _this.subarticles = response.data.data.slice(1, 5); + _this.article = response.data[0]; + _this.subarticles = response.data.slice(1, 5); })["catch"](function (error) { console.log(error); }); } }); +/***/ }), + +/***/ "./node_modules/babel-loader/lib/index.js?!./node_modules/vue-loader/lib/index.js?!./resources/js/pages/Article.vue?vue&type=script&lang=js&": +/*!*************************************************************************************************************************************************************!*\ + !*** ./node_modules/babel-loader/lib??ref--4-0!./node_modules/vue-loader/lib??vue-loader-options!./resources/js/pages/Article.vue?vue&type=script&lang=js& ***! + \*************************************************************************************************************************************************************/ +/*! exports provided: default */ +/***/ (function(module, __webpack_exports__, __webpack_require__) { + +"use strict"; +__webpack_require__.r(__webpack_exports__); +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +/* harmony default export */ __webpack_exports__["default"] = ({ + data: function data() { + return { + article: [] + }; + }, + mounted: function mounted() { + var _this = this; + + axios.get("/api/article/".concat(this.$route.params.id)).then(function (response) { + _this.article = response.data.data; + }); + } +}); + +/***/ }), + +/***/ "./node_modules/babel-loader/lib/index.js?!./node_modules/vue-loader/lib/index.js?!./resources/js/pages/Home.vue?vue&type=script&lang=js&": +/*!**********************************************************************************************************************************************************!*\ + !*** ./node_modules/babel-loader/lib??ref--4-0!./node_modules/vue-loader/lib??vue-loader-options!./resources/js/pages/Home.vue?vue&type=script&lang=js& ***! + \**********************************************************************************************************************************************************/ +/*! exports provided: default */ +/***/ (function(module, __webpack_exports__, __webpack_require__) { + +"use strict"; +__webpack_require__.r(__webpack_exports__); +/* harmony import */ var _components_TodaysPick__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ../components/TodaysPick */ "./resources/js/components/TodaysPick.vue"); +/* harmony import */ var _components_DiscoverTopics__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ../components/DiscoverTopics */ "./resources/js/components/DiscoverTopics.vue"); +/* harmony import */ var _components_RecentStories__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ../components/RecentStories */ "./resources/js/components/RecentStories.vue"); +// +// +// +// +// +// +// + + + +/* harmony default export */ __webpack_exports__["default"] = ({ + components: { + TodaysPick: _components_TodaysPick__WEBPACK_IMPORTED_MODULE_0__["default"], + DiscoverTopics: _components_DiscoverTopics__WEBPACK_IMPORTED_MODULE_1__["default"], + RecentStories: _components_RecentStories__WEBPACK_IMPORTED_MODULE_2__["default"] + } +}); + +/***/ }), + +/***/ "./node_modules/css-loader/index.js?!./node_modules/vue-loader/lib/loaders/stylePostLoader.js!./node_modules/postcss-loader/src/index.js?!./node_modules/vue-loader/lib/index.js?!./resources/js/pages/Article.vue?vue&type=style&index=0&lang=css&": +/*!********************************************************************************************************************************************************************************************************************************************************************!*\ + !*** ./node_modules/css-loader??ref--6-1!./node_modules/vue-loader/lib/loaders/stylePostLoader.js!./node_modules/postcss-loader/src??ref--6-2!./node_modules/vue-loader/lib??vue-loader-options!./resources/js/pages/Article.vue?vue&type=style&index=0&lang=css& ***! + \********************************************************************************************************************************************************************************************************************************************************************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +exports = module.exports = __webpack_require__(/*! ../../../node_modules/css-loader/lib/css-base.js */ "./node_modules/css-loader/lib/css-base.js")(false); +// imports + + +// module +exports.push([module.i, ".image-dark {\n -webkit-filter: brightness(50%);\n filter: brightness(50%);\n}\n", ""]); + +// exports + + +/***/ }), + +/***/ "./node_modules/css-loader/lib/css-base.js": +/*!*************************************************!*\ + !*** ./node_modules/css-loader/lib/css-base.js ***! + \*************************************************/ +/*! no static exports found */ +/***/ (function(module, exports) { + +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ +// css base code, injected by the css-loader +module.exports = function(useSourceMap) { + var list = []; + + // return the list of modules as css string + list.toString = function toString() { + return this.map(function (item) { + var content = cssWithMappingToString(item, useSourceMap); + if(item[2]) { + return "@media " + item[2] + "{" + content + "}"; + } else { + return content; + } + }).join(""); + }; + + // import a list of modules into the list + list.i = function(modules, mediaQuery) { + if(typeof modules === "string") + modules = [[null, modules, ""]]; + var alreadyImportedModules = {}; + for(var i = 0; i < this.length; i++) { + var id = this[i][0]; + if(typeof id === "number") + alreadyImportedModules[id] = true; + } + for(i = 0; i < modules.length; i++) { + var item = modules[i]; + // skip already imported module + // this implementation is not 100% perfect for weird media query combinations + // when a module is imported multiple times with different media queries. + // I hope this will never occur (Hey this way we have smaller bundles) + if(typeof item[0] !== "number" || !alreadyImportedModules[item[0]]) { + if(mediaQuery && !item[2]) { + item[2] = mediaQuery; + } else if(mediaQuery) { + item[2] = "(" + item[2] + ") and (" + mediaQuery + ")"; + } + list.push(item); + } + } + }; + return list; +}; + +function cssWithMappingToString(item, useSourceMap) { + var content = item[1] || ''; + var cssMapping = item[3]; + if (!cssMapping) { + return content; + } + + if (useSourceMap && typeof btoa === 'function') { + var sourceMapping = toComment(cssMapping); + var sourceURLs = cssMapping.sources.map(function (source) { + return '/*# sourceURL=' + cssMapping.sourceRoot + source + ' */' + }); + + return [content].concat(sourceURLs).concat([sourceMapping]).join('\n'); + } + + return [content].join('\n'); +} + +// Adapted from convert-source-map (MIT) +function toComment(sourceMap) { + // eslint-disable-next-line no-undef + var base64 = btoa(unescape(encodeURIComponent(JSON.stringify(sourceMap)))); + var data = 'sourceMappingURL=data:application/json;charset=utf-8;base64,' + base64; + + return '/*# ' + data + ' */'; +} + + /***/ }), /***/ "./node_modules/lodash/lodash.js": @@ -19754,6 +19968,545 @@ process.umask = function() { return 0; }; /* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./../webpack/buildin/global.js */ "./node_modules/webpack/buildin/global.js"), __webpack_require__(/*! ./../process/browser.js */ "./node_modules/process/browser.js"))) +/***/ }), + +/***/ "./node_modules/style-loader/index.js!./node_modules/css-loader/index.js?!./node_modules/vue-loader/lib/loaders/stylePostLoader.js!./node_modules/postcss-loader/src/index.js?!./node_modules/vue-loader/lib/index.js?!./resources/js/pages/Article.vue?vue&type=style&index=0&lang=css&": +/*!************************************************************************************************************************************************************************************************************************************************************************************************!*\ + !*** ./node_modules/style-loader!./node_modules/css-loader??ref--6-1!./node_modules/vue-loader/lib/loaders/stylePostLoader.js!./node_modules/postcss-loader/src??ref--6-2!./node_modules/vue-loader/lib??vue-loader-options!./resources/js/pages/Article.vue?vue&type=style&index=0&lang=css& ***! + \************************************************************************************************************************************************************************************************************************************************************************************************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + + +var content = __webpack_require__(/*! !../../../node_modules/css-loader??ref--6-1!../../../node_modules/vue-loader/lib/loaders/stylePostLoader.js!../../../node_modules/postcss-loader/src??ref--6-2!../../../node_modules/vue-loader/lib??vue-loader-options!./Article.vue?vue&type=style&index=0&lang=css& */ "./node_modules/css-loader/index.js?!./node_modules/vue-loader/lib/loaders/stylePostLoader.js!./node_modules/postcss-loader/src/index.js?!./node_modules/vue-loader/lib/index.js?!./resources/js/pages/Article.vue?vue&type=style&index=0&lang=css&"); + +if(typeof content === 'string') content = [[module.i, content, '']]; + +var transform; +var insertInto; + + + +var options = {"hmr":true} + +options.transform = transform +options.insertInto = undefined; + +var update = __webpack_require__(/*! ../../../node_modules/style-loader/lib/addStyles.js */ "./node_modules/style-loader/lib/addStyles.js")(content, options); + +if(content.locals) module.exports = content.locals; + +if(false) {} + +/***/ }), + +/***/ "./node_modules/style-loader/lib/addStyles.js": +/*!****************************************************!*\ + !*** ./node_modules/style-loader/lib/addStyles.js ***! + \****************************************************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +var stylesInDom = {}; + +var memoize = function (fn) { + var memo; + + return function () { + if (typeof memo === "undefined") memo = fn.apply(this, arguments); + return memo; + }; +}; + +var isOldIE = memoize(function () { + // Test for IE <= 9 as proposed by Browserhacks + // @see http://browserhacks.com/#hack-e71d8692f65334173fee715c222cb805 + // Tests for existence of standard globals is to allow style-loader + // to operate correctly into non-standard environments + // @see https://github.com/webpack-contrib/style-loader/issues/177 + return window && document && document.all && !window.atob; +}); + +var getTarget = function (target, parent) { + if (parent){ + return parent.querySelector(target); + } + return document.querySelector(target); +}; + +var getElement = (function (fn) { + var memo = {}; + + return function(target, parent) { + // If passing function in options, then use it for resolve "head" element. + // Useful for Shadow Root style i.e + // { + // insertInto: function () { return document.querySelector("#foo").shadowRoot } + // } + if (typeof target === 'function') { + return target(); + } + if (typeof memo[target] === "undefined") { + var styleTarget = getTarget.call(this, target, parent); + // Special case to return head of iframe instead of iframe itself + if (window.HTMLIFrameElement && styleTarget instanceof window.HTMLIFrameElement) { + try { + // This will throw an exception if access to iframe is blocked + // due to cross-origin restrictions + styleTarget = styleTarget.contentDocument.head; + } catch(e) { + styleTarget = null; + } + } + memo[target] = styleTarget; + } + return memo[target] + }; +})(); + +var singleton = null; +var singletonCounter = 0; +var stylesInsertedAtTop = []; + +var fixUrls = __webpack_require__(/*! ./urls */ "./node_modules/style-loader/lib/urls.js"); + +module.exports = function(list, options) { + if (typeof DEBUG !== "undefined" && DEBUG) { + if (typeof document !== "object") throw new Error("The style-loader cannot be used in a non-browser environment"); + } + + options = options || {}; + + options.attrs = typeof options.attrs === "object" ? options.attrs : {}; + + // Force single-tag solution on IE6-9, which has a hard limit on the # of