First release

src/lib/cache.js

@@ -0,0 +1,66 @@
+class InstaCache {
+	/**
+	 * @property {number} ttl time to keep each resource in milliseconds
+	 */
+	constructor(ttl) {
+		this.ttl = ttl
+		/** @type {Map<string, {data: any, time: number}>} */
+		this.cache = new Map()
+	}
+
+	clean() {
+		for (const key of this.cache.keys()) {
+			const value = this.cache.get(key)
+			if (Date.now() > value.time + this.ttl) this.cache.delete(key)
+		}
+	}
+
+	/**
+	 * @param {string} key
+	 */
+	get(key) {
+		return this.cache.get(key).data
+	}
+
+	/**
+	 * @param {string} key
+	 * @param {any} data
+	 */
+	set(key, data) {
+		this.cache.set(key, {data, time: Date.now()})
+	}
+
+	/**
+	 * @param {string} key
+	 * @param {() => Promise<T>} callback
+	 * @returns {Promise<T>}
+	 * @template T
+	 */
+	getOrFetch(key, callback) {
+		this.clean()
+		if (this.cache.has(key)) return Promise.resolve(this.get(key))
+		else {
+			const pending = callback().then(result => {
+				this.set(key, result)
+				return result
+			})
+			this.set(key, pending)
+			return pending
+		}
+	}
+
+	/**
+	 * @param {string} key
+	 * @param {() => Promise<T>} callback
+	 * @returns {Promise<T>}
+	 * @template T
+	 */
+	getOrFetchPromise(key, callback) {
+		return this.getOrFetch(key, callback).then(result => {
+			this.cache.delete(key)
+			return result
+		})
+	}
+}
+
+module.exports = InstaCache

src/lib/collectors.js

@@ -0,0 +1,43 @@
+const constants = require("./constants")
+const {request} = require("./utils/request")
+const {extractSharedData} = require("./utils/body")
+const InstaCache = require("./cache")
+const {User} = require("./structures")
+require("./testimports")(constants, request, extractSharedData, InstaCache, User)
+
+const cache = new InstaCache(600e3)
+
+function fetchUser(username) {
+	return cache.getOrFetch("user/"+username, () => {
+		return request(`https://www.instagram.com/${username}/`).then(res => res.text()).then(text => {
+			const sharedData = extractSharedData(text)
+			const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
+			return user
+		})
+	})
+}
+
+/**
+ * @param {string} userID
+ * @param {string} after
+ * @returns {Promise<import("./types").PagedEdges<import("./types").GraphImage>>}
+ */
+function fetchTimelinePage(userID, after) {
+	const p = new URLSearchParams()
+	p.set("query_hash", constants.external.timeline_query_hash)
+	p.set("variables", JSON.stringify({
+		id: userID,
+		first: constants.external.timeline_fetch_first,
+		after: after
+	}))
+	return cache.getOrFetchPromise("page/"+after, () => {
+		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
+			/** @type {import("./types").PagedEdges<import("./types").GraphImage>} */
+			const timeline = root.data.user.edge_owner_to_timeline_media
+			return timeline
+		})
+	})
+}
+
+module.exports.fetchUser = fetchUser
+module.exports.fetchTimelinePage = fetchTimelinePage

src/lib/constants.js

@@ -0,0 +1,12 @@
+module.exports = {
+	image_cache_control: `public, max-age=${7*24*60*60}`,
+
+	external: {
+		timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
+		timeline_fetch_first: 12
+	},
+
+	symbols: {
+		NO_MORE_PAGES: Symbol("NO_MORE_PAGES")
+	}
+}

src/lib/structures/Timeline.js

@@ -0,0 +1,45 @@
+const constants = require("../constants")
+const TimelineImage = require("./TimelineImage")
+const collectors = require("../collectors")
+require("../testimports")(constants, TimelineImage)
+
+function transformEdges(edges) {
+	return edges.map(e => new TimelineImage(e.node))
+}
+
+class Timeline {
+	/**
+	 * @param {import("./User")} user
+	 */
+	constructor(user) {
+		this.user = user
+		this.pages = []
+		this.addPage(this.user.data.edge_owner_to_timeline_media)
+		this.page_info = this.user.data.edge_owner_to_timeline_media.page_info
+	}
+
+	hasNextPage() {
+		return this.page_info.has_next_page
+	}
+
+	fetchNextPage() {
+		if (!this.hasNextPage()) return constants.symbols.NO_MORE_PAGES
+		return collectors.fetchTimelinePage(this.user.data.id, this.page_info.end_cursor).then(page => {
+			this.addPage(page)
+			return this.pages.slice(-1)[0]
+		})
+	}
+
+	async fetchUpToPage(index) {
+		while (this.pages[index] === undefined && this.hasNextPage()) {
+			await this.fetchNextPage()
+		}
+	}
+
+	addPage(page) {
+		this.pages.push(transformEdges(page.edges))
+		this.page_info = page.page_info
+	}
+}
+
+module.exports = Timeline

src/lib/structures/TimelineImage.js

@@ -0,0 +1,41 @@
+class GraphImage {
+	/**
+	 * @param {import("../types").GraphImage} data
+	 */
+	constructor(data) {
+		this.data = data
+	}
+
+	/**
+	 * @param {number} size
+	 */
+	getSuggestedThumbnail(size) {
+		let found = null
+		for (const tr of this.data.thumbnail_resources) {
+			found = tr
+			if (tr.config_width >= size) break
+		}
+		return found
+	}
+
+	getSrcset() {
+		return this.data.thumbnail_resources.map(tr => {
+			const p = new URLSearchParams()
+			p.set("width", String(tr.config_width))
+			p.set("url", tr.src)
+			return `/imageproxy?${p.toString()} ${tr.config_width}w`
+		}).join(", ")
+	}
+
+	getCaption() {
+		if (this.data.edge_media_to_caption.edges[0]) return this.data.edge_media_to_caption.edges[0].node.text
+		else return null
+	}
+
+	getAlt() {
+		// For some reason, pages 2+ don't contain a11y data. Instagram web client falls back to image caption.
+		return this.data.accessibility_caption || this.getCaption() || "No image description available."
+	}
+}
+
+module.exports = GraphImage

src/lib/structures/User.js

@@ -0,0 +1,21 @@
+const Timeline = require("./Timeline")
+require("../testimports")(Timeline)
+
+class User {
+	/**
+	 * @param {import("../types").GraphUser} data
+	 */
+	constructor(data) {
+		this.data = data
+		this.following = data.edge_follow.count
+		this.followedBy = data.edge_followed_by.count
+		this.posts = data.edge_owner_to_timeline_media.count
+		this.timeline = new Timeline(this)
+	}
+
+	export() {
+		return this.data
+	}
+}
+
+module.exports = User

src/lib/structures/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	User: require("./User")
+}

src/lib/testimports.js

@@ -0,0 +1,7 @@
+module.exports = function(...items) {
+	items.forEach(item => {
+		if (item === undefined || (item && item.constructor && item.constructor.name == "Object" && Object.keys(item).length == 0)) {
+			throw new Error("Bad import: item looks like this: "+JSON.stringify(item))
+		}
+	})
+}

src/lib/types.js

@@ -0,0 +1,64 @@
+/**
+ * @typedef GraphEdgeCount
+ * @property {number} count
+ */
+
+/**
+ * @typedef GraphEdgesText
+ * @type {{edges: {node: {text: string}}[]}}
+ */
+
+/**
+ * @typedef PagedEdges<T>
+ * @property {number} count
+ * @property {{has_next_page: boolean, end_cursor: string}} page_info
+ * @property {{node: T}[]} edges
+ * @template T
+ */
+
+/**
+ * @typedef GraphUser
+ * @property {string} biography
+ * @property {string} external_url
+ * @property {GraphEdgeCount} edge_followed_by
+ * @property {GraphEdgeCount} edge_follow
+ * @property {string} full_name
+ * @property {string} id
+ * @property {boolean} is_business_account
+ * @property {boolean} is_joined_recently
+ * @property {boolean} is_verified
+ * @property {string} profile_pic_url
+ * @property {string} profile_pic_url_hd
+ * @property {string} username
+ *
+ * @property {any} edge_felix_video_timeline
+ * @property {PagedEdges<GraphImage>} edge_owner_to_timeline_media
+ * @property {any} edge_saved_media
+ * @property {any} edge_media_collections
+ */
+
+/**
+ * @typedef Thumbnail
+ * @property {string} src
+ * @property {number} config_width
+ * @property {number} config_height
+ */
+
+/**
+ * @typedef GraphImage
+ * @property {string} id
+ * @property {GraphEdgesText} edge_media_to_caption
+ * @property {string} shortcode
+ * @property {GraphEdgeCount} edge_media_to_comment
+ * @property {number} taken_at_timestamp No milliseconds
+ * @property {GraphEdgeCount} edge_liked_by
+ * @property {GraphEdgeCount} edge_media_preview_like
+ * @property {{width: number, height: number}} dimensions
+ * @property {string} display_url
+ * @property {{id: string, username: string}} owner
+ * @property {string} thumbnail_src
+ * @property {Thumbnail[]} thumbnail_resources
+ * @property {string} accessibility_caption
+ */
+
+module.exports = {}

src/lib/utils/body.js

@@ -0,0 +1,17 @@
+const {Parser} = require("./parser/parser")
+
+/**
+ * @param {string} text
+ */
+function extractSharedData(text) {
+	const parser = new Parser(text)
+	parser.seek("window._sharedData = ", {moveToMatch: true, useEnd: true})
+	parser.store()
+	const end = parser.seek(";</script>")
+	parser.restore()
+	const sharedDataString = parser.slice(end - parser.cursor)
+	const sharedData = JSON.parse(sharedDataString)
+	return sharedData
+}
+
+module.exports.extractSharedData = extractSharedData

src/lib/utils/parser/.gitrepo

@@ -0,0 +1,12 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
+;
+[subrepo]
+	remote = /home/cloud/Code/common/parser
+	branch = master
+	commit = 4a8e58ed4996fb8ef0382e184fd54a9aed931ad7
+	parent = 8e4464598c33a6e8e4af6117fc0fa50600c83f05
+	method = merge
+	cmdver = 0.4.0

src/lib/utils/parser/parser.js

@@ -0,0 +1,160 @@
+/**
+ * @typedef GetOptions
+ * @property {string} [split] Characters to split on
+ * @property {string} [mode] "until" or "between"; choose where to get the content from
+ * @property {function} [transform] Transformation to apply to result before returning
+ */
+
+const tf = {
+	lc: s => s.toLowerCase()
+}
+
+class Parser {
+	constructor(string) {
+		this.string = string;
+		this.substore = [];
+		this.cursor = 0;
+		this.cursorStore = [];
+		this.mode = "until";
+		this.transform = s => s;
+		this.split = " ";
+	}
+
+	/**
+	 * Return all the remaining text from the buffer, without updating the cursor
+	 * @return {String}
+	 */
+	remaining() {
+		return this.string.slice(this.cursor);
+	}
+
+	/**
+	 * Have we reached the end of the string yet?
+	 * @return {boolean}
+	 */
+	hasRemaining() {
+		return this.cursor < this.string.length
+	}
+
+	/**
+	 * @param {GetOptions} [options]
+	 * @returns {String}
+	 */
+	get(options = {}) {
+		["mode", "split", "transform"].forEach(o => {
+			if (!options[o]) options[o] = this[o];
+		});
+		if (options.mode == "until") {
+			let next = this.string.indexOf(options.split, this.cursor+options.split.length);
+			if (next == -1) {
+				let result = this.remaining();
+				this.cursor = this.string.length;
+				return result;
+			} else {
+				let result = this.string.slice(this.cursor, next);
+				this.cursor = next + options.split.length;
+				return options.transform(result);
+			}
+		} else if (options.mode == "between") {
+			let start = this.string.indexOf(options.split, this.cursor);
+			let end = this.string.indexOf(options.split, start+options.split.length);
+			let result = this.string.slice(start+options.split.length, end);
+			this.cursor = end + options.split.length;
+			return options.transform(result);
+		}
+	}
+
+	/**
+	 * Get a number of chars from the buffer.
+	 * @param {number} length Number of chars to get
+	 * @param {boolean} [move] Whether to update the cursor
+	 */
+	slice(length, move = false) {
+		let result = this.string.slice(this.cursor, this.cursor+length);
+		if (move) this.cursor += length;
+		return result;
+	}
+
+	/**
+	 * Repeatedly swallow a character.
+	 * @param {String} char
+	 */
+	swallow(char) {
+		let before = this.cursor;
+		while (this.string[this.cursor] == char) this.cursor++;
+		return this.cursor - before;
+	}
+
+	/**
+	 * Push the current cursor position to the store
+	 */
+	store() {
+		this.cursorStore.push(this.cursor);
+	}
+
+	/**
+	 * Pop the previous cursor position from the store
+	 */
+	restore() {
+		this.cursor = this.cursorStore.pop();
+	}
+
+	/**
+	 * Run a get operation, test against an input, return success or failure, and restore the cursor.
+	 * @param {String} value The value to test against
+	 * @param {Object} options Options for get
+	 */
+	test(value, options) {
+		this.store();
+		let next = this.get(options);
+		let result = next == value;
+		this.restore();
+		return result;
+	}
+
+	/**
+	 * Run a get operation, test against an input, and throw an error if it doesn't match.
+	 * @param {String} value
+	 * @param {GetOptions} [options]
+	 */
+	expect(value, options = {}) {
+		let next = this.get(options);
+		if (next != value) throw new Error("Expected "+value+", got "+next);
+	}
+
+	/**
+	 * Seek past the next occurance of the string.
+	 * @param {string} toFind
+	 * @param {{moveToMatch?: boolean, useEnd?: boolean}} options both default to false
+	 */
+	seek(toFind, options = {}) {
+		if (options.moveToMatch === undefined) options.moveToMatch = false
+		if (options.useEnd === undefined) options.useEnd = false
+		let index = this.string.indexOf(toFind, this.cursor)
+		if (index !== -1) {
+			if (options.useEnd) index += toFind.length
+			if (options.moveToMatch) this.cursor = index
+		}
+		return index
+	}
+
+	/**
+	 * Replace the current string, adding the old one to the substore.
+	 * @param {string} string
+	 */
+	unshiftSubstore(string) {
+		this.substore.unshift({string: this.string, cursor: this.cursor, cursorStore: this.cursorStore})
+		this.string = string
+		this.cursor = 0
+		this.cursorStore = []
+	}
+
+	/**
+	 * Replace the current string with the first entry from the substore.
+	 */
+	shiftSubstore() {
+		Object.assign(this, this.substore.shift())
+	}
+}
+
+module.exports.Parser = Parser

src/lib/utils/request.js

@@ -0,0 +1,11 @@
+const fetch = require("node-fetch").default
+
+function request(url) {
+	return fetch(url, {
+		headers: {
+			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+		}
+	})
+}
+
+module.exports.request = request