const axios = require('axios').default; const mysql = require('mysql2'); const slugify = require('slugify'); const GhostAdminAPI = require('@tryghost/admin-api'); const Downloader = require('nodejs-file-downloader'); const prom = require('prom-client'); const config = require('./config.js'); const db_conn = mysql.createConnection(config.db_conn).promise(); const api = new GhostAdminAPI(config.api); async function getInstagramUrl(url) { return axios.get(url, { headers: { "Accept-Encoding": "gzip,deflate,compress" } }); } function processInstagramPosts(posts) { var postData = []; for (post of posts) { postData.push([ post.id, post.id, post.caption, post.media_type, post.media_url, post.permalink, post.timestamp.replace(/\+0000$/, "") ]); if ('children' in post) { for (child of post.children.data) { const childId = child.id; var caption = null; postData.push([ post.id, child.id, caption, child.media_type, child.media_url, child.permalink, child.timestamp ]); } } } return postData; } async function pullInstagramPosts() { const url = "https://graph.instagram.com/me/media?fields=id,caption,media_type,media_url,permalink,thumbnail_url,timestamp,children{id,media_type,media_url,permalink,thumbnail_url,timestamp}&access_token=" + config.instagramToken; var data = await getInstagramUrl(url); data = data.data; var postData = []; const paginate = process.argv[3] || false; while (data != null) { postData.push(...processInstagramPosts(data.data)); if (paginate && 'paging' in data && 'next' in data.paging) { data = await getInstagramUrl(data.paging.next); data = data.data; } else { data = null; } } const insertSql = "INSERT IGNORE INTO `ig_posts` (`parent_instagram_id`, `instagram_id`, `caption`, `media_type`, `media_url`, `permalink`, `datetime`) VALUES ?"; const foundPosts = new prom.Gauge({ name: 'instasync_instagram_posts', help: 'Total posts found on Instagram' }); foundPosts.set(postData.length); console.log("Inserting " + postData.length + " posts"); await db_conn.query(insertSql, [postData]); } //Helper function that returns an appropriate mobiledoc card for the post type function getAppropriateCard(post, uploadedMediaUrl) { if (post.media_type == "VIDEO") { return ["html",{"html":"
"}]; } else { return ["image",{"src": uploadedMediaUrl,"alt":"","title":""}]; } } async function doWork() { await pullInstagramPosts(); //Get all the Instagram photos/videos that we have get to post to the blog const [posts, _] = await db_conn.query("SELECT * FROM ig_posts WHERE post_id IS NULL AND instagram_id = parent_instagram_id UNION SELECT * FROM ig_posts WHERE post_id IS NULL AND instagram_id != parent_instagram_id"); const processedPosts = {}; var cardIndex = 0; var lastParentId = null; const toProcess = new prom.Gauge({ name: 'instasync_processed_posts', help: 'Total IG posts to post to Ghost' }); toProcess.set(posts.length); console.log("Processing " + posts.length + " posts"); const downloadCounter = new prom.Counter({ name: 'instasync_downloads', help: 'Total IG posts downloaded' }); for (post of posts) { //Download the media to our server const parts = post.media_url.split('/'); // HACK: despite the .heic extensions, these are JPEG files right now // This will probably change in the future and break things. const fileName = parts[parts.length - 1].split('?')[0].replace('.heic', '.jpg'); const downloader = new Downloader({ url: post.media_url, directory: "./ig-images", fileName: fileName, cloneFiles: false, skipExistingFileName: true }) await downloader.download();//Downloader.download() returns a promise. //Upload it to the Ghost blog file directory downloadCounter.inc(); console.log("Uploading " + fileName); var upload; if (fileName.endsWith('mp4')) { upload = await api.media.upload({ file: "ig-images/" + fileName, }) } else { upload = await api.images.upload({ file: "ig-images/" + fileName, }) } //This is the return uploaded media url const uploadedMediaUrl = upload.url; //If the parent Instagram ID differs to the last one then we are working in relation to a different parent post if (post.parent_instagram_id != lastParentId) { //Reset our media index cardIndex = 0; lastParentId = post.parent_instagram_id; } const CARD_SECTION = 10; if (post.parent_instagram_id in processedPosts) { processedPosts[post.parent_instagram_id]["media_urls"].push(post.media_url); processedPosts[post.parent_instagram_id]["rowIds"].push(post.instagram_id); //Mobiledoc data processedPosts[post.parent_instagram_id]["cards"].push(getAppropriateCard(post, uploadedMediaUrl)); processedPosts[post.parent_instagram_id]["sections"].push([CARD_SECTION, cardIndex]); cardIndex++; //This is the first media for this parent ID } else { //const caption = post.caption != null ? post.caption.replace(/(?:\r\n|\r|\n)/g, '