318 lines
11 KiB
JavaScript
318 lines
11 KiB
JavaScript
const axios = require('axios').default;
|
|
const mysql = require('mysql2');
|
|
const slugify = require('slugify');
|
|
const GhostAdminAPI = require('@tryghost/admin-api');
|
|
const Downloader = require('nodejs-file-downloader');
|
|
const prom = require('prom-client');
|
|
const config = require('./config.js');
|
|
|
|
const db_conn = mysql.createConnection(config.db_conn).promise();
|
|
const api = new GhostAdminAPI(config.api);
|
|
|
|
async function getInstagramUrl(url) {
|
|
return axios.get(url, {
|
|
headers: { "Accept-Encoding": "gzip,deflate,compress" }
|
|
});
|
|
}
|
|
|
|
function processInstagramPosts(posts) {
|
|
var postData = [];
|
|
|
|
for (post of posts) {
|
|
|
|
postData.push([
|
|
post.id,
|
|
post.id,
|
|
post.caption,
|
|
post.media_type,
|
|
post.media_url,
|
|
post.permalink,
|
|
post.timestamp.replace(/\+0000$/, "")
|
|
]);
|
|
|
|
if ('children' in post) {
|
|
|
|
for (child of post.children.data) {
|
|
|
|
const childId = child.id;
|
|
|
|
var caption = null;
|
|
|
|
postData.push([
|
|
post.id,
|
|
child.id,
|
|
caption,
|
|
child.media_type,
|
|
child.media_url,
|
|
child.permalink,
|
|
child.timestamp
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
return postData;
|
|
}
|
|
|
|
async function pullInstagramPosts() {
|
|
|
|
const url = "https://graph.instagram.com/me/media?fields=id,caption,media_type,media_url,permalink,thumbnail_url,timestamp,children{id,media_type,media_url,permalink,thumbnail_url,timestamp}&access_token=" + config.instagramToken;
|
|
|
|
var data = await getInstagramUrl(url);
|
|
data = data.data;
|
|
var postData = [];
|
|
const paginate = process.argv[3] || false;
|
|
|
|
while (data != null) {
|
|
postData.push(...processInstagramPosts(data.data));
|
|
if (paginate && 'paging' in data && 'next' in data.paging) {
|
|
data = await getInstagramUrl(data.paging.next);
|
|
data = data.data;
|
|
} else {
|
|
data = null;
|
|
}
|
|
}
|
|
|
|
const insertSql = "INSERT IGNORE INTO `ig_posts` (`parent_instagram_id`, `instagram_id`, `caption`, `media_type`, `media_url`, `permalink`, `datetime`) VALUES ?";
|
|
|
|
const foundPosts = new prom.Gauge({
|
|
name: 'instasync_instagram_posts',
|
|
help: 'Total posts found on Instagram'
|
|
});
|
|
foundPosts.set(postData.length);
|
|
console.log("Inserting " + postData.length + " posts");
|
|
await db_conn.query(insertSql, [postData]);
|
|
}
|
|
|
|
|
|
//Helper function that returns an appropriate mobiledoc card for the post type
|
|
function getAppropriateCard(post, uploadedMediaUrl) {
|
|
|
|
if (post.media_type == "VIDEO") {
|
|
|
|
return ["html",{"html":"<p><video width=\"100%\" controls><source src=\"" + uploadedMediaUrl + "\" type=\"video/mp4\">Your browser does not support the video tag.</video></p>"}];
|
|
|
|
} else {
|
|
|
|
return ["image",{"src": uploadedMediaUrl,"alt":"","title":""}];
|
|
}
|
|
}
|
|
|
|
|
|
async function doWork() {
|
|
|
|
await pullInstagramPosts();
|
|
|
|
//Get all the Instagram photos/videos that we have get to post to the blog
|
|
const [posts, _] = await db_conn.query("SELECT * FROM ig_posts WHERE post_id IS NULL AND instagram_id = parent_instagram_id UNION SELECT * FROM ig_posts WHERE post_id IS NULL AND instagram_id != parent_instagram_id");
|
|
|
|
const processedPosts = {};
|
|
|
|
var cardIndex = 0;
|
|
var lastParentId = null;
|
|
|
|
const toProcess = new prom.Gauge({
|
|
name: 'instasync_processed_posts',
|
|
help: 'Total IG posts to post to Ghost'
|
|
});
|
|
toProcess.set(posts.length);
|
|
console.log("Processing " + posts.length + " posts");
|
|
const downloadCounter = new prom.Counter({
|
|
name: 'instasync_downloads',
|
|
help: 'Total IG posts downloaded'
|
|
});
|
|
for (post of posts) {
|
|
|
|
//Download the media to our server
|
|
const parts = post.media_url.split('/');
|
|
// HACK: despite the .heic extensions, these are JPEG files right now
|
|
// This will probably change in the future and break things.
|
|
var fileName = parts[parts.length - 1].split('?')[0].replace('.heic', '.jpg');
|
|
// as of June 2024, no extension seems to mean "video"
|
|
if (fileName.indexOf('.') == -1) {
|
|
fileName += '.mp4';
|
|
}
|
|
|
|
const downloader = new Downloader({
|
|
url: post.media_url,
|
|
directory: "./ig-images",
|
|
fileName: fileName,
|
|
cloneFiles: false,
|
|
skipExistingFileName: true
|
|
})
|
|
|
|
await downloader.download();//Downloader.download() returns a promise.
|
|
|
|
//Upload it to the Ghost blog file directory
|
|
downloadCounter.inc();
|
|
console.log("Uploading " + fileName);
|
|
var upload;
|
|
if (fileName.endsWith('mp4')) {
|
|
upload = await api.media.upload({
|
|
file: "ig-images/" + fileName,
|
|
})
|
|
} else {
|
|
upload = await api.images.upload({
|
|
file: "ig-images/" + fileName,
|
|
})
|
|
}
|
|
|
|
//This is the return uploaded media url
|
|
const uploadedMediaUrl = upload.url;
|
|
|
|
//If the parent Instagram ID differs to the last one then we are working in relation to a different parent post
|
|
if (post.parent_instagram_id != lastParentId) {
|
|
|
|
//Reset our media index
|
|
cardIndex = 0;
|
|
lastParentId = post.parent_instagram_id;
|
|
}
|
|
|
|
const CARD_SECTION = 10;
|
|
if (post.parent_instagram_id in processedPosts) {
|
|
processedPosts[post.parent_instagram_id]["media_urls"].push(post.media_url);
|
|
processedPosts[post.parent_instagram_id]["rowIds"].push(post.instagram_id);
|
|
|
|
//Mobiledoc data
|
|
processedPosts[post.parent_instagram_id]["cards"].push(getAppropriateCard(post, uploadedMediaUrl));
|
|
processedPosts[post.parent_instagram_id]["sections"].push([CARD_SECTION, cardIndex]);
|
|
|
|
cardIndex++;
|
|
//This is the first media for this parent ID
|
|
} else {
|
|
|
|
//const caption = post.caption != null ? post.caption.replace(/(?:\r\n|\r|\n)/g, '<br>') : "";
|
|
const caption = post.caption != null ? post.caption : "";
|
|
const captionMobileDoc = ["html",{"html": caption}];
|
|
|
|
var captionLines = caption.split("\n")
|
|
captionLines = captionLines.filter(function(e){return e});
|
|
|
|
var mobileDocCards = [];
|
|
var mobileDocSections = [];
|
|
|
|
for (line of captionLines) {
|
|
|
|
mobileDocSections.push([1,"p",[[0,[],0,line]]]);
|
|
}
|
|
|
|
processedPosts[post.parent_instagram_id] = {};
|
|
processedPosts[post.parent_instagram_id]["featureImage"] = uploadedMediaUrl;
|
|
if (post.media_type == "VIDEO") {
|
|
processedPosts[post.parent_instagram_id]["featureImage"] = null;
|
|
|
|
mobileDocCards.push(getAppropriateCard(post, uploadedMediaUrl));
|
|
mobileDocSections.push([CARD_SECTION, cardIndex]);
|
|
cardIndex++;
|
|
}
|
|
|
|
var indexOfFullStop = caption.indexOf('.');
|
|
var indexOfNewLine = caption.indexOf('\n');
|
|
indexOfFullStop = indexOfFullStop !== -1 ? indexOfFullStop : 1000;
|
|
indexOfNewLine = indexOfNewLine !== -1 ? indexOfNewLine : 1000;
|
|
|
|
const indexOf = Math.min(indexOfFullStop, indexOfNewLine);
|
|
|
|
const postTitle = caption.substr(0, indexOf !== 1000 ? indexOf : caption.length);
|
|
const postSlug = slugify("Photo " + postTitle);
|
|
|
|
var postTags = [];
|
|
var matchedTags = caption.match(/#[A-Za-z0-9\-]+/gi);
|
|
matchedTags = matchedTags != null && matchedTags.length > 0 ? matchedTags.map(tag => tag.replace("#", "")) : [];
|
|
postTags.push("photo-post");
|
|
postTags = postTags.concat(matchedTags);
|
|
|
|
var publishedAt = post.datetime.toISOString();
|
|
|
|
processedPosts[post.parent_instagram_id]["publishedAt"] = publishedAt;
|
|
processedPosts[post.parent_instagram_id]["postTitle"] = postTitle;
|
|
processedPosts[post.parent_instagram_id]["postCaption"] = post.caption;
|
|
processedPosts[post.parent_instagram_id]["postSlug"] = postSlug;
|
|
processedPosts[post.parent_instagram_id]["postTags"] = postTags;
|
|
processedPosts[post.parent_instagram_id]["permalink"] = post.permalink;
|
|
processedPosts[post.parent_instagram_id]["caption"] = post.caption;
|
|
processedPosts[post.parent_instagram_id]["media_urls"] = [post.media_url];
|
|
processedPosts[post.parent_instagram_id]["rowIds"] = [post.instagram_id];
|
|
|
|
//Mobiledoc data
|
|
processedPosts[post.parent_instagram_id]["cards"] = mobileDocCards;
|
|
processedPosts[post.parent_instagram_id]["sections"] = mobileDocSections;
|
|
}
|
|
}
|
|
|
|
//We will now submit the posts to the blog..
|
|
|
|
const submittedPosts = new prom.Gauge({
|
|
name: 'instasync_submitted_posts',
|
|
help: 'Total IG posts submitted to Ghost'
|
|
});
|
|
submittedPosts.set(posts.length);
|
|
console.log("Submitting " + posts.length + " posts");
|
|
for (instagramParentId of Object.keys(processedPosts)) {
|
|
|
|
const processedPost = processedPosts[instagramParentId];
|
|
|
|
let featureImage = processedPost["featureImage"];
|
|
let publishedAt = processedPost["publishedAt"];
|
|
let postTitle = processedPost["postTitle"];
|
|
let postCaption = processedPost["postCaption"];
|
|
let postSlug = processedPost["postSlug"];
|
|
let postTags = processedPost["postTags"];
|
|
let permalink = processedPost["permalink"];
|
|
|
|
let rowIds = processedPost["rowIds"];
|
|
let cards = processedPost["cards"];
|
|
let sections = processedPost["sections"];
|
|
|
|
console.log(cards);
|
|
console.log(sections);
|
|
|
|
const mobiledoc = {
|
|
version: "0.3.2",
|
|
atoms: [],
|
|
markups: [],
|
|
cards: cards,
|
|
sections: sections,
|
|
}
|
|
|
|
const res = await api.posts
|
|
.add(
|
|
{
|
|
title: postTitle,
|
|
slug: postSlug.substring(0, 190),
|
|
tags: postTags,
|
|
meta_description: postCaption.substring(0, 500),
|
|
meta_title: postTitle,
|
|
feature_image: featureImage,
|
|
status: "published",
|
|
published_at: publishedAt,
|
|
created_at: publishedAt,
|
|
updated_at: publishedAt,
|
|
authors: ["hello@alo.land"],
|
|
mobiledoc: JSON.stringify(mobiledoc)
|
|
},
|
|
)
|
|
|
|
|
|
const postId = res.id;
|
|
|
|
console.log("POSTED: " + postId);
|
|
|
|
//Mark the media rows submitted as part of this post as complete
|
|
const updateSql = "UPDATE ig_posts SET post_id = ? WHERE instagram_id IN (?)";
|
|
const updateResponse = await db_conn.query(updateSql, [postId, rowIds]);
|
|
|
|
console.log("MARKED AS POSTED");
|
|
}
|
|
let promClient = new prom.Pushgateway('http://pushgateway.service.consul:9091');
|
|
await promClient
|
|
.push({ jobName: 'instasync' })
|
|
.catch(err => {
|
|
console.log('Error: ${err}');
|
|
});
|
|
console.log("Done");
|
|
}
|
|
|
|
doWork()
|
|
.then(result => { process.exit() });
|