Pulled in the twitter content backup functionality from curvyandtrans.com

Sadly, lost the images from one of Emmy_Zje's deleted tweets.

build/twitter-client.js

@@ -0,0 +1,107 @@
+const fs = require('fs-extra');
+const { resolve } = require('./resolve');
+const { chunk, uniq, difference } = require('lodash');
+const Twitter = require('twitter-lite');
+const log = require('fancy-log');
+const tweetparse = require('./lib/tweetparse');
+const { hasOwn } = require('./lib/util');
+
+module.exports = exports = async function () {
+  const tc = new TwitterClient();
+  await tc.initialize();
+  return tc;
+};
+
+
+class TwitterClient {
+
+  async initialize () {
+    const [ lookup, backup, cache ] = await Promise.all([
+      fs.readJson(resolve('twitter-config.json')).catch(() => null)
+        .then(makeFetcher),
+      fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
+      fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
+    ]);
+
+    this._lookup = lookup;
+    this._backupData = backup;
+    this._cache = cache;
+    this._presentTweets = Object.keys(cache);
+  }
+
+  async write () {
+    return Promise.all([
+      fs.writeFile(resolve('twitter-cache.json'),  JSON.stringify(this._cache,  null, 2)),
+      fs.writeFile(resolve('twitter-backup.json'), JSON.stringify(this._backupData, null, 2)),
+    ]);
+  }
+
+  async get (tweetids) {
+    if (!Array.isArray(tweetids)) tweetids = [ tweetids ];
+
+    tweetids = uniq(tweetids.map(parseTweetId));
+
+    let tweetsNeeded = this._missing(tweetids);
+
+    while (tweetsNeeded.length) {
+      log('Fetching tweets: ' + tweetsNeeded.join(', '));
+      const arriving = await Promise.all(chunk(tweetsNeeded, 99).map(this._lookup));
+      const tweetsRequested = tweetsNeeded;
+      tweetsNeeded = [];
+      const loaded = [];
+      for (const tweet of arriving.flat(1)) {
+        if (tweet.quoted_status_id_str && !this._cache[tweet.quoted_status_id_str]) {
+          tweetsNeeded.push(tweet.quoted_status_id_str);
+        }
+
+        this._backupData[tweet.id_str] = tweet;
+        this._cache[tweet.id_str] = tweetparse(tweet);
+        loaded.push(tweet.id_str);
+      }
+
+      const absent = difference(tweetsRequested, loaded);
+      for (const id of absent) {
+        if (!hasOwn(this._backupData, id)) {
+          log.error('Could not find tweet ' + id);
+          continue;
+        }
+        const tweet = this._backupData[id];
+
+        if (tweet) {
+          log('Pulled tweet from backup ' + id);
+          this._cache[id] = tweetparse(this._backupData[id]);
+        } else {
+          this._cache[id] = false;
+        }
+      }
+    }
+
+    return tweetids.map((id) => this._cache[id] || null);
+  }
+
+  _missing (tweetids) {
+    return difference(tweetids, this._presentTweets);
+  }
+}
+
+
+function makeFetcher (config) {
+  if (!config) return () => [];
+  const client = new Twitter(config);
+  return (tweetids) => client
+    .get('statuses/lookup', { id: tweetids.join(','), tweet_mode: 'extended' })
+    // .then((r) => { console.log({r}); return r; })
+    .catch((e) => { log.error(e); return []; });
+}
+
+const tweeturl = /https?:\/\/twitter\.com\/(?:#!\/)?(?:\w+)\/status(?:es)?\/(\d+)/i;
+const tweetidcheck = /^\d+$/;
+function parseTweetId (tweetid) {
+  // we can't trust an id that isn't a string
+  if (typeof tweetid !== 'string') return false;
+
+  const match = tweetid.match(tweeturl);
+  if (match) return match[1];
+  if (tweetid.match(tweetidcheck)) return tweetid;
+  return false;
+}