Adding terraform config

This currently just redirects to the old bible, but at least everything is up and running

terraform/files/decorate/index.js

@@ -0,0 +1,92 @@
+/* eslint no-console:0 */
+/* global URLSearchParams */
+
+const AWS = require('aws-sdk');
+const zlib = require('zlib');
+const util = require('util');
+const path = require('path');
+const {URL} = require('url');
+const s3 = new AWS.S3();
+const { parse: parseLog } = require('cloudfront-log-parser');
+const parseUA = require('ua-parser-js');
+const format = require('date-fns/format');
+const gunzip = util.promisify(zlib.gunzip);
+const gzip = util.promisify(zlib.gzip);
+
+function url (input) {
+  const { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username } = new URL(input);
+  return { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username };
+}
+
+exports.handler = async (event) => {
+  // Read options from the event.
+  console.log('Reading options from event:\n', JSON.stringify(event, null, 2));
+
+  const Bucket = event.Records[0].s3.bucket.name;
+  const inputKey    = event.Records[0].s3.object.key;
+
+  const file = path.parse(inputKey);
+  const outputKey = path.format({ ...file, dir: 'Converted', ext: '.json.gz' });
+
+  const response = await s3.getObject({ Bucket, Key: inputKey }).promise();
+  const input = (await gunzip(response.Body)).toString('utf8');
+
+  const entries = parseLog(input, { format: 'web' });
+
+  console.log(`Found ${entries.length} rows`);
+
+  const results = entries.map((row) => {
+    // filter out OPTIONS calls
+    if (row['cs-method'] === 'OPTIONS') return null;
+
+    // I only care about the pixel hits, nothing else.
+    if (row['cs-uri-stem'] !== '/i') return null;
+
+    // this isn't an analytics event
+    if (row['cs-referer'] === '-') return null;
+
+    row = Object.fromEntries(Object.entries(row).map(([ k, v ]) => [ k.replace(/-/g, '_'), v ]));
+
+    const query = (row.cs_uri_query === '-')
+      ? {}
+      : Object.fromEntries(new URLSearchParams(row.cs_uri_query))
+    ;
+
+    // we didn't get analytics data from this load, ignore it
+    if (!query.start) return null;
+
+    const useragent = parseUA(row.cs_user_agent);
+    const { referer } = query;
+
+    const sessionStart = Number(query.start);
+    const sessionEnd = query.end === 'null' ? 0 : Number(query.end);
+    const duration = sessionEnd > sessionStart ? Math.floor((sessionEnd - sessionStart) / 1000) : null;
+
+    return JSON.stringify({
+      dts: `${row.date} ${row.time}`,
+      url: url(row.cs_referer),
+      referer: url(query.referer),
+      client_start: format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss'),
+      client_end: sessionEnd ? format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss') : null,
+      duration,
+      useragent,
+      query,
+      original: row,
+    });
+  }).filter(Boolean);
+
+  if (!results.length) {
+    console.log('No results to save');
+    return;
+  }
+
+  console.log('Writing new file to ' + outputKey);
+
+  await s3.putObject({
+    Bucket,
+    Key: outputKey,
+    Body: await gzip(Buffer.from(results.join('\n'))),
+    ContentType: 'application/gzip',
+  }).promise();
+
+};

terraform/files/decorate/package-lock.json

@@ -0,0 +1,23 @@
+{
+  "name": "decorate",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "cloudfront-log-parser": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/cloudfront-log-parser/-/cloudfront-log-parser-1.1.0.tgz",
+      "integrity": "sha512-g1lxh8aW5ZrQ7/loX2/vLzz4SWefQhSvZw++wgoIx3aEugXHKyfYaWOXGS4pNNp9hi7JcXITxLYqBI2FY+jtgA=="
+    },
+    "date-fns": {
+      "version": "2.9.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.9.0.tgz",
+      "integrity": "sha512-khbFLu/MlzLjEzy9Gh8oY1hNt/Dvxw3J6Rbc28cVoYWQaC1S3YI4xwkF9ZWcjDLscbZlY9hISMr66RFzZagLsA=="
+    },
+    "ua-parser-js": {
+      "version": "0.7.21",
+      "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz",
+      "integrity": "sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ=="
+    }
+  }
+}

terraform/files/decorate/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "decorate",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Jocelyn Badgley <joc@twipped.com> (http://twipped.com/)",
+  "license": "MIT",
+  "dependencies": {
+    "cloudfront-log-parser": "~1.1.0",
+    "date-fns": "~2.9.0",
+    "ua-parser-js": "~0.7.21"
+  },
+  "engines": {
+    "node": ">=12.14.0"
+  }
+}

terraform/files/index_redirect.js

@@ -0,0 +1,22 @@
+// This is a solution for the problem with s3 websites and subfolders www.site.com/folder/ will not auto
+//   redirect to www.site.com/folder/index.html like most other modern web servers
+// This should be deployed as a Lambda@Edge connected to the CloudFront Distribution
+// Only Node.js 10.x Runtime supports Lambda@Edge for right now, we have to wait for AWS to support 12x and beyond
+
+exports.handler = async (event, context) => {
+  /*
+   * Expand S3 request to have index.html if it ends in /
+   */
+  const request = event.Records[0].cf.request;
+  if ((request.uri !== '/') /* Not the root object, which redirects properly */
+    && (request.uri.endsWith('/') /* Folder with slash */
+      || (request.uri.lastIndexOf('.') < request.uri.lastIndexOf('/')) /* Most likely a folder, it has no extension (heuristic) */
+    )) {
+    if (request.uri.endsWith('/')) {
+      request.uri = request.uri.concat('index.html');
+    } else {
+      request.uri = request.uri.concat('/index.html');
+    }
+  }
+  return request;
+};

terraform/files/table.sql

@@ -0,0 +1,23 @@
+CREATE EXTERNAL TABLE gdbible.events (
+  dts TIMESTAMP,
+  page STRING,
+  client_start TIMESTAMP,
+  client_end TIMESTAMP,
+  duration INT,
+  useragent struct<
+    browser: struct< name:STRING, version:STRING, major: INT >,
+    os: struct< name:STRING, version:STRING >
+  >,
+  `query` struct<
+    tid: STRING,
+    page_height:INT,
+    viewport_height:INT,
+    max_scroll:INT,
+    viewed:INT,
+    language:STRING,
+    referrer:STRING
+  >,
+  original struct<c_ip: STRING>
+)
+ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
+LOCATION 's3://gdbible-analytics/Converted'