adds periodic restarting of dynos #2 and staggers error restarts

Author: Rory Campbell

config.json

@@ -0,0 +1,83 @@
+const rp = require('request-promise')
+const moment = require('moment')
+const { restartAllDynos, sleep } = require('./heroku')
+
+const config = {
+  allowedTimeoutRatio: 0.05,
+  minErrorCount: 3,
+  intervalMs: 10000,
+  sampleMs: 60000
+}
+
+const getLogs = async () => {
+  const options = {
+    uri: 'https://papertrailapp.com/api/v1/events/search.json?q=heroku/router&limit=2000',
+    headers: {
+      'X-Papertrail-Token': process.env.PAPERTRAIL_TOKEN
+    },
+    json: true
+  }
+
+  const { events } = await rp(options)
+
+  return events
+}
+
+const getStatusCode = (log) => {
+  try {
+    const pair = log.message.split(' status=')
+    return Number(pair[1].slice(0, 3))
+  } catch (e) {
+    return null
+  }
+}
+
+const shouldRestart = ({ allowedTimeoutRatio, sampleMs, minErrorCount }, now, logs) => {
+  const nowMoment = moment(now)
+
+  const recentLogs = logs.filter(l => {
+    const logMoment = moment(l.received_at)
+
+    return nowMoment.diff(logMoment) < sampleMs
+  })
+
+  const statusCodes = recentLogs.map(getStatusCode).filter(sc => sc)
+
+  const errors = statusCodes.filter(sc => sc >= 500).length
+  const timeouts = statusCodes.filter(sc => sc >= 503).length
+
+  const errorRatio = errors / statusCodes.length
+  const timeoutRatio = timeouts / statusCodes.length
+
+  console.log('datetime', nowMoment.format())
+  console.log('request count', statusCodes.length)
+  console.log('errors', errors)
+  console.log('error ratio', errorRatio)
+  console.log('timeouts', timeouts)
+  console.log('timeout ratio', timeoutRatio)
+
+  return timeouts >= minErrorCount &&
+        timeoutRatio > allowedTimeoutRatio
+}
+
+const go = async () => {
+  try {
+    const { intervalMs } = config
+
+    const logs = await getLogs()
+
+    const now = Date.now()
+    if (shouldRestart(config, now, logs)) {
+      await restartAllDynos(intervalMs)
+    } else {
+      console.log('No restart required')
+    }
+    await sleep(intervalMs)
+
+    process.exit(0)
+  } catch (err) {
+    console.error('errorRestarter error: ', err)
+  }
+}
+
+go()

errorRestarter.js

@@ -0,0 +1,33 @@
+require('dotenv').config()
+
+const Heroku = require('heroku-client')
+const heroku = new Heroku({ token: process.env.HEROKU_TARGET_APP_API_TOKEN })
+
+const getDynos = () => heroku.get(`/apps/${process.env.APP_NAME}/dynos`)
+const restartDyno = ({ id }) => heroku.delete(`/apps/${process.env.APP_NAME}/dynos/${id}`)
+
+const restartAllDynos = async (delay) => {
+  console.log(`Restarting dynos for ${process.env.APP_NAME}`)
+  const dynos = getDynos()
+  await asyncForEach(dynos, async (dyno) => {
+    await restartDyno(dyno)
+    await sleep(delay)
+  })
+  console.log('Dynos restarted')
+}
+
+async function asyncForEach (array, callback) {
+  for (let index = 0; index < array.length; index++) {
+    await callback(array[index], index, array)
+  }
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+module.exports = {
+  getDynos,
+  restartDyno,
+  restartAllDynos,
+  sleep,
+  asyncForEach
+}

heroku.js

@@ -1,13 +1,27 @@
-var forever = require('forever-monitor');
+var forever = require('forever-monitor')
 
-const file = 'monitor.js';
- 
-var child = new (forever.Monitor)(file, {
+// restarts all dynos when error rate goes too high
+const errorRestarterFile = 'errorRestarter.js'
+
+var errorRestarterChild = new (forever.Monitor)(errorRestarterFile, {
+  args: []
+})
+
+errorRestarterChild.on('exit', function () {
+  console.log(`${errorRestarterFile} has exited`)
+})
+
+errorRestarterChild.start()
+
+// restarts a proportion of dynos every set period of time
+const periodicRestarterFile = 'periodicRestarter.js'
+
+var periodicRestarterChild = new (forever.Monitor)(periodicRestarterFile, {
   args: []
-});
+})
 
-child.on('exit', function () {
-  console.log(`${file} has exited`);
-});
+periodicRestarterChild.on('exit', function () {
+  console.log(`${periodicRestarterFile} has exited`)
+})
 
-child.start();
+periodicRestarterChild.start()

index.js

@@ -3,6 +3,9 @@
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
+  "engines": {
+    "node": "12.18.0"
+  },
   "scripts": {
     "start": "node index.js",
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -13,7 +16,8 @@
     "dotenv": "^8.2.0",
     "forever-monitor": "^2.0.0",
     "heroku-client": "^3.1.0",
-    "moment": "^2.24.0",
+    "lodash": "^4.17.15",
+    "moment": "^2.27.0",
     "request": "^2.88.2",
     "request-promise": "^4.2.5"
   }

monitor.js

@@ -0,0 +1,34 @@
+const { getDynos, sleep, restartDyno } = require('./heroku')
+const _ = require('lodash')
+const INTERVAL = 15 * 60 * 1000 // 15 mins
+const RATIO = 0.1
+
+const every15MinsRestart10pcOfDynos = async () => {
+  let dynos = await getDynos()
+
+  dynos = _.sortBy(dynos, d => new Date(d.created_at))
+
+  if (!dynos.length) {
+    console.log('No dynos')
+    await sleep(10000)
+    every15MinsRestart10pcOfDynos()
+  }
+
+  const timeToSleep = INTERVAL / (RATIO * dynos.length)
+  console.log('timeToSleep', timeToSleep)
+
+  await sleep(timeToSleep)
+
+  const dyno = dynos[0]
+
+  console.log('restarting dyno', dyno)
+  await restartDyno(dyno)
+
+  every15MinsRestart10pcOfDynos()
+}
+
+try {
+  every15MinsRestart10pcOfDynos(0)
+} catch (err) {
+  console.log('err', err)
+}