Automated Data Collection

.github/workflows/event-update.yml

@@ -0,0 +1,35 @@
+name: 'Event Data Updater'
+run-name: Event Data Update
+on:
+  # Enables workflow to be called manually from GitHub Actions
+  workflow_dispatch:
+  # Run every Wednesday at 09:00 UTC (during weekly maintenance)
+  schedule:
+    - cron: '0 9 * * 3'
+jobs:
+  Update-Event-Data:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps: 
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+      - name: Configure date
+        run: |
+          echo "ACTION_DATE=$(date --rfc-3339=date)" >> ${GITHUB_ENV}
+      - name: Pull remote changes
+        run: |
+          git pull
+      - name: Install npm packages
+        run: |
+          npm i
+      - name: Compile TS files
+        run: |
+          npx tsc ./bin/AutoScraper.ts
+      - name: Gather Event Data
+        run: node ./bin/event-update.js
+      - name: Commit changes to data
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with: 
+          file_pattern: 'data/*.json'
+          commit_message: "Event data update: ${{env.ACTION_DATE}}"

bin/AutoScraper.ts

@@ -0,0 +1,79 @@
+const fs = require('fs')
+const https = require('https')
+
+interface CalendarData {
+  [type: string]: {
+    [month: string]: {
+      [day: string]: {
+        [eventNo: string]: {
+          [eventId: string]: string[]
+        }
+      }
+    }
+  }
+}
+
+export class AutoScraper {
+  private static readonly calendarURL: string = 'https://lostarkcodex.com/us/eventcalendar/'
+  private static bufferData: string = ''
+  private static calendarData: CalendarData = {}
+
+  public static scrapeCalendarData(): void {
+    https.get(this.calendarURL, (res: any) => {
+      res.setEncoding('utf8')
+      res.on('data', (chunk: string) => {
+        this.bufferData += chunk
+      })
+      res.on('end', () => {
+        this.extractCalendarData()
+        this.cleanCalendarData()
+        this.writeData()
+      })
+    })
+  }
+
+  // Strip everything except the calendar_data variable
+  private static extractCalendarData(): void {
+    this.bufferData = this.bufferData
+      .replace(/\r\n/gmi, ' ')
+      .replace(/.*var calendar_data=/i, '')
+      .replace(/(?<=;).*/i, '')
+      .replace(';', '')
+
+    try {
+      JSON.parse(this.bufferData)
+    } catch (err: unknown) {
+      console.error(err);
+      this.bufferData = '{}'
+    }
+  }
+
+  // Removes duplicates from entries
+  private static cleanCalendarData(): void {
+    this.calendarData = JSON.parse(this.bufferData)
+    Object.keys(this.calendarData).forEach(
+      type => Object.keys(this.calendarData[type]).forEach(
+        month => Object.keys(this.calendarData[type][month]).forEach(
+          day => Object.keys(this.calendarData[type][month][day]).forEach(
+            event => Object.keys(this.calendarData[type][month][day][event]).forEach(
+              // Overwrite current array with one without duplicates
+              id => this.calendarData[type][month][day][event][id] = Array.from(new Set(this.calendarData[type][month][day][event][id]))
+            )
+          )
+        )
+      )
+    )
+  }
+
+  // If data was successfully formed, backs up the previous data file, then creates a new data file
+  private static writeData(): void {
+    if (Object.values(this.calendarData).length > 0) {
+      fs.rename('./data/data.json', './data/data.old.json', (err: Error) => {
+        if (err) console.error(err)
+        fs.writeFileSync('./data/data.json', JSON.stringify(this.calendarData), 'utf8')
+      })
+    }
+  }
+}
+
+module.exports = AutoScraper

bin/event-update.js

@@ -0,0 +1,4 @@
+// This file is the launchpad for GitHub Actions. Add functions as required.
+const AutoScraper = require('./AutoScraper');
+
+AutoScraper.scrapeCalendarData();