Apply hexagonal architecture

Author: stojanovic

README.md

@@ -20,15 +20,12 @@ This project has the following folder structure:
 ├── package.json
 ├── samconfig.toml # AWS SAM config file, generated by SAM
 ├── src # Source code for all functions
-│   ├── parse-covid-csv # Fuction source code
-│   │   ├── events # Events for local testing
-│   │   │   ├── context.ts
-│   │   │   └── event.json
-│   │   └── lambda.ts # Function source code
-│   └── tests
-│       └── parse-covid-csv
-│           └── lambda.test.ts
-├── template.yaml # Main CloudFormation file
+│   └── parse-covid-csv # Fuction source code
+│       ├── events # Events for local testing
+│       │   ├── context.ts
+│       │   └── event.json
+│       └── lambda.ts # Function source code
+├── template.yaml # Main CloudFormation file 
 ├── tsconfig.json
 ├── webpack.config.js # Webpack config
 └── yarn.lock
@@ -82,16 +79,8 @@ sam deploy
 
 ### Testing
 
-To run the application tests, run the following command:
-
-```bash
-npm test
-```
-
-This will run the jest tests for the sample serverless component, particularly the [`/src/tests/parse-covid-csv/lambda.test.ts`](./src/tests/parse-covid-csv/lambda.test.ts).
-
-Note. If you notice error logs during the test run, please do not get concerned, as the tests are also evaluating the error cases and it is recommended to trace the particular error logs.
+This section will be added tomorrow.
 
 ## License
 
-MIT, see [LICENSE](LICENSE).
+MIT, see [LICENSE](LICENSE).

WHO-COVID-19-global-data.csv

@@ -0,0 +1,24 @@
+import { DocumentClient } from 'aws-sdk/clients/dynamodb'
+import { v4 as uuidv4 } from 'uuid'
+const documentClient = new DocumentClient()
+
+// This Database Adapter's sole responsibility is handling the communication protocol with AWS DynamoDB
+// 1. Accepts an item (JSON), tableName, to know where to store the data and 
+//    as an optional parameter it accepts the database client library, where we by default set the DynamoDBs Document Client
+//    In this case the reason behind it is because we want to make it easier to test and mock. 
+//    This approach enables simple JavaScript objects or spy objects to be passed, without having to extensively mock the DocumentClient
+export default {
+  saveItem: async function saveItem(item: any, tableName: string, db = documentClient): Promise<void> {
+    // 2. We generate the item ID
+    item.id = uuidv4()
+    try {
+      // 3. Try to store the file into our current database
+      await db.put({
+        TableName: tableName,
+        Item: item
+      }).promise()
+    } catch(e) {
+      throw JSON.stringify({message: `Error saving ${JSON.stringify(e)}` })
+    }
+  }
+}

adapters/database-adapter.ts

@@ -0,0 +1,20 @@
+import { Readable } from 'stream'
+import S3 from 'aws-sdk/clients/s3'
+const s3 = new S3()
+
+// This File Storage Adapter's sole responsibility is handling the communication protocol with AWS S3
+// 1. Accepts a file key (identifier), the bucket where its stored, and
+//    as an optional parameter it accepts the file storage client library, where we by default set the AWS S3 client instance
+//    As with the database adapter, in this case the reason behind it is because we want to make it easier to test and mock.
+//    This approach enables simple JavaScript objects or spy objects to be passed, without having to extensively mock the S3 Client library
+export default {
+  getFileStream: async function getFileStream(key: string, bucket: string, filestorageLibrary = s3): Promise<Readable> {
+    console.log('getting file stream', bucket, key)
+    // 2. We simply call the file storage library to get us the object based on its file information details 
+    return filestorageLibrary.getObject({
+      Bucket: bucket,
+      Key: key
+    }).createReadStream()
+    // We return the File Read Stream, as we don't have to download the file, but by passing the stream we can just attach it to our csv parser library
+  }
+} 

adapters/filestorage-adapter.ts

@@ -1,13 +1,13 @@
 {
-  "name": "aws-sam-ts-template",
-  "version": "1.0.0",
+  "name": "sample-testing-app",
+  "version": "2.0.0",
   "description": "",
   "main": "index.js",
   "scripts": {
     "watch": "NODE_ENV=dev webpack --watch --mode=development",
     "build": "NODE_ENV=prod BUNDLE_ANALYZER=false webpack --mode=production --progress",
     "build-analyze": "NODE_ENV=prod BUNDLE_ANALYZER=true webpack --mode=production --progress",
-    "lint": "eslint -c .eslintrc.js --ext ts  src",
+    "lint": "eslint -c .eslintrc.js --ext ts  functions",
     "pretest": "npm run lint",
     "test": "jest",
     "coverage": "jest --coverage"
@@ -21,17 +21,17 @@
     "@types/uuid": "^8.0.0",
     "aws-lambda": "^1.0.5",
     "aws-sdk": "^2.639.0",
+    "aws-xray-sdk": "^2.5.0",
     "neat-csv": "^5.2.0",
     "source-map-support": "^0.5.16",
     "tslint": "^6.1.0",
-    "uuid": "^8.1.0"
+    "uuid": "^8.2.0"
   },
   "devDependencies": {
     "@types/jest": "^25.1.4",
     "@typescript-eslint/eslint-plugin": "^2.23.0",
     "@typescript-eslint/eslint-plugin-tslint": "^2.23.0",
     "@typescript-eslint/parser": "^2.23.0",
-    "aws-sdk-mock": "^5.1.0",
     "eslint": "^6.8.0",
     "eslint-plugin-prefer-arrow": "^1.1.7",
     "eslint-plugin-prefer-arrow-functions": "^3.0.1",

package-lock.json

@@ -1,51 +1,32 @@
 // Allow CloudWatch to read source maps
 import 'source-map-support/register'
-import { DocumentClient } from 'aws-sdk/clients/dynamodb'
-import { S3Event, S3EventRecord } from 'aws-lambda'
-import S3 from 'aws-sdk/clients/s3'
-import path from 'path'
-import os from 'os'
-import util from 'util'
-import * as stream from 'stream'
-import * as fs from 'fs'
-import neatCsv from 'neat-csv'
-import { v4 as uuidv4 } from 'uuid'
-const pipeline = util.promisify(stream.pipeline)
+import AWSXRay from 'aws-xray-sdk-core'
+import { S3Event } from 'aws-lambda'
+import https from 'https'
+
+// Instead of keeping all of our logic in one file,
+// it is much easier to handle the logic by storing it the /src folder
+import processCSV from './src/main'
+
+if (process.env.AWS_EXECUTION_ENV) {
+  AWSXRay.captureHTTPsGlobal(https, true)
+}
+
+// Here we are introducing a File Event Port, per Hexagonal architecture principles
+import parseFileEvent from './src/parse-file-event'
+
+// Since we are going to reference a File structure, { key, bucket }, quite a lot,
+// it is much better to define a proper interface than to rely on "any" a type
+import IFile from './src/IFile'
+
 const TABLE_NAME = process.env.TABLE_NAME || ''
-const documentClient = new DocumentClient()
-const s3 = new S3()
+
 export async function handler(event: S3Event): Promise<any> {
-  const s3Records = event.Records
-  await Promise.all(s3Records.map(async (record: S3EventRecord) => processCSV(record)))
-}
-async function processCSV(record: S3EventRecord): Promise<any> {
-  const downloadPath = path.join(os.tmpdir(), record.s3.object.key)
-  try {
-    const readable = s3.getObject({
-      Bucket: record.s3.bucket.name,
-      Key: record.s3.object.key,
-    }).createReadStream()
-    const writable = fs.createWriteStream(downloadPath, { encoding: 'utf8' })
-    await pipeline(
-      readable,
-      writable
-    )
-  } catch (e) {
-    console.log(e)
-    throw e
-  }
-  const readCsv = fs.createReadStream(downloadPath)
-  const jsonData = await neatCsv(readCsv)
-  await Promise.all(jsonData.map(async (entry: any) => {
-    entry.id = uuidv4()
-    try {
-      return await documentClient.put({
-        TableName: TABLE_NAME,
-        Item: entry,
-      }).promise()
-    } catch (e) {
-      console.error(e)
-      throw e
-    }
-  }))
+
+  // With it we are parsing an external event to a format more suitable for our business logic
+  const receivedFiles: IFile[] = parseFileEvent(event)
+
+  // Simply, we are going through all event Files,
+  // and passing them to our main business logic for processing
+  return await Promise.all(receivedFiles.map(async (file: IFile) => processCSV(file, TABLE_NAME)))
 }

package.json

@@ -0,0 +1,4 @@
+export default interface IFile {
+  bucket: string
+  key: string
+}

src/parse-covid-csv/lambda.ts

@@ -0,0 +1,44 @@
+import IFile from './IFile'
+
+// Importing the adapters to other external services we are using
+// *** Note! Take a look at the adapter file names, we aren't referencing S3 or DynamoDB anywhere
+import fileStorageAdapter from '../../../adapters/filestorage-adapter'
+import databaseAdapter from '../../../adapters/database-adapter'
+
+// This is the NPM library we are going to use to parse a CSV file into a JSON structure
+import neatCsv from 'neat-csv'
+
+// This file represents our main business logic where it needs to
+// get and process a CSV file and store its data into a database
+// Take a note of the parameters passed:
+// 1. storedFile - representing the file location info (where its stored and by which key)
+// 2. tableName - what is the name of the database table  where we want to store the file contents
+
+// Now, particularly these are the more important for us:
+// 3. fileAdapter - the adapter responsible(!) for the implementation protocol with our current file storage service
+// 4. dbAdapter - the adapter responsible(!) for the implementation protocol with our current database service
+//
+//  Note!
+//    The reason for setting these adapters as optional paramters and also defining their default values is
+//    because then we can easily override the parameters with another of our choosing.
+//    Making it easier to:
+//     - Migrate to another database or fileStorage service. Say we run a Lambda and want to use Azure CosmosDB,
+//       just change the implementation of the dbAdapter
+//     - Ease mocking when testing. Just call the "processCSV" function with simple JavaScript objects, or spy objects.
+
+
+// Now to explain the core business logic within
+export default async function processCSV(storedFile: IFile, tableName: string, fileAdapter = fileStorageAdapter, dbAdapter = databaseAdapter): Promise<any> {
+  const { key, bucket } = storedFile
+
+  // 1. Retrieving a File from its Storage
+  const fileStream = await fileAdapter.getFileStream(key, bucket)
+
+  // 2. Invoking the CSV library and parsing it to JSON.
+  //    The "jsonData" is now an Array of JSON objects, each JSON object is one CSV row
+  const jsonData = await neatCsv(fileStream)
+
+  // 3. Going over each parsed CSV row (which is now in the JSON format)
+  //    And calling the dbAdapter to store it.
+  return await Promise.all(jsonData.map(async (entry: any) => await dbAdapter.saveItem(entry, tableName)))
+}

src/parse-covid-csv/src/IFile.ts

@@ -0,0 +1,17 @@
+import { S3Event, S3EventRecord } from 'aws-lambda'
+import IFile from './IFile'
+
+// This File Event Parser is solely responsible for handling the data structure of Amazon S3 events
+// 1. Accepts an S3 event
+export default function parseFileEvent(event: S3Event): IFile[] {
+  // 2. Extracts S3 Records, which are basically files that the S3 event wanted to notify us about
+  const s3Records: S3EventRecord[] = event.Records
+  // 2. Extracts File Information of objects from S3 Records,
+  //    which are defined by the bucket they belong to, the key as their identifier and their size
+  const extractObject = (record: S3EventRecord) => {
+    const { bucket, object } = record.s3
+    return { bucket: bucket.name, key: object.key, size: object.size } 
+  }
+  // Goes through each S3 record and at the end returns an array of the extracted file information objects
+  return s3Records.map(extractObject) 
+}