Infrastructure as Code in CLI

.github/workflows/release.yml

@@ -31,11 +31,19 @@ jobs:
         uses: actions/checkout@v2
       - name: Setup NodeJS and NPM modules
         uses: ./.github/actions/npm-setup
-      - name: Build package
-        run: yarn build nx-aws-cache
-      - uses: ./.github/actions/npm-publish
+      - name: Build packages
+        run: yarn run build:all
+      - name: Publish @nx-aws-plugin/nx-aws-cache
+        uses: ./.github/actions/npm-publish
         with:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
           RELEASE_VERSION: ${{ needs.repoMeta.outputs.tag-name }}
           RELEASE_TYPE: ${{ needs.repoMeta.outputs.release-type }}
           SRC_FOLDER: dist/packages/nx-aws-cache
+      - name: Publish @nx-aws-plugin/nx-aws-cache-iac
+        uses: ./.github/actions/npm-publish
+        with:
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          RELEASE_VERSION: ${{ needs.repoMeta.outputs.tag-name }}
+          RELEASE_TYPE: ${{ needs.repoMeta.outputs.release-type }}
+          SRC_FOLDER: dist/apps/nx-aws-cache-iac

.gitignore

@@ -4,6 +4,8 @@
 /dist
 /tmp
 /out-tsc
+/cdk-out
+/cdk.out
 **/dist
 
 # dependencies

README.md

@@ -24,6 +24,10 @@ npm run nx generate @nx-aws-plugin/nx-aws-cache:init
 
 This will make the necessary changes to nx.json in your workspace to use nx-aws-cache runner.
 
+## Setup Infrastructure and Secrets automatically
+
+Read the [README.md of @nx-aws-plugin/nx-aws-cache-iac](https://github.com/bojanbass/nx-aws/blob/master/apps/nx-aws-cache-iac/README.md)
+
 ## Plugin settings
 
 There are two ways to set-up plugin options, using `nx.json` or `Environment variables`. Here is a list of all possible options:

package.json

@@ -27,10 +27,14 @@
     "nx": "nx",
     "start": "nx serve",
     "build": "nx build",
+    "build:all": "nx run-many --target=build --all",
     "build:actions": "yarn tsc --project ./.github/actions/tsconfig.json",
     "test": "nx test",
+    "test:all": "nx run-many --target=test --all",
     "lint": "nx workspace-lint && nx lint",
+    "lint:all": "nx run-many --target=lint --all",
     "e2e": "nx e2e",
+    "e2e:all": "nx run-many --target=e2e --all",
     "affected:apps": "nx affected:apps",
     "affected:libs": "nx affected:libs",
     "affected:build": "nx affected:build",
@@ -54,7 +58,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.478.0",
-    "@aws-sdk/credential-providers": "3.474.0",
+    "@aws-sdk/credential-providers": "3.478.0",
     "@aws-sdk/lib-storage": "^3.478.0",
     "@aws-sdk/property-provider": "^3.374.0",
     "@swc/helpers": "~0.5.3",
@@ -64,8 +68,10 @@
   "devDependencies": {
     "@actions/core": "1.10.1",
     "@actions/github": "5.1.1",
+    "@aws-sdk/client-secrets-manager": "^3.507.0",
     "@aws-sdk/util-stream-node": "^3.374.0",
     "@nx/devkit": "16.2.2",
+    "@nx/esbuild": "16.2.2",
     "@nx/eslint-plugin": "16.2.2",
     "@nx/jest": "16.2.2",
     "@nx/js": "16.2.2",
@@ -77,7 +83,10 @@
     "@types/tar": "6.1.10",
     "@typescript-eslint/eslint-plugin": "5.18.0",
     "@typescript-eslint/parser": "5.18.0",
+    "aws-cdk-lib": "^2.126.0",
     "aws-sdk-client-mock": "^3.0.0",
+    "constructs": "^10.3.0",
+    "esbuild": "^0.20.0",
     "eslint": "8.13.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-import": "2.25.4",

packages/nx-aws-cache-iac/.eslintrc.json

@@ -0,0 +1,12 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "rules": {
+    "@typescript-eslint/ban-types": "off",
+    "@typescript-eslint/no-non-null-assertion": "off",
+    "no-new": "off",
+    "no-console": "off",
+    "import/no-internal-modules": "off",
+    "max-lines-per-function": "off"
+  }
+}

packages/nx-aws-cache-iac/README.md

@@ -0,0 +1,42 @@
+# NX AWS Cache Infrastructure as Code
+
+The plugin does not enforce any specific infrastructure. It does require you to have certain elements (e.g. an S3 bucket, a user with access to it, etc.). You can create such infrastructure manually or you can use the provided IaC (Infrastructure as Code) to create it automatically.
+
+In order to ease the process of creating the infrastructure, this package provides a CLI that will create the infrastructure for you. It contains the IaC for creating the infrastructure and you can run it by just calling a command. It uses AWS CDK (Cloud Development Kit) to create the infrastructure. The infrastructure is defined in the `lib/nx-aws-cache-iac-stack.ts` file.
+
+The CLI will create a new Stack in your AWS account. The Stack will create a new S3 bucket and an IAM user with access to it. The credentials for the user will be stored in the AWS Secrets Manager.
+
+## How to deploy the infrastructure
+
+This command will deploy a new Stack that creates a S3 bucket for the cache and an IAM user with access to it.
+This user will have the credentials stored in the AWS Secrets Manager.
+
+```bash
+# Login into AWS, then run
+npx @nx-aws-plugin/nx-aws-cache-iac cdk deploy
+
+# NOTE that you can also run `diff` and `destroy`
+```
+
+## How to retrieve secrets
+
+To download the credentials for the IAM user, run the following command which will store the credentials in the `.env.local` file.
+
+```bash
+# Login into AWS, then run
+npx @nx-aws-plugin/nx-aws-cache-iac config-to-env
+```
+
+## How to use the cache
+
+Follow the instructions of the project. Note that the environment variables are already set in the `.env.local` file. After you source such file, you only have to set the runner in the `nx.json` file like this:
+
+```json
+{
+  "tasksRunnerOptions": {
+    "default": {
+      "runner": "@nx-aws-plugin/nx-aws-cache"
+    }
+  }
+}
+```

packages/nx-aws-cache-iac/bin/cdk-app.ts

@@ -0,0 +1,5 @@
+import { App } from 'aws-cdk-lib';
+import { AppStack } from '../src/nx-aws-cache';
+
+const app = new App();
+new AppStack(app, 'NxAwsCache');

packages/nx-aws-cache-iac/bin/cdk.ts

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+import { spawn } from 'child_process';
+import path from 'path';
+
+const main = async () => {
+  const [, , command] = process.argv;
+
+  if (command !== 'deploy' && command !== 'destroy' && command !== 'diff') {
+    console.error('Command must be one of `deploy`, `destroy`, or `diff`');
+    return;
+  }
+
+  const appPath = path.resolve(__dirname, 'cdk-app.cjs');
+
+  const spawnedProcess = spawn('npx', ['cdk@latest', command, '--all', '--app', `node ${appPath}`], {
+    cwd: __dirname,
+    stdio: 'inherit',
+  });
+
+  const promise = new Promise<void>((resolve, reject) => {
+    spawnedProcess.on('close', (code) =>
+      code === 0 ? resolve() : reject(new Error(`Process exited with code ${code}`)),
+    );
+  });
+
+  await promise;
+
+  if (command === 'deploy')
+    console.info(
+      'Deployment complete. You can now run `npx @nx-aws-plugin/nx-aws-cache-iac config-to-env` to configure your Nx AWS Cache.',
+    );
+};
+
+main();

packages/nx-aws-cache-iac/bin/config-to-env.ts

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+
+import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parse } from 'dotenv';
+import { exit } from 'node:process';
+
+const getAWSSecretConfiguration = async (value: string) => {
+  const secretsmanager = new SecretsManagerClient({});
+
+  const getSecretCmd = new GetSecretValueCommand({
+    SecretId: value,
+  });
+
+  const secretString = (await secretsmanager.send(getSecretCmd)).SecretString!;
+
+  return JSON.parse(secretString) as Record<string, string>;
+};
+
+export const createEnvFileFromPairs = (
+  environmentPairs: Record<string, string>,
+  directory: string,
+  fileName: string,
+) => {
+  const dotenvPath = resolve(directory, fileName);
+  console.log(` 📚 Exporting vars into ${dotenvPath}`);
+
+  const encoding = 'utf8';
+
+  let currentData: { [value: string]: string } = {};
+
+  try {
+    if (existsSync(dotenvPath)) currentData = parse(readFileSync(dotenvPath, { encoding }));
+  } catch (error: unknown) {
+    throw new Error(`Failed to read ${dotenvPath}`);
+  }
+
+  // NOTE: we don't override existing values
+  const combined = { ...currentData, ...environmentPairs };
+
+  let fileData = Object.entries(combined)
+    .map(([key, value]) => {
+      try {
+        JSON.parse(value);
+
+        return `${key}='${value}'`;
+      } catch (error) {
+        return `${key}=${JSON.stringify(value)}`;
+      }
+    })
+    .join('\n');
+
+  fileData += '\n';
+
+  const newEntries = Math.abs(Object.entries(currentData).length - Object.entries(combined).length);
+
+  console.log(newEntries > 0 ? ` ✅ Added ${newEntries} new entries` : ' ✅ No new entries');
+
+  writeFileSync(dotenvPath, fileData, { encoding });
+};
+
+async function main() {
+  try {
+    const secretName = '/nx-aws-cache/configuration';
+    const secrets = await getAWSSecretConfiguration(secretName);
+
+    createEnvFileFromPairs(secrets, process.cwd(), '.env.local');
+  } catch (err) {
+    console.error('Failed:', err);
+    exit(1);
+  }
+}
+
+main();

packages/nx-aws-cache-iac/cdk.json

@@ -0,0 +1,6 @@
+{
+  "versionReporting": false,
+  "output": "../../dist/cdk",
+  "requireApproval": "never",
+  "context": {}
+}

packages/nx-aws-cache-iac/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@nx-aws-plugin/nx-aws-cache-iac",
+  "version": "3.0.0",
+  "description": "Creates the infrastructure for the AWS S3 Cache plugin for Nx",
+  "keywords": [
+    "AWS",
+    "S3",
+    "Tasks",
+    "Runner",
+    "Cache",
+    "Workspace",
+    "Nrwl",
+    "Nx",
+    "Monorepo",
+    "CDK",
+    "IaC"
+  ],
+  "license": "MIT",
+  "author": "Bojan Bratuz",
+  "files": [
+    "./cdk.cjs",
+    "./cdk-app.cjs",
+    "./config-to-env.cjs"
+  ],
+  "bin": {
+    "cdk": "./cdk.cjs",
+    "config-to-env": "./config-to-env.cjs"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bojanbass/nx-aws"
+  },
+  "homepage": "https://github.com/bojanbass/nx-aws"
+}

packages/nx-aws-cache-iac/project.json

@@ -0,0 +1,38 @@
+{
+  "name": "nx-aws-cache-iac",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "nx-aws-cache-iac/src",
+  "projectType": "application",
+  "targets": {
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "main": "packages/nx-aws-cache-iac/bin/cdk.ts",
+        "additionalEntryPoints": [
+          "packages/nx-aws-cache-iac/bin/config-to-env.ts",
+          "packages/nx-aws-cache-iac/bin/cdk-app.ts"
+        ],
+        "assets": ["README.md"],
+        "tsConfig": "packages/nx-aws-cache-iac/tsconfig.lib.json",
+        "outputPath": "dist/packages/nx-aws-cache-iac",
+        "platform": "node",
+        "format": ["cjs"],
+        "skipTypeCheck": true,
+        "thirdParty": true,
+        "bundle": true
+      }
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": [
+          "packages/nx-aws-cache-iac/**/*.ts",
+          "packages/nx-aws-cache-iac/*.json"
+        ]
+      }
+    }
+  },
+  "tags": []
+}