New Serverless Pattern - Database Activity Streams - Java Lambda with SAM - OpenSearch

das-lambda-java-sam/BootStrapFromCloudShell.sh

@@ -0,0 +1,56 @@
+# Creating script for bootstrapping of secrets needed for DAS to work
+#!/bin/bash
+
+username=$1
+# Variable name to check
+region=$AWS_REGION
+# Default value if the variable is not set
+default_value="us-east-1"
+
+# Check if the variable is set
+if [ -n "$region" ]; then
+  # Variable exists, use its value
+  value=$region
+  echo "Variable \$AWS_REGION exists and its value is: $value"
+else
+  # Variable does not exist, set it to the default value
+  region=$default_value
+  echo "Variable \$AWS_REGION does not exist. Setting it to default value: $default_value"
+fi
+
+echo "Creating user $username"
+# prompt for password
+read -s -p "Enter password: " password
+echo    # Move to the next line after password input
+
+echo "Creating user $username"
+aws iam create-user --user-name $username --output json --no-cli-pager
+aws iam create-login-profile --user-name $username --password $password --password-reset-required --output json --no-cli-pager
+aws iam attach-user-policy --user-name $username --policy-arn arn:aws:iam::aws:policy/AdministratorAccess --output json --no-cli-pager
+
+echo "Creating access key..."
+ACCESSKEY=$(aws iam create-access-key --user-name $username --output json --no-cli-pager)
+ACCESSKEYID=$(echo $ACCESSKEY | jq -r '.AccessKey.AccessKeyId')
+SECRETKEYID=$(echo $ACCESSKEY | jq -r '.AccessKey.SecretAccessKey')
+
+echo "Checking if secrets exist..."
+aws secretsmanager delete-secret --secret-id aws-access-key-id --force-delete-without-recovery --region $region
+aws secretsmanager delete-secret --secret-id aws-secret-access-key --force-delete-without-recovery --region $region
+
+sleep 10
+
+echo "Creating secrets..."
+aws secretsmanager create-secret --name aws-access-key-id --secret-string {\"aws-access-key-id\":\"$ACCESSKEYID\"}
+aws secretsmanager create-secret --name aws-secret-access-key --secret-string {\"aws-secret-access-key\":\"$SECRETKEYID\"}
+
+echo "Configuring profile..."
+aws configure set aws_access_key_id $ACCESSKEYID --profile $username
+aws configure set aws_secret_access_key $SECRETKEYID --profile $username
+aws configure set region $AWS_REGION --profile $username
+
+echo "Your access Key:"
+echo $ACCESSKEYID
+echo "Your secret Key:"
+echo $SECRETKEYID
+echo "Your region:"
+echo $region

das-lambda-java-sam/BootStrapFromCloudShellNoConsoleAccess.sh

@@ -0,0 +1,52 @@
+# Creating script for bootstrapping of secrets needed for DAS to work
+#!/bin/bash
+
+username=$1
+# Variable name to check
+region=$AWS_REGION
+# Default value if the variable is not set
+default_value="us-east-1"
+
+# Check if the variable is set
+if [ -n "$region" ]; then
+  # Variable exists, use its value
+  value=$region
+  echo "Variable \$AWS_REGION exists and its value is: $value"
+else
+  # Variable does not exist, set it to the default value
+  region=$default_value
+  echo "Variable \$AWS_REGION does not exist. Setting it to default value: $default_value"
+fi
+
+echo "Creating user $username"
+aws iam create-user --user-name $username --output json --no-cli-pager
+aws iam attach-user-policy --user-name $username --policy-arn arn:aws:iam::aws:policy/AdministratorAccess --output json --no-cli-pager
+
+# improvement: check for access keys and loop thru it to delete them
+
+echo "Creating access key..."
+ACCESSKEY=$(aws iam create-access-key --user-name $username --output json --no-cli-pager)
+ACCESSKEYID=$(echo $ACCESSKEY | jq -r '.AccessKey.AccessKeyId')
+SECRETKEYID=$(echo $ACCESSKEY | jq -r '.AccessKey.SecretAccessKey')
+
+echo "Checking if secrets exist..."
+aws secretsmanager delete-secret --secret-id aws-access-key-id --force-delete-without-recovery --region $region --no-cli-pager
+aws secretsmanager delete-secret --secret-id aws-secret-access-key --force-delete-without-recovery --region $region --no-cli-pager
+
+sleep 10
+
+echo "Creating secrets..."
+aws secretsmanager create-secret --name aws-access-key-id --secret-string {\"aws-access-key-id\":\"$ACCESSKEYID\"} --region $region --no-cli-pager
+aws secretsmanager create-secret --name aws-secret-access-key --secret-string {\"aws-secret-access-key\":\"$SECRETKEYID\"} --region $region --no-cli-pager
+
+echo "Configuring profile..."
+aws configure set aws_access_key_id $ACCESSKEYID --profile $username
+aws configure set aws_secret_access_key $SECRETKEYID --profile $username
+aws configure set region $AWS_REGION --profile $username
+
+echo "Your access Key:"
+echo $ACCESSKEYID
+echo "Your secret Key:"
+echo $SECRETKEYID
+echo "Your region:"
+echo $region

das-lambda-java-sam/CreateAWSCLIProfileOnLocalMachine.sh

@@ -0,0 +1,8 @@
+# Creating script for getting AWS Keys needed to set-up CLI profile on local machine
+#!/bin/bash
+
+aws configure set aws_access_key_id $2 --profile $1
+
+aws configure set aws_secret_access_key $3 --profile $1
+
+aws configure set region $4 --profile $1

das-lambda-java-sam/Diagrams/DasToOpenSearch.drawio

@@ -0,0 +1,134 @@
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" version="26.2.13">
+  <diagram name="Page-1" id="42p7qGAsqCyJ2PC4ZeZP">
+    <mxGraphModel dx="1138" dy="824" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-16" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-2" target="yU0m2aVtfWTGo2ubRJdk-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-2" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#C925D1;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.aurora;" vertex="1" parent="1">
+          <mxGeometry x="40" y="120" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-3" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#8C4FFF;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.kinesis_data_streams;" vertex="1" parent="1">
+          <mxGeometry x="160" y="120" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-4" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#ED7100;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.lambda;" vertex="1" parent="1">
+          <mxGeometry x="280" y="120" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-38" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-5" target="yU0m2aVtfWTGo2ubRJdk-6">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-5" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#7AA116;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.s3;" vertex="1" parent="1">
+          <mxGeometry x="400" y="120" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-40" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-6" target="yU0m2aVtfWTGo2ubRJdk-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-6" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#E7157B;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.sqs;" vertex="1" parent="1">
+          <mxGeometry x="520" y="120" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-41" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-7" target="yU0m2aVtfWTGo2ubRJdk-8">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-7" value="" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#8C4FFF;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.opensearch_ingestion;" vertex="1" parent="1">
+          <mxGeometry x="640" y="120" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-8" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#8C4FFF;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.elasticsearch_service;" vertex="1" parent="1">
+          <mxGeometry x="760" y="120" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-24" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;elbow=vertical;startArrow=none;endFill=1;strokeColor=#545B64;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" relative="1" as="geometry">
+            <mxPoint x="240" y="159.5" as="sourcePoint" />
+            <mxPoint x="280" y="159.5" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="240" y="159.5" />
+              <mxPoint x="240" y="159.5" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-26" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;elbow=vertical;startArrow=none;endFill=1;strokeColor=#545B64;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" relative="1" as="geometry">
+            <mxPoint x="240" y="159.5" as="sourcePoint" />
+            <mxPoint x="280" y="159.5" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="240" y="159.5" />
+              <mxPoint x="240" y="159.5" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-30" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;elbow=vertical;startArrow=none;endFill=1;strokeColor=#545B64;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" relative="1" as="geometry">
+            <mxPoint x="360" y="160" as="sourcePoint" />
+            <mxPoint x="400" y="160" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="360" y="160" />
+              <mxPoint x="360" y="160" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-45" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-7" target="yU0m2aVtfWTGo2ubRJdk-5">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="670" y="61" as="sourcePoint" />
+            <mxPoint x="448.98800000000006" y="60" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="670" y="60" />
+              <mxPoint x="439" y="60" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-46" value="" style="sketch=0;points=[[0,0,0],[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0,0],[0,1,0],[0.25,1,0],[0.5,1,0],[0.75,1,0],[1,1,0],[0,0.25,0],[0,0.5,0],[0,0.75,0],[1,0.25,0],[1,0.5,0],[1,0.75,0]];outlineConnect=0;fontColor=#232F3E;fillColor=#DD344C;strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4.resourceIcon;resIcon=mxgraph.aws4.key_management_service;" vertex="1" parent="1">
+          <mxGeometry x="160" y="10" width="78" height="78" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-59" value="" style="endArrow=classic;startArrow=classic;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-4">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="280" y="110" as="sourcePoint" />
+            <mxPoint x="240" y="50" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-61" value="" style="endArrow=classic;startArrow=classic;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="100" y="122" as="sourcePoint" />
+            <mxPoint x="160" y="60" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-62" value="" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#8C4FFF;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.opensearch_dashboards;" vertex="1" parent="1">
+          <mxGeometry x="760" y="280" width="78" height="59" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-74" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;" edge="1" parent="1" source="yU0m2aVtfWTGo2ubRJdk-62">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="802" y="280" as="sourcePoint" />
+            <mxPoint x="800" y="200" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-75" value="Aurora Postgres&lt;div&gt;Database&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
+          <mxGeometry x="25" y="198" width="110" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-76" value="DAS Kinesis&lt;div&gt;Data Streams&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="169" y="210" width="60" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-77" value="KMS" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="170" y="88" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-78" value="Lambda&lt;div&gt;Function&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="289" y="200" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-79" value="S3 Bucket" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="409" y="198" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-81" value="SQS" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="529" y="198" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-82" value="OpenSearch&lt;div&gt;Ingestion&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="649" y="203" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-83" value="OpenSearch" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="769" y="88" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="yU0m2aVtfWTGo2ubRJdk-84" value="OpenSearch&lt;div&gt;Dashboard&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="769" y="350" width="60" height="19" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

das-lambda-java-sam/GetAWSCLIProfileDetailsFromCloudshell.sh

@@ -0,0 +1,9 @@
+# Creating script for getting AWS Keys needed to set-up CLI profile on local machine
+#!/bin/bash
+
+AWS_ACCESS_KEY=$(aws configure get aws_access_key_id --profile $1)
+echo $AWS_ACCESS_KEY
+AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile $1)
+echo $AWS_SECRET_ACCESS_KEY
+AWS_CLI_REGION=$(aws configure get region --profile $1)
+echo $AWS_CLI_REGION