From 445870ec348e232cebbcc0aae5d83dcdca04ace6 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 19:58:39 -0400 Subject: [PATCH 01/30] environment --- .gitignore | 2 ++ multidocker/.env | 18 ------------------ 2 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 multidocker/.env diff --git a/.gitignore b/.gitignore index 1269488..8c1dd8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ data +.env +local.conf diff --git a/multidocker/.env b/multidocker/.env deleted file mode 100644 index 8bb9281..0000000 --- a/multidocker/.env +++ /dev/null @@ -1,18 +0,0 @@ -NODE_VERSION=16 -ALPINE_VERSION=3.17 -RSYNC_PASSWORD= -GATSBY_DATACORE_URL=/ -#GATSBY_ASSETS_URL=https://assets.datacore.app/ -GATSBY_ASSETS_URL=/assets/ -WEBSITE_BRANCH=master -SERVER_BRANCH=master -ASSET_BRANCH=master -BOT_BRANCH=master -NGINX_CONF=test.conf -#NGINX_CONF=test-with-site-server.conf -BUILD_TYPE=development -PM2_INFO_URL= -PM2_ERROR_URL= -BOT_TOKEN= -IMGFLIP_PASSWORD= -IMGFLIP_USERNAME= \ No newline at end of file From bf8a6fd51a9e88e11be32bc182b6874748461594 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:07:31 -0400 Subject: [PATCH 02/30] New default env, and add missing args to gittower --- multidocker/.env.defaults | 50 ++++++++++++++++++++++++++++++++++ multidocker/docker-compose.yml | 3 ++ 2 files changed, 53 insertions(+) create mode 100644 multidocker/.env.defaults diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults new file mode 100644 index 0000000..ba2f0e8 --- /dev/null +++ b/multidocker/.env.defaults @@ -0,0 +1,50 @@ +NODE_VERSION=16 +ALPINE_VERSION=3.17 + +RSYNC_PASSWORD= + +WEBSITE_BRANCH=master +SERVER_BRANCH=master +ASSET_BRANCH=master +BOT_BRANCH=master + +# Docker only seems to like the absolute path for this file. +#NGINX_CONF=/app/DockerImage/multidocker/contexts/nginx/local.conf +#NGINX_CONF=/app/DockerImage/multidocker/contexts/nginx/test-with-site-server.conf + +BUILD_TYPE=development +PM2_INFO_URL= +PM2_ERROR_URL= +BOT_TOKEN= +IMGFLIP_PASSWORD= +IMGFLIP_USERNAME= + +# Shared Assets +ASSETS_DIR=/asset-server/build/out +SQLITE_DB=./datacore.db + +# Website Environment +GATSBY_DATACORE_URL=/ +GATSBY_ASSETS_URL=/assets/ + +# MongoDB Environment +MONGO_INITDB_DATABASE=user +MONGO_INITDB_ROOT_USERNAME=admin +MONGO_INITDB_ROOT_PASSWORD=secret + +# Site Server Environment +STT_BOT_USERNAME= +STT_BOT_PASSWORD= +JWT_SECRET= +DB_CONNECTION_STRING=sqlite:/app/exdata/profiles.sqlite +MONGO_CONN_STRING=mongodb://admin:secret@datacoredb +MONGO_DB_NAME=datacore +MONGO_PROFILE_COLLECTION=playerProfiles +MONGO_TRACKED_VOYAGES_COLLECTION=trackedVoyages +MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments +MONGO_FBB_SOLVES_COLLECTION=solves +MONGO_FBB_TRIALS_COLLECTION=trials +MONGO_FBB_COLLECTION=bossBattles +CORS_ORIGIN=http://datacore.app/ +PROFILE_DATA_PATH=./static/profiles +LOG_PATH= diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 005a609..853a4d2 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -13,6 +13,9 @@ services: image: datacore/gittower build: context: contexts/gittower + args: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} deploy: restart_policy: condition: on-failure From 84c8845d38f1c04d17e7916c40562737f9d43880 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:20:32 -0400 Subject: [PATCH 03/30] Fix up environments for stack w/ mongo --- multidocker/docker-compose.yml | 89 ++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 853a4d2..03d8af2 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: '3' volumes: sitebuild: assets: @@ -7,6 +7,7 @@ volumes: nginx-logs: site-logs: bot-logs: + datacore-mongo: services: gittower: @@ -36,29 +37,22 @@ services: NODE_VERSION: PM2_ERROR_URL: PM2_INFO_URL: - + profiles: ["all", "fullstack", "site-server", "website"] + website: - image: datacore/website build: context: contexts/website additional_contexts: source: https://github.com/stt-datacore/website.git#${WEBSITE_BRANCH} args: - ALPINE_VERSION: - NODE_VERSION: - x-develop: - watch: - - action: sync - path: ./source/src - target: /app/src - - action: rebuild - path: ./source/package.json - environment: - GATSBY_ASSETS_URL: - GATSBY_DATACORE_URL: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} + GATSBY_ASSETS_URL: ${GATSBY_ASSETS_URL} + GATSBY_DATACORE_URL: ${GATSBY_DATACORE_URL} deploy: restart_policy: condition: on-failure + container_name: DCWebsite volumes: - sitebuild:/sitebuild:rw @@ -79,10 +73,34 @@ services: restart_policy: condition: on-failure volumes: - - assets:/assets:rw" + - ${ASSETS_DIR}:/assets:rw" environment: - OUT_PATH=/assets/ - profiles: [ "all", "asset-server", "fullstack"] + profiles: [ "all", "asset-server", "website", "fullstack" ] + + datacoredb: + image: mongo:4.4.6 + container_name: datacoredb + volumes: + - datacore-mongo:/data/db + environment: + - MONGO_INITDB_DATABASE=${MONGO_INITDB_DATABASE} + - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} + profiles: ["all", "site-server", "bot", "fullstack"] + + local-dev-db: + image: mongo:4.4.6 + container_name: local-dev-db + volumes: + - datacore-mongo:/data/db + ports: + - '27017-27019:27017-27019' + environment: + - MONGO_INITDB_DATABASE=${MONGO_INITDB_DATABASE} + - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} + profiles: ["local-mongo"] site-server: image: datacore/site-server @@ -94,6 +112,22 @@ services: target: ${BUILD_TYPE} args: $NODE_VERSION: + environment: + - PROFILE_DATA_PATH=${PROFILE_DATA_PATH} + - DB_CONNECTION_STRING=${DB_CONNECTION_STRING} + - LOG_PATH=${LOG_PATH} + - STT_BOT_USERNAME=${STT_BOT_USERNAME} + - STT_BOT_PASSWORD=${STT_BOT_PASSWORD} + - JWT_SECRET=${JWT_SECRET} + - MONGO_CONN_STRING=${MONGO_CONN_STRING} + - MONGO_DB_NAME=${MONGO_DB_NAME} + - MONGO_PROFILE_COLLECTION=${MONGO_PROFILE_COLLECTION} + - MONGO_TRACKED_VOYAGES_COLLECTION=${MONGO_TRACKED_VOYAGES_COLLECTION} + - MONGO_TRACKED_ASSIGNMENTS_COLLECTION=${MONGO_TRACKED_ASSIGNMENTS_COLLECTION} + - MONGO_FBB_SOLVES_COLLECTION=${MONGO_FBB_SOLVES_COLLECTION} + - MONGO_FBB_TRIALS_COLLECTION=${MONGO_FBB_TRIALS_COLLECTION} + - MONGO_FBB_COLLECTION=${MONGO_FBB_COLLECTION} + - CORS_ORIGIN=${CORS_ORIGIN} container_name: DCSiteServer deploy: restart_policy: @@ -103,13 +137,10 @@ services: volumes: - site-logs:/data/logs:rw - profiles:/data/profiles:rw" - - ./datacore.db:/database - environment: - - PROFILE_DATA_PATH=/data/profiles - - DB_CONNECTION_STRING=sqlite:/database - - LOG_PATH=/data/logs/ + - ${SQLITE_DB}:/database depends_on: - pm2-discord + - datacoredb profiles: ["all", "site-server", "fullstack"] nginx: @@ -123,11 +154,13 @@ services: - "442:442" volumes: - sitebuild:/sitebuild:ro - - assets:/assets:ro + - ${ASSETS_DIR}:/assets:ro - profiles:/profiles:ro - - ./contexts/nginx/${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw - nginx-logs:/var/log/nginx:rw - + - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw + depends_on: + - site-server + - datacoredb profiles: [ "all", "website", "asset-server", "site-server", "fullstack" ] node_exporter: @@ -236,6 +269,12 @@ services: - BOT_TOKEN=${BOT_TOKEN} - ASSETS_URL=${GATSBY_ASSETS_URL} - DATACORE_URL=${GATSBY_DATACORE_URL} + - MONGO_CONN_STRING=${MONGO_CONN_STRING} + - MONGO_DB_NAME=${MONGO_DB_NAME} + - MONGO_PROFILE_COLLECTION=${MONGO_PROFILE_COLLECTION} + - MONGO_TRACKED_VOYAGES_COLLECTION=${MONGO_TRACKED_VOYAGES_COLLECTION} + - MONGO_TRACKED_ASSIGNMENTS_COLLECTION=${MONGO_TRACKED_ASSIGNMENTS_COLLECTION} depends_on: - cpp-image-analysis + - datacoredb profiles: ["all", "bot"] \ No newline at end of file From e977f28a48e6ec2f0e957a047ba16714ca28e365 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:43:38 -0400 Subject: [PATCH 04/30] add missing ARGS; correct DB entry --- multidocker/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 03d8af2..e7d4869 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -65,8 +65,8 @@ services: additional_contexts: source: https://github.com/stt-datacore/asset-server.git#${ASSET_BRANCH} args: - ALPINE_VERSION: - NODE_VERSION: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} container_name: DCAssetServer deploy: @@ -137,7 +137,7 @@ services: volumes: - site-logs:/data/logs:rw - profiles:/data/profiles:rw" - - ${SQLITE_DB}:/database + - ${SQLITE_DB}:/datacore.db depends_on: - pm2-discord - datacoredb From 0261588b15adccec6d4bc4a41910e66419ec0231 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:44:17 -0400 Subject: [PATCH 05/30] Put cron job echo in quotes (doesn't work for me otherwise) Add missing docker-compose package (doesn't work for me otherwise) --- multidocker/contexts/gittower/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multidocker/contexts/gittower/Dockerfile b/multidocker/contexts/gittower/Dockerfile index 7851bbf..2475d65 100644 --- a/multidocker/contexts/gittower/Dockerfile +++ b/multidocker/contexts/gittower/Dockerfile @@ -1,7 +1,7 @@ ARG ALPINE_VERSION FROM alpine:${ALPINE_VERSION} -RUN apk add docker git bash -RUN echo */30 * * * * ./runjobs > crontab +RUN apk add docker git bash docker-compose +RUN echo "*/5 * * * * /app/runjobs" > crontab RUN crontab ./crontab RUN mkdir /app WORKDIR /app From 64d8ce29caf2a4781b4a724906a9cd2f89a4b576 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:44:28 -0400 Subject: [PATCH 06/30] add notification echoes --- multidocker/contexts/gittower/gittower | 1 + multidocker/contexts/gittower/runjobs | 1 + 2 files changed, 2 insertions(+) diff --git a/multidocker/contexts/gittower/gittower b/multidocker/contexts/gittower/gittower index d91098c..eef25fe 100755 --- a/multidocker/contexts/gittower/gittower +++ b/multidocker/contexts/gittower/gittower @@ -1,3 +1,4 @@ #!/bin/bash ./setup-jobs +echo gittower waiting ... crond -f -l 8 \ No newline at end of file diff --git a/multidocker/contexts/gittower/runjobs b/multidocker/contexts/gittower/runjobs index d392fd2..620cce9 100755 --- a/multidocker/contexts/gittower/runjobs +++ b/multidocker/contexts/gittower/runjobs @@ -13,3 +13,4 @@ do git fetch git status|grep "is behind" && reload ${dir##*/} done +echo gittower waiting ... From 09208d288b73547acb705e54535b65039935bce4 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:45:57 -0400 Subject: [PATCH 07/30] Change to alpine:latest because compose commands unsupported --- multidocker/contexts/gittower/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multidocker/contexts/gittower/Dockerfile b/multidocker/contexts/gittower/Dockerfile index 2475d65..8577acb 100644 --- a/multidocker/contexts/gittower/Dockerfile +++ b/multidocker/contexts/gittower/Dockerfile @@ -1,5 +1,6 @@ ARG ALPINE_VERSION -FROM alpine:${ALPINE_VERSION} +# FROM alpine:${ALPINE_VERSION} +FROM alpine:latest RUN apk add docker git bash docker-compose RUN echo "*/5 * * * * /app/runjobs" > crontab RUN crontab ./crontab From 737b10e9ffd4300eebfb3ed9330e1757cbc05fbc Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 3 Oct 2023 20:59:13 -0400 Subject: [PATCH 08/30] add missing args to website docker file --- multidocker/contexts/website/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/multidocker/contexts/website/Dockerfile b/multidocker/contexts/website/Dockerfile index 4609668..6cfa291 100644 --- a/multidocker/contexts/website/Dockerfile +++ b/multidocker/contexts/website/Dockerfile @@ -1,6 +1,6 @@ ARG NODE_VERSION ARG ALPINE_VERSION -FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} +FROM node:$NODE_VERSION-alpine$ALPINE_VERSION RUN apk add build-base python3 git rsync WORKDIR /app # Sentry requires the github sha @@ -9,4 +9,11 @@ COPY --from=source package.json yarn.lock ./ RUN yarn install RUN npx update-browserslist-db@latest COPY --from=source . . -CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ && sleep infinity \ No newline at end of file + +ARG GATSBY_DATACORE_URL +RUN echo GATSBY_DATACORE_URL=$GATSBY_DATACORE_URL > .env +ARG GATSBY_ASSETS_URL +RUN echo GATSBY_ASSETS_URL=$GATSBY_ASSETS_URL >> .env +RUN cat .env && sleep 5s + +CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ From d39c5e5d378b6309ae1da5831a018a18563b7036 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Thu, 5 Oct 2023 21:52:16 -0400 Subject: [PATCH 09/30] fix profile sqlite mount --- multidocker/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index e7d4869..894b088 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -73,7 +73,7 @@ services: restart_policy: condition: on-failure volumes: - - ${ASSETS_DIR}:/assets:rw" + - ${ASSETS_DIR}:/assets:rw environment: - OUT_PATH=/assets/ profiles: [ "all", "asset-server", "website", "fullstack" ] @@ -113,7 +113,7 @@ services: args: $NODE_VERSION: environment: - - PROFILE_DATA_PATH=${PROFILE_DATA_PATH} + - PROFILE_DATA_PATH=/profiles - DB_CONNECTION_STRING=${DB_CONNECTION_STRING} - LOG_PATH=${LOG_PATH} - STT_BOT_USERNAME=${STT_BOT_USERNAME} @@ -136,7 +136,7 @@ services: - "4420:4420" volumes: - site-logs:/data/logs:rw - - profiles:/data/profiles:rw" + - ${PROFILE_DATA_PATH}:/profiles:rw" - ${SQLITE_DB}:/datacore.db depends_on: - pm2-discord From 617dbe0d363c47e6446d513ece23bdcf3fced4ef Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 00:51:51 -0400 Subject: [PATCH 10/30] Configuration changes for bot --- multidocker/.env.defaults | 21 +++++-- .../contexts/cpp-image-analysis/Dockerfile | 3 +- multidocker/contexts/pm2-discord/Dockerfile | 8 +-- multidocker/contexts/site-server/Dockerfile | 2 +- multidocker/docker-compose.yml | 57 ++++++++++--------- 5 files changed, 54 insertions(+), 37 deletions(-) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index ba2f0e8..fa95762 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -13,9 +13,12 @@ BOT_BRANCH=master #NGINX_CONF=/app/DockerImage/multidocker/contexts/nginx/test-with-site-server.conf BUILD_TYPE=development + +# PM2 PM2_INFO_URL= PM2_ERROR_URL= -BOT_TOKEN= + +# ImgFlip IMGFLIP_PASSWORD= IMGFLIP_USERNAME= @@ -23,10 +26,18 @@ IMGFLIP_USERNAME= ASSETS_DIR=/asset-server/build/out SQLITE_DB=./datacore.db +# Website Environment +GATSBY_DATACORE_URL=/ +GATSBY_ASSETS_URL=/assets/ # Website Environment GATSBY_DATACORE_URL=/ GATSBY_ASSETS_URL=/assets/ +# Common +PROFILE_DATA_PATH=/site-server/static/profiles +CORS_ORIGIN=http://datacore.app/ +LOG_PATH=/logs + # MongoDB Environment MONGO_INITDB_DATABASE=user MONGO_INITDB_ROOT_USERNAME=admin @@ -36,7 +47,7 @@ MONGO_INITDB_ROOT_PASSWORD=secret STT_BOT_USERNAME= STT_BOT_PASSWORD= JWT_SECRET= -DB_CONNECTION_STRING=sqlite:/app/exdata/profiles.sqlite +DB_CONNECTION_STRING=sqlite:/datacore.db MONGO_CONN_STRING=mongodb://admin:secret@datacoredb MONGO_DB_NAME=datacore MONGO_PROFILE_COLLECTION=playerProfiles @@ -45,6 +56,6 @@ MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments MONGO_FBB_SOLVES_COLLECTION=solves MONGO_FBB_TRIALS_COLLECTION=trials MONGO_FBB_COLLECTION=bossBattles -CORS_ORIGIN=http://datacore.app/ -PROFILE_DATA_PATH=./static/profiles -LOG_PATH= + +# Bot +BOT_TOKEN= diff --git a/multidocker/contexts/cpp-image-analysis/Dockerfile b/multidocker/contexts/cpp-image-analysis/Dockerfile index 21867f9..c42642c 100644 --- a/multidocker/contexts/cpp-image-analysis/Dockerfile +++ b/multidocker/contexts/cpp-image-analysis/Dockerfile @@ -50,4 +50,5 @@ WORKDIR /usr/src/dcimageanalysis/build CMD ./imserver FROM development as production -CMD pm2-runtime start "./imserver --jsonpath=/website/public_web/structured/ --trainpath=/data/train/ --datapath=/cpp-image-analysis/data/ --asseturl=https://assets.datacore.app/" --name "imageAnalysis" + +CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=/traindata/train/ --datapath=/traindata/data/ --asseturl=https://assets.datacore.app/" --name "imageAnalysis" diff --git a/multidocker/contexts/pm2-discord/Dockerfile b/multidocker/contexts/pm2-discord/Dockerfile index 2dd4978..dee34e6 100644 --- a/multidocker/contexts/pm2-discord/Dockerfile +++ b/multidocker/contexts/pm2-discord/Dockerfile @@ -1,7 +1,5 @@ ARG ALPINE_VERSION ARG NODE_VERSION -ARG PM2_ERROR_URL -ARG PM2_INFO_URL FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} RUN npm install pm2 -g @@ -20,5 +18,7 @@ RUN pm2 set pm2-discord-webhook:stop true RUN pm2 set pm2-discord-webhook:restart true # Set destinations for webhooks (URLs sourced from env) -RUN pm2 set pm2-discord-webhook:webhook_url_logs ${PM2_INFO_URL} -RUN pm2 set pm2-discord-webhook:webhook_url_errors ${PM2_ERROR_URL} \ No newline at end of file +ARG PM2_INFO_URL +RUN pm2 set pm2-discord-webhook:webhook_url_logs $PM2_INFO_URL +ARG PM2_ERROR_URL +RUN pm2 set pm2-discord-webhook:webhook_url_errors $PM2_ERROR_URL \ No newline at end of file diff --git a/multidocker/contexts/site-server/Dockerfile b/multidocker/contexts/site-server/Dockerfile index 7cdf0f0..b4b72b9 100644 --- a/multidocker/contexts/site-server/Dockerfile +++ b/multidocker/contexts/site-server/Dockerfile @@ -2,7 +2,7 @@ FROM datacore/pm2-discord as development RUN apk add python3 build-base sqlite RUN mkdir /app WORKDIR /app -COPY --from=source package.json package-lock.json . +COPY --from=source package.json package-lock.json ./ RUN npm install COPY --from=source . . RUN npm run build diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 894b088..2e3709d 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -33,11 +33,11 @@ services: build: context: contexts/pm2-discord args: - ALPINE_VERSION: - NODE_VERSION: - PM2_ERROR_URL: - PM2_INFO_URL: - profiles: ["all", "fullstack", "site-server", "website"] + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} + PM2_ERROR_URL: ${PM2_ERROR_URL} + PM2_INFO_URL: ${PM2_INFO_URL} + profiles: ["all", "fullstack", "site-server", "website", "bot"] website: build: @@ -115,7 +115,7 @@ services: environment: - PROFILE_DATA_PATH=/profiles - DB_CONNECTION_STRING=${DB_CONNECTION_STRING} - - LOG_PATH=${LOG_PATH} + - LOG_PATH=/data/logs - STT_BOT_USERNAME=${STT_BOT_USERNAME} - STT_BOT_PASSWORD=${STT_BOT_PASSWORD} - JWT_SECRET=${JWT_SECRET} @@ -135,7 +135,7 @@ services: ports: - "4420:4420" volumes: - - site-logs:/data/logs:rw + - ${LOG_PATH}:/data/logs:rw - ${PROFILE_DATA_PATH}:/profiles:rw" - ${SQLITE_DB}:/datacore.db depends_on: @@ -155,7 +155,7 @@ services: volumes: - sitebuild:/sitebuild:ro - ${ASSETS_DIR}:/assets:ro - - profiles:/profiles:ro + - ${PROFILE_DATA_PATH}:/profiles:ro - nginx-logs:/var/log/nginx:rw - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw depends_on: @@ -208,12 +208,12 @@ services: ports: - "1337:22" volumes: - - "sitebuild:/data/sitebuild:rw" - - "assets:/data/assets:rw" - - "profiles:/data/profiles:rw" - - "traindata:/data/traindata:rw" - - "nginx-logs:/logs/nginx:ro" - - "site-logs:/logs/site-server:ro" + - sitebuild:/sitebuild:rw + - ${ASSETS_DIR}:/data/assets:ro + - ${PROFILE_DATA_PATH}:/profiles:rw + - traindata:/data/traindata:rw + - nginx-logs:/logs/nginx:ro + - site-logs:/logs/site-server:ro environment: - ROOT_PASSWORD=${RSYNC_PASSWORD} profiles: [ "all", "rsync", "fullstack" ] @@ -230,12 +230,12 @@ services: deploy: restart_policy: condition: on-failure - ports: - - "5000:5000" + # ports: + # - 5000:5000 volumes: - - "sitebuild:/sitebuild:ro" - - "./contexts/cvdata:/data:ro" - - "traindata:/traindata:rw" + - sitebuild:/sitebuild:ro + - traindata:/traindata:rw + - ./contexts/cpp-image-analysis/cvdata:/traindata/data:ro depends_on: - pm2-discord profiles: [ "all", "bot" ] @@ -254,26 +254,31 @@ services: restart_policy: condition: on-failure volumes: - - "profiles:/data/profiles:rw" - "sitebuild:/sitebuild:ro" - - "bot-logs:/data/logs:rw" - - "assets:/assets:ro" + - ${ASSETS_DIR}:/data/assets:ro + - bot-logs:/data/logs:rw + - ${PROFILE_DATA_PATH}:/profiles:rw" + - ${SQLITE_DB}:/datacore.db environment: - - PROFILE_DATA_PATH=/data/profiles - - DB_CONNECTION_STRING=sqlite:/data/datacore.db + - PROFILE_DATA_PATH=/profiles/ + - DB_CONNECTION_STRING=sqlite:/datacore.db - LOG_PATH=/data/logs/ - DC_DATA_PATH=/sitebuild/structured - CONFIG_PATH=/data/bot_config.json - INGFLIP_USERNAME=${INGFLIP_USERNAME} - INGFLIP_PASSWORD=${INGFLIP_PASSWORD} - BOT_TOKEN=${BOT_TOKEN} - - ASSETS_URL=${GATSBY_ASSETS_URL} - - DATACORE_URL=${GATSBY_DATACORE_URL} + - ASSETS_URL=${BOT_ASSETS_URL} + - DATACORE_URL=${BOT_DATACORE_URL} + - IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 - MONGO_CONN_STRING=${MONGO_CONN_STRING} - MONGO_DB_NAME=${MONGO_DB_NAME} - MONGO_PROFILE_COLLECTION=${MONGO_PROFILE_COLLECTION} - MONGO_TRACKED_VOYAGES_COLLECTION=${MONGO_TRACKED_VOYAGES_COLLECTION} - MONGO_TRACKED_ASSIGNMENTS_COLLECTION=${MONGO_TRACKED_ASSIGNMENTS_COLLECTION} + - MONGO_FBB_SOLVES_COLLECTION=${MONGO_FBB_SOLVES_COLLECTION} + - MONGO_FBB_TRIALS_COLLECTION=${MONGO_FBB_TRIALS_COLLECTION} + - MONGO_FBB_COLLECTION=${MONGO_FBB_COLLECTION} depends_on: - cpp-image-analysis - datacoredb From 0f994df3a0f3baf958b41896fadae260cb3a44ee Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 01:25:37 -0400 Subject: [PATCH 11/30] Configuration tweaking --- multidocker/contexts/cpp-image-analysis/Dockerfile | 5 ++++- multidocker/docker-compose.yml | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/multidocker/contexts/cpp-image-analysis/Dockerfile b/multidocker/contexts/cpp-image-analysis/Dockerfile index c42642c..cc955db 100644 --- a/multidocker/contexts/cpp-image-analysis/Dockerfile +++ b/multidocker/contexts/cpp-image-analysis/Dockerfile @@ -3,6 +3,9 @@ FROM datacore/pm2-discord as development ENV LANG=C.UTF-8 +WORKDIR /traindata +RUN mkdir train + WORKDIR /usr/src/dcimageanalysis COPY --from=source . . @@ -51,4 +54,4 @@ CMD ./imserver FROM development as production -CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=/traindata/train/ --datapath=/traindata/data/ --asseturl=https://assets.datacore.app/" --name "imageAnalysis" +CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=/traindata/train/ --datapath=/usr/src/dcimageanalysis/data/ --asseturl=http://NginxServer/assets/" --name "imageAnalysis" diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 2e3709d..a9ddf45 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -235,9 +235,10 @@ services: volumes: - sitebuild:/sitebuild:ro - traindata:/traindata:rw - - ./contexts/cpp-image-analysis/cvdata:/traindata/data:ro + # - ./contexts/cpp-image-analysis/cvdata:/traindata/data:ro depends_on: - pm2-discord + - nginx profiles: [ "all", "bot" ] bot: From 9bce404f2e1564aec1823217c4f1e40b2d7f3a26 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 10:23:43 -0400 Subject: [PATCH 12/30] correctly configure volumes and image analysis. --- multidocker/.env.defaults | 2 ++ .../contexts/cpp-image-analysis/Dockerfile | 8 ++++++-- multidocker/datacore.db | Bin 36864 -> 0 bytes multidocker/docker-compose.yml | 18 +++++++++++------- 4 files changed, 19 insertions(+), 9 deletions(-) delete mode 100644 multidocker/datacore.db diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index fa95762..3273a04 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -59,3 +59,5 @@ MONGO_FBB_COLLECTION=bossBattles # Bot BOT_TOKEN= +BOT_DATACORE_URL=https://datacore.app/ +BOT_ASSETS_URL=https://assets.datacore.app/ \ No newline at end of file diff --git a/multidocker/contexts/cpp-image-analysis/Dockerfile b/multidocker/contexts/cpp-image-analysis/Dockerfile index cc955db..5b6adcf 100644 --- a/multidocker/contexts/cpp-image-analysis/Dockerfile +++ b/multidocker/contexts/cpp-image-analysis/Dockerfile @@ -3,13 +3,17 @@ FROM datacore/pm2-discord as development ENV LANG=C.UTF-8 +WORKDIR /usr/src/dcimageanalysis +COPY --from=source . . + WORKDIR /traindata RUN mkdir train +RUN cp /usr/src/dcimageanalysis/data . -r WORKDIR /usr/src/dcimageanalysis -COPY --from=source . . RUN apk add --no-cache --virtual .build \ + curl \ build-base \ pkgconfig \ cmake \ @@ -54,4 +58,4 @@ CMD ./imserver FROM development as production -CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=/traindata/train/ --datapath=/usr/src/dcimageanalysis/data/ --asseturl=http://NginxServer/assets/" --name "imageAnalysis" +CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=$CPP_TRAIN_PATH --datapath=$CPP_DATA_PATH --asseturl=$ASSETS_URL" --name "imageAnalysis" diff --git a/multidocker/datacore.db b/multidocker/datacore.db deleted file mode 100644 index 919c95008704951879de593e0943d5fb6febfe43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36864 zcmeI5&u?5ua>vQFwO(fvCpVo840y2()5i4s{d5YKq7x=ErBKu&hG9q}awJboa_kv$ zy@sIxYs9CV0_2a$JwX1PT=Ea(^L>=6YpA9nI*u1uG+~w=>(#5S`gV17b-$kR{eQSK zJDZN4%oit+*v+%<+oe=+kd;Y|87y&?_t%+Wbwt({A)aU z?&IV6{EO4MoH*Fs`vjog4U#*aEYBVc9z^ry)N;Dmb%5^ge6+K_v$wr-fAr`9_m3cXzb4eXt9I{~p{ER-^5$``cT$chCsz1GGOD$bWuq0P_301CaL? z^Cz?8>GH+Dudf36f_)tzj~*+6F2Q&>`SNTsJKdX{0NkZ}%d@ks!^7!vc`*NC3ecBs zUxDo7^CwTX=ci9*Palo`?*839pjm!CUz}|dmUm`LqUbsI$8^DSxc@&o;NCBOdGr1E z*FPEHUw(DWD)QlS`qlaL6bM%Td~KyOF5P_h6f8G+^1bgC>u=n=d2{{y?Lp@dRs;FP z%JVDn_DUoCc>Z*DDox358}8nlESF!CVqWUCquEla=gOzu{o9BxW+$`L$=Q7ILy9zz zE(OOq*6q)aH5y4lmQEN-A8kFjb1-`6R15UucZiP3?=VkP*lTWCSt-8G($zpD+Si|F8e$|E%#}{>cbr1Tq2{fs8;#AR~|w z$OvQvG6ETaj6g=^jR@3$`8i< z2ca5;?p8g%H4YylH+p}}<@no;L(WxCPLCcgznC2#KjZ-CbmP{y8z<+-XR|MlXVb;T zt+2WA<#ch#;p>O9Q=Oq)A~OE49IyUV&o(b9TTD)$P9Jj0^A$A~^;Ie+zrT)(<>5?6 zYkwM-R+U}bB|J)Mcy*7YVS0GJn4Nv|6M3|%?b!s1-lDvn_!d3+a~f&ODNj9OYrLiuMZvB`Xxgc4rOE$J{N|xUEvHdb_k($S{OSQ z;(Z_bsKwj~4Q*K!5vnxqYYtPcJ8!->8a8uA<9s>Qyflv)258%a6{wdqxL#Dwy9~|K?q}p zvMHS9rlaR@cqDltW=$Ts~)Ufp0Y~lcv{Abrs(o1~D_c-it?k>&73nF@7J4 zl(T)AJjK)3wQJLStykD~jHT`yhk#ia>Mn^))A}Urhya`A-MDgRXUvwqEz{l6 zCeO~ap>XJ$jUCY*+cdU1)kSr*1q7nF*-}?txk-odI2mDe>(B8R00fu|Qd1u4-uI~S zt9tBgF)f`#CHm8hE5|n-SlBr~O$ouHM*$=x9@v&UCM)c8A2AeCfx4olWXF z?i=AXcGIM_H)ZFvJ7?=OkYM2AG+K!s2ZA|URmr1YI?CpNkIfRN4l{GMS~))#o$7-N zYh!m^>3niFHogN*dEZp7%_z>HJ*Pd|*fgC(*j#ODpW94()dvTdvB{8$UQ+ILwwP|t zWBUPYTH3mBCK>>B-8(HN4ckSSbEx*G6Ab`xaW+QrQ2L=4`Q&OhKFpZVj+28-{1lFD z?KYU8n?+@+q6E&Vx!~qvDq-L z{Ja`5E9=e~8GQ{a@npsm!s#qAZhgjz`Y8-u&__LOSG#u2{+8?<g-FP+KoG4CD2whRTBxI(C&^JzxsSn6IBcuMhFI{*?J-z^p+|?2| z@_qSk+`?gQec=Ij*0sK%kNnjsCGtxmDFJzCf}i`XQF^X%XhnXpQn|*!pmIy*g4Ez> z9W!d6MHfSccUSol6UB?dXBmT#m5$FQriL8gz%?};!4+YNTDs0x4+f*3BO@kV=Q}Po zN!gat-CR*6bX_{2{oHAWQ!9?0LuR2m)OAEOvMkcIHOqozcl2biXuaugi?MSms;4$_ zY7B!=G~P^;PukcN>AI)%v%VovS3YvB>sUiL1C1S+9NbiBhJ=3QBN9=5H5o;R_+&x~ zmn+t2sP~^KFg2_$QdaUYj$x8vr{T?uPk5NS_~q<3JE>>gxHdKuhhJlx7P6WjPG%2a zx~PfbOiONjm^fL+^|k}EPEQ+M75?qv(9v&8N2CP=apf}2S!qC#-CGS{aswK@6lC$ zez!KYqE7WU%XpWAqd0SKud2pj zU~Gvy+YKA7;*#>A_W3aCVJxIF86Oq&<$;q=X(z&*l4wlLr`)5j*56&5PE6yKHDYX& zBEUi8`nyR2AJ~#~@}{R6C)wJdaMoMDGM-%jnNx`SVQ=E_lR?bKB7M>_Yl@KGepShn z#F$Nrh^150BtLW7uGBj#;!cfuMXSDoF@8Z(ObyMzZ0yFAbuAyqbZk@8b;&&h=0A^( ziL;}TF2C4Za8YkwQG%gst6$A_iqp`!9h*Z2+lnK5!qJ=?r~Y#N`EDiu(SwWL$9osouoo&IP!2>S^p z`h*(SBW61vXZK-g@>SvAgITN8pnftn@q#sk!yw8Rz2C8m&GIH81%gTnmxxThK#vPF z)6eqTuMbTgzE5bzr_@Mqdli)rDU+u*@ku1|K|c*AuS8=kM0pxc@$1sV;KY01>Bg7& zE=UZ1ddnpR6KBot(Xj|5!2OEE_`DN!YK$G4GPyp|g2y=(<%#@c=dOM+ftjhoe&}4C zG`RE)4|VI)pgp4fF4u(ddByH5vg9jEU3fS^={N?p{J6*dmz)d)?*WrwD z%E^vV-uT+RaK@N&*7+U;)9-hmq7p`-Qxf^fR6%j#5~;z?ayzAi^8~JwOf@fsQtlAK z$stlvT}JPtm*vVEL*ug{;BB;43tptT&@fa~grp*s_h^nCo_)p_M?GBfe*&EzJ)E7K z@PBlj954BHm9SZCmYdCHyV-5BHe;ya9eZ&sgi%0&K&?S6F_%+fkxJO+Vs>Fw#v%iM zS*A0AvT$Me;wN=Co3?B(5N6ccY_PId&4vpv6j%{c?oyJHzDU4qt5AkA!+ef&6$JbaILx*zh z6I)5b7YQN&&(Ib@smd`3!qiS8{7Fs};n`BH5o01eThbK}o-MHoq9^DQ9_4POyL`i$ z2zM)LC7$9&4O|d-HGo0TObreJFX~WQV+%(Fp1xcLf+}VU5gyZqBZP)p32RjeLAaw+ z7)fYhwJpM99UTJE4Z4C?*q64n8@Mi4_7-kOvl{Hd7?!thsp2H{4*(1wrz z^leTMM6@_**oGiGfCdC%^0$btq(DK#Q6L`cB&Y8}q6S5Q4uU~YpcD>4y-tNRiWiPx z6(W@giI4Ao+9XWLI0ewPs`l7W?KX$Wf^9FJ(EdQH1xgFh5GAG-w}!pUuWJ@CqKpAiO#ubSG1`V{p&TZKa*$CDg%2W7av~FlrU0T^8uWUW zX+)C+RNAz4%A`U^ib|5e6lI8uN}g9R1I12NW6*ARN)gONBxI=B2PzDC|9~W+!Zj2P z>{bnw5ze)cfzlQe2pZU_mV8M-;Fi3GN>Z_-canOx(PUY0oNb`GtI;niJ*|*|Vgq#G z5LCSV))YrF)U$(C+IoLc#?3irwPO~ohfTE4Q+Iix3X;4;M}`eibdqO~f$DB49D2==Y;TcpM0+T~H&Cig4klA- za&BilP^wU_sXxgKlGh8oi`5(yYel7jy=kc8L0bs$HB>}tR}`I8a-w?O0%b&qN~R*_ z*)SzawQVZ-My*4!3M!rI3+=6luW#ei{v3QLdZ1WH7a_wQ5l=yNw`Pn`))AlDAmN&& z8P{c17L??Pg8bqu`H1t4ug)i@XT!H0TD4CqO0l&jO-K{EDK^{JNq#p5_v8a)(R9P= ZZB`||?a!XknJ;}CE|5L{HryY+`@ct(4toFq diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index a9ddf45..305f551 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -141,7 +141,7 @@ services: depends_on: - pm2-discord - datacoredb - profiles: ["all", "site-server", "fullstack"] + profiles: ["all", "site-server", "fullstack", "bot"] nginx: image: nginx:alpine${ALPINE_VERSION} @@ -161,7 +161,7 @@ services: depends_on: - site-server - datacoredb - profiles: [ "all", "website", "asset-server", "site-server", "fullstack" ] + profiles: [ "all", "website", "asset-server", "site-server", "fullstack", "bot" ] node_exporter: image: prom/node-exporter @@ -223,22 +223,26 @@ services: build: context: contexts/cpp-image-analysis additional_contexts: - source: https://github.com/stt-datacore/cpp-image-analysis.git + source: https://github.com/stt-datacore/cpp-image-analysis.git#${CPP_BRANCH} args: ALPINE_VERSION: container_name: DCImageAnalysis deploy: restart_policy: condition: on-failure - # ports: - # - 5000:5000 + environment: + - CPP_TRAIN_PATH=/traindata/train/ + - CPP_DATA_PATH=/traindata/data/ + - ASSETS_URL=${BOT_ASSETS_URL} + ports: + - 5000:5000 volumes: - sitebuild:/sitebuild:ro - traindata:/traindata:rw - # - ./contexts/cpp-image-analysis/cvdata:/traindata/data:ro depends_on: - pm2-discord - nginx + - website profiles: [ "all", "bot" ] bot: @@ -250,7 +254,7 @@ services: target: ${BUILD_TYPE} args: NODE_VERSION: - container_name: DCBot + container_name: DCBot deploy: restart_policy: condition: on-failure From 6aa33d59c164e3a739eb610f99b70b5705dfa632 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 12:45:35 -0400 Subject: [PATCH 13/30] Update configuration --- multidocker/.env.defaults | 4 +++- multidocker/contexts/gittower/setup-jobs | 3 ++- multidocker/docker-compose.yml | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 3273a04..5778f43 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -60,4 +60,6 @@ MONGO_FBB_COLLECTION=bossBattles # Bot BOT_TOKEN= BOT_DATACORE_URL=https://datacore.app/ -BOT_ASSETS_URL=https://assets.datacore.app/ \ No newline at end of file +BOT_ASSETS_URL=https://assets.datacore.app/ +DEFAULT_GUILD= +NODE_ENV=development \ No newline at end of file diff --git a/multidocker/contexts/gittower/setup-jobs b/multidocker/contexts/gittower/setup-jobs index 8c3a2bd..0f41f04 100755 --- a/multidocker/contexts/gittower/setup-jobs +++ b/multidocker/contexts/gittower/setup-jobs @@ -4,4 +4,5 @@ BASE_URL=https://github.com/stt-datacore/ ./watch website ${BASE_URL}website.git ${WEBSITE_BRANCH} ./watch site-server ${BASE_URL}site-server.git ${SERVER_BRANCH} -./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH} \ No newline at end of file +./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH} +./watch bot ${BASE_URL}bot.git ${BOT_BRANCH} diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 305f551..960fe35 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -275,6 +275,8 @@ services: - BOT_TOKEN=${BOT_TOKEN} - ASSETS_URL=${BOT_ASSETS_URL} - DATACORE_URL=${BOT_DATACORE_URL} + - DEFAULT_GUILD=${DEFAULT_GUILD} + - NODE_ENV=${NODE_ENV} - IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 - MONGO_CONN_STRING=${MONGO_CONN_STRING} - MONGO_DB_NAME=${MONGO_DB_NAME} From 933241023116582f2b155e24226b9ed5bd46525b Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 22:22:12 -0400 Subject: [PATCH 14/30] Finish merging new env pattern --- multidocker/.env.defaults | 31 +++++++- multidocker/conf.defaults/bot.env | 22 ++++++ .../conf.defaults/cpp-image-analysis.env | 3 + multidocker/conf.defaults/site-server.env | 6 +- multidocker/conf.defaults/website.env | 4 +- multidocker/contexts/website/Dockerfile | 6 -- multidocker/docker-compose.yml | 71 +++++-------------- 7 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 multidocker/conf.defaults/bot.env create mode 100644 multidocker/conf.defaults/cpp-image-analysis.env diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 539d0f3..9880886 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -1,9 +1,38 @@ NODE_VERSION=16 ALPINE_VERSION=3.17 + WEBSITE_BRANCH=master SERVER_BRANCH=master ASSET_BRANCH=master BOT_BRANCH=master -BUILD_TYPE=development +CPP_BRANCH=master + NGINX_CONF=development.conf +BUILD_TYPE=development + +# PM2 +PM2_INFO_URL= +PM2_ERROR_URL= + +# Shared Assets +ASSETS_DIR=/app/asset-server/build/out +SQLITE_DB=/app/site-server/static/profiles.sqlite + +# Common +PROFILE_DATA_PATH=/app/site-server/static/profiles +LOG_PATH=/app/logs + +# MongoDB Environment +MONGO_INITDB_DATABASE=user +MONGO_INITDB_ROOT_USERNAME=admin +MONGO_INITDB_ROOT_PASSWORD=secret +DB_CONNECTION_STRING=sqlite:/datacore.db +MONGO_CONN_STRING=mongodb://admin:secret@datacoredb +MONGO_DB_NAME=datacore +MONGO_PROFILE_COLLECTION=playerProfiles +MONGO_TRACKED_VOYAGES_COLLECTION=trackedVoyages +MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments +MONGO_FBB_SOLVES_COLLECTION=solves +MONGO_FBB_TRIALS_COLLECTION=trials +MONGO_FBB_COLLECTION=bossBattles diff --git a/multidocker/conf.defaults/bot.env b/multidocker/conf.defaults/bot.env new file mode 100644 index 0000000..b173bfa --- /dev/null +++ b/multidocker/conf.defaults/bot.env @@ -0,0 +1,22 @@ + +# Bot +BOT_TOKEN= +DATACORE_URL=https://datacore.app/ +ASSETS_URL=https://assets.datacore.app/ +DEFAULT_GUILD= + +# Seperate configuration for bot, because certain dev scenarios +# require that this be different +NODE_ENV=development + +IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 + +PROFILE_DATA_PATH=/profiles/ +DB_CONNECTION_STRING=sqlite:/datacore.db +LOG_PATH=/data/logs/ +DC_DATA_PATH=/sitebuild/structured +CONFIG_PATH=/data/bot_config.json + +# ImgFlip +IMGFLIP_PASSWORD= +IMGFLIP_USERNAME= diff --git a/multidocker/conf.defaults/cpp-image-analysis.env b/multidocker/conf.defaults/cpp-image-analysis.env new file mode 100644 index 0000000..06dcc64 --- /dev/null +++ b/multidocker/conf.defaults/cpp-image-analysis.env @@ -0,0 +1,3 @@ +CPP_TRAIN_PATH=/traindata/train/ +CPP_DATA_PATH=/traindata/data/ + diff --git a/multidocker/conf.defaults/site-server.env b/multidocker/conf.defaults/site-server.env index 68c054f..ce5e9cc 100644 --- a/multidocker/conf.defaults/site-server.env +++ b/multidocker/conf.defaults/site-server.env @@ -1,5 +1,7 @@ +# Site Server Environment STT_BOT_USERNAME= STT_BOT_PASSWORD= JWT_SECRET= -DB_CONNECTION_STRING=sqlite://database -CORS_ORIGIN=https://datacore.app/ \ No newline at end of file +CORS_ORIGIN=https://datacore.app/ +PROFILE_DATA_PATH=/profiles +LOG_PATH=/data/logs diff --git a/multidocker/conf.defaults/website.env b/multidocker/conf.defaults/website.env index 66914ad..749a347 100644 --- a/multidocker/conf.defaults/website.env +++ b/multidocker/conf.defaults/website.env @@ -1,3 +1,3 @@ +# Website Environment GATSBY_DATACORE_URL=/ -#GATSBY_ASSETS_URL=https://assets.datacore.app/ -GATSBY_ASSETS_URL=/assets/ \ No newline at end of file +GATSBY_ASSETS_URL=/assets/ diff --git a/multidocker/contexts/website/Dockerfile b/multidocker/contexts/website/Dockerfile index 6cfa291..79a95ec 100644 --- a/multidocker/contexts/website/Dockerfile +++ b/multidocker/contexts/website/Dockerfile @@ -10,10 +10,4 @@ RUN yarn install RUN npx update-browserslist-db@latest COPY --from=source . . -ARG GATSBY_DATACORE_URL -RUN echo GATSBY_DATACORE_URL=$GATSBY_DATACORE_URL > .env -ARG GATSBY_ASSETS_URL -RUN echo GATSBY_ASSETS_URL=$GATSBY_ASSETS_URL >> .env -RUN cat .env && sleep 5s - CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index e46416f..7fe96c8 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -37,6 +37,8 @@ services: NODE_VERSION: ${NODE_VERSION} PM2_ERROR_URL: ${PM2_ERROR_URL} PM2_INFO_URL: ${PM2_INFO_URL} + env_file: + - .env profiles: ["all", "fullstack", "site-server", "website", "bot"] website: @@ -45,11 +47,10 @@ services: additional_contexts: source: https://github.com/stt-datacore/website.git#${WEBSITE_BRANCH} args: - ALPINE_VERSION: ${ALPINE_VERSION} - NODE_VERSION: ${NODE_VERSION} - GATSBY_ASSETS_URL: ${GATSBY_ASSETS_URL} - GATSBY_DATACORE_URL: ${GATSBY_DATACORE_URL} + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} env_file: + - .env - ./conf/website.env deploy: restart_policy: @@ -85,10 +86,8 @@ services: container_name: datacoredb volumes: - datacore-mongo:/data/db - environment: - - MONGO_INITDB_DATABASE=${MONGO_INITDB_DATABASE} - - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME} - - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} + env_file: + - .env profiles: ["all", "site-server", "bot", "fullstack"] local-dev-db: @@ -98,10 +97,8 @@ services: - datacore-mongo:/data/db ports: - '27017-27019:27017-27019' - environment: - - MONGO_INITDB_DATABASE=${MONGO_INITDB_DATABASE} - - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME} - - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} + env_file: + - .env profiles: ["local-mongo"] site-server: @@ -114,22 +111,6 @@ services: target: ${BUILD_TYPE} args: $NODE_VERSION: - environment: - - PROFILE_DATA_PATH=/profiles - - DB_CONNECTION_STRING=${DB_CONNECTION_STRING} - - LOG_PATH=/data/logs - - STT_BOT_USERNAME=${STT_BOT_USERNAME} - - STT_BOT_PASSWORD=${STT_BOT_PASSWORD} - - JWT_SECRET=${JWT_SECRET} - - MONGO_CONN_STRING=${MONGO_CONN_STRING} - - MONGO_DB_NAME=${MONGO_DB_NAME} - - MONGO_PROFILE_COLLECTION=${MONGO_PROFILE_COLLECTION} - - MONGO_TRACKED_VOYAGES_COLLECTION=${MONGO_TRACKED_VOYAGES_COLLECTION} - - MONGO_TRACKED_ASSIGNMENTS_COLLECTION=${MONGO_TRACKED_ASSIGNMENTS_COLLECTION} - - MONGO_FBB_SOLVES_COLLECTION=${MONGO_FBB_SOLVES_COLLECTION} - - MONGO_FBB_TRIALS_COLLECTION=${MONGO_FBB_TRIALS_COLLECTION} - - MONGO_FBB_COLLECTION=${MONGO_FBB_COLLECTION} - - CORS_ORIGIN=${CORS_ORIGIN} container_name: DCSiteServer deploy: restart_policy: @@ -141,7 +122,8 @@ services: - ${PROFILE_DATA_PATH}:/profiles:rw" - ${SQLITE_DB}:/datacore.db env_file: - - ./conf/site-server.env + - .env + - ./conf/site-server.env depends_on: - pm2-discord - datacoredb @@ -234,10 +216,10 @@ services: deploy: restart_policy: condition: on-failure - environment: - - CPP_TRAIN_PATH=/traindata/train/ - - CPP_DATA_PATH=/traindata/data/ - - ASSETS_URL=${BOT_ASSETS_URL} + env_file: + - .env + - ./conf/bot.env + - ./conf/cpp-image-analysis.env ports: - 5000:5000 volumes: @@ -268,29 +250,8 @@ services: - bot-logs:/data/logs:rw - ${PROFILE_DATA_PATH}:/profiles:rw" - ${SQLITE_DB}:/datacore.db - environment: - - PROFILE_DATA_PATH=/profiles/ - - DB_CONNECTION_STRING=sqlite:/datacore.db - - LOG_PATH=/data/logs/ - - DC_DATA_PATH=/sitebuild/structured - - CONFIG_PATH=/data/bot_config.json - - INGFLIP_USERNAME=${INGFLIP_USERNAME} - - INGFLIP_PASSWORD=${INGFLIP_PASSWORD} - - BOT_TOKEN=${BOT_TOKEN} - - ASSETS_URL=${BOT_ASSETS_URL} - - DATACORE_URL=${BOT_DATACORE_URL} - - DEFAULT_GUILD=${DEFAULT_GUILD} - - NODE_ENV=${NODE_ENV} - - IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 - - MONGO_CONN_STRING=${MONGO_CONN_STRING} - - MONGO_DB_NAME=${MONGO_DB_NAME} - - MONGO_PROFILE_COLLECTION=${MONGO_PROFILE_COLLECTION} - - MONGO_TRACKED_VOYAGES_COLLECTION=${MONGO_TRACKED_VOYAGES_COLLECTION} - - MONGO_TRACKED_ASSIGNMENTS_COLLECTION=${MONGO_TRACKED_ASSIGNMENTS_COLLECTION} - - MONGO_FBB_SOLVES_COLLECTION=${MONGO_FBB_SOLVES_COLLECTION} - - MONGO_FBB_TRIALS_COLLECTION=${MONGO_FBB_TRIALS_COLLECTION} - - MONGO_FBB_COLLECTION=${MONGO_FBB_COLLECTION} env_file: + - .env - ./conf/bot.env depends_on: - cpp-image-analysis From 511f34400cfe68099b56c6abc3d2073cffa1038d Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sat, 7 Oct 2023 22:28:41 -0400 Subject: [PATCH 15/30] clean up volumes --- multidocker/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 7fe96c8..09fb71c 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -119,7 +119,7 @@ services: - "4420:4420" volumes: - ${LOG_PATH}:/data/logs:rw - - ${PROFILE_DATA_PATH}:/profiles:rw" + - ${PROFILE_DATA_PATH}:/profiles:rw - ${SQLITE_DB}:/datacore.db env_file: - .env @@ -245,10 +245,10 @@ services: restart_policy: condition: on-failure volumes: - - "sitebuild:/sitebuild:ro" + - sitebuild:/sitebuild:ro - ${ASSETS_DIR}:/data/assets:ro - bot-logs:/data/logs:rw - - ${PROFILE_DATA_PATH}:/profiles:rw" + - ${PROFILE_DATA_PATH}:/profiles:rw - ${SQLITE_DB}:/datacore.db env_file: - .env From 86f6e525257418d2324f8ccbcf768ad87fbc13a5 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Wed, 11 Oct 2023 22:06:21 -0400 Subject: [PATCH 16/30] Migrate everything into docker volumes --- multidocker/.env.defaults | 6 +----- multidocker/conf.defaults/bot.env | 2 +- multidocker/docker-compose.yml | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 9880886..6567580 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -18,16 +18,12 @@ PM2_ERROR_URL= ASSETS_DIR=/app/asset-server/build/out SQLITE_DB=/app/site-server/static/profiles.sqlite -# Common -PROFILE_DATA_PATH=/app/site-server/static/profiles -LOG_PATH=/app/logs - # MongoDB Environment MONGO_INITDB_DATABASE=user MONGO_INITDB_ROOT_USERNAME=admin MONGO_INITDB_ROOT_PASSWORD=secret -DB_CONNECTION_STRING=sqlite:/datacore.db +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db MONGO_CONN_STRING=mongodb://admin:secret@datacoredb MONGO_DB_NAME=datacore MONGO_PROFILE_COLLECTION=playerProfiles diff --git a/multidocker/conf.defaults/bot.env b/multidocker/conf.defaults/bot.env index b173bfa..45f73d0 100644 --- a/multidocker/conf.defaults/bot.env +++ b/multidocker/conf.defaults/bot.env @@ -12,7 +12,7 @@ NODE_ENV=development IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 PROFILE_DATA_PATH=/profiles/ -DB_CONNECTION_STRING=sqlite:/datacore.db +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db LOG_PATH=/data/logs/ DC_DATA_PATH=/sitebuild/structured CONFIG_PATH=/data/bot_config.json diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 09fb71c..5472c73 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -8,6 +8,7 @@ volumes: site-logs: bot-logs: datacore-mongo: + sqlite: services: gittower: @@ -76,7 +77,7 @@ services: restart_policy: condition: on-failure volumes: - - ${ASSETS_DIR}:/assets:rw + - assets:/assets:rw environment: - OUT_PATH=/assets/ profiles: [ "all", "asset-server", "website", "fullstack" ] @@ -118,9 +119,9 @@ services: ports: - "4420:4420" volumes: - - ${LOG_PATH}:/data/logs:rw - - ${PROFILE_DATA_PATH}:/profiles:rw - - ${SQLITE_DB}:/datacore.db + - site-logs:/data/logs:rw + - profiles:/profiles:rw + - sqlite:/sqlite env_file: - .env - ./conf/site-server.env @@ -140,8 +141,8 @@ services: - "442:442" volumes: - sitebuild:/sitebuild:ro - - ${ASSETS_DIR}:/assets:ro - - ${PROFILE_DATA_PATH}:/profiles:ro + - assets:/assets:ro + - profiles:/profiles:ro - nginx-logs:/var/log/nginx:rw - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw depends_on: @@ -195,8 +196,8 @@ services: - "1337:22" volumes: - sitebuild:/sitebuild:rw - - ${ASSETS_DIR}:/data/assets:ro - - ${PROFILE_DATA_PATH}:/profiles:rw + - assets:/data/assets:ro + - profiles:/profiles:rw - traindata:/data/traindata:rw - nginx-logs:/logs/nginx:ro - site-logs:/logs/site-server:ro @@ -246,10 +247,10 @@ services: condition: on-failure volumes: - sitebuild:/sitebuild:ro - - ${ASSETS_DIR}:/data/assets:ro + - assets:/data/assets:ro - bot-logs:/data/logs:rw - - ${PROFILE_DATA_PATH}:/profiles:rw - - ${SQLITE_DB}:/datacore.db + - profiles:/profiles:rw + - sqlite:/sqlite env_file: - .env - ./conf/bot.env From 52b78f94dbb75f4495a8c08bb1121d77ebc961b1 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Fri, 27 Oct 2023 09:09:34 -0400 Subject: [PATCH 17/30] remove local-up db (unused) --- multidocker/docker-compose.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index 5472c73..c1cbc83 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -89,18 +89,9 @@ services: - datacore-mongo:/data/db env_file: - .env - profiles: ["all", "site-server", "bot", "fullstack"] - - local-dev-db: - image: mongo:4.4.6 - container_name: local-dev-db - volumes: - - datacore-mongo:/data/db ports: - '27017-27019:27017-27019' - env_file: - - .env - profiles: ["local-mongo"] + profiles: ["all", "site-server", "bot", "fullstack"] site-server: image: datacore/site-server From 4dd52a3dff0e46d748049e48680b6db3a59ebee0 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 07:58:00 -0400 Subject: [PATCH 18/30] Update default env --- multidocker/.env.defaults | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 6567580..9d23c3b 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -15,8 +15,8 @@ PM2_INFO_URL= PM2_ERROR_URL= # Shared Assets -ASSETS_DIR=/app/asset-server/build/out -SQLITE_DB=/app/site-server/static/profiles.sqlite +# ASSETS_DIR=/app/asset-server/build/out +# SQLITE_DB=/app/site-server/static/profiles.sqlite # MongoDB Environment MONGO_INITDB_DATABASE=user @@ -29,6 +29,7 @@ MONGO_DB_NAME=datacore MONGO_PROFILE_COLLECTION=playerProfiles MONGO_TRACKED_VOYAGES_COLLECTION=trackedVoyages MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments +MONGO_TELEMETRY_VOYAGE_COLLECTION=telemetry MONGO_FBB_SOLVES_COLLECTION=solves MONGO_FBB_TRIALS_COLLECTION=trials MONGO_FBB_COLLECTION=bossBattles From a3c471906bb4f6ba06e4a0d663172377d91803ec Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 13:35:03 -0400 Subject: [PATCH 19/30] Update docker file to install node 16 (in a less deprecated way) --- Dockerfile | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b1118d..87de87c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,15 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - -RUN apt-get install -y nodejs +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +# RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - +# RUN apt-get install -y nodejs +ARG node_version=v16.20.2 +RUN cd /opt \ + && curl -LO https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz \ + && tar xJf node-${node_version}-linux-x64.tar.xz \ + && rm node-${node_version}-linux-x64.tar.xz +ENV PATH=/opt/node-${node_version}-linux-x64/bin:${PATH} WORKDIR / @@ -50,9 +57,9 @@ COPY dataorig /dataorig/ WORKDIR / #RUN git clone https://github.com/stt-datacore/website.git #- run script should do this RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git -RUN git clone https://github.com/stt-datacore/bot.git +RUN git clone -b discord.js-upgrade-2 https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git -RUN git clone https://github.com/stt-datacore/site-server.git +RUN git clone -b upgrade-node16-only https://github.com/stt-datacore/site-server.git # Build asset parser WORKDIR /asset-server @@ -95,6 +102,7 @@ RUN make -j$(nproc) #RUN mv /asset-server/out/* /dataorig/asset-server/ #RUN rm -rf /asset-server/out #RUN ln -s /data/asset-server /asset-server/out - +RUN echo Node Version +RUN node --version COPY ./runme.sh /runme.sh ENTRYPOINT /runme.sh From 22a0315e3238237a60cf9634cf092f63f0988a70 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 14:41:07 -0400 Subject: [PATCH 20/30] Update environment with new collection --- multidocker/.env.defaults | 1 + 1 file changed, 1 insertion(+) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 9d23c3b..dc56308 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -33,3 +33,4 @@ MONGO_TELEMETRY_VOYAGE_COLLECTION=telemetry MONGO_FBB_SOLVES_COLLECTION=solves MONGO_FBB_TRIALS_COLLECTION=trials MONGO_FBB_COLLECTION=bossBattles +MONGO_DISCORD_USERS_COLLECTION=discordUsers \ No newline at end of file From 78f845fae0d542b1c7ebc076fac4dad16bba5b04 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 14:59:34 -0400 Subject: [PATCH 21/30] update docker file --- Dockerfile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 87de87c..a885435 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,15 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +## begin node + +## old node install commands + # RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - # RUN apt-get install -y nodejs + +## new node install commands + ARG node_version=v16.20.2 RUN cd /opt \ && curl -LO https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz \ @@ -36,6 +42,8 @@ RUN cd /opt \ && rm node-${node_version}-linux-x64.tar.xz ENV PATH=/opt/node-${node_version}-linux-x64/bin:${PATH} +## end node + WORKDIR / # Update npm and install pm2 @@ -56,6 +64,9 @@ COPY dataorig /dataorig/ # Clone repos WORKDIR / #RUN git clone https://github.com/stt-datacore/website.git #- run script should do this + +# experimental and easy to revert + RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git RUN git clone -b discord.js-upgrade-2 https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git From 7d070d0b64a76a327f05d93746a2b44a2c9f6f98 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 15:00:41 -0400 Subject: [PATCH 22/30] change commented line back to original --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a885435..ef2788a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ## old node install commands -# RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - +# RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - # RUN apt-get install -y nodejs ## new node install commands From 3c71b6956e0428f2d83b52b31ca39da3850e1f5b Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 29 Oct 2023 15:18:19 -0400 Subject: [PATCH 23/30] Finalize new docker image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef2788a..11a32a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,9 +68,9 @@ WORKDIR / # experimental and easy to revert RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git -RUN git clone -b discord.js-upgrade-2 https://github.com/stt-datacore/bot.git +RUN git clone https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git -RUN git clone -b upgrade-node16-only https://github.com/stt-datacore/site-server.git +RUN git clone https://github.com/stt-datacore/site-server.git # Build asset parser WORKDIR /asset-server From 3eed4534f39cff127d4f462e9653216ffe7d9ca9 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Mon, 4 Dec 2023 17:24:03 -0500 Subject: [PATCH 24/30] Correct the bot call --- multidocker/contexts/bot/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidocker/contexts/bot/Dockerfile b/multidocker/contexts/bot/Dockerfile index 638c778..8785bf5 100644 --- a/multidocker/contexts/bot/Dockerfile +++ b/multidocker/contexts/bot/Dockerfile @@ -11,4 +11,4 @@ COPY bot_config.json /data/ CMD npm run start FROM development as production -CMD pm2-runtime --name server start npm -- start \ No newline at end of file +CMD pm2-runtime --name bot start npm -- start \ No newline at end of file From 8535c875dec2be4144ce7c19a2e2558cc373139a Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 23 Apr 2024 23:14:38 -0400 Subject: [PATCH 25/30] Upgrade site platform to node 18 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 11a32a6..c980dfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ## new node install commands -ARG node_version=v16.20.2 +ARG node_version=v18.20.2 RUN cd /opt \ && curl -LO https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz \ && tar xJf node-${node_version}-linux-x64.tar.xz \ From 6b60b9be9af4ed9b58ba5d2848dbe0c31d6a5170 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 23 Apr 2024 23:35:35 -0400 Subject: [PATCH 26/30] add set yarn version to website build --- multidocker/contexts/website/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/multidocker/contexts/website/Dockerfile b/multidocker/contexts/website/Dockerfile index 79a95ec..e76432d 100644 --- a/multidocker/contexts/website/Dockerfile +++ b/multidocker/contexts/website/Dockerfile @@ -6,6 +6,7 @@ WORKDIR /app # Sentry requires the github sha RUN git clone https://github.com/stt-datacore/website.git .git --bare --depth 1 COPY --from=source package.json yarn.lock ./ +RUN yarn set version 3.6.3 RUN yarn install RUN npx update-browserslist-db@latest COPY --from=source . . From b7ed669cf5e6eb1ecb61b76253104e6e76d722f8 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 23 Apr 2024 23:35:46 -0400 Subject: [PATCH 27/30] set default node version to 18 --- multidocker/.env.defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index dc56308..4ae4ca2 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -1,4 +1,4 @@ -NODE_VERSION=16 +NODE_VERSION=18 ALPINE_VERSION=3.17 WEBSITE_BRANCH=master From 8d0b1a875ff799c79a214a4af6b35352952418e5 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Mon, 2 Sep 2024 20:27:55 -0400 Subject: [PATCH 28/30] change site-server branch. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c980dfd..959dc6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ WORKDIR / RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git RUN git clone https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git -RUN git clone https://github.com/stt-datacore/site-server.git +RUN git clone https://github.com/stt-datacore/site-server.git -b sqlite-overhaul # Build asset parser WORKDIR /asset-server From 7e93059ebbbe93fd9f0c18fd64ad8d74c18227b3 Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Tue, 10 Sep 2024 19:48:57 -0400 Subject: [PATCH 29/30] restore to master for site-server merge. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 959dc6a..c980dfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ WORKDIR / RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git RUN git clone https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git -RUN git clone https://github.com/stt-datacore/site-server.git -b sqlite-overhaul +RUN git clone https://github.com/stt-datacore/site-server.git # Build asset parser WORKDIR /asset-server From 4e6ccd140e8fdcc8df6846d91448434ab7fb358e Mon Sep 17 00:00:00 2001 From: Nathaniel Moschkin Date: Sun, 19 Jan 2025 13:55:31 -0500 Subject: [PATCH 30/30] Multidocker cleanup. --- .gitignore | 2 ++ multidocker/.env | 18 ----------- multidocker/conf-defaults/bot.env | 18 +++++++++++ .../conf-defaults/cpp-image-analysis.env | 3 ++ multidocker/conf-defaults/rsync.env | 1 + multidocker/conf-defaults/site-server.env | 7 +++++ multidocker/conf-defaults/website.env | 3 ++ multidocker/contexts/nginx/local.conf | 31 +++++++++++++++++++ 8 files changed, 65 insertions(+), 18 deletions(-) delete mode 100644 multidocker/.env create mode 100644 multidocker/conf-defaults/bot.env create mode 100644 multidocker/conf-defaults/cpp-image-analysis.env create mode 100644 multidocker/conf-defaults/rsync.env create mode 100644 multidocker/conf-defaults/site-server.env create mode 100644 multidocker/conf-defaults/website.env create mode 100644 multidocker/contexts/nginx/local.conf diff --git a/.gitignore b/.gitignore index 1269488..e705be2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ data +.env +multidocker/conf \ No newline at end of file diff --git a/multidocker/.env b/multidocker/.env deleted file mode 100644 index 8bb9281..0000000 --- a/multidocker/.env +++ /dev/null @@ -1,18 +0,0 @@ -NODE_VERSION=16 -ALPINE_VERSION=3.17 -RSYNC_PASSWORD= -GATSBY_DATACORE_URL=/ -#GATSBY_ASSETS_URL=https://assets.datacore.app/ -GATSBY_ASSETS_URL=/assets/ -WEBSITE_BRANCH=master -SERVER_BRANCH=master -ASSET_BRANCH=master -BOT_BRANCH=master -NGINX_CONF=test.conf -#NGINX_CONF=test-with-site-server.conf -BUILD_TYPE=development -PM2_INFO_URL= -PM2_ERROR_URL= -BOT_TOKEN= -IMGFLIP_PASSWORD= -IMGFLIP_USERNAME= \ No newline at end of file diff --git a/multidocker/conf-defaults/bot.env b/multidocker/conf-defaults/bot.env new file mode 100644 index 0000000..3b62071 --- /dev/null +++ b/multidocker/conf-defaults/bot.env @@ -0,0 +1,18 @@ + +# Bot +BOT_TOKEN= +DATACORE_URL=https://datacore.app/ +ASSETS_URL=https://assets.datacore.app/ +NODE_ENV=production + +IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 + +PROFILE_DATA_PATH=/profiles/ +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db +LOG_PATH=/data/logs/ +DC_DATA_PATH=/sitebuild/structured +CONFIG_PATH=/data/bot_config.json + +# ImgFlip +IMGFLIP_PASSWORD= +IMGFLIP_USERNAME= diff --git a/multidocker/conf-defaults/cpp-image-analysis.env b/multidocker/conf-defaults/cpp-image-analysis.env new file mode 100644 index 0000000..06dcc64 --- /dev/null +++ b/multidocker/conf-defaults/cpp-image-analysis.env @@ -0,0 +1,3 @@ +CPP_TRAIN_PATH=/traindata/train/ +CPP_DATA_PATH=/traindata/data/ + diff --git a/multidocker/conf-defaults/rsync.env b/multidocker/conf-defaults/rsync.env new file mode 100644 index 0000000..1b185d5 --- /dev/null +++ b/multidocker/conf-defaults/rsync.env @@ -0,0 +1 @@ +RSYNC_PASSWORD=insecure \ No newline at end of file diff --git a/multidocker/conf-defaults/site-server.env b/multidocker/conf-defaults/site-server.env new file mode 100644 index 0000000..ce5e9cc --- /dev/null +++ b/multidocker/conf-defaults/site-server.env @@ -0,0 +1,7 @@ +# Site Server Environment +STT_BOT_USERNAME= +STT_BOT_PASSWORD= +JWT_SECRET= +CORS_ORIGIN=https://datacore.app/ +PROFILE_DATA_PATH=/profiles +LOG_PATH=/data/logs diff --git a/multidocker/conf-defaults/website.env b/multidocker/conf-defaults/website.env new file mode 100644 index 0000000..749a347 --- /dev/null +++ b/multidocker/conf-defaults/website.env @@ -0,0 +1,3 @@ +# Website Environment +GATSBY_DATACORE_URL=/ +GATSBY_ASSETS_URL=/assets/ diff --git a/multidocker/contexts/nginx/local.conf b/multidocker/contexts/nginx/local.conf new file mode 100644 index 0000000..419ccb3 --- /dev/null +++ b/multidocker/contexts/nginx/local.conf @@ -0,0 +1,31 @@ +server { + listen 80; + client_max_body_size 100M; + #server_name datacore.local; + + location / { + root /sitebuild; + expires 1h; + } + + location /assets/ { + root /assets/; + access_log off; + expires max; + } + + location /assets/data/ { + root /assets/data/; + expires 1h; + } + + location /profiles/ { + root /profiles/; + expires 1h; + } + + location /api/ { + proxy_pass http://DCSiteServer:4420; + expires -1; + } +} \ No newline at end of file