diff --git a/.env.default b/.env.default index 6dee5305b..eea1aab03 100644 --- a/.env.default +++ b/.env.default @@ -1,5 +1,4 @@ -COMPOSE_FILE=./docker/docker-compose.yml:./docker/docker-compose.override.yml -COMPOSE_PROJECT_NAME=projectname +PROJECT_NAME=projectname PROFILE_NAME=druxxy THEME_NAME=NA SITE_NAME=Example @@ -29,3 +28,4 @@ REDIS_PORT=6379 REDIS_PASSWD=1234567890 # See readme for basicauth convention RA_BASIC_AUTH= +ORCHESTRATOR=k3s diff --git a/Makefile b/Makefile index 53c5a6761..b9afe19ff 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -# Add utility functions and scripts to the container -include scripts/makefile/*.mk - .PHONY: all fast allfast provision si exec exec0 down clean dev drush info phpcs phpcbf hooksymlink clang cinsp compval watchdogval drupalrectorval upgradestatusval behat sniffers tests front front-install front-build clear-front lintval lint storybook back behatdl behatdi browser_driver browser_driver_stop statusreportval contentgen newlineeof localize local-settings redis-settings content patchval diff .DEFAULT_GOAL := help @@ -10,9 +7,22 @@ include scripts/makefile/*.mk # Prepare enviroment variables from defaults $(shell false | cp -i \.env.default \.env 2>/dev/null) -$(shell false | cp -i \.\/docker\/docker-compose\.override\.yml\.default \.\/docker\/docker-compose\.override\.yml 2>/dev/null) include .env +# Select orchestrator related commands +# include Makefile_$(ORCHESTRATOR).mk +ifeq ($(ORCHESTRATOR), k3s) +include Makefile_k3s.mk +else +include Makefile_docker-compose.mk +endif + +# Include utility functions and scripts +include scripts/makefile/*.mk + +# Sanitize PROJECT_NAME input +COMPOSE_PROJECT_NAME := $(shell echo "$(PROJECT_NAME)" | tr -cd '[a-zA-Z0-9]' | tr '[:upper:]' '[:lower:]') + # Get user/group id to manage permissions between host and containers LOCAL_UID := $(shell id -u) LOCAL_GID := $(shell id -g) @@ -21,10 +31,6 @@ LOCAL_GID := $(shell id -g) CUID ?= $(LOCAL_UID) CGID ?= $(LOCAL_GID) -# Define network name. -COMPOSE_NET_NAME := $(COMPOSE_PROJECT_NAME)_front - -SDC_SERVICES=$(shell docker-compose config --services) # Determine database data directory if defined DB_MOUNT_DIR=$(shell cd docker && realpath $(DB_DATA_DIR))/ ifeq ($(findstring mysql,$(SDC_SERVICES)),mysql) @@ -37,43 +43,45 @@ endif # Define current directory only once CURDIR=$(shell pwd) -# Execute php container as regular user -php = docker-compose exec -T --user $(CUID):$(CGID) php ${1} -# Execute php container as root user -php-0 = docker-compose exec -T --user 0:0 php ${1} - +# Variables ADDITIONAL_PHP_PACKAGES := tzdata graphicsmagick # php7-intl php7-redis wkhtmltopdf gnu-libiconv php7-pdo_pgsql postgresql-client postgresql-contrib DC_MODULES := project_default_content better_normalizers default_content hal serialization MG_MODULES := migrate_generator migrate migrate_plus migrate_source_csv migrate_tools + ## Full site install from the scratch -all: | provision back front si localize hooksymlink info +all: | provision back front si localize hooksymlink info xx # Install for CI deploy:review. Back & Front tasks are run in a dedicated previous step in order to leverage CI cache all_ci: | provision si localize hooksymlink info # Full site install from the scratch with DB in ram (makes data NOT persistant) -allfast: | fast provision back front si localize hooksymlink info +allfast: | fast ## Update .env to build DB in ram (makes data NOT persistant) fast: $(shell sed -i "s|^#DB_URL=sqlite:///dev/shm/d8.sqlite|DB_URL=sqlite:///dev/shm/d8.sqlite|g" .env) $(shell sed -i "s|^DB_URL=sqlite:./../.cache/d8.sqlite|#DB_URL=sqlite:./../.cache/d8.sqlite|g" .env) + $(info - Your .env file was updated. Run `make all` again.) + exit 1 -## Provision enviroment -provision: -# Check if enviroment variables has been defined +enforce-project-name: ifeq ($(strip $(COMPOSE_PROJECT_NAME)),projectname) - $(info Project name can not be default, please enter project name.) - $(eval COMPOSE_PROJECT_NAME = $(strip $(shell read -p "Project name: " REPLY;echo -n $$REPLY))) - $(shell sed -i -e '/COMPOSE_PROJECT_NAME=/ s/=.*/=$(COMPOSE_PROJECT_NAME)/' .env) - $(info Please review your project settings and run `make all` again.) + $(eval COMPOSE_PROJECT_NAME = $(strip $(shell read -p "Please enter project name: " REPLY;echo -n $$REPLY))) + $(shell sed -i -e '/PROJECT_NAME=/ s/=.*/=$(COMPOSE_PROJECT_NAME)/' .env) + $(info - Run `make all` again.) exit 1 endif + +## Provision enviroment +provision: +# Check if enviroment variables has been defined + make -s enforce-project-name ifdef DB_MOUNT_DIR $(shell [ ! -d $(DB_MOUNT_DIR) ] && mkdir -p $(DB_MOUNT_DIR) && chmod 777 $(DB_MOUNT_DIR)) endif - make -s down + make -s down 2> /dev/null + make -s install-orchestrator @echo "Build and run containers..." - docker-compose up -d --remove-orphans + make -s up # Set composer2 as default $(call php-0, ln -fs composer2 /usr/bin/composer) ifneq ($(strip $(ADDITIONAL_PHP_PACKAGES)),) @@ -120,8 +128,8 @@ endif ifneq ($(strip $(MG_MODULES)),) $(call php, drush en $(MG_MODULES) -y) $(call php, drush migrate_generator:generate_migrations /var/www/html/content --update) - $(call php, drush migrate:import --all --group=migrate_generator_group) - $(call php, drush migrate_generator:clean_migrations migrate_generator_group) + $(call php, drush migrate:import --all --group=mgg) + $(call php, drush migrate_generator:clean_migrations mgg) $(call php, drush pmu $(MG_MODULES) -y) endif @@ -134,8 +142,8 @@ ifneq ("$(wildcard settings/settings.local.php)","") $(call php, drush cr) endif -REDIS_IS_INSTALLED := $(shell grep "redis.connection" web/sites/default/settings.php | tail -1 | wc -l || echo "0") redis-settings: + REDIS_IS_INSTALLED := $(shell grep "redis.connection" web/sites/default/settings.php | tail -1 | wc -l || echo "0") ifeq ($(REDIS_IS_INSTALLED), 1) @echo "Redis settings already installed, nothing to do" else @@ -153,62 +161,62 @@ localize: $(call php, drush locale:import:all /var/www/html/translations/ --type=customized --override=all) @echo "Localization finished" +xx: +ifeq ($(shell $(call PROJECT_IS_UP)),true) + @echo "it's up!" +endif + ## Display project's information info: - $(info ) - $(info Containers for "$(COMPOSE_PROJECT_NAME)" info:) - $(eval CONTAINERS = $(shell docker ps -f name=$(COMPOSE_PROJECT_NAME) --format "{{ .ID }}" -f 'label=traefik.enable=true')) - $(foreach CONTAINER, $(CONTAINERS),$(info http://$(shell printf '%-19s \n' $(shell docker inspect --format='{{(index .NetworkSettings.Networks "$(COMPOSE_NET_NAME)").IPAddress}}:{{index .Config.Labels "traefik.port"}} {{range $$p, $$conf := .NetworkSettings.Ports}}{{$$p}}{{end}} {{.Name}}' $(CONTAINER) | rev | sed "s/pct\//,pct:/g" | sed "s/,//" | rev | awk '{ print $0}')) )) - $(info ) ifdef REVIEW_DOMAIN $(eval BASE_URL := $(MAIN_DOMAIN_NAME)) else - $(eval BASE_URL := $(shell docker inspect --format='{{(index .NetworkSettings.Networks "$(COMPOSE_NET_NAME)").IPAddress}}:{{index .Config.Labels "traefik.port"}}' $(COMPOSE_PROJECT_NAME)_web)) + $(eval BASE_URL := $(LOCAL_IP)) endif - $(info Login as System Admin: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(ADMIN_NAME)" /admin/content/ | awk -F "default" '{print $$2}'))))) - $(info Login as Contributor: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(TESTER_NAME)" /admin/content/ | awk -F "default" '{print $$2}'))))) + $(info ) + $(info Containers for "$(COMPOSE_PROJECT_NAME)":) + $(info ) + $(info Login as System Admin: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(ADMIN_NAME)" /admin/content/ | awk -F "default" '{print \$$2}'))))) + $(info Login as Contributor: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(TESTER_NAME)" /admin/content/ | awk -F "default" '{print \$$2}'))))) $(info ) ifneq ($(shell diff .env .env.default -q),) @echo -e "\x1b[33mWARNING\x1b[0m - .env and .env.default files differ. Use 'make diff' to see details." endif -ifneq ($(shell diff docker/docker-compose.override.yml docker/docker-compose.override.yml.default -q),) - @echo -e "\x1b[33mWARNING\x1b[0m - docker/docker-compose.override.yml and docker/docker-compose.override.yml.default files differ. Use 'make diff' to see details." -endif ## Output diff between local and versioned files diff: diff -u0 --color .env .env.default || true; echo "" - diff -u0 --color docker/docker-compose.override.yml docker/docker-compose.override.yml.default || true; echo "" - - -## Run shell in PHP container as regular user -exec: - docker-compose exec --user $(CUID):$(CGID) php ash - -## Run shell in PHP container as root -exec0: - docker-compose exec --user 0:0 php ash +#TODO: Re-test that later on DEFER EVALUATION down: +ifeq ($(shell $(call PROJECT_IS_UP)),true) @echo "Removing network & containers for $(COMPOSE_PROJECT_NAME)" - @docker-compose down -v --remove-orphans --rmi local - @if [ ! -z "$(shell docker ps -f 'name=$(COMPOSE_PROJECT_NAME)_chrome' --format '{{.Names}}')" ]; then \ - echo 'Stoping browser driver.' && make -s browser_driver_stop; fi + make -s down-containers +else + @echo "No container to down" +endif + make -s down-test-browser + -DIRS = web/core web/libraries web/modules/contrib web/profiles/contrib web/sites web/themes/contrib vendor -## Totally remove project build folder, docker containers and network -clean: info + +## Totally remove project build folder, containers and network +clean: + $(eval DIRS = web/core web/libraries web/modules/contrib web/profiles/contrib web/sites web/themes/contrib vendor) + $(info Cleaning project "$(COMPOSE_PROJECT_NAME)"...) make -s down - $(eval SCAFFOLD = $(shell docker run --rm -v $(CURDIR):/mnt -w /mnt --user $(CUID):$(CGID) $(IMAGE_PHP) composer run-script list-scaffold-files | grep -P '^(?!>)')) - @docker run --rm --user 0:0 -v $(CURDIR):/mnt -w /mnt -e RMLIST="$(addprefix web/,$(SCAFFOLD)) $(DIRS)" $(IMAGE_PHP) sh -c 'for i in $$RMLIST; do rm -fr $$i && echo "Removed $$i"; done' + make -s scaffold-list + $(eval RMLIST = $(addprefix web/,$(SCAFFOLD)) $(DIRS)) + for i in $(RMLIST); do rm -rf $$i && echo "Removed $$i"; done ifdef DB_MOUNT_DIR @echo "Clean-up database data from $(DB_MOUNT_DIR) ..." - docker run --rm --user 0:0 -v $(shell dirname $(DB_MOUNT_DIR)):/mnt $(IMAGE_PHP) sh -c "rm -fr /mnt/`basename $(DB_MOUNT_DIR)`" + $(shell rm -fr $(DB_MOUNT_DIR)) + #TODO: what was doccomp equivalent? endif ifeq ($(CLEAR_FRONT_PACKAGES), yes) make clear-front endif + make -s uninstall-orchestrator ## Enable development mode and disable caching dev: @@ -232,3 +240,7 @@ dev: drush: $(call php, $(filter-out "$@",$(MAKECMDGOALS))) $(info "To pass arguments use double dash: "make drush en devel -- -y"") + + +## TODO: CHECK And FIX TEST COMMANDS + diff --git a/Makefile_docker-compose.mk b/Makefile_docker-compose.mk new file mode 100644 index 000000000..2e8b89eae --- /dev/null +++ b/Makefile_docker-compose.mk @@ -0,0 +1,128 @@ +$(shell false | cp -i \.\/docker\/docker-compose\.override\.yml\.default \.\/docker\/docker-compose\.override\.yml 2>/dev/null) + + + +# Define network name +COMPOSE_NET_NAME := $(COMPOSE_PROJECT_NAME)_front +# Define docker-compose files +COMPOSE_FILE=./docker/docker-compose.yml:./docker/docker-compose.override.yml +# List docker-compose services +SDC_SERVICES=$(shell docker-compose config --services) + +LOCAL_IP = $(shell docker inspect --format='{{(index .NetworkSettings.Networks "$(COMPOSE_NET_NAME)").IPAddress}}:{{index .Config.Labels "traefik.port"}}' $(COMPOSE_PROJECT_NAME)_web) + +PROJECT_IS_UP = + +# Execute php container as regular user +php = docker-compose exec -T --user $(CUID):$(CGID) php ${1} +# Execute php container as root user +php-0 = docker-compose exec -T --user 0:0 php ${1} + + + +## Run shell in PHP container as regular user +exec: + docker-compose exec --user $(CUID):$(CGID) php ash + +## Run shell in PHP container as root +exec0: + docker-compose exec --user 0:0 php ash + + + +## Output diff between local and versioned files +diff: + diff -u0 --color .env .env.default || true; echo "" + diff -u0 --color docker/docker-compose.override.yml docker/docker-compose.override.yml.default || true; echo "" + + +down-test-browser: + @if [ ! -z "$(shell docker ps -f 'name=$(COMPOSE_PROJECT_NAME)_chrome' --format '{{.Names}}')" ]; then \ + echo 'Stoping browser driver.' && make -s browser_driver_stop; fi + + +provision: + docker-compose up -d --remove-orphans + + +## Display project's information +info: + $(info ) + $(info Containers for "$(COMPOSE_PROJECT_NAME)" info:) + $(eval CONTAINERS = $(shell docker ps -f name=$(COMPOSE_PROJECT_NAME) --format "{{ .ID }}" -f 'label=traefik.enable=true')) + $(foreach CONTAINER, $(CONTAINERS),$(info http://$(shell printf '%-19s \n' $(shell docker inspect --format='{{(index .NetworkSettings.Networks "$(COMPOSE_NET_NAME)").IPAddress}}:{{index .Config.Labels "traefik.port"}} {{range $$p, $$conf := .NetworkSettings.Ports}}{{$$p}}{{end}} {{.Name}}' $(CONTAINER) | rev | sed "s/pct\//,pct:/g" | sed "s/,//" | rev | awk '{ print $0}')) )) + $(info ) +ifdef REVIEW_DOMAIN + $(eval BASE_URL := $(MAIN_DOMAIN_NAME)) +else + $(eval BASE_URL := $(shell docker inspect --format='{{(index .NetworkSettings.Networks "$(COMPOSE_NET_NAME)").IPAddress}}:{{index .Config.Labels "traefik.port"}}' $(COMPOSE_PROJECT_NAME)_web)) +endif + $(info Login as System Admin: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(ADMIN_NAME)" /admin/content/ | awk -F "default" '{print $$2}'))))) + $(info Login as Contributor: http://$(shell printf '%-19s \n' $(shell echo "$(BASE_URL)"$(shell $(call php, drush user:login --name="$(TESTER_NAME)" /admin/content/ | awk -F "default" '{print $$2}'))))) + $(info ) +ifneq ($(shell diff .env .env.default -q),) + @echo -e "\x1b[33mWARNING\x1b[0m - .env and .env.default files differ. Use 'make diff' to see details." +endif +ifneq ($(shell diff docker/docker-compose.override.yml docker/docker-compose.override.yml.default -q),) + @echo -e "\x1b[33mWARNING\x1b[0m - docker/docker-compose.override.yml and docker/docker-compose.override.yml.default files differ. Use 'make diff' to see details." +endif + + +down: + @docker-compose down -v --remove-orphans --rmi local + @if [ ! -z "$(shell docker ps -f 'name=$(COMPOSE_PROJECT_NAME)_chrome' --format '{{.Names}}')" ]; then \ + echo 'Stoping browser driver.' && make -s browser_driver_stop; fi + + +## Totally remove project build folder, docker containers and network +clean: + make -s down + $(eval SCAFFOLD = $(shell docker run --rm -v $(CURDIR):/mnt -w /mnt --user $(CUID):$(CGID) $(IMAGE_PHP) composer run-script list-scaffold-files | grep -P '^(?!>)')) + @docker run --rm --user 0:0 -v $(CURDIR):/mnt -w /mnt -e RMLIST="$(addprefix web/,$(SCAFFOLD)) $(DIRS)" $(IMAGE_PHP) sh -c 'for i in $$RMLIST; do rm -fr $$i && echo "Removed $$i"; done' +ifdef DB_MOUNT_DIR + @echo "Clean-up database data from $(DB_MOUNT_DIR) ..." + docker run --rm --user 0:0 -v $(shell dirname $(DB_MOUNT_DIR)):/mnt $(IMAGE_PHP) sh -c "rm -fr /mnt/`basename $(DB_MOUNT_DIR)`" +endif +ifeq ($(CLEAR_FRONT_PACKAGES), yes) + make clear-front +endif + + +# Execute front container function. +frontexec = docker run \ + --rm \ + --init \ + -u $(CUID):$(CGID) \ + -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ + --workdir /app \ + $(IMAGE_FRONT) ${1} + + +# Execute front container function on localhost:FRONT_PORT. Needed for dynamic storybook. +frontexec-with-port = docker run \ + --rm \ + --init \ + -p $(FRONT_PORT):$(FRONT_PORT) \ + -u $(CUID):$(CGID) \ + -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ + --workdir /app \ + $(IMAGE_FRONT) ${1} + +# Execute front container with TTY. Needed for storybook components creation. +frontexec-with-interactive = docker run \ + --rm \ + --init \ + -u $(CUID):$(CGID) \ + -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ + --workdir /app \ + -it \ + $(IMAGE_FRONT) ${1} + + + +testfront: + $(call frontexec,ls -lah) + +xxx: + @echo "im in docker-compose.mk" + diff --git a/Makefile_k3s.mk b/Makefile_k3s.mk new file mode 100644 index 000000000..a31fb85bb --- /dev/null +++ b/Makefile_k3s.mk @@ -0,0 +1,124 @@ + + +# Used to give a random name to Kubernetes pods executed on the fly by "kubectl run" +RANDOM_STRING ?= $(shell cat /dev/urandom | tr -dc 'a-fA-F0-9' | tr '[:upper:]' '[:lower:]' | fold -w 10 | head -n 1) + +IMAGE_HELM=alpine/helm:3.8.0 +KUBECTL_IS_INSTALLED := $(shell [ -e "$(shell which kubectl 2> /dev/null)" ] && echo true || echo false) +HELM_IS_INSTALLED := $(shell [ -e "$(shell which helm 2> /dev/null)" ] && echo true || echo false) +JQ_IS_INSTALLED := $(shell [ -e "$(shell which jq 2> /dev/null)" ] || [ -e "$(shell which gojq 2> /dev/null)" ] && echo true || echo false) +SDC_SERVICES=$(shell kubectl get pods -l name=$(COMPOSE_PROJECT_NAME) -o jsonpath="{.items[*].spec.containers[*].name}" 2>/dev/null) +LOCAL_IP = $(shell kubectl get pods -l name=$(COMPOSE_PROJECT_NAME) --template '{{range .items}}{{.status.podIP}}{{"\n"}}{{end}}') + + +PROJECT_IS_UP = kubectl get deployment $(COMPOSE_PROJECT_NAME) -o go-template='{{ if eq .status.readyReplicas .status.replicas }}{{ "true" }}{{ end }}' &>/dev/null && echo true || echo false + +# Check if k3s is present, install it if not +lookfork3s: +ifeq ($(KUBECTL_IS_INSTALLED), false) + @echo "Downloading and installing container orchestrator..." + curl -sfL https://get.k3s.io | K3S_NODE_NAME=sdc K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION="v1.22.5+k3s1" sh - + #curl -sfL https://get.k3s.io | K3S_NODE_NAME=sdc K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION="v1.23.3+k3s1" sh - + @echo + @echo "- If your command fail here, run it again." + @echo +endif + +install-orchestrator: + make -s lookfork3s + for i in {1..50}; do echo "Waiting for default service account..." && kubectl -n default get serviceaccount default -o name &> /dev/null && break || sleep 3; done; echo "Default service account is up !" + +up: + if [ $(HELM_IS_INSTALLED) = false ]; then \ + kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_HELM) --rm -i --quiet --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)", "type": "" } }, { "name": "host-k3s-config", "hostPath": { "path": "/etc/rancher/k3s/k3s.yaml", "type": "" } } ], "containers": [ { "name": "test", "image": "$(IMAGE_HELM)", "command": [ "helm","upgrade","--install","--kubeconfig=/etc/rancher/k3s/k3s.yaml","$(COMPOSE_PROJECT_NAME)","./helm/","--set","projectName=$(COMPOSE_PROJECT_NAME),projectPath=$(CURDIR),imagePhp=$(IMAGE_PHP),imageNginx=$(IMAGE_NGINX),userGroup=$(CGID)" ], "workingDir": "/app", "resources": {}, "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" }, { "name": "host-k3s-config", "mountPath": "/etc/rancher/k3s/k3s.yaml" } ], "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "hostNetwork": true, "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) }, "schedulerName": "default-scheduler", "enableServiceLinks": true }, "status": {} }'; \ + else helm upgrade --install --kubeconfig="/etc/rancher/k3s/k3s.yaml" $(COMPOSE_PROJECT_NAME) ./helm/ --set projectName="$(COMPOSE_PROJECT_NAME)",projectPath="$(CURDIR)",imagePhp="$(IMAGE_PHP)",imageNginx="$(IMAGE_NGINX)",userGroup="$(CGID)"; fi; + for i in {1..50}; do echo "Waiting for PHP container..." && kubectl exec -it deploy/$(COMPOSE_PROJECT_NAME) -c php -- "whoami" &> /dev/null && break || sleep 3; done; echo "Container is up !" + $(call php-0, chown -R $(CUID):$(CGID) .) + + +down-containers: + if [ $(HELM_IS_INSTALLED) = false ]; then kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_HELM) --rm -i --quiet --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)", "type": "" } }, { "name": "host-k3s-config", "hostPath": { "path": "/etc/rancher/k3s/k3s.yaml", "type": "" } } ], "containers": [ { "name": "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)", "image": "$(IMAGE_HELM)", "command": [ "helm","uninstall","--wait","--kubeconfig=/etc/rancher/k3s/k3s.yaml","$(COMPOSE_PROJECT_NAME)" ], "workingDir": "/app", "resources": {}, "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" }, { "name": "host-k3s-config", "mountPath": "/etc/rancher/k3s/k3s.yaml" } ], "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "hostNetwork": true, "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) }, "schedulerName": "default-scheduler", "enableServiceLinks": true }, "status": {} }'; else helm uninstall --kubeconfig=/etc/rancher/k3s/k3s.yaml --wait $(COMPOSE_PROJECT_NAME); fi; + + +uninstall-orchestrator: +ifeq ($(KUBECTL_IS_INSTALLED), true) + /usr/local/bin/k3s-killall.sh + /usr/local/bin/k3s-uninstall.sh +endif + + +killall: +ifeq ($(KUBECTL_IS_INSTALLED), true) + /usr/local/bin/k3s-killall.sh + /usr/local/bin/k3s-uninstall.sh +endif + +# Execute php container as regular user +php = kubectl exec -it deploy/$(COMPOSE_PROJECT_NAME) -c php -- su -s /bin/ash www-data -c "${1}" +# Execute php container as root user +php-0 = kubectl exec -it deploy/$(COMPOSE_PROJECT_NAME) -c php -- ${1} + +## Run shell in PHP container as regular user +exec: + kubectl exec -it deploy/$(COMPOSE_PROJECT_NAME) -c php -- su -s /bin/ash www-data -c ash + +## Run shell in PHP container as root +exec0: + kubectl exec -it deploy/$(COMPOSE_PROJECT_NAME) -c php -- ash + +scaffold-list: + $(eval SCAFFOLD = $(shell kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_PHP) --rm -i --quiet --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)", "type": "" } } ], "containers": [ { "name": "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)", "image": "$(IMAGE_PHP)", "command": [ "composer", "run-script", "list-scaffold-files" ], "workingDir": "/app", "resources": {}, "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" } ], "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "hostNetwork": true, "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) }, "schedulerName": "default-scheduler", "enableServiceLinks": true }, "status": {} }' | grep -P '^(?!>)')) + +## Display logs for all services +logs: + kubectl logs -f deploy/$(COMPOSE_PROJECT_NAME) --all-containers=true + + + +h: +ifeq ($(HELM_IS_INSTALLED), true) + @echo "Helm is installed" +else + @echo "Helm is not installed" +endif + + +k: +ifeq ($(KUBECTL_IS_INSTALLED), true) + @echo "Kubernetes is installed" +else + @echo "Kubernetes is not installed" +endif + + +j: +ifeq ($(JQ_IS_INSTALLED), true) + @echo "Jq is installed" +else + @echo "Jq is not installed" +endif + + + + + +# Convert list of commands to json array format expected by "kubectl run --overrides" commands +jsonarrayconverter = if [ $(JQ_IS_INSTALLED) = false ]; then kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=stedolan/jq --restart=Never --quiet -i --rm --command -- jq -c -n --arg groups "${1}" '$$groups | split(" ")' 2> /dev/null; else jq -c -n --arg groups "${1}" '$$groups | split(" ")'; fi; + +# Execute front container function +frontexec = make -s lookfork3s; kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_FRONT) --rm -i --quiet --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)/web/themes/custom/$(THEME_NAME)" } } ], "containers": [ { "name": "frontexec", "image": "$(IMAGE_FRONT)", "command": $(shell $(call jsonarrayconverter,${1})), "workingDir": "/app", "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" } ], "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) } } }' + +# Execute front container function on localhost:FRONT_PORT. Needed for dynamic storybook +frontexec-with-port = make -s lookfork3s; kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_FRONT) --rm -i --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)/web/themes/custom/$(THEME_NAME)" } } ], "containers": [ { "name": "frontexec-with-port", "image": "$(IMAGE_FRONT)", "command": $(shell $(call jsonarrayconverter,${1})), "stdin": true, "tty": true, "ports": [ { "containerPort": $(FRONT_PORT), "protocol": "TCP" } ], "workingDir": "/app", "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" } ], "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) } } }' + +# Execute front container with TTY. Needed for storybook components creation +frontexec-with-interactive = make -s lookfork3s; kubectl run "$(COMPOSE_PROJECT_NAME)-$(RANDOM_STRING)" --image=$(IMAGE_FRONT) --rm -i --overrides='{ "kind": "Pod", "apiVersion": "v1", "spec": { "volumes": [ { "name": "host-volume", "hostPath": { "path": "$(CURDIR)/web/themes/custom/$(THEME_NAME)" } } ], "containers": [ { "name": "frontexec-with-interactive", "image": "$(IMAGE_FRONT)", "command": $(shell $(call jsonarrayconverter,${1})), "stdin": true, "tty": true, "workingDir": "/app", "volumeMounts": [ { "name": "host-volume", "mountPath": "/app" } ], "terminationMessagePolicy": "FallbackToLogsOnError", "imagePullPolicy": "IfNotPresent" } ], "restartPolicy": "Never", "securityContext": { "runAsUser": $(CUID), "runAsGroup": $(CGID) } } }' + + + +testfront: + $(call frontexec,ls -lah) + +xxx: + @echo "im in k3s.mk" + diff --git a/composer.json b/composer.json index 9bf9a030c..d7a36370b 100644 --- a/composer.json +++ b/composer.json @@ -50,6 +50,14 @@ "sort-packages": true, "preferred-install": { "*": "dist" + }, + "allow-plugins": { + "composer/installers": true, + "cweagans/composer-patches": true, + "drupal/core-project-message": true, + "drupal/core-composer-scaffold": true, + "drupal/core-vendor-hardening": true, + "skilldlabs/drupal-cleanup": true } }, "autoload": { @@ -115,9 +123,6 @@ }, "drupal/default_content": { "Do not reimport existing entities": "https://www.drupal.org/files/issues/do_not_reimport-2698425-56.patch" - }, - "drupal/migrate_generator": { - "Ability to remove config of generated migrations": "https://git.drupalcode.org/project/migrate_generator/commit/20dc65e.patch" } } } diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 000000000..19c5b70f8 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: sdc +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl new file mode 100644 index 000000000..c54488b5b --- /dev/null +++ b/helm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "sdc.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sdc.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sdc.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "sdc.labels" -}} +helm.sh/chart: {{ include "sdc.chart" . }} +{{ include "sdc.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sdc.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sdc.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "sdc.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "sdc.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml new file mode 100644 index 000000000..851aae478 --- /dev/null +++ b/helm/templates/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config +data: + upstream: | + upstream upstream { + server localhost:9000; + } \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml new file mode 100644 index 000000000..b45b4c656 --- /dev/null +++ b/helm/templates/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.projectName }} + labels: + name: {{ .Values.projectName }} +spec: + revisionHistoryLimit: 0 + replicas: 1 + selector: + matchLabels: + name: {{ .Values.projectName }} + strategy: + type: Recreate + template: + metadata: + labels: + name: {{ .Values.projectName }} + spec: + volumes: + - name: codebase + hostPath: + path: {{ .Values.projectPath }} + type: Directory + - name: nginx-config + configMap: + name: nginx-config + - name: cache-js + emptyDir: + medium: Memory + - name: cache-css + emptyDir: + medium: Memory + - name: cache-php + emptyDir: + medium: Memory + dnsConfig: + options: + - name: ndots + value: "2" + containers: + - name: php + image: {{ .Values.imagePhp }} + env: + - name: COMPOSER_MEMORY_LIMIT + value: "-1" + ports: + - containerPort: 9000 + protocol: TCP + volumeMounts: + - mountPath: /var/www/html + name: codebase + - mountPath: /var/www/html/web/sites/default/files/css + name: cache-css + - mountPath: /var/www/html/web/sites/default/files/js + name: cache-js + - mountPath: /var/www/html/web/sites/default/files/php + name: cache-php + - name: nginx + image: {{ .Values.imageNginx }} + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/www/html + name: codebase + - mountPath: /etc/nginx/upstream + name: nginx-config + subPath: upstream + - mountPath: /var/www/html/web/sites/default/files/css + name: cache-css + - mountPath: /var/www/html/web/sites/default/files/js + name: cache-js + - name: mailhog + image: skilldlabs/mailhog + restartPolicy: Always + nodeSelector: + kubernetes.io/hostname: sdc diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml new file mode 100644 index 000000000..d62e40fb1 --- /dev/null +++ b/helm/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ .Values.projectName }} + labels: + mrn: {{ .Values.projectName }} + annotations: + kubernetes.io/tls-acme: "true" + kubernetes.io/ingress.class: web-ingress + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/from-to-www-redirect: "true" +spec: + rules: + - host: local.skilld.cloud + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: {{ .Values.projectName }} + port: + number: 80 + tls: + - hosts: + - local.skilld.cloud + secretName: {{ .Values.projectName }}-tls +{{- end }} diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml new file mode 100644 index 000000000..6930cad3d --- /dev/null +++ b/helm/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.projectName }} +spec: + ports: + - name: http + port: 80 + protocol: TCP + - name: mailhog + port: 8025 + protocol: TCP + selector: + mrn: {{ .Values.projectName }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 000000000..e786346f9 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,7 @@ +# Default values for sdc. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +ingress: + enabled: false + diff --git a/scripts/makefile/front.mk b/scripts/makefile/front.mk index 75b5084b3..6224976e5 100644 --- a/scripts/makefile/front.mk +++ b/scripts/makefile/front.mk @@ -1,48 +1,18 @@ FRONT_PORT?=65200 -# Execute front container function. -frontexec = docker run \ - --rm \ - --init \ - -u $(CUID):$(CGID) \ - -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ - --workdir /app \ - $(IMAGE_FRONT) ${1} - -# Execute front container function on localhost:FRONT_PORT. Needed for dynamic storybook. -frontexec-with-port = docker run \ - --rm \ - --init \ - -p $(FRONT_PORT):$(FRONT_PORT) \ - -u $(CUID):$(CGID) \ - -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ - --workdir /app \ - $(IMAGE_FRONT) ${1} - -# Execute front container with TTY. Needed for storybook components creation. -frontexec-with-interactive = docker run \ - --rm \ - --init \ - -u $(CUID):$(CGID) \ - -v $(CURDIR)/web/themes/custom/$(THEME_NAME):/app \ - --workdir /app \ - -it \ - $(IMAGE_FRONT) ${1} +## Install frontend dependencies & build assets +front: | front-install front-build clear-front: @echo "Clean of node_modules and compiled dist... To skip this action please set CLEAR_FRONT_PACKAGES=no in .env file" - $(call frontexec, rm -rf /app/node_modules /app/dist) - -## Install frontend dependencies & build assets -front: | front-install front-build + $(call frontexec,rm -rf /app/node_modules /app/dist) front-install: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Installing yarn dependencies..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn install --ignore-optional --check-files --prod); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn install --ignore-optional --check-files --prod); \ else \ echo "- Theme directory defined in .env file was not found. Skipping front-install."; \ fi @@ -50,10 +20,9 @@ front-install: front-build: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Building front assets..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn build --stats=verbose); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn build --stats=verbose); \ else \ echo "- Theme directory defined in .env file was not found. Skipping front-build."; \ fi @@ -61,10 +30,9 @@ front-build: lintval: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Running theme linters..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn run lint); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn run lint); \ else \ echo "- Theme directory defined in .env file was not found. Skipping theme linters."; \ fi @@ -72,11 +40,10 @@ lintval: lint: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Running theme linters with fix..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn install --ignore-optional --check-files --prod); \ - $(call frontexec, yarn lint-fix); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn install --ignore-optional --check-files --prod); \ + $(call frontexec,yarn lint-fix); \ else \ echo "- Theme directory defined in .env file was not found. Skipping theme linters with fix."; \ fi @@ -84,12 +51,11 @@ lint: storybook: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Running dynamic storybook..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn install --ignore-optional --check-files); \ - $(call frontexec, yarn run build); \ - $(call frontexec-with-port, yarn storybook -p $(FRONT_PORT)); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn install --ignore-optional --check-files); \ + $(call frontexec,yarn run build); \ + $(call frontexec-with-port,yarn storybook -p $(FRONT_PORT)); \ else \ echo "- Theme directory defined in .env file was not found. Skipping dynamic storybook."; \ fi @@ -97,17 +63,15 @@ storybook: build-storybook: @if [ -d $(CURDIR)/web/themes/custom/$(THEME_NAME) ]; then \ echo "- Theme directory found. Exporting static storybook..."; \ - docker pull $(IMAGE_FRONT); \ - $(call frontexec, node -v); \ - $(call frontexec, yarn -v); \ - $(call frontexec, yarn install --ignore-optional --check-files); \ - $(call frontexec, yarn run build); \ - $(call frontexec, yarn run build-storybook); \ + $(call frontexec,node -v); \ + $(call frontexec,yarn -v); \ + $(call frontexec,yarn install --ignore-optional --check-files); \ + $(call frontexec,yarn run build); \ + $(call frontexec,yarn run build-storybook); \ else \ echo "- Theme directory defined in .env file was not found. Skipping dynamic storybook."; \ fi create-component: @echo "Create component CLI dialog... It assumed that you already have 'make storybook' or 'make build-storybook' finished" - docker pull $(IMAGE_FRONT) - $(call frontexec-with-interactive, yarn cc) + $(call frontexec-with-interactive,yarn cc)