From 29d371ffd855969f4d5b2a1396d04cc0c1b73c1b Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Wed, 9 Mar 2022 15:53:14 +0100 Subject: [PATCH 1/4] Set correct ip in x-forwarded-for header --- .../templates/vcl/subroutines/recv.vcl.j2 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 index d8ac8731..7f0d7c22 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 @@ -91,9 +91,16 @@ if (req.method != "GET" && {% endif %} if (req.restarts == 0) { - if (req.http.x-forwarded-for) { - # leave only real client ip - always first ip - set req.http.X-Forwarded-For = regsub(req.http.X-Forwarded-For, "^([^,]+),?.*$", "\1"); + # Trust headers only when request is from localhost (nginx) + if (client.ip == "127.0.0.1") { + if (req.http.x-real-ip) { + set req.http.X-Forwarded-For = req.http.x-real-ip; + } else if (req.http.x-forwarded-for) { + # leave only real client ip - always last ip omiting 127.0.0.1 as it nginx address added by varnish + set req.http.X-Forwarded-For = regsub(req.http.X-Forwarded-For, "([^ ,]+)(, ?127\.0\.0\.1)?$", "\1"); + }else { + set req.http.X-Forwarded-For = client.ip; + } } else { set req.http.X-Forwarded-For = client.ip; } From b4079ec90b45b6e767d3b02d4ff3544e1219af18 Mon Sep 17 00:00:00 2001 From: Filip Sobalski Date: Wed, 13 Jan 2021 15:28:48 +0100 Subject: [PATCH 2/4] [SERVICE] [FEATURE] Shlink link shortener deployment --- group_vars/service.shlink.defaults.yml | 0 inventory/aws_ec2.yml | 20 ++- roles/cs.shlink/defaults/main.yml | 115 +++++++++++++ roles/cs.shlink/handlers/main.yml | 9 ++ roles/cs.shlink/tasks/main.yml | 195 +++++++++++++++++++++++ roles/cs.shlink/templates/shlink.service | 55 +++++++ roles/cs.shlink/templates/shlink.sh | 23 +++ roles/cs.shlink/vars/main.yml | 28 ++++ site.maintenance.service.shlink.yml | 121 ++++++++++++++ 9 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 group_vars/service.shlink.defaults.yml create mode 100644 roles/cs.shlink/defaults/main.yml create mode 100644 roles/cs.shlink/handlers/main.yml create mode 100644 roles/cs.shlink/tasks/main.yml create mode 100644 roles/cs.shlink/templates/shlink.service create mode 100644 roles/cs.shlink/templates/shlink.sh create mode 100644 roles/cs.shlink/vars/main.yml create mode 100644 site.maintenance.service.shlink.yml diff --git a/group_vars/service.shlink.defaults.yml b/group_vars/service.shlink.defaults.yml new file mode 100644 index 00000000..e69de29b diff --git a/inventory/aws_ec2.yml b/inventory/aws_ec2.yml index f5f0acb1..5f5b2c46 100644 --- a/inventory/aws_ec2.yml +++ b/inventory/aws_ec2.yml @@ -23,6 +23,14 @@ keyed_groups: prefix: "app" separator: "_" + - key: tags.Environment + prefix: "" + separator: "" + + - key: tags.Project + prefix: "" + separator: "" + - key: tags.Role prefix: "" separator: "" @@ -65,4 +73,14 @@ keyed_groups: - key: tags.TraitImmutable prefix: "" - separator: "" \ No newline at end of file + separator: "" + + - key: tags.RoleService + prefix: "" + separator: "" + parent_group: "service" + + - key: tags.ServiceInstance + prefix: "" + separator: "" + parent_group: "service" \ No newline at end of file diff --git a/roles/cs.shlink/defaults/main.yml b/roles/cs.shlink/defaults/main.yml new file mode 100644 index 00000000..354cb4cb --- /dev/null +++ b/roles/cs.shlink/defaults/main.yml @@ -0,0 +1,115 @@ +# The suffix used for naming all instance resources +# Set it if deploying multiple instances on the same host +shlink_instance: ~ + +# Use https +shlink_short_domain_https: yes + +# The domain name that the app is hosted at +shlink_short_domain: ~ + +# Base URL path of the app +shlink_base_path: '' + +# Redirect to this URL on access to the base path +shlink_base_url_redirect_to: ~ + +# Redirect to this URL when shortcode is not in the DB +shlink_missing_url_redirect_to: ~ + +# Redirect to this URL when target URL is invalid +shlink_404_redirect_to: ~ + +# Redirect code - 301 or 302 +shlink_redirect_status_code: 302 + +# How long to keep redirs in cache +shlink_redirect_cache_lifetime: 30 + +# Minimum amount of visits needed to keep the shortened entry +shlink_delete_short_url_threshold: 3 + +# Initial length of short codes +shlink_shortcode_length: 4 + +# Whether to validate target URLs +shlink_validate_url: no + +# Disable tracking when this query parameter is present +shlink_disable_track_param: _shlnt + +# Whether to anonymize remote IPs +# Warning! If disabled then you must ensure GDPR compliance somehow which +# might not be possible legally without displaying some opt-in, so better +# to keep anonymization enabled at all times. +shlink_anonymize_remote_addr: yes + +# Number of workers +shlink_web_worker_num: 16 +shlink_task_worker_num: 16 + +# Target docker container tag +shlink_version: 2.4.2 + +# GeoLite DB license key +shlink_geolite_license_key: ~ + +# If defined then an external MySQL DB will be used +shlink_mysql_db_host: ~ +shlink_mysql_db_name: shlink +shlink_mysql_db_user: shlink +shlink_mysql_db_pass: ~ +shlink_mysql_db_port: 3306 + +# Use local SQLite DB if MySQL DB hostname is not set +shlink_sqlite_db: "{{ not (shlink_mysql_db_host | default(false, true)) }}" + +shlink_user: "shlink{{ shlink_instance_suffix }}" +shlink_group: "{{ shlink_user }}" +shlink_home: "/home/{{ shlink_user }}" + +# Port bound to the host - not that container is ran rootless +# you cannot bind to ports < 1024! This is a backend only so +# you need a reverse-proxy in front anyway. +shlink_http_bind_port: 8080 + +# The IP to publish the http port to +shlink_http_bind_host: 0.0.0.0 + +# The internal port exposed by the container +shlink_http_container_port: 8888 + +# The shlink backend configuration build. See reference: +# - Env vars described: https://shlink.io/documentation/install-docker-image/#supported-env-vars +# - Available JSON params: https://shlink.io/documentation/install-docker-image/#provide-config-via-volumes +shlink_config_default: + anonymize_remote_addr: "{{ shlink_anonymize_remote_addr }}" + + disable_track_param: "{{ shlink_disable_track_param }}" + + short_domain_host: "{{ shlink_short_domain }}" + short_domain_schema: "{{ shlink_short_domain_https | ternary('https', 'http') }}" + + validate_url: "{{ shlink_validate_url }}" + + base_path: "{{ shlink_base_path }}" + base_url_redirect_to: "{{ shlink_base_url_redirect_to }}" + invalid_short_url_redirect_to: "{{ shlink_missing_url_redirect_to }}" + regular_404_redirect_to: "{{ shlink_404_redirect_to }}" + + web_worker_num: "{{ shlink_web_worker_num }}" + task_worker_num: "{{ shlink_task_worker_num }}" + default_short_codes_length: "{{ shlink_shortcode_length }}" + geolite_license_key: "{{ shlink_geolite_license_key }}" + + redirect_status_code: "{{ shlink_redirect_status_code }}" + redirect_cache_lifetime: "{{ shlink_redirect_cache_lifetime }}" + + delete_short_url_threshold: "{{ shlink_delete_short_url_threshold }}" + + port: "{{ shlink_http_container_port }}" + + db_config: "{{ shlink_sqlite_db | ternary(shlink_db_config_sqlite, shlink_db_config_mysql) | dict2items | rejectattr('value', 'none') | list | items2dict }}" + +# Extra parameters to override / add to the default JSON config +shlink_config_custom: {} diff --git a/roles/cs.shlink/handlers/main.yml b/roles/cs.shlink/handlers/main.yml new file mode 100644 index 00000000..34b03629 --- /dev/null +++ b/roles/cs.shlink/handlers/main.yml @@ -0,0 +1,9 @@ +- name: Reload systemctl daemon + systemd: + daemon_reload: yes + daemon_reexec: yes + +- name: Restart shlink + service: + name: "{{ shlink_service_name }}.service" + state: restarted diff --git a/roles/cs.shlink/tasks/main.yml b/roles/cs.shlink/tasks/main.yml new file mode 100644 index 00000000..f2a95f32 --- /dev/null +++ b/roles/cs.shlink/tasks/main.yml @@ -0,0 +1,195 @@ +### +# *** Installs shlink URL shortener *** +# +# The app is running as a rootless container as user `shlink_user`. +# All app data is stored in these user's home directory. +# +# No HTTPS termination is provided - this you must provide an HTTPS-terminating +# reverse proxy to this instance's `shlink_http_bind_port`. +### + +- name: Install packages for running rootless podman containers + yum: + state: present + name: + - podman + - slirp4netns + - fuse + - fuse-overlayfs + +- name: Enable user namespaces via sysctl + sysctl: + state: present + reload: yes + name: user.max_user_namespaces + value: "28633" + +- name: Create user + user: + state: present + name: "{{ shlink_user }}" + group: "{{ shlink_group }}" + home: "{{ shlink_home }}" + move_home: yes + +- name: Create group + group: + name: "{{ shlink_group }}" + +- name: Create user + user: + state: present + name: "{{ shlink_user }}" + group: "{{ shlink_group }}" + home: "{{ shlink_home }}" + +- name: Create basic dirs + file: + state: directory + path: "{{ item }}" + mode: "0755" + owner: "{{ shlink_user }}" + group: "{{ shlink_group }}" + loop: + - "{{ shlink_conf_volume_src }}" + - "{{ shlink_home }}/bin" + +- name: Create data dirs + file: + state: directory + path: "{{ shlink_data_volume_src }}/{{ item }}" + mode: "0750" + owner: "{{ shlink_user }}" + group: "{{ shlink_group }}" + loop: "{{ shlink_data_mounts }}" + +- name: Install CLI wrapper script + template: + src: shlink.sh + dest: "{{ shlink_home }}/.local/bin/shlink" + mode: "0755" + owner: "{{ shlink_user }}" + group: "{{ shlink_group }}" + +- name: Add user local bindir to PATH + lineinfile: + state: present + create: yes + dest: "{{ shlink_home }}/{{ item }}" + line: "export PATH=\"{{ shlink_home }}/.local/bin/:$PATH\"" + mode: "0644" + owner: "{{ shlink_user }}" + group: "{{ shlink_group }}" + loop: + - .profile + - .bash_profile + - .zsh_profile + + +- name: Compute effective backend config + set_fact: + shlink_json_config: >- + {{ + shlink_config_default + | combine( + shlink_config_custom | default({}, true) + ) + | dict2items + | rejectattr('value', 'none') | list + | items2dict + }} + +- name: Print backend configuration + debug: + msg: | + ============================================ + = Effective Backend Configuration = + ============================================ + + {{ shlink_json_config | to_nice_yaml }} + +- name: Configure backend app + copy: + dest: "{{ shlink_conf_volume_src }}/shlink.config.json" + content: "{{ shlink_json_config | to_nice_json }}" + owner: "{{ shlink_user }}" + group: "{{ shlink_group }}" + mode: "0640" + notify: Restart shlink + +- name: Configure backend systemd service + template: + src: shlink.service + dest: "/etc/systemd/system/{{ shlink_service_name }}.service" + register: shlink_service_configure + notify: Restart shlink + +- name: Reload systemctl daemon + when: shlink_service_configure is changed + systemd: + daemon_reload: yes + daemon_reexec: yes + +- name: Enable and start backend systemd service + systemd: + force: yes + enabled: yes + state: started + name: "{{ shlink_service_name }}.service" + +- meta: flush_handlers + +- name: Generate master API key + shell: >- + {{ shlink_home }}/.local/bin/shlink \ + api-key:gen \ + --no-ansi \ + --no-interaction \ + | grep "key:" \ + | sed -E 's/^.*"([a-f0-9_-]+)".*$/\1/g' \ + | tee {{ shlink_home }}/.shlink-api-key + args: + creates: "{{ shlink_home }}/.shlink-api-key" + register: shlink_gen_master_key + become: yes + become_user: "{{ shlink_user }}" + +- name: Print master API key + when: shlink_gen_master_key is changed + debug: + msg: | + ============================================ + = !!! Auto-generated Master API Key !!! = + ============================================ + + This key is generated only on initial setup + and stored in: + {{ shlink_home }}/.shlink-api-key + + It will not be shown again. + + Key: {{ shlink_gen_master_key.stdout }} + + +- name: Print installation info + debug: + msg: | + ============================================ + = Deployment Maintenance Information = + ============================================ + + You can access the shlink CLI by logging in + as `{{shlink_user}}` user with `shlink` command. + + You can also do this directly via SSH: + $ ssh {{shlink_user}}@{{ansible_host}} shlink list + + You can directly access the podman container only + as the `{{shlink_user}}` user: + $ podman container logs {{shlink_container_name}} + + Manage the service as `root` user via systemd: + $ systemctl status {{shlink_service_name}} + + Tail the container logs: + $ journalctl -u {{shlink_service_name}} -f \ No newline at end of file diff --git a/roles/cs.shlink/templates/shlink.service b/roles/cs.shlink/templates/shlink.service new file mode 100644 index 00000000..ef6f3edf --- /dev/null +++ b/roles/cs.shlink/templates/shlink.service @@ -0,0 +1,55 @@ +[Unit] +Description=shlink URL shortener {% if shlink_instance | default(false, true) %} at {{ shlink_instance }}{% endif %} +After=syslog.target network.target +Wants=network-online.target + +[Service] +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=shlink + +User={{ shlink_user }} +Group={{ shlink_group }} +WorkingDirectory={{ shlink_home }} + +Type=simple + +# Try to update image, pulls can fail due to docker.io rate-limits when not logged in +ExecStartPre=-/usr/bin/podman \ + pull \ + --quiet \ + {{ shlink_container_image }} + +# Work around podman bug which sometimes thinks exited container is still running +ExecStartPre=-/usr/bin/podman \ + rm \ + --force \ + {{ shlink_container_name }} + +ExecStart=/usr/bin/podman \ + run \ + --rm \ + --pull missing \ + --volume {{ shlink_conf_volume_src }}:{{ shlink_conf_mountpoint }} \ + {% for mount in shlink_data_mounts -%} + --volume {{ shlink_data_volume_src }}/{{ mount }}:{{ shlink_data_mountpoint_base }}/{{ mount }} \ + {% endfor -%} + --publish {{ shlink_http_bind_host }}:{{ shlink_http_bind_port }}:{{ shlink_http_container_port }} \ + --name {{ shlink_container_name }} \ + {{ shlink_container_image }} + +# Clean-up any old image versions +ExecStartPost=-/usr/bin/podman \ + image prune + +ExecStop=/usr/bin/podman \ + stop \ + --timeout 15 \ + {{ shlink_container_name }} + +Restart=on-failure +RestartSec=30 +TimeoutStopSec=20 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/cs.shlink/templates/shlink.sh b/roles/cs.shlink/templates/shlink.sh new file mode 100644 index 00000000..d309fd85 --- /dev/null +++ b/roles/cs.shlink/templates/shlink.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +### +# This is a wrapper to conveniently use shlink cli from outside the container +### + +/usr/bin/podman exec \ + --interactive \ + --tty \ + "{{ shlink_container_name }}" \ + shlink "$@" + +code=$? + +if [ $code -ne 0 ] ; then + if [ $code -eq 125 ] ; then + echo -e "" >&2 + echo -e "-> Before running this command ensure shlink systemd service is running!" >&2 + echo -e "-> Current shlink service status: \n\n$(systemctl status shlink 2>&1 || true)" >&2 + fi + + exit $code +fi \ No newline at end of file diff --git a/roles/cs.shlink/vars/main.yml b/roles/cs.shlink/vars/main.yml new file mode 100644 index 00000000..ebf7b9de --- /dev/null +++ b/roles/cs.shlink/vars/main.yml @@ -0,0 +1,28 @@ +shlink_instance_suffix: "{{ shlink_instance | default(false, true) | ternary('_' ~ shlink_instance, '') }}" +shlink_service_name: "shlink{{ shlink_instance_suffix }}" + +shlink_container_name: "shlink{{ shlink_instance_suffix }}" +shlink_container_image: "docker.io/shlinkio/shlink:{{ shlink_version }}" + +shlink_conf_volume_src: "{{ shlink_home }}/conf" +shlink_conf_mountpoint: "/etc/shlink/config/params" + +shlink_sqlite_dbdir: persistent + +shlink_data_volume_src: "{{ shlink_home }}/data" +shlink_data_mountpoint_base: "/etc/shlink/data" +shlink_data_mounts: + - "{{ shlink_sqlite_dbdir }}" + - log + +shlink_db_config_mysql: + driver: pdo_mysql + host: "{{ shlink_mysql_db_host }}" + dbname: "{{ shlink_mysql_db_name }}" + user: "{{ shlink_mysql_db_user }}" + password: "{{ shlink_mysql_db_pass }}" + port: "{{ shlink_mysql_db_port }}" + +shlink_db_config_sqlite: + driver: pdo_sqlite + path: "{{ shlink_data_mountpoint_base }}/{{ shlink_sqlite_dbdir }}/database.sqlite" \ No newline at end of file diff --git a/site.maintenance.service.shlink.yml b/site.maintenance.service.shlink.yml new file mode 100644 index 00000000..d77d4afc --- /dev/null +++ b/site.maintenance.service.shlink.yml @@ -0,0 +1,121 @@ +### +# *** This playbook deploys `shlink` URL shortener *** +# +# You must provide values for at least these vars in your project config: +# shlink_short_domain: +# shlink_geolite_license_key: +# +# Refer to `cs.shlink` role defaults for further configuration options. +### + +- hosts: all + connection: local + gather_facts: no + tags: always + tasks: + - name: Create group targeting our specific host + group_by: + key: shlink_service_node + when: >- + inventory_hostname + in ( + groups['shlink'] | default([]) + | intersect(groups[mageops_project] | default([])) + | intersect(groups[mageops_environment] | default([])) + ) + +- hosts: shlink_service_node + gather_facts: yes + vars: + ### Base shlink MageSuite config + + shlink_user: shlink + shlink_group: shlink + shlink_home: "/home/{{ shlink_user }}" + shlink_http_bind_port: 8080 + shlink_http_bind_host: 127.0.0.1 + + ### Base MageOps params + + mageops_app_type: "shlink" + mageops_app_user: "{{ shlink_user }}" + mageops_app_group: "{{ shlink_group }}" + mageops_node_role: "{{ mageops_app_type }}" + + ### Base system config + + packages_install: "{{ mageops_packages_common }}" + + firewall_enable: not (aws_use | default(false)) + firewall_public_services: + - http + - https + + nginx_blacklist_vhost_check_include_file: ~ + + https_termination_hosts: + - name: "{{ shlink_short_domain }}" + server_name: "{{ shlink_short_domain }}" + letsencrypt: yes + https_termination_proxy_http_port: yes + + https_termination_upstream_host: 127.0.0.1 + https_termination_upstream_port: "{{ shlink_http_bind_port }}" + https_termination_upstream: "{{ https_termination_upstream_host }}:{{ https_termination_upstream_port }}" + + https_termination_crt_acme_email: filip.sobalski@creativestyle.pl + + pre_tasks: + - name: Automatically configure MySQL host as RDS instance + when: >- + aws_use + and shlink_mysql_db_name | default(false, true) + and shlink_mysql_db_pass | default(false, true) + and not shlink_mysql_db_host | default(false, true) + delegate_to: localhost + delegate_facts: no + become: no + block: + - name: Compute AWS rds facts + include_role: + name: cs.aws-rds-facts + + - name: Set shlink MySQL host + set_fact: + shlink_mysql_db_host: "{{ aws_rds_instance_host }}" + + roles: + - role: pinkeen.selinux-disable + - role: geerlingguy.ntp + - role: cs.switch-to-dnf + - role: cs.cron + - role: cs.swap + - role: cs.earlyoom + - role: cs.packages + - role: cs.provisioning-migrations + + - role: cs.firewalld + when: firewall_enable + + - role: cs.mageops-cli-user + mageops_cli_user: root + + - role: cs.mageops-cli-user + mageops_cli_user: "{{ mageops_app_user }}" + mageops_cli_user_group: "{{ mageops_app_group }}" + mageops_cli_user_uid: "{{ mageops_app_uid }}" + mageops_cli_user_gid: "{{ mageops_app_gid }}" + + - role: cs.mageops-cli-profile + - role: cs.mageops-authorize-keys + mageops_ssh_authorize_app: yes + + - role: cs.monitoring + monitoring_node_exporter_enabled: yes + + - role: cs.shlink + - role: cs.nginx-https-termination + + + + From 01550f3dda1d913ac6ce3e4e3649e8187205d387 Mon Sep 17 00:00:00 2001 From: Filip Sobalski Date: Tue, 16 Mar 2021 15:18:45 +0100 Subject: [PATCH 3/4] [shlink] Add option to configure multiple domains --- site.maintenance.service.shlink.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/site.maintenance.service.shlink.yml b/site.maintenance.service.shlink.yml index d77d4afc..4561b693 100644 --- a/site.maintenance.service.shlink.yml +++ b/site.maintenance.service.shlink.yml @@ -53,10 +53,8 @@ nginx_blacklist_vhost_check_include_file: ~ - https_termination_hosts: - - name: "{{ shlink_short_domain }}" - server_name: "{{ shlink_short_domain }}" - letsencrypt: yes + https_termination_hosts: [] + https_termination_proxy_http_port: yes https_termination_upstream_host: 127.0.0.1 @@ -66,6 +64,22 @@ https_termination_crt_acme_email: filip.sobalski@creativestyle.pl pre_tasks: + - name: Initialize short domain list + set_fact: + shlink_domains: "{{ [shlink_short_domain] + shlink_short_domains_extra | default([]) }}" + + - name: Compute vhost configuration + set_fact: + https_termination_hosts: "{{ https_termination_hosts + [host] }}" + vars: + host: + name: "{{ domain }}" + server_name: "{{ domain }}" + letsencrypt: yes + loop: "{{ shlink_domains }}" + loop_control: + loop_var: domain + - name: Automatically configure MySQL host as RDS instance when: >- aws_use From a7cc9593136c79c415df2db00509db1768f96fe4 Mon Sep 17 00:00:00 2001 From: Filip Sobalski Date: Tue, 11 May 2021 14:02:22 +0200 Subject: [PATCH 4/4] Bump shlink version to current latest (2.4.2 => 2.6.2) --- roles/cs.shlink/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cs.shlink/defaults/main.yml b/roles/cs.shlink/defaults/main.yml index 354cb4cb..bfc9ddbc 100644 --- a/roles/cs.shlink/defaults/main.yml +++ b/roles/cs.shlink/defaults/main.yml @@ -49,7 +49,7 @@ shlink_web_worker_num: 16 shlink_task_worker_num: 16 # Target docker container tag -shlink_version: 2.4.2 +shlink_version: 2.6.2 # GeoLite DB license key shlink_geolite_license_key: ~