diff --git a/defaults/main.yml b/defaults/main.yml index ff1b125..7902ba4 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,4 +1,5 @@ --- +add_mail_admins_logging_handler: false onadata_system_user: "onadata" onadata_system_group: "{{ onadata_system_user }}" onadata_system_user_home: "/home/{{ onadata_system_user }}" @@ -6,7 +7,7 @@ onadata_system_wide_dependencies: - binutils - libproj-dev - gdal-bin - - memcached + # - memcached - libmemcached-dev - build-essential - git @@ -27,7 +28,7 @@ onadata_system_wide_dependencies: onadata_java_version: 11 onadata_java_package: "openjdk-{{ onadata_java_version }}-jre-headless" onadata_java_home: "/usr/lib/jvm/java-{{ onadata_java_version }}-openjdk-amd64" -onadata_oauth2_pkce_required: False +onadata_oauth2_pkce_required: false onadata_google_flow_scopes: - "https://www.googleapis.com/auth/spreadsheets" - "https://www.googleapis.com/auth/docs" @@ -44,20 +45,21 @@ onadata_python_packages: - python3-pip - python-celery-common - python3-sphinx -onadata_setuptools_version: +onadata_setuptools_version: ~ onadata_pip_git_packages: [] onadata_pip_paths: - "{{ onadata_checkout_path }}/requirements/base.pip" - "{{ onadata_checkout_path }}/requirements/ses.pip" - "{{ onadata_checkout_path }}/requirements/s3.pip" + - "{{ onadata_checkout_path }}/requirements/azure.pip" onadata_python_source_version: "3.10" onadata_python_version: "python{{ onadata_python_source_version }}" onadata_version: "v3.8.6" onadata_git_url: "https://github.com/onaio/onadata.git" onadata_custom_template_git_url: "https://github.com/onaio/onadata-template.git" onadata_enable_custom_templates: false -onadata_git_key_content: -onadata_git_key_ssh_file: +onadata_git_key_content: ~ +onadata_git_key_ssh_file: ~ onadata_git_key_filename: "id_ed25519" onadata_custom_template_version: "master" onadata_service_name: "{{ onadata_system_user }}" @@ -69,17 +71,17 @@ onadata_log_path: "/var/log/{{ onadata_service_name }}" onadata_recreate_virtual_env: false onadata_local_settings_path: "{{ onadata_checkout_path }}/onadata/preset/local_settings.py" onadata_settings_template_path: "onadata_checkout_path/onadata/preset/local_settings.py.j2" -onadata_login_url: +onadata_login_url: ~ onadata_init_commands: [] - # - migrate --noinput - # - collectstatic --noinput + # - migrate --noinput + # - collectstatic --noinput onadata_static_path: "{{ onadata_checkout_path }}/onadata/static/" onadata_media_path: "{{ onadata_system_user_home }}/media" onadata_media_path_mode: "0755" onadata_media_server_port: "443" onadata_media_server_protocol: "https" onadata_media_server_domain: "{{ onadata_domain }}" -onadata_media_url: "{{ onadata_media_server_protocol }}://{{ onadata_media_server_domain }}:{{ onadata_media_server_port }}/media/" +onadata_media_url: "/media/" onadata_settings_module: "onadata.preset.local_settings" onadata_wsgi_module: "onadata.apps.main.wsgi:application" onadata_enable_celery: true @@ -89,12 +91,12 @@ onadata_domain: "example.com" # by a new worker. Value should be in KiB # See: https://docs.celeryproject.org/en/stable/userguide/workers.html#max-memory-per-child-setting onadata_use_celery_5_imports: true -onadata_celery_worker_max_memory: 390625 # 400MB in Kibibyte +onadata_celery_worker_max_memory: 390625 # 400MB in Kibibyte onadata_celeryd_nodes: "{{ onadata_domain }} export-node publish-xls-form-node google-export xlsx-exports csv-exports kml-exports osm-exports csv-zip-exports sav-zip-exports external-exports zip-exports osm-exports exports permissions-async" onadata_celeryd_opts: "-O fair --concurrency=8 --autoscale=6,1 -Q:{{ onadata_domain }} celery -Q:export-node exports -Q:publish-xls-form-node publish_xlsform -Q:google-export google_export -Q:xlsx-exports xlsx_exports -Q:csv-exports csv_exports -Q:kml-exports kml_exports -Q:osm-exports osm-exports -Q:csv-zip-exports csv_zip_exports -Q:sav-zip-exports sav_zip_exports -Q:external-exports external_exports -Q:zip-exports zip_exports -Q:osm-exports osm_exports -Q:exports exports -Q:permissions-async permissions_async --max-memory-per-child={{ onadata_celery_worker_max_memory }}" # Task hard time limit in seconds. Celery kills & replaces a worker processing a task that exceeds set time # See: https://docs.celeryproject.org/en/stable/userguide/configuration.html#std:setting-task_time_limit -onadata_celery_task_time_limit: 3600 # 1 hour +onadata_celery_task_time_limit: 3600 # 1 hour # The maximum number of tasks a worker can execute before it's replaced with a new worker # See: https://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-worker_max_tasks_per_child onadata_celery_worker_max_tasks: 100 @@ -111,15 +113,15 @@ onadata_celerybeat_extra_env_vars: [] onadata_celeryd_extra_env_vars: [] onadata_pgsql_db: "onadata" onadata_pgsql_user: "onadata" -onadata_pgsql_password: +onadata_pgsql_password: ~ onadata_pgsql_host: "127.0.0.1" onadata_pgsql_port: 5432 onadata_disable_server_side_cursors: "False" onadata_pgsql_replicas: [] -onadata_kpi_formbuilder_url: +onadata_kpi_formbuilder_url: ~ onadata_django_secret_key: "secret" onadata_rabbitmq_user: onadata -onadata_rabbitmq_password: +onadata_rabbitmq_password: ~ onadata_rabbitmq_host: 127.0.0.1 onadata_rabbitmq_port: 5672 onadata_celery_broker_url: "amqp://{{ onadata_rabbitmq_user }}:{{ onadata_rabbitmq_password }}@{{ onadata_rabbitmq_host }}:{{ onadata_rabbitmq_port }}/" @@ -130,21 +132,21 @@ onadata_celery_result_backend: "cache+memcached://{{ onadata_memcached_uri }}" onadata_cache_backend: "memcached" onadata_memcached_uri: "127.0.0.1:11211" onadata_redis_url: "redis://127.0.0.1:6379/7" -onadata_aws_access_key: -onadata_aws_secret_key: -onadata_smtp_host: -onadata_smtp_port: -onadata_smtp_login: -onadata_smtp_password: +onadata_aws_access_key: ~ +onadata_aws_secret_key: ~ +onadata_smtp_host: ~ +onadata_smtp_port: ~ +onadata_smtp_login: ~ +onadata_smtp_password: ~ onadata_smtp_use_tls: "True" onadata_smtp_from: "noreply@example.com" # Whether Ona Data is to use AWS's services such as S3 onadata_use_aws: false # Whether OnaData is to use Azure's services such as Blob storage onadata_use_azure: false -onadata_azure_account_name: -onadata_azure_account_key: -onadata_azure_container: +onadata_azure_account_name: ~ +onadata_azure_account_key: ~ +onadata_azure_container: ~ onadata_s3_bucket: "{{ onadata_domain | replace('.', '-') }}-onadata" onadata_s3_region: "eu-west-1" onadata_email_admins: @@ -160,16 +162,16 @@ onadata_cors_expose_headers: [] onadata_use_x_forwarded_host: "False" onadata_use_x_forwarded_port: "False" onadata_csrf_trusted_origins: [] -onadata_google_site_verification: -onadata_google_analytics_property_id: +onadata_google_site_verification: ~ +onadata_google_analytics_property_id: ~ # Whether to run Ona Data in read-only mode (no writes to database) onadata_read_only_mode: "False" -onadata_jwt_secret_key: +onadata_jwt_secret_key: ~ onadata_enketo_auth_cookie_domain: ".example.com" onadata_enketo_login_url: "https://{{ onadata_domain }}/login" onadata_enketo_url: "https://enketo.{{ onadata_domain }}" -onadata_enketo_api_salt: -onadata_enketo_api_token: +onadata_enketo_api_salt: ~ +onadata_enketo_api_token: ~ # Whether Enketo is configured to work offline onadata_enketo_offline: "True" onadata_enketo_auth_cookie: "__enketo" @@ -209,6 +211,9 @@ onadata_celery_task_route: permissions_async: - "onadata.libs.utils.project_utils.set_project_perms_to_xform_async" - "onadata.libs.utils.project_utils.propagate_project_permissions_async" + - "onadata.apps.api.tasks.add_org_user_and_share_projects_async" + - "onadata.apps.api.tasks.remove_org_user_async" + - "onadata.apps.api.tasks.share_project_async" # Cache control maximum age in seconds onadata_cache_control_max_age: 30 @@ -240,10 +245,11 @@ onadata_wsgi_cheaper_overload: 30 # Pricing library onadata_include_pricing: false -onadata_pricing_version: "v0.8.0" -onadata_zoho_auth_token: -onadata_zoho_api_token: -onadata_zoho_org_id: +onadata_load_pricing_account_fixtures: false +onadata_pricing_version: "v2.1.4" +onadata_zoho_auth_token: ~ +onadata_zoho_api_token: ~ +onadata_zoho_org_id: ~ onadata_zoho_api_url: "https://subscriptions.zoho.com/api/v1" # Email Verification @@ -253,9 +259,9 @@ onadata_verification_url: "https://{{ onadata_domain }}/email-verification-confi # Google Integrations (exports) onadata_include_google_export: false -onadata_google_client_id: -onadata_google_client_secret: -onadata_google_project_id: +onadata_google_client_id: ~ +onadata_google_client_secret: ~ +onadata_google_project_id: ~ onadata_google_client_email: "google-integration@{{ onadata_domain|replace('.', '-') }}.iam.gserviceaccount.com" onadata_google_export_version: "v0.8.0" @@ -310,14 +316,14 @@ onadata_odk_key_lifetime: 10 onadata_odk_token_length: 7 # Authentication settings -onadata_api_disable_digest_auth: False +onadata_api_disable_digest_auth: false # Git key # Whether this role should delegate the clean up of the git key to # ansible django or delay the clean up after the installation of the git # packages -onadata_unmanaged_git_key: True -onadata_csp_cookie_secure: True +onadata_unmanaged_git_key: true +onadata_csp_cookie_secure: true onadata_csp_img_src: [] onadata_csp_script_src: [] onadata_csp_style_src: [] @@ -325,12 +331,12 @@ onadata_csp_frame_src: [] onadata_csp_font_src: [] onadata_csp_connect_src: [] onadata_csrf_cookie_samesite: "Strict" -onadata_crf_cookie_secure: True -onadata_crf_use_sessions: True -onadata_session_cookie_secure: True +onadata_crf_cookie_secure: true +onadata_crf_use_sessions: true +onadata_session_cookie_secure: true onadata_session_cookie_samesite: "Strict" # throttling -onadata_enable_requests_throttling: False +onadata_enable_requests_throttling: false onadata_throttling_rate: 50 onadata_throttling_user_agents: [] diff --git a/tasks/init.yml b/tasks/init.yml index ee04643..7227cfb 100644 --- a/tasks/init.yml +++ b/tasks/init.yml @@ -18,4 +18,4 @@ virtualenv: "{{ onadata_venv_path }}" fixtures: "{{ onadata_venv_path }}/src/pricing/pricing/fixtures/initial_accounts.json" settings: "{{ onadata_settings_module }}" - when: onadata_include_pricing \ No newline at end of file + when: onadata_load_pricing_account_fixtures diff --git a/tasks/install.yml b/tasks/install.yml index 2e8fe8c..504ee49 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -25,10 +25,32 @@ when: - not onadata_unmanaged_git_key +- name: Install docs.pip + become: true + become_user: "{{ onadata_system_user }}" + shell: "source {{ onadata_venv_path }}/bin/activate && python -m pip install --no-cache-dir -r requirements/docs.pip" + args: + executable: /bin/bash + chdir: "{{ onadata_codebase_path }}" + - name: Generate Docs become: true become_user: "{{ onadata_system_user }}" - shell: "source {{ onadata_venv_path }}/bin/activate && /usr/bin/make html" + shell: "source {{ onadata_venv_path }}/bin/activate && make html" args: executable: /bin/bash chdir: "{{ onadata_codebase_path }}/docs" + +- name: Fix libsassl erorr + become: true + shell: "ln -sfn /usr/lib/x86_64-linux-gnu/sasl2/ /usr/lib64/sasl2" + args: + executable: /bin/bash + +- name: Add /var/log/celery path + become: true + ansible.builtin.file: + mode: 0755 + path: /var/log/celery + state: directory + owner: "{{ onadata_system_user }}" diff --git a/templates/onadata_checkout_path/onadata/preset/local_settings.py.j2 b/templates/onadata_checkout_path/onadata/preset/local_settings.py.j2 index 44a06e3..cd7cfa6 100644 --- a/templates/onadata_checkout_path/onadata/preset/local_settings.py.j2 +++ b/templates/onadata_checkout_path/onadata/preset/local_settings.py.j2 @@ -164,6 +164,14 @@ AZURE_ACCOUNT_KEY = '{{ onadata_azure_account_key }}' AZURE_CONTAINER = '{{ onadata_azure_container }}' {% endif %} + +{% if DEFAULT_FILE_STORAGE is defined %}: +STORAGES = { + 'default': {'BACKEND': DEFAULT_FILE_STORAGE or 'storages.backends.s3boto3.S3Boto3Storage'}, + 'staticfiles': {'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'} +} +{% endif %} + ADMINS = ( {% for cur_admin in onadata_email_admins %} ('{{ cur_admin.name }}', '{{ cur_admin.email }}'), @@ -257,7 +265,7 @@ if {{ onadata_enketo_offline }}: ENKETO_AUTH_COOKIE = '{{ onadata_enketo_auth_cookie }}' ENKETO_META_UID_COOKIE = '{{ onadata_enketo_meta_uid_cookie }}' -if {{ onadata_use_aws }}: +if {{ DEFAULT_FILE_STORAGE is defined }}: # source http://dryan.me/articles/elb-django-allowed-hosts/ # add ec2 private ip - ensures load balancer has access import requests @@ -437,9 +445,10 @@ from sentry_sdk.integrations.django import DjangoIntegration sentry_sdk.init( dsn='{{ onadata_sentry_dsn }}', integrations=[DjangoIntegration()], - traces_sample_rate=1.0, + traces_sample_rate=0.001, send_default_pii=False, release='{{ onadata_version }}', + environment='{{ onadata_s3_bucket }}' ) RAVEN_CONFIG = { 'dsn': '{{ onadata_sentry_dsn }}', @@ -480,12 +489,14 @@ LOGGING = { }, }, 'handlers': { + {% if add_mail_admins_logging_handler %} 'mail_admins': { 'level': 'ERROR', 'filters': ['require_debug_false', 'skip_suspicious_operations'], 'class': 'django.utils.log.AdminEmailHandler', 'email_backend': '{{ onadata_mail_admins_email_backend }}' }, + {% endif %} 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler',