Add install docs command and update make html command

defaults/main.yml

@@ -1,12 +1,13 @@
 ---
+add_mail_admins_logging_handler: false
 onadata_system_user: "onadata"
 onadata_system_group: "{{ onadata_system_user }}"
 onadata_system_user_home: "/home/{{ onadata_system_user }}"
 onadata_system_wide_dependencies:
   - binutils
   - libproj-dev
   - gdal-bin
-  - memcached
+  # - memcached
   - libmemcached-dev
   - build-essential
   - git
@@ -27,7 +28,7 @@ onadata_system_wide_dependencies:
 onadata_java_version: 11
 onadata_java_package: "openjdk-{{ onadata_java_version }}-jre-headless"
 onadata_java_home: "/usr/lib/jvm/java-{{ onadata_java_version }}-openjdk-amd64"
-onadata_oauth2_pkce_required: False
+onadata_oauth2_pkce_required: false
 onadata_google_flow_scopes:
   - "https://www.googleapis.com/auth/spreadsheets"
   - "https://www.googleapis.com/auth/docs"
@@ -44,20 +45,21 @@ onadata_python_packages:
   - python3-pip
   - python-celery-common
   - python3-sphinx
-onadata_setuptools_version:
+onadata_setuptools_version: ~
 onadata_pip_git_packages: []
 onadata_pip_paths:
   - "{{ onadata_checkout_path }}/requirements/base.pip"
   - "{{ onadata_checkout_path }}/requirements/ses.pip"
   - "{{ onadata_checkout_path }}/requirements/s3.pip"
+  - "{{ onadata_checkout_path }}/requirements/azure.pip"
 onadata_python_source_version: "3.10"
 onadata_python_version: "python{{ onadata_python_source_version }}"
 onadata_version: "v3.8.6"
 onadata_git_url: "https://github.com/onaio/onadata.git"
 onadata_custom_template_git_url: "https://github.com/onaio/onadata-template.git"
 onadata_enable_custom_templates: false
-onadata_git_key_content:
-onadata_git_key_ssh_file:
+onadata_git_key_content: ~
+onadata_git_key_ssh_file: ~
 onadata_git_key_filename: "id_ed25519"
 onadata_custom_template_version: "master"
 onadata_service_name: "{{ onadata_system_user }}"
@@ -69,17 +71,17 @@ onadata_log_path: "/var/log/{{ onadata_service_name }}"
 onadata_recreate_virtual_env: false
 onadata_local_settings_path: "{{ onadata_checkout_path }}/onadata/preset/local_settings.py"
 onadata_settings_template_path: "onadata_checkout_path/onadata/preset/local_settings.py.j2"
-onadata_login_url:
+onadata_login_url: ~
 onadata_init_commands: []
-  # - migrate --noinput
-  # - collectstatic --noinput
+    # - migrate --noinput
+    # - collectstatic --noinput
 onadata_static_path: "{{ onadata_checkout_path }}/onadata/static/"
 onadata_media_path: "{{ onadata_system_user_home }}/media"
 onadata_media_path_mode: "0755"
 onadata_media_server_port: "443"
 onadata_media_server_protocol: "https"
 onadata_media_server_domain: "{{ onadata_domain }}"
-onadata_media_url: "{{ onadata_media_server_protocol }}://{{ onadata_media_server_domain  }}:{{ onadata_media_server_port }}/media/"
+onadata_media_url: "/media/"
 onadata_settings_module: "onadata.preset.local_settings"
 onadata_wsgi_module: "onadata.apps.main.wsgi:application"
 onadata_enable_celery: true
@@ -89,12 +91,12 @@ onadata_domain: "example.com"
 # by a new worker. Value should be in KiB
 # See: https://docs.celeryproject.org/en/stable/userguide/workers.html#max-memory-per-child-setting
 onadata_use_celery_5_imports: true
-onadata_celery_worker_max_memory: 390625 # 400MB in Kibibyte
+onadata_celery_worker_max_memory: 390625  # 400MB in Kibibyte
 onadata_celeryd_nodes: "{{ onadata_domain }} export-node publish-xls-form-node google-export xlsx-exports csv-exports kml-exports osm-exports csv-zip-exports sav-zip-exports external-exports zip-exports osm-exports exports permissions-async"
 onadata_celeryd_opts: "-O fair --concurrency=8 --autoscale=6,1 -Q:{{ onadata_domain }} celery -Q:export-node exports -Q:publish-xls-form-node publish_xlsform -Q:google-export google_export -Q:xlsx-exports xlsx_exports -Q:csv-exports csv_exports -Q:kml-exports kml_exports -Q:osm-exports osm-exports -Q:csv-zip-exports csv_zip_exports -Q:sav-zip-exports sav_zip_exports -Q:external-exports external_exports -Q:zip-exports zip_exports -Q:osm-exports osm_exports -Q:exports exports -Q:permissions-async permissions_async --max-memory-per-child={{ onadata_celery_worker_max_memory }}"
 # Task hard time limit in seconds. Celery kills & replaces a worker processing a task that exceeds set time
 # See: https://docs.celeryproject.org/en/stable/userguide/configuration.html#std:setting-task_time_limit
-onadata_celery_task_time_limit: 3600 # 1 hour
+onadata_celery_task_time_limit: 3600  # 1 hour
 # The maximum number of tasks a worker can execute before it's replaced with a new worker
 # See: https://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-worker_max_tasks_per_child
 onadata_celery_worker_max_tasks: 100
@@ -111,15 +113,15 @@ onadata_celerybeat_extra_env_vars: []
 onadata_celeryd_extra_env_vars: []
 onadata_pgsql_db: "onadata"
 onadata_pgsql_user: "onadata"
-onadata_pgsql_password:
+onadata_pgsql_password: ~
 onadata_pgsql_host: "127.0.0.1"
 onadata_pgsql_port: 5432
 onadata_disable_server_side_cursors: "False"
 onadata_pgsql_replicas: []
-onadata_kpi_formbuilder_url:
+onadata_kpi_formbuilder_url: ~
 onadata_django_secret_key: "secret"
 onadata_rabbitmq_user: onadata
-onadata_rabbitmq_password:
+onadata_rabbitmq_password: ~
 onadata_rabbitmq_host: 127.0.0.1
 onadata_rabbitmq_port: 5672
 onadata_celery_broker_url: "amqp://{{ onadata_rabbitmq_user }}:{{ onadata_rabbitmq_password }}@{{ onadata_rabbitmq_host }}:{{ onadata_rabbitmq_port }}/"
@@ -130,21 +132,21 @@ onadata_celery_result_backend: "cache+memcached://{{ onadata_memcached_uri }}"
 onadata_cache_backend: "memcached"
 onadata_memcached_uri: "127.0.0.1:11211"
 onadata_redis_url: "redis://127.0.0.1:6379/7"
-onadata_aws_access_key:
-onadata_aws_secret_key:
-onadata_smtp_host:
-onadata_smtp_port:
-onadata_smtp_login:
-onadata_smtp_password:
+onadata_aws_access_key: ~
+onadata_aws_secret_key: ~
+onadata_smtp_host: ~
+onadata_smtp_port: ~
+onadata_smtp_login: ~
+onadata_smtp_password: ~
 onadata_smtp_use_tls: "True"
 onadata_smtp_from: "[email protected]"
 # Whether Ona Data is to use AWS's services such as S3
 onadata_use_aws: false
 # Whether OnaData is to use Azure's services such as Blob storage
 onadata_use_azure: false
-onadata_azure_account_name:
-onadata_azure_account_key:
-onadata_azure_container:
+onadata_azure_account_name: ~
+onadata_azure_account_key: ~
+onadata_azure_container: ~
 onadata_s3_bucket: "{{ onadata_domain | replace('.', '-') }}-onadata"
 onadata_s3_region: "eu-west-1"
 onadata_email_admins:
@@ -160,16 +162,16 @@ onadata_cors_expose_headers: []
 onadata_use_x_forwarded_host: "False"
 onadata_use_x_forwarded_port: "False"
 onadata_csrf_trusted_origins: []
-onadata_google_site_verification:
-onadata_google_analytics_property_id:
+onadata_google_site_verification: ~
+onadata_google_analytics_property_id: ~
 # Whether to run Ona Data in read-only mode (no writes to database)
 onadata_read_only_mode: "False"
-onadata_jwt_secret_key:
+onadata_jwt_secret_key: ~
 onadata_enketo_auth_cookie_domain: ".example.com"
 onadata_enketo_login_url: "https://{{ onadata_domain }}/login"
 onadata_enketo_url: "https://enketo.{{ onadata_domain }}"
-onadata_enketo_api_salt:
-onadata_enketo_api_token:
+onadata_enketo_api_salt: ~
+onadata_enketo_api_token: ~
 # Whether Enketo is configured to work offline
 onadata_enketo_offline: "True"
 onadata_enketo_auth_cookie: "__enketo"
@@ -209,6 +211,9 @@ onadata_celery_task_route:
   permissions_async:
     - "onadata.libs.utils.project_utils.set_project_perms_to_xform_async"
     - "onadata.libs.utils.project_utils.propagate_project_permissions_async"
+    - "onadata.apps.api.tasks.add_org_user_and_share_projects_async"
+    - "onadata.apps.api.tasks.remove_org_user_async"
+    - "onadata.apps.api.tasks.share_project_async"
 
 # Cache control maximum age in seconds
 onadata_cache_control_max_age: 30
@@ -240,10 +245,11 @@ onadata_wsgi_cheaper_overload: 30
 
 # Pricing library
 onadata_include_pricing: false
-onadata_pricing_version: "v0.8.0"
-onadata_zoho_auth_token:
-onadata_zoho_api_token:
-onadata_zoho_org_id:
+onadata_load_pricing_account_fixtures: false
+onadata_pricing_version: "v2.1.4"
+onadata_zoho_auth_token: ~
+onadata_zoho_api_token: ~
+onadata_zoho_org_id: ~
 onadata_zoho_api_url: "https://subscriptions.zoho.com/api/v1"
 
 # Email Verification
@@ -253,9 +259,9 @@ onadata_verification_url: "https://{{ onadata_domain }}/email-verification-confi
 
 # Google Integrations (exports)
 onadata_include_google_export: false
-onadata_google_client_id:
-onadata_google_client_secret:
-onadata_google_project_id:
+onadata_google_client_id: ~
+onadata_google_client_secret: ~
+onadata_google_project_id: ~
 onadata_google_client_email: "google-integration@{{ onadata_domain|replace('.', '-') }}.iam.gserviceaccount.com"
 onadata_google_export_version: "v0.8.0"
 
@@ -310,27 +316,27 @@ onadata_odk_key_lifetime: 10
 onadata_odk_token_length: 7
 
 # Authentication settings
-onadata_api_disable_digest_auth: False
+onadata_api_disable_digest_auth: false
 
 # Git key
 # Whether this role should delegate the clean up of the git key to
 # ansible django or delay the clean up after the installation of the git
 # packages
-onadata_unmanaged_git_key: True
-onadata_csp_cookie_secure: True
+onadata_unmanaged_git_key: true
+onadata_csp_cookie_secure: true
 onadata_csp_img_src: []
 onadata_csp_script_src: []
 onadata_csp_style_src: []
 onadata_csp_frame_src: []
 onadata_csp_font_src: []
 onadata_csp_connect_src: []
 onadata_csrf_cookie_samesite: "Strict"
-onadata_crf_cookie_secure: True
-onadata_crf_use_sessions: True
-onadata_session_cookie_secure: True
+onadata_crf_cookie_secure: true
+onadata_crf_use_sessions: true
+onadata_session_cookie_secure: true
 onadata_session_cookie_samesite: "Strict"
 
 # throttling
-onadata_enable_requests_throttling: False
+onadata_enable_requests_throttling: false
 onadata_throttling_rate: 50
 onadata_throttling_user_agents: []

tasks/init.yml

@@ -18,4 +18,4 @@
     virtualenv: "{{ onadata_venv_path }}"
     fixtures: "{{ onadata_venv_path }}/src/pricing/pricing/fixtures/initial_accounts.json"
     settings: "{{ onadata_settings_module }}"
-  when: onadata_include_pricing
+  when: onadata_load_pricing_account_fixtures

tasks/install.yml

@@ -25,10 +25,32 @@
   when:
     - not onadata_unmanaged_git_key
 
+- name: Install docs.pip
+  become: true
+  become_user: "{{ onadata_system_user }}"
+  shell: "source {{ onadata_venv_path }}/bin/activate && python -m pip install --no-cache-dir -r requirements/docs.pip"
+  args:
+    executable: /bin/bash
+    chdir: "{{ onadata_codebase_path }}"
+
 - name: Generate Docs
   become: true
   become_user: "{{ onadata_system_user }}"
-  shell: "source {{ onadata_venv_path }}/bin/activate && /usr/bin/make html"
+  shell: "source {{ onadata_venv_path }}/bin/activate && make html"
   args:
     executable: /bin/bash
     chdir: "{{ onadata_codebase_path }}/docs"
+
+- name: Fix libsassl erorr
+  become: true
+  shell: "ln -sfn /usr/lib/x86_64-linux-gnu/sasl2/ /usr/lib64/sasl2"
+  args:
+    executable: /bin/bash
+
+- name: Add /var/log/celery path
+  become: true
+  ansible.builtin.file:
+    mode: 0755
+    path: /var/log/celery
+    state: directory
+    owner: "{{ onadata_system_user }}"

templates/onadata_checkout_path/onadata/preset/local_settings.py.j2

@@ -164,6 +164,14 @@ AZURE_ACCOUNT_KEY = '{{ onadata_azure_account_key }}'
 AZURE_CONTAINER = '{{ onadata_azure_container }}'
 {% endif %}
 
+
+{% if DEFAULT_FILE_STORAGE is defined %}:
+STORAGES = {
+      'default': {'BACKEND': DEFAULT_FILE_STORAGE or 'storages.backends.s3boto3.S3Boto3Storage'},
+      'staticfiles': {'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'}
+}
+{% endif %}
+
 ADMINS = (
 {% for cur_admin in onadata_email_admins %}
     ('{{ cur_admin.name }}', '{{ cur_admin.email }}'),
@@ -257,7 +265,7 @@ if {{ onadata_enketo_offline }}:
 ENKETO_AUTH_COOKIE = '{{ onadata_enketo_auth_cookie }}'
 ENKETO_META_UID_COOKIE = '{{ onadata_enketo_meta_uid_cookie }}'
 
-if {{ onadata_use_aws }}:
+if {{ DEFAULT_FILE_STORAGE is defined }}:
     # source http://dryan.me/articles/elb-django-allowed-hosts/
     # add ec2 private ip - ensures load balancer has access
     import requests
@@ -437,9 +445,10 @@ from sentry_sdk.integrations.django import DjangoIntegration
 sentry_sdk.init(
     dsn='{{ onadata_sentry_dsn }}',
     integrations=[DjangoIntegration()],
-    traces_sample_rate=1.0,
+    traces_sample_rate=0.001,
     send_default_pii=False,
     release='{{ onadata_version }}',
+    environment='{{ onadata_s3_bucket }}'
 )
 RAVEN_CONFIG = {
     'dsn': '{{ onadata_sentry_dsn }}',
@@ -480,12 +489,14 @@ LOGGING = {
         },
     },
     'handlers': {
+        {% if add_mail_admins_logging_handler %}
         'mail_admins': {
             'level': 'ERROR',
             'filters': ['require_debug_false', 'skip_suspicious_operations'],
             'class': 'django.utils.log.AdminEmailHandler',
             'email_backend': '{{ onadata_mail_admins_email_backend }}'
         },
+        {% endif %}
         'console': {
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',