[ENG-8513] No DB transaction Notifications

.github/workflows/test-build.yml

@@ -11,6 +11,7 @@ env:
   ELASTICSEARCH6_ARCHIVE: elasticsearch-6.3.1.tar.gz
   OSF_DB_PORT: 5432
   OSF_DB_PASSWORD: postgres
+  GITHUB_ACTIONS: true
 
 jobs:
   build-cache:
@@ -167,3 +168,36 @@ jobs:
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
+
+  mailhog:
+    runs-on: ubuntu-22.04
+    permissions:
+      checks: write
+    needs: build-cache
+    services:
+      postgres:
+        image: postgres
+
+        env:
+          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+      mailhog:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ./.github/actions/start-build
+    - name: Run tests
+      run: poetry run python3 -m invoke test-ci-mailhog -n 1 --junit
+    - name: Upload report
+      if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+      uses: ./.github/actions/gen-report

README-docker-compose.md

@@ -271,10 +271,8 @@
       docker compose run --rm web python3 -m scripts.parse_citation_styles
       ```
 - Start ember_osf_web
-  - Needed for quickfiles feature:
-      ```bash
-      docker compose up -d ember_osf_web
-      ```
+  - Needed for ember app:
+    - `docker-compose up -d ember_osf_web`
 - OPTIONAL: Register OAuth Scopes
   - Needed for things such as the ember-osf dummy app
       ```bash

addons/base/views.py

@@ -21,7 +21,6 @@
 
 from addons.base import exceptions as addon_errors
 from addons.base.models import BaseStorageAddon
-from addons.osfstorage.models import OsfStorageFile
 from addons.osfstorage.models import OsfStorageFileNode
 from addons.osfstorage.utils import enqueue_update_analytics
 
@@ -34,9 +33,7 @@
 from framework.exceptions import HTTPError
 from framework.flask import redirect
 from framework.sentry import log_exception
-from framework.routing import proxy_url
 from framework.transactions.handlers import no_auto_transaction
-from website import mails
 from website import settings
 from addons.base import signals as file_signals
 from addons.base.utils import format_last_known_metadata, get_mfr_url
@@ -54,7 +51,7 @@
     DraftRegistration,
     Guid,
     FileVersionUserMetadata,
-    FileVersion
+    FileVersion, NotificationType
 )
 from osf.metrics import PreprintView, PreprintDownload
 from osf.utils import permissions
@@ -483,7 +480,7 @@ def _construct_payload(auth, resource, credentials, waterbutler_settings):
 
 @must_be_signed
 @no_auto_transaction
-@must_be_valid_project(quickfiles_valid=True, preprints_valid=True)
+@must_be_valid_project(preprints_valid=True)
 def create_waterbutler_log(payload, **kwargs):
     with transaction.atomic():
         try:
@@ -578,20 +575,24 @@ def create_waterbutler_log(payload, **kwargs):
                     params=payload
                 )
 
-            if payload.get('email') is True or payload.get('errors'):
-                mails.send_mail(
-                    user.username,
-                    mails.FILE_OPERATION_FAILED if payload.get('errors')
-                    else mails.FILE_OPERATION_SUCCESS,
-                    action=payload['action'],
-                    source_node=source_node,
-                    destination_node=destination_node,
-                    source_path=payload['source']['materialized'],
-                    source_addon=payload['source']['addon'],
-                    destination_addon=payload['destination']['addon'],
-                    osf_support_email=settings.OSF_SUPPORT_EMAIL
+            if payload.get('email') or payload.get('errors'):
+                if payload.get('email'):
+                    notification_type = NotificationType.Type.FILE_OPERATION_SUCCESS
+                if payload.get('errors'):
+                    notification_type = NotificationType.Type.FILE_OPERATION_FAILED
+
+                NotificationType.objects.get(name=notification_type).emit(
+                    user=user,
+                    event_context={
+                        'action': payload['action'],
+                        'source_node': source_node,
+                        'destination_node': destination_node,
+                        'source_path': payload['source']['materialized'],
+                        'source_addon': payload['source']['addon'],
+                        'destination_addon': payload['destination']['addon'],
+                        'osf_support_email': settings.OSF_SUPPORT_EMAIL
+                    }
                 )
-
             if payload.get('errors'):
                 # Action failed but our function succeeded
                 # Bail out to avoid file_signals
@@ -603,22 +604,52 @@ def create_waterbutler_log(payload, **kwargs):
     metadata = payload.get('metadata') or payload.get('destination')
 
     target_node = AbstractNode.load(metadata.get('nid'))
-    if target_node and not target_node.is_quickfiles and payload['action'] != 'download_file':
+    if target_node and payload['action'] != 'download_file':
         update_storage_usage_with_size(payload)
 
     with transaction.atomic():
-        file_signals.file_updated.send(target=node, user=user, event_type=action, payload=payload)
+        file_signals.file_updated.send(target=node, user=user, payload=payload)
+
+    with transaction.atomic():
+        match action:
+            case NotificationType.Type.FILE_ADDED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.FILE_ADDED)
+            case NotificationType.Type.FILE_REMOVED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.FILE_REMOVED)
+            case NotificationType.Type.FILE_UPDATED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.FILE_UPDATED)
+            case NotificationType.Type.ADDON_FILE_RENAMED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.ADDON_FILE_RENAMED)
+            case NotificationType.Type.ADDON_FILE_COPIED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.ADDON_FILE_COPIED)
+            case NotificationType.Type.ADDON_FILE_REMOVED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.ADDON_FILE_REMOVED)
+            case NotificationType.Type.ADDON_FILE_MOVED:
+                notification = NotificationType.objects.get(name=NotificationType.Type.ADDON_FILE_MOVED)
+            case _:
+                raise NotImplementedError(f'action {action} not implemented')
+
+        notification.emit(
+            user=user,
+            event_context={
+                'profile_image_url': user.profile_image_url(),
+                'localized_timestamp': str(timezone.now()),
+                'user_fullname': user.fullname,
+                'url': node.absolute_url,
+            }
+        )
 
     return {'status': 'success'}
 
 
 @file_signals.file_updated.connect
-def addon_delete_file_node(self, target, user, event_type, payload):
+def addon_delete_file_node(self, target, user, payload):
     """ Get addon BaseFileNode(s), move it into the TrashedFileNode collection
     and remove it from StoredFileNode.
     Required so that the guids of deleted addon files are not re-pointed when an
     addon file or folder is moved or renamed.
     """
+    event_type = payload['action']
     if event_type == 'file_removed' and payload.get('provider', None) != 'osfstorage':
         provider = payload['provider']
         path = payload['metadata']['path']
@@ -1006,13 +1037,17 @@ def persistent_file_download(auth, **kwargs):
     file = BaseFileNode.active.filter(_id=id_or_guid).first()
     if not file:
         guid = Guid.load(id_or_guid)
-        if guid:
-            file = guid.referent
-        else:
+        if not guid:
             raise HTTPError(http_status.HTTP_404_NOT_FOUND, data={
                 'message_short': 'File Not Found',
                 'message_long': 'The requested file could not be found.'
             })
+
+        file = guid.referent
+        if type(file) is Preprint:
+            referent, _ = Guid.load_referent(id_or_guid)
+            file = referent.primary_file
+
     if not file.is_file:
         raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data={
             'message_long': 'Downloading folders is not permitted.'
@@ -1032,16 +1067,6 @@ def persistent_file_download(auth, **kwargs):
     )
 
 
-def addon_view_or_download_quickfile(**kwargs):
-    fid = kwargs.get('fid', 'NOT_AN_FID')
-    file_ = OsfStorageFile.load(fid)
-    if not file_:
-        raise HTTPError(http_status.HTTP_404_NOT_FOUND, data={
-            'message_short': 'File Not Found',
-            'message_long': 'The requested file could not be found.'
-        })
-    return proxy_url(f'/project/{file_.target._id}/files/osfstorage/{fid}/')
-
 def addon_view_file(auth, node, file_node, version):
     # TODO: resolve circular import issue
     from addons.wiki import settings as wiki_settings

addons/boa/tasks.py

@@ -14,10 +14,9 @@
 from addons.boa.boa_error_code import BoaErrorCode
 from framework import sentry
 from framework.celery_tasks import app as celery_app
-from osf.models import OSFUser
+from osf.models import OSFUser, NotificationType
 from osf.utils.fields import ensure_str, ensure_bytes
 from website import settings as osf_settings
-from website.mails import send_mail, ADDONS_BOA_JOB_COMPLETE, ADDONS_BOA_JOB_FAILURE
 
 logger = logging.getLogger(__name__)
 
@@ -184,18 +183,21 @@ async def submit_to_boa_async(host, username, password, user_guid, project_guid,
 
     logger.info('Successfully uploaded query output to OSF.')
     logger.debug('Task ends <<<<<<<<')
-    await sync_to_async(send_mail)(
-        to_addr=user.username,
-        mail=ADDONS_BOA_JOB_COMPLETE,
-        fullname=user.fullname,
-        query_file_name=query_file_name,
-        query_file_full_path=file_full_path,
-        output_file_name=output_file_name,
-        job_id=boa_job.id,
-        project_url=project_url,
-        boa_job_list_url=boa_settings.BOA_JOB_LIST_URL,
-        boa_support_email=boa_settings.BOA_SUPPORT_EMAIL,
-        osf_support_email=osf_settings.OSF_SUPPORT_EMAIL,
+    NotificationType.objects.get(
+        name=NotificationType.Type.ADDONS_BOA_JOB_COMPLETE
+    ).emit(
+        user=user,
+        event_context={
+            'fullname': user.fullname,
+            'query_file_name': query_file_name,
+            'query_file_full_path': file_full_path,
+            'output_file_name': output_file_name,
+            'job_id': boa_job.id,
+            'project_url': project_url,
+            'boa_job_list_url': boa_settings.BOA_JOB_LIST_URL,
+            'boa_support_email': boa_settings.BOA_SUPPORT_EMAIL,
+            'osf_support_email': osf_settings.OSF_SUPPORT_EMAIL,
+        }
     )
     return BoaErrorCode.NO_ERROR
 
@@ -209,22 +211,25 @@ def handle_boa_error(message, code, username, fullname, project_url, query_file_
         sentry.log_message(message, skip_session=True)
     except Exception:
         pass
-    send_mail(
-        to_addr=username,
-        mail=ADDONS_BOA_JOB_FAILURE,
-        fullname=fullname,
-        code=code,
-        message=message,
-        query_file_name=query_file_name,
-        file_size=file_size,
-        max_file_size=boa_settings.MAX_SUBMISSION_SIZE,
-        query_file_full_path=query_file_full_path,
-        output_file_name=output_file_name,
-        job_id=job_id,
-        max_job_wait_hours=boa_settings.MAX_JOB_WAITING_TIME / 3600,
-        project_url=project_url,
-        boa_job_list_url=boa_settings.BOA_JOB_LIST_URL,
-        boa_support_email=boa_settings.BOA_SUPPORT_EMAIL,
-        osf_support_email=osf_settings.OSF_SUPPORT_EMAIL,
+    NotificationType.objects.get(
+        name=NotificationType.Type.ADDONS_BOA_JOB_FAILURE
+    ).emit(
+        destination_address=username,
+        event_context={
+            'fullname': fullname,
+            'code': code,
+            'query_file_name': query_file_name,
+            'file_size': file_size,
+            'max_file_size': boa_settings.MAX_SUBMISSION_SIZE,
+            'query_file_full_path': query_file_full_path,
+            'output_file_name': output_file_name,
+            'job_id': job_id,
+            'max_job_wait_hours': boa_settings.MAX_JOB_WAITING_TIME / 3600,
+            'project_url': project_url,
+            'boa_job_list_url': boa_settings.BOA_JOB_LIST_URL,
+            'boa_support_email': boa_settings.BOA_SUPPORT_EMAIL,
+            'osf_support_email': osf_settings.OSF_SUPPORT_EMAIL,
+
+        }
     )
     return code

addons/boa/tests/test_tasks.py

@@ -9,10 +9,11 @@
 from addons.boa import settings as boa_settings
 from addons.boa.boa_error_code import BoaErrorCode
 from addons.boa.tasks import submit_to_boa, submit_to_boa_async, handle_boa_error
+from osf.models import NotificationType
 from osf_tests.factories import AuthUserFactory, ProjectFactory
 from tests.base import OsfTestCase
+from tests.utils import capture_notifications
 from website import settings as osf_settings
-from website.mails import ADDONS_BOA_JOB_COMPLETE, ADDONS_BOA_JOB_FAILURE
 
 DEFAULT_REFRESH_JOB_INTERVAL = boa_settings.REFRESH_JOB_INTERVAL
 DEFAULT_MAX_JOB_WAITING_TIME = boa_settings.MAX_JOB_WAITING_TIME
@@ -53,42 +54,25 @@ def test_boa_error_code(self):
         assert BoaErrorCode.JOB_TIME_OUT_ERROR == 7
 
     def test_handle_boa_error(self):
-        with mock.patch('addons.boa.tasks.send_mail', return_value=None) as mock_send_mail, \
-                mock.patch('addons.boa.tasks.sentry.log_message', return_value=None) as mock_sentry_log_message, \
-                mock.patch('addons.boa.tasks.logger.error', return_value=None) as mock_logger_error:
-            return_value = handle_boa_error(
-                self.error_message,
-                BoaErrorCode.UNKNOWN,
-                self.user_username,
-                self.user_fullname,
-                self.project_url,
-                self.file_full_path,
-                query_file_name=self.query_file_name,
-                file_size=self.file_size,
-                output_file_name=self.output_file_name,
-                job_id=self.job_id
-            )
-            mock_send_mail.assert_called_with(
-                to_addr=self.user_username,
-                mail=ADDONS_BOA_JOB_FAILURE,
-                fullname=self.user_fullname,
-                code=BoaErrorCode.UNKNOWN,
-                message=self.error_message,
-                query_file_name=self.query_file_name,
-                file_size=self.file_size,
-                max_file_size=boa_settings.MAX_SUBMISSION_SIZE,
-                query_file_full_path=self.file_full_path,
-                output_file_name=self.output_file_name,
-                job_id=self.job_id,
-                max_job_wait_hours=self.max_job_wait_hours,
-                project_url=self.project_url,
-                boa_job_list_url=boa_settings.BOA_JOB_LIST_URL,
-                boa_support_email=boa_settings.BOA_SUPPORT_EMAIL,
-                osf_support_email=osf_settings.OSF_SUPPORT_EMAIL,
-            )
-            mock_sentry_log_message.assert_called_with(self.error_message, skip_session=True)
-            mock_logger_error.assert_called_with(self.error_message)
-            assert return_value == BoaErrorCode.UNKNOWN
+        with mock.patch('addons.boa.tasks.sentry.log_message', return_value=None) as mock_sentry_log_message:
+            with mock.patch('addons.boa.tasks.logger.error', return_value=None) as mock_logger_error:
+                with capture_notifications() as notifications:
+                    return_value = handle_boa_error(
+                        self.error_message,
+                        BoaErrorCode.UNKNOWN,
+                        self.user_username,
+                        self.user_fullname,
+                        self.project_url,
+                        self.file_full_path,
+                        file_size=self.file_size,
+                        output_file_name=self.output_file_name,
+                        job_id=self.job_id
+                    )
+                assert len(notifications) == 1
+                assert notifications[0]['type'] == NotificationType.Type.ADDONS_BOA_JOB_FAILURE
+                mock_sentry_log_message.assert_called_with(self.error_message, skip_session=True)
+                mock_logger_error.assert_called_with(self.error_message)
+                assert return_value == BoaErrorCode.UNKNOWN
 
 
 class TestSubmitToBoa(OsfTestCase):

addons/osfstorage/tests/test_views.py

@@ -24,7 +24,7 @@
 from framework.auth import cas
 
 from osf import features
-from osf.models import Tag, QuickFilesNode
+from osf.models import Tag
 from osf.models import files as models
 from addons.osfstorage.apps import osf_storage_root
 from addons.osfstorage import utils