Skip to content

Task/removeoldkeys #373

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Task/removeoldkeys #373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e13969c
remove notification counts v1
jzbahrai Jun 12, 2025
6ea183f
update utils
jzbahrai Jun 12, 2025
7412cf6
Merge branch 'main' into task/removeoldkeys
jzbahrai Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/waffles/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ docopt==0.6.2
Flask==2.3.3
markupsafe==2.1.5
setuptools==75.6.0 # required for distutils in Python 3.12
git+https://github.com/cds-snc/[email protected].4#egg=notifications-utils
git+https://github.com/cds-snc/[email protected].5#egg=notifications-utils
88 changes: 1 addition & 87 deletions notifications_utils/clients/redis/annual_limit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,8 @@
"""
This module stores daily notification counts and annual limit statuses for a service in Redis using a hash structure:

# TODO: Remove the first key once all services have been migrated to the new Redis structure
annual-limit: {
{service_id}: {
notifications: {
sms_delivered: int,
email_delivered: int,
sms_failed: int,
email_failed: int
},
status: {
near_sms_limit: Datetime,
near_email_limit: Datetime,
Expand Down Expand Up @@ -38,11 +31,6 @@

from notifications_utils.clients.redis.redis_client import RedisClient

# TODO: Remove the first 4 keys once all services have been migrated to the new Redis structure
SMS_DELIVERED = "sms_delivered"
EMAIL_DELIVERED = "email_delivered"
SMS_FAILED = "sms_failed"
EMAIL_FAILED = "email_failed"
SMS_DELIVERED_TODAY = "sms_delivered_today"
EMAIL_DELIVERED_TODAY = "email_delivered_today"
SMS_FAILED_TODAY = "sms_failed_today"
Expand All @@ -51,7 +39,6 @@
TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY = "total_sms_fiscal_year_to_yesterday"
TOTAL_EMAIL_FISCAL_YEAR_TO_YESTERDAY = "total_email_fiscal_year_to_yesterday"

NOTIFICATION_FIELDS = [SMS_DELIVERED, EMAIL_DELIVERED, SMS_FAILED, EMAIL_FAILED]
NOTIFICATION_FIELDS_V2 = [
SMS_DELIVERED_TODAY,
EMAIL_DELIVERED_TODAY,
Expand All @@ -69,14 +56,6 @@
STATUS_FIELDS = [NEAR_SMS_LIMIT, NEAR_EMAIL_LIMIT, OVER_SMS_LIMIT, OVER_EMAIL_LIMIT]


# TODO: Remove this once all services have been migrated to the new Redis structure
def annual_limit_notifications_key(service_id):
"""
Generates the Redis hash key for storing daily metrics of a service.
"""
return f"annual-limit:{service_id}:notifications"


def annual_limit_notifications_v2_key(service_id):
"""
Generates the Redis hash key for storing daily metrics of a service.
Expand Down Expand Up @@ -144,22 +123,14 @@ def increment_notification_count(self, service_id: str, field: str):
service_id (str): _description_
field (str): _description_
"""
# TODO: Remove the else
if field in NOTIFICATION_FIELDS_V2:
self._redis_client.increment_hash_value(annual_limit_notifications_v2_key(service_id), field)
else:
self._redis_client.increment_hash_value(annual_limit_notifications_key(service_id), field)

def get_notification_count(self, service_id: str, field: str):
"""
Retrieves the specified daily notification count for a service. (e.g. SMS_DELIVERED, EMAIL_FAILED, SMS_DELIVERED_TODAY etc.)
"""
count = self._redis_client.get_hash_field(annual_limit_notifications_v2_key(service_id), field)
if count:
return int(count.decode("utf-8"))
# TODO: Remove this once all services have been migrated to the new Redis structure
# TODO: Change the above to return 0 if count is None
count = self._redis_client.get_hash_field(annual_limit_notifications_key(service_id), field)
return 0 if not count else int(count.decode("utf-8"))

def get_all_notification_counts(self, service_id: str):
Expand All @@ -177,10 +148,7 @@ def get_all_notification_counts(self, service_id: str):
int,
NOTIFICATION_FIELDS_V2,
)
# TODO: Remove this once all services have been migrated to the new Redis structure
return prepare_byte_dict(
self._redis_client.get_all_from_hash(annual_limit_notifications_key(service_id)), int, NOTIFICATION_FIELDS
)
return {}

def reset_all_notification_counts(self, service_ids=None):
"""Resets all daily notification metrics.
Expand All @@ -196,31 +164,14 @@ def reset_all_notification_counts(self, service_ids=None):
)
# We also want to remove the seeded_at field from the notifications_v2 hash
self._redis_client.delete_hash_fields(hashes=hashes, fields=NOTIFICATION_FIELDS_V2 + [SEEDED_AT])
# TODO: Remove the else once all services have been migrated to the new Redis structure
hashes = (
annual_limit_notifications_key("*")
if not service_ids
else [annual_limit_notifications_key(service_id) for service_id in service_ids]
)
self._redis_client.delete_hash_fields(hashes=hashes, fields=NOTIFICATION_FIELDS)

def seed_annual_limit_notifications(self, service_id: str, mapping: dict):
"""Seeds annual limit notifications for a service.
# TODO: Update the docstring once all services have been migrated to the new Redis structure
Args:
service_id (str): Service to seed annual limit notifications for.
mapping (dict): A dict used to map notification counts to their respective fields formatted as follows

Examples:
`mapping` format:

{
"sms_delivered": int,
"email_delivered": int,
"sms_failed": int,
"email_failed": int,
}
as we added notifications_v2, the mapping can also be:
{
"sms_delivered_today": int,
"email_delivered_today": int,
Expand Down Expand Up @@ -252,28 +203,6 @@ def seed_annual_limit_notifications(self, service_id: str, mapping: dict):
v2_values = prepare_byte_dict(self._redis_client.get_all_from_hash(annual_limit_notifications_v2_key(service_id)), str)
current_app.logger.info(f"[alimit-debug-redis] Finished setting values, result is: {v2_values}")

# Create a legacy mapping from either original legacy keys or mapped from V2 keys
legacy_mapping = {}

# First try to use any legacy fields directly present in the mapping
for k in NOTIFICATION_FIELDS:
if k in mapping:
legacy_mapping[k] = mapping[k]

# If legacy fields aren't present, map from V2 fields
if SMS_DELIVERED_TODAY in mapping and SMS_DELIVERED not in legacy_mapping:
legacy_mapping[SMS_DELIVERED] = mapping[SMS_DELIVERED_TODAY]
if EMAIL_DELIVERED_TODAY in mapping and EMAIL_DELIVERED not in legacy_mapping:
legacy_mapping[EMAIL_DELIVERED] = mapping[EMAIL_DELIVERED_TODAY]
if SMS_FAILED_TODAY in mapping and SMS_FAILED not in legacy_mapping:
legacy_mapping[SMS_FAILED] = mapping[SMS_FAILED_TODAY]
if EMAIL_FAILED_TODAY in mapping and EMAIL_FAILED not in legacy_mapping:
legacy_mapping[EMAIL_FAILED] = mapping[EMAIL_FAILED_TODAY]

# Store legacy fields if we have any
if legacy_mapping:
self._redis_client.bulk_set_hash_fields(key=annual_limit_notifications_key(service_id), mapping=legacy_mapping)

current_app.logger.info(f"[alimit-debug-redis] Setting seeded_at for service {service_id}")
self.set_seeded_at(service_id)
current_app.logger.info(
Expand All @@ -293,16 +222,12 @@ def set_seeded_at(self, service_id):
self._redis_client.set_hash_value(
annual_limit_notifications_v2_key(service_id), SEEDED_AT, datetime.utcnow().strftime("%Y-%m-%d")
)
# TODO: Remove the below once all services have been migrated to the new Redis structure
# Setting the seeded at in status for backward compatibility
self._redis_client.set_hash_value(annual_limit_status_key(service_id), SEEDED_AT, datetime.utcnow().strftime("%Y-%m-%d"))

def clear_notification_counts(self, service_id: str):
"""
Clears all daily notification metrics for a service.
"""
self._redis_client.expire(annual_limit_notifications_v2_key(service_id), -1)
self._redis_client.expire(annual_limit_notifications_key(service_id), -1)

def set_annual_limit_status(self, service_id: str, field: str, value: datetime):
"""
Expand Down Expand Up @@ -351,23 +276,15 @@ def clear_annual_limit_statuses(self, service_id: str):
# Helper methods for daily metrics
def increment_sms_delivered(self, service_id: str):
self.increment_notification_count(service_id, SMS_DELIVERED_TODAY)
# TODO: remove the below line
self.increment_notification_count(service_id, SMS_DELIVERED)

def increment_sms_failed(self, service_id: str):
self.increment_notification_count(service_id, SMS_FAILED_TODAY)
# TODO: remove the below line
self.increment_notification_count(service_id, SMS_FAILED)

def increment_email_delivered(self, service_id: str):
self.increment_notification_count(service_id, EMAIL_DELIVERED_TODAY)
# TODO: remove the below line
self.increment_notification_count(service_id, EMAIL_DELIVERED)

def increment_email_failed(self, service_id: str):
self.increment_notification_count(service_id, EMAIL_FAILED_TODAY)
# TODO: remove the below line
self.increment_notification_count(service_id, EMAIL_FAILED)

# Helper methods for annual limits statuses
def set_nearing_sms_limit(self, service_id: str):
Expand Down Expand Up @@ -412,11 +329,8 @@ def delete_all_annual_limit_hashes(self, service_ids=None):
"""
if not service_ids:
self._redis_client.delete_cache_keys_by_pattern(annual_limit_notifications_v2_key("*"))
# TODO: Remove the line below
self._redis_client.delete_cache_keys_by_pattern(annual_limit_notifications_key("*"))
self._redis_client.delete_cache_keys_by_pattern(annual_limit_status_key("*"))
else:
for service_id in service_ids:
self._redis_client.delete(annual_limit_notifications_v2_key(service_id))
self._redis_client.delete(annual_limit_notifications_key(service_id))
self._redis_client.delete(annual_limit_status_key(service_id))
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "notifications-utils"
version = "53.2.4"
version = "53.2.5"
description = "Shared python code for Notification - Provides logging utils etc."
authors = ["Canadian Digital Service"]
license = "MIT license"
Expand Down
Loading
Loading