From f28762e332ce678ee6332ebc9b5dd4045b899ac8 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Thu, 7 Dec 2023 11:46:15 +0800 Subject: [PATCH 01/11] add upload-link handler --- dtable_events/app/app.py | 3 + .../tasks/dtable_upload_link_handler.py | 114 ++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 dtable_events/tasks/dtable_upload_link_handler.py diff --git a/dtable_events/app/app.py b/dtable_events/app/app.py index f1c13e9d..a08f30b0 100644 --- a/dtable_events/app/app.py +++ b/dtable_events/app/app.py @@ -11,6 +11,7 @@ from dtable_events.tasks.ldap_syncer import LDAPSyncer from dtable_events.tasks.dtable_asset_trash_cleaner import DTableAssetTrashCleaner from dtable_events.tasks.license_expiring_notices_sender import LicenseExpiringNoticesSender +from dtable_events.tasks.dtable_upload_link_handler import DTableUploadLinkHandler from dtable_events.notification_rules.handler import NotificationRuleHandler from dtable_events.notification_rules.dtable_notification_rules_scanner import DTableNofiticationRulesScanner from dtable_events.automations.handler import AutomationRuleHandler @@ -59,6 +60,7 @@ def __init__(self, config, task_mode): self._workflow_schedule_scanner = WorkflowSchedulesScanner(config) self._dtable_asset_trash_cleaner = DTableAssetTrashCleaner(config) self._license_expiring_notices_sender = LicenseExpiringNoticesSender() + self._dtable_upload_link_handler = DTableUploadLinkHandler(config) # convert pdf manager conver_page_to_pdf_manager.init(config) @@ -91,5 +93,6 @@ def serve_forever(self): self._workflow_schedule_scanner.start() # default True self._dtable_asset_trash_cleaner.start() # always True self._license_expiring_notices_sender.start() # always True + self._dtable_upload_link_handler.start() # always True # convert pdf manager conver_page_to_pdf_manager.start() # always True diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py new file mode 100644 index 00000000..59676a9b --- /dev/null +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +import logging +import time +import json +import stat +from datetime import datetime, timedelta +from threading import Thread, Event + +from apscheduler.schedulers.blocking import BlockingScheduler + +from seaserv import seafile_api + +from dtable_events.app.event_redis import RedisClient, redis_cache +from dtable_events.db import init_db_session_class +from dtable_events.utils import uuid_str_to_36_chars + +logger = logging.getLogger(__name__) + + +class DTableUploadLinkHandler(Thread): + def __init__(self, config): + Thread.__init__(self) + self._finished = Event() + self._redis_client = RedisClient(config) + self.interval_hours = 1 + + def get_cache_key(self, hour): + return f'public_form_upload_link:{hour}' + + def handle_upload_link(self, event_data): + dtable_uuid = event_data.get('dtable_uuid') + repo_id = event_data.get('repo_id') + parent_dir = event_data.get('parent_dir') or '' + if 'public/forms' not in parent_dir: + return + cache_key = self.get_cache_key((datetime.now().hour+1)%24) + dtable_uuids_cache = redis_cache.get(cache_key) + if dtable_uuids_cache: + try: + dtable_uuids = json.loads(dtable_uuids_cache) + except Exception as e: + logger.warning('key: %s cache invalid', cache_key) + dtable_uuids = [(dtable_uuid, repo_id)] + else: + dtable_uuids.append((dtable_uuid, repo_id)) + else: + dtable_uuids = [(dtable_uuid, repo_id)] + redis_cache.set(cache_key, json.dumps(dtable_uuids), 12*60*60) + + def listen_redis(self): + logger.info('Starting handle dtable upload link...') + subscriber = self._redis_client.get_subscriber('upload-link') + while not self._finished.is_set(): + try: + message = subscriber.get_message() + if message is not None: + data = json.loads(message['data']) + try: + self.handle_upload_link(data) + except Exception as e: + logger.error('Handle dtable upload link: %s' % e) + else: + time.sleep(0.5) + except Exception as e: + logger.error('Failed get message from redis: %s' % e) + subscriber = self._redis_client.get_subscriber('upload-link') + + def handle_form_timeout_images(self): + sched = BlockingScheduler() + + @sched.scheduled_job('cron', day_of_week='*', hour='*') + def handle(): + handle_hour = (datetime.now() - timedelta(hours=self.interval_hours)).hour + cache_key = self.get_cache_key(handle_hour) + dtable_uuids_cache = redis_cache.get(cache_key) + if not dtable_uuids_cache: + return + try: + dtable_uuids = json.loads(dtable_uuids_cache) + except Exception as e: + logger.warning('cache: %s loads dtable uuids error: %s', cache_key, e) + return + now_timestamp = datetime.now().timestamp() + logger.debug('start to check key: %s dtable_uuids: %s ', cache_key, len(dtable_uuids)) + for dtable_uuid, repo_id in dtable_uuids: + try: + public_forms_path = f'/asset/{uuid_str_to_36_chars(dtable_uuid)}/public/forms' + logger.debug('start to scan repo: %s dtable: %s path: %s', repo_id, dtable_uuid, public_forms_path) + dirents = [] + limit, offset = 1000, 0 + while True: + step_dirents = seafile_api.list_dir_by_path(repo_id, public_forms_path, offset, limit) + if not step_dirents: + break + dirents.extend(step_dirents) + if len(step_dirents) < limit: + break + offset += limit + logger.debug('repo: %s dtable: %s path: %s dirents: %s', repo_id, dtable_uuid, public_forms_path, len(dirents)) + for dirent in dirents: + if stat.S_ISDIR(dirent.mode): + continue + if dirent.mtime and now_timestamp - dirent.mtime > self.interval_hours * 60 * 60: + seafile_api.del_file(repo_id, public_forms_path, dirent.obj_name, '') + except Exception as e: + logger.exception('scan repo: %s dtable: %s path: %s error: %s', repo_id, dtable_uuid, public_forms_path, e) + redis_cache.delete(cache_key) + + sched.start() + + def run(self): + logger.info('start to listen upload-link redis and handle form timeout images...') + Thread(target=self.listen_redis, daemon=True).start() + Thread(target=self.handle_form_timeout_images, daemon=True).start() From 41566f7f48047a8bca0dbc185c9e7b06d7a92919 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Thu, 7 Dec 2023 16:46:12 +0800 Subject: [PATCH 02/11] fix seafile del file --- dtable_events/tasks/dtable_upload_link_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 59676a9b..9f7cea44 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -101,7 +101,7 @@ def handle(): if stat.S_ISDIR(dirent.mode): continue if dirent.mtime and now_timestamp - dirent.mtime > self.interval_hours * 60 * 60: - seafile_api.del_file(repo_id, public_forms_path, dirent.obj_name, '') + seafile_api.del_file(repo_id, public_forms_path, json.dumps([dirent.obj_name]), '') except Exception as e: logger.exception('scan repo: %s dtable: %s path: %s error: %s', repo_id, dtable_uuid, public_forms_path, e) redis_cache.delete(cache_key) From f7fb8a36129b7c6f3cf974088ac59aa64d5584c0 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Thu, 7 Dec 2023 16:59:53 +0800 Subject: [PATCH 03/11] opt upload-link-handle interval --- dtable_events/tasks/dtable_upload_link_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 9f7cea44..ba96094f 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -22,7 +22,8 @@ def __init__(self, config): Thread.__init__(self) self._finished = Event() self._redis_client = RedisClient(config) - self.interval_hours = 1 + self.interval_hours = 6 + self.cache_timeout = 12 * 60 * 60 def get_cache_key(self, hour): return f'public_form_upload_link:{hour}' @@ -45,7 +46,7 @@ def handle_upload_link(self, event_data): dtable_uuids.append((dtable_uuid, repo_id)) else: dtable_uuids = [(dtable_uuid, repo_id)] - redis_cache.set(cache_key, json.dumps(dtable_uuids), 12*60*60) + redis_cache.set(cache_key, json.dumps(dtable_uuids), self.cache_timeout) def listen_redis(self): logger.info('Starting handle dtable upload link...') From 7d042981e26fd423949a1db0f0ba83fa07541330 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Fri, 8 Dec 2023 12:11:11 +0800 Subject: [PATCH 04/11] opt store form cache --- dtable_events/tasks/dtable_upload_link_handler.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index ba96094f..c2171b81 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -12,7 +12,7 @@ from dtable_events.app.event_redis import RedisClient, redis_cache from dtable_events.db import init_db_session_class -from dtable_events.utils import uuid_str_to_36_chars +from dtable_events.utils import uuid_str_to_36_chars, uuid_str_to_32_chars logger = logging.getLogger(__name__) @@ -29,7 +29,7 @@ def get_cache_key(self, hour): return f'public_form_upload_link:{hour}' def handle_upload_link(self, event_data): - dtable_uuid = event_data.get('dtable_uuid') + dtable_uuid = uuid_str_to_32_chars(event_data.get('dtable_uuid')) repo_id = event_data.get('repo_id') parent_dir = event_data.get('parent_dir') or '' if 'public/forms' not in parent_dir: @@ -43,7 +43,13 @@ def handle_upload_link(self, event_data): logger.warning('key: %s cache invalid', cache_key) dtable_uuids = [(dtable_uuid, repo_id)] else: - dtable_uuids.append((dtable_uuid, repo_id)) + flag = True + for tmp_dtable_uuid, _ in dtable_uuids: + if dtable_uuid == tmp_dtable_uuid: + flag = False + break + if flag: + dtable_uuids.append((dtable_uuid, repo_id)) else: dtable_uuids = [(dtable_uuid, repo_id)] redis_cache.set(cache_key, json.dumps(dtable_uuids), self.cache_timeout) From 973386af813a3879cdd76e1cabbc7097ea2ec088 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Fri, 8 Dec 2023 18:07:25 +0800 Subject: [PATCH 05/11] handle upload-link-flags with database --- .../tasks/dtable_upload_link_handler.py | 152 +++++++----------- 1 file changed, 61 insertions(+), 91 deletions(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index c2171b81..536e4297 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- import logging -import time -import json import stat +from collections import defaultdict from datetime import datetime, timedelta -from threading import Thread, Event +from threading import Thread from apscheduler.schedulers.blocking import BlockingScheduler from seaserv import seafile_api -from dtable_events.app.event_redis import RedisClient, redis_cache from dtable_events.db import init_db_session_class -from dtable_events.utils import uuid_str_to_36_chars, uuid_str_to_32_chars +from dtable_events.utils import uuid_str_to_36_chars logger = logging.getLogger(__name__) @@ -20,102 +18,74 @@ class DTableUploadLinkHandler(Thread): def __init__(self, config): Thread.__init__(self) - self._finished = Event() - self._redis_client = RedisClient(config) + self.session_class = init_db_session_class(config) self.interval_hours = 6 - self.cache_timeout = 12 * 60 * 60 - def get_cache_key(self, hour): - return f'public_form_upload_link:{hour}' - - def handle_upload_link(self, event_data): - dtable_uuid = uuid_str_to_32_chars(event_data.get('dtable_uuid')) - repo_id = event_data.get('repo_id') - parent_dir = event_data.get('parent_dir') or '' - if 'public/forms' not in parent_dir: - return - cache_key = self.get_cache_key((datetime.now().hour+1)%24) - dtable_uuids_cache = redis_cache.get(cache_key) - if dtable_uuids_cache: - try: - dtable_uuids = json.loads(dtable_uuids_cache) - except Exception as e: - logger.warning('key: %s cache invalid', cache_key) - dtable_uuids = [(dtable_uuid, repo_id)] - else: - flag = True - for tmp_dtable_uuid, _ in dtable_uuids: - if dtable_uuid == tmp_dtable_uuid: - flag = False - break - if flag: - dtable_uuids.append((dtable_uuid, repo_id)) - else: - dtable_uuids = [(dtable_uuid, repo_id)] - redis_cache.set(cache_key, json.dumps(dtable_uuids), self.cache_timeout) - - def listen_redis(self): - logger.info('Starting handle dtable upload link...') - subscriber = self._redis_client.get_subscriber('upload-link') - while not self._finished.is_set(): + def handle_flags(self, session): + now = datetime.now() + flag_time = (now - timedelta(hours=self.interval_hours)).replace(minute=0, second=0, microsecond=0) + offset, limit = 0, 1000 + while True: + sql = "SELECT dtable_uuid, repo_id FROM dtable_form_upload_link_flags WHERE flag_time=:flag_time LIMIT :offset, :limit" try: - message = subscriber.get_message() - if message is not None: - data = json.loads(message['data']) - try: - self.handle_upload_link(data) - except Exception as e: - logger.error('Handle dtable upload link: %s' % e) - else: - time.sleep(0.5) + results = list(session.execute(sql, {'flag_time': flag_time, 'offset': offset, 'limit': limit})) except Exception as e: - logger.error('Failed get message from redis: %s' % e) - subscriber = self._redis_client.get_subscriber('upload-link') + logger.error('query upload flags flag_time: %s error: %s', flag_time, e) + break + logger.debug('flag_time: %s offset: %s limit: %s query results: %s', flag_time, offset, limit, len(results)) + repo_id_dtable_uuids_dict = defaultdict(list) + for dtable_uuid, repo_id in results: + repo_id_dtable_uuids_dict[repo_id].append(dtable_uuid) + for repo_id, dtable_uuids in repo_id_dtable_uuids_dict.items(): + logger.debug('repo: %s dtable_uuids: %s', repo_id, len(dtable_uuids)) + try: + repo = seafile_api.get_repo(repo_id) + if not repo: + continue + for dtable_uuid in dtable_uuids: + public_forms_path = f'/asset/{uuid_str_to_36_chars(dtable_uuid)}/public/forms' + dir_id = seafile_api.get_dir_id_by_path(repo_id, public_forms_path) + if not dir_id: + continue + f_offset, f_limit= 0, 1000 + to_delete_files = [] + while True: + dirents = seafile_api.list_dir_by_path(repo_id, public_forms_path, f_offset, f_limit) + if not dirents: + break + for dirent in dirents: + if stat.S_ISDIR(dirent.mode): + continue + if (now.timestamp() - datetime.fromtimestamp(dirent.mtime)) > self.interval_hours * 60 * 60: + to_delete_files.append(dirent.obj_name) + if len(dirents) < f_limit: + break + f_offset += f_limit + logger.debug('repo: %s dtable: %s to delete files: %s', repo_id, dtable_uuid, len(to_delete_files)) + for file in to_delete_files: + seafile_api.del_file(repo_id, public_forms_path, file, '') + except Exception as e: + logger.exception('repo: %s handle upload flags error: %s', repo_id, e) + offset += limit + sql = "DELETE FROM dtable_form_upload_link_flags WHERE flag_time <= :flag_time" + try: + session.execute(sql, {'flag_time': flag_time}) + session.commit() + except Exception as e: + logger.error('delete upload flags old data flag time: %s error: %s', flag_time, e) - def handle_form_timeout_images(self): + def run(self): + logger.info('start to handle upload flags...') sched = BlockingScheduler() @sched.scheduled_job('cron', day_of_week='*', hour='*') def handle(): - handle_hour = (datetime.now() - timedelta(hours=self.interval_hours)).hour - cache_key = self.get_cache_key(handle_hour) - dtable_uuids_cache = redis_cache.get(cache_key) - if not dtable_uuids_cache: - return + session = self.session_class() try: - dtable_uuids = json.loads(dtable_uuids_cache) + self.handle_flags(session) except Exception as e: - logger.warning('cache: %s loads dtable uuids error: %s', cache_key, e) - return - now_timestamp = datetime.now().timestamp() - logger.debug('start to check key: %s dtable_uuids: %s ', cache_key, len(dtable_uuids)) - for dtable_uuid, repo_id in dtable_uuids: - try: - public_forms_path = f'/asset/{uuid_str_to_36_chars(dtable_uuid)}/public/forms' - logger.debug('start to scan repo: %s dtable: %s path: %s', repo_id, dtable_uuid, public_forms_path) - dirents = [] - limit, offset = 1000, 0 - while True: - step_dirents = seafile_api.list_dir_by_path(repo_id, public_forms_path, offset, limit) - if not step_dirents: - break - dirents.extend(step_dirents) - if len(step_dirents) < limit: - break - offset += limit - logger.debug('repo: %s dtable: %s path: %s dirents: %s', repo_id, dtable_uuid, public_forms_path, len(dirents)) - for dirent in dirents: - if stat.S_ISDIR(dirent.mode): - continue - if dirent.mtime and now_timestamp - dirent.mtime > self.interval_hours * 60 * 60: - seafile_api.del_file(repo_id, public_forms_path, json.dumps([dirent.obj_name]), '') - except Exception as e: - logger.exception('scan repo: %s dtable: %s path: %s error: %s', repo_id, dtable_uuid, public_forms_path, e) - redis_cache.delete(cache_key) + logger.exception('handle upload flags error: %s', e) + finally: + session.close() sched.start() - - def run(self): - logger.info('start to listen upload-link redis and handle form timeout images...') - Thread(target=self.listen_redis, daemon=True).start() - Thread(target=self.handle_form_timeout_images, daemon=True).start() From 75d38abc254dc2712f33db8978992451f4c808d2 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Fri, 8 Dec 2023 18:16:51 +0800 Subject: [PATCH 06/11] fix query upload-link-flags --- dtable_events/tasks/dtable_upload_link_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 536e4297..7b4afff4 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -66,6 +66,8 @@ def handle_flags(self, session): seafile_api.del_file(repo_id, public_forms_path, file, '') except Exception as e: logger.exception('repo: %s handle upload flags error: %s', repo_id, e) + if len(results) < limit: + break offset += limit sql = "DELETE FROM dtable_form_upload_link_flags WHERE flag_time <= :flag_time" try: From aee1df6da352fe3f479d18b10fa22198c3c69682 Mon Sep 17 00:00:00 2001 From: AlexHappy <1223408988@qq.com> Date: Mon, 26 Feb 2024 01:41:48 +0800 Subject: [PATCH 07/11] update upload public/forms/temp --- dtable_events/tasks/dtable_upload_link_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 7b4afff4..0ba842a9 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -6,6 +6,7 @@ from threading import Thread from apscheduler.schedulers.blocking import BlockingScheduler +from sqlalchemy import text from seaserv import seafile_api @@ -28,7 +29,7 @@ def handle_flags(self, session): while True: sql = "SELECT dtable_uuid, repo_id FROM dtable_form_upload_link_flags WHERE flag_time=:flag_time LIMIT :offset, :limit" try: - results = list(session.execute(sql, {'flag_time': flag_time, 'offset': offset, 'limit': limit})) + results = list(session.execute(text(sql), {'flag_time': flag_time, 'offset': offset, 'limit': limit})) except Exception as e: logger.error('query upload flags flag_time: %s error: %s', flag_time, e) break @@ -43,7 +44,7 @@ def handle_flags(self, session): if not repo: continue for dtable_uuid in dtable_uuids: - public_forms_path = f'/asset/{uuid_str_to_36_chars(dtable_uuid)}/public/forms' + public_forms_path = f'/asset/{uuid_str_to_36_chars(dtable_uuid)}/public/forms/temp' dir_id = seafile_api.get_dir_id_by_path(repo_id, public_forms_path) if not dir_id: continue @@ -71,7 +72,7 @@ def handle_flags(self, session): offset += limit sql = "DELETE FROM dtable_form_upload_link_flags WHERE flag_time <= :flag_time" try: - session.execute(sql, {'flag_time': flag_time}) + session.execute(text(sql), {'flag_time': flag_time}) session.commit() except Exception as e: logger.error('delete upload flags old data flag time: %s error: %s', flag_time, e) From 00212614c2107f4dea561c035703b5ceb61c27b5 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Wed, 28 Feb 2024 13:46:08 +0800 Subject: [PATCH 08/11] opt select flag uuids --- dtable_events/tasks/dtable_upload_link_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 0ba842a9..0e912178 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -27,7 +27,7 @@ def handle_flags(self, session): flag_time = (now - timedelta(hours=self.interval_hours)).replace(minute=0, second=0, microsecond=0) offset, limit = 0, 1000 while True: - sql = "SELECT dtable_uuid, repo_id FROM dtable_form_upload_link_flags WHERE flag_time=:flag_time LIMIT :offset, :limit" + sql = "SELECT dtable_uuid, repo_id FROM dtable_form_upload_link_flags WHERE flag_time<=:flag_time LIMIT :offset, :limit" try: results = list(session.execute(text(sql), {'flag_time': flag_time, 'offset': offset, 'limit': limit})) except Exception as e: From 723763353edae83a33d74b9c694cfd0cecefa7c0 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Wed, 28 Feb 2024 17:50:45 +0800 Subject: [PATCH 09/11] fix timestamp sub --- dtable_events/tasks/dtable_upload_link_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 0e912178..9b35fb48 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -57,7 +57,7 @@ def handle_flags(self, session): for dirent in dirents: if stat.S_ISDIR(dirent.mode): continue - if (now.timestamp() - datetime.fromtimestamp(dirent.mtime)) > self.interval_hours * 60 * 60: + if (now.timestamp() - dirent.mtime) > self.interval_hours * 60 * 60: to_delete_files.append(dirent.obj_name) if len(dirents) < f_limit: break From 0c4022434b91e444898bb46b3364e7f821eb51ac Mon Sep 17 00:00:00 2001 From: r350178982 <32759763+r350178982@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:09:36 +0800 Subject: [PATCH 10/11] fix-del-files --- dtable_events/tasks/dtable_upload_link_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 9b35fb48..686cda02 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -4,6 +4,7 @@ from collections import defaultdict from datetime import datetime, timedelta from threading import Thread +import json from apscheduler.schedulers.blocking import BlockingScheduler from sqlalchemy import text @@ -63,8 +64,7 @@ def handle_flags(self, session): break f_offset += f_limit logger.debug('repo: %s dtable: %s to delete files: %s', repo_id, dtable_uuid, len(to_delete_files)) - for file in to_delete_files: - seafile_api.del_file(repo_id, public_forms_path, file, '') + seafile_api.del_file(repo_id, public_forms_path, json.dumps(to_delete_files), '') except Exception as e: logger.exception('repo: %s handle upload flags error: %s', repo_id, e) if len(results) < limit: From ce470c629729e8d3d03175c0549fa60eab245401 Mon Sep 17 00:00:00 2001 From: AlexCXC <1223408988@qq.com> Date: Mon, 4 Mar 2024 09:55:47 +0800 Subject: [PATCH 11/11] opt batch delete timeout files --- dtable_events/tasks/dtable_upload_link_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dtable_events/tasks/dtable_upload_link_handler.py b/dtable_events/tasks/dtable_upload_link_handler.py index 686cda02..fa30ceca 100644 --- a/dtable_events/tasks/dtable_upload_link_handler.py +++ b/dtable_events/tasks/dtable_upload_link_handler.py @@ -64,7 +64,9 @@ def handle_flags(self, session): break f_offset += f_limit logger.debug('repo: %s dtable: %s to delete files: %s', repo_id, dtable_uuid, len(to_delete_files)) - seafile_api.del_file(repo_id, public_forms_path, json.dumps(to_delete_files), '') + del_step = 1000 + for i in range(0, len(to_delete_files), del_step): + seafile_api.del_file(repo_id, public_forms_path, json.dumps(to_delete_files[i: i+del_step]), '') except Exception as e: logger.exception('repo: %s handle upload flags error: %s', repo_id, e) if len(results) < limit: