From 26881ac6310b41ab986d8f2657adffd44320107e Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Sat, 29 Mar 2025 19:02:15 +0530 Subject: [PATCH 1/9] feat: Add model to store background task details --- .../db/migrations/0003_backgroundtasklog.py | 30 ++++++++++++++++ backend/djangoindia/db/models/common.py | 26 ++++++++++++++ backend/djangoindia/utils/common.py | 34 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 backend/djangoindia/db/migrations/0003_backgroundtasklog.py create mode 100644 backend/djangoindia/db/models/common.py create mode 100644 backend/djangoindia/utils/common.py diff --git a/backend/djangoindia/db/migrations/0003_backgroundtasklog.py b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py new file mode 100644 index 00000000..c88abdc0 --- /dev/null +++ b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.5 on 2025-03-29 13:28 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('db', '0002_alter_eventregistration_options_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='BackgroundTaskLog', + fields=[ + ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=256)), + ('args', models.JSONField(blank=True, default=dict, null=True)), + ('kwargs', models.JSONField(blank=True, default=dict, null=True)), + ('status', models.CharField(choices=[('started', 'Started'), ('in_progress', 'In Progress'), ('failed', 'Failed'), ('complete', 'Complete')], default='started', max_length=32)), + ('error', models.CharField(blank=True, max_length=512, null=True)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/backend/djangoindia/db/models/common.py b/backend/djangoindia/db/models/common.py new file mode 100644 index 00000000..8e922d7c --- /dev/null +++ b/backend/djangoindia/db/models/common.py @@ -0,0 +1,26 @@ +from .base import BaseModel +from django.db import models + + +class BackgroundTaskLog(BaseModel): + """ + model to hold the information about the background task that runs. + """ + class StatusChoices(models.TextChoices): + STARTED = "started" + IN_PROGRESS = "in_progress" + FAILED = "failed" + COMPLETE = "complete" + + name = models.CharField(max_length=256, null=False, blank=False) + args = models.JSONField(default=dict, null=True, blank=True) + kwargs = models.JSONField(default=dict, null=True, blank=True) + status = models.CharField(max_length=32, null=False, blank=False,default=StatusChoices.STARTED, + choices=StatusChoices.choices) + error = models.CharField(max_length=512, null=True, blank=True) + + + def validate_status(self): + status_values = BackgroundTaskLog.StatusChoices.names + if self.status not in status_values: + raise ValueError(f"Invalid status choice it should be one of {','.join(status_values)}") \ No newline at end of file diff --git a/backend/djangoindia/utils/common.py b/backend/djangoindia/utils/common.py new file mode 100644 index 00000000..decf936c --- /dev/null +++ b/backend/djangoindia/utils/common.py @@ -0,0 +1,34 @@ +from functools import wraps + +from djangoindia.db.models.common import BackgroundTaskLog + + +def log_bg_task(func): + """ + A decorator to log background task details. + + Usage: + ------- + >>> from djangoindia.utils.common import log_bg_task + >>> @log_bg_task + >>> def my_celery_task(*args, **kwargs): + >>> # Task logic here + >>> pass + """ + + @wraps(func) + def wrapper(*args, **kwargs): + task_log = BackgroundTaskLog.objects.create( + name=func.__name__, + status="started", + args=list(args), + kwargs=kwargs + ) + task_log.status = "in_progress" + result = func(*args, **kwargs) + task_log.status = "complete" + # if task doesn't return error/expection result will be None + task_log.error = str(result) + task_log.save(update_fields=["status", "error"]) + return result + return wrapper From faf1b88465f517b39bb4e54e375f0767484206f2 Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Sat, 29 Mar 2025 19:07:08 +0530 Subject: [PATCH 2/9] fix: blank line at the end --- backend/djangoindia/db/models/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/djangoindia/db/models/common.py b/backend/djangoindia/db/models/common.py index 8e922d7c..b8d1ff8e 100644 --- a/backend/djangoindia/db/models/common.py +++ b/backend/djangoindia/db/models/common.py @@ -23,4 +23,4 @@ class StatusChoices(models.TextChoices): def validate_status(self): status_values = BackgroundTaskLog.StatusChoices.names if self.status not in status_values: - raise ValueError(f"Invalid status choice it should be one of {','.join(status_values)}") \ No newline at end of file + raise ValueError(f"Invalid status choice it should be one of {','.join(status_values)}") From 0e4e5eb7fa02227967e68caedb4c2399efe92854 Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Mon, 31 Mar 2025 09:39:42 +0530 Subject: [PATCH 3/9] feat: addressed review comments. code refactor --- .../db/migrations/0003_backgroundtasklog.py | 12 ++++++---- backend/djangoindia/db/models/__init__.py | 2 ++ backend/djangoindia/db/models/common.py | 23 +++++++------------ backend/djangoindia/utils/common.py | 19 ++++++++++----- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/backend/djangoindia/db/migrations/0003_backgroundtasklog.py b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py index c88abdc0..4c437c49 100644 --- a/backend/djangoindia/db/migrations/0003_backgroundtasklog.py +++ b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.5 on 2025-03-29 13:28 +# Generated by Django 4.2.5 on 2025-03-31 03:53 from django.db import migrations, models import uuid @@ -18,10 +18,12 @@ class Migration(migrations.Migration): ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), ('name', models.CharField(max_length=256)), - ('args', models.JSONField(blank=True, default=dict, null=True)), - ('kwargs', models.JSONField(blank=True, default=dict, null=True)), - ('status', models.CharField(choices=[('started', 'Started'), ('in_progress', 'In Progress'), ('failed', 'Failed'), ('complete', 'Complete')], default='started', max_length=32)), - ('error', models.CharField(blank=True, max_length=512, null=True)), + ('start_datetime', models.DateTimeField()), + ('end_datetime', models.DateTimeField(blank=True, null=True)), + ('args', models.JSONField(blank=True, default=dict, help_text='default arguments present in task', null=True)), + ('kwargs', models.JSONField(blank=True, default=dict, help_text='Additional arguments', null=True)), + ('status', models.CharField(choices=[('successfully', 'Successfull'), ('FAILURE', 'Failure')], max_length=32)), + ('log', models.CharField(blank=True, max_length=512, null=True)), ], options={ 'abstract': False, diff --git a/backend/djangoindia/db/models/__init__.py b/backend/djangoindia/db/models/__init__.py index aeff3c02..c9cf4272 100644 --- a/backend/djangoindia/db/models/__init__.py +++ b/backend/djangoindia/db/models/__init__.py @@ -4,6 +4,7 @@ from .update import Update from .user import SocialLoginConnection, User from .volunteer import Volunteer +from .common import BackgroundTaskLog __all__ = [ @@ -20,4 +21,5 @@ "Sponsor", "EventUserRegistration", "EventCommunication", + "BackgroundTaskLog", ] diff --git a/backend/djangoindia/db/models/common.py b/backend/djangoindia/db/models/common.py index b8d1ff8e..eee4af14 100644 --- a/backend/djangoindia/db/models/common.py +++ b/backend/djangoindia/db/models/common.py @@ -7,20 +7,13 @@ class BackgroundTaskLog(BaseModel): model to hold the information about the background task that runs. """ class StatusChoices(models.TextChoices): - STARTED = "started" - IN_PROGRESS = "in_progress" - FAILED = "failed" - COMPLETE = "complete" + SUCCESSFULL = "successfully" + FAILURE = "FAILURE" name = models.CharField(max_length=256, null=False, blank=False) - args = models.JSONField(default=dict, null=True, blank=True) - kwargs = models.JSONField(default=dict, null=True, blank=True) - status = models.CharField(max_length=32, null=False, blank=False,default=StatusChoices.STARTED, - choices=StatusChoices.choices) - error = models.CharField(max_length=512, null=True, blank=True) - - - def validate_status(self): - status_values = BackgroundTaskLog.StatusChoices.names - if self.status not in status_values: - raise ValueError(f"Invalid status choice it should be one of {','.join(status_values)}") + start_datetime = models.DateTimeField(null=False) + end_datetime = models.DateTimeField(null=True, blank=True) + args = models.JSONField(default=dict, null=True, blank=True, help_text="default arguments present in task") + kwargs = models.JSONField(default=dict, null=True, blank=True, help_text="Additional arguments") + status = models.CharField(max_length=32, null=False, blank=False, choices=StatusChoices.choices) + log = models.CharField(max_length=512, null=True, blank=True) \ No newline at end of file diff --git a/backend/djangoindia/utils/common.py b/backend/djangoindia/utils/common.py index decf936c..0726b2d3 100644 --- a/backend/djangoindia/utils/common.py +++ b/backend/djangoindia/utils/common.py @@ -1,5 +1,6 @@ from functools import wraps +from django.utils import timezone from djangoindia.db.models.common import BackgroundTaskLog @@ -20,15 +21,21 @@ def log_bg_task(func): def wrapper(*args, **kwargs): task_log = BackgroundTaskLog.objects.create( name=func.__name__, - status="started", + start_datetime=timezone.now(), args=list(args), kwargs=kwargs ) - task_log.status = "in_progress" - result = func(*args, **kwargs) - task_log.status = "complete" + try: + result = func(*args, **kwargs) + except Exception as ex: + result = str(ex) + if result is None: + task_log.status = BackgroundTaskLog.StatusChoices.SUCCESSFULL + else: + task_log.status = BackgroundTaskLog.StatusChoices.FAILURE # if task doesn't return error/expection result will be None - task_log.error = str(result) - task_log.save(update_fields=["status", "error"]) + task_log.log = str(result) if result else None + task_log.end_datetime = timezone.now() + task_log.save() return result return wrapper From 51a497bcc02d95bea848ea8f8ea6c3ebc4e514e2 Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Mon, 31 Mar 2025 09:44:05 +0530 Subject: [PATCH 4/9] fix: fix blank line --- backend/djangoindia/db/models/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/djangoindia/db/models/common.py b/backend/djangoindia/db/models/common.py index eee4af14..9fc4811b 100644 --- a/backend/djangoindia/db/models/common.py +++ b/backend/djangoindia/db/models/common.py @@ -16,4 +16,4 @@ class StatusChoices(models.TextChoices): args = models.JSONField(default=dict, null=True, blank=True, help_text="default arguments present in task") kwargs = models.JSONField(default=dict, null=True, blank=True, help_text="Additional arguments") status = models.CharField(max_length=32, null=False, blank=False, choices=StatusChoices.choices) - log = models.CharField(max_length=512, null=True, blank=True) \ No newline at end of file + log = models.CharField(max_length=512, null=True, blank=True) From c6c96ec6aa597bf4875fb9b07ce3a498e1fbaccf Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Tue, 15 Apr 2025 20:59:30 +0530 Subject: [PATCH 5/9] add more status choices. fix tasks address review comments --- .../db/migrations/0003_backgroundtasklog.py | 4 ++-- backend/djangoindia/utils/common.py | 17 +++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/backend/djangoindia/db/migrations/0003_backgroundtasklog.py b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py index 4c437c49..b681a35e 100644 --- a/backend/djangoindia/db/migrations/0003_backgroundtasklog.py +++ b/backend/djangoindia/db/migrations/0003_backgroundtasklog.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.5 on 2025-03-31 03:53 +# Generated by Django 4.2.5 on 2025-04-15 15:26 from django.db import migrations, models import uuid @@ -22,7 +22,7 @@ class Migration(migrations.Migration): ('end_datetime', models.DateTimeField(blank=True, null=True)), ('args', models.JSONField(blank=True, default=dict, help_text='default arguments present in task', null=True)), ('kwargs', models.JSONField(blank=True, default=dict, help_text='Additional arguments', null=True)), - ('status', models.CharField(choices=[('successfully', 'Successfull'), ('FAILURE', 'Failure')], max_length=32)), + ('status', models.CharField(choices=[('successful', 'Successfull'), ('failure', 'Failure'), ('pending', 'Pending'), ('started', 'Started')], default='pending', max_length=32)), ('log', models.CharField(blank=True, max_length=512, null=True)), ], options={ diff --git a/backend/djangoindia/utils/common.py b/backend/djangoindia/utils/common.py index 0726b2d3..f1cc381d 100644 --- a/backend/djangoindia/utils/common.py +++ b/backend/djangoindia/utils/common.py @@ -27,15 +27,12 @@ def wrapper(*args, **kwargs): ) try: result = func(*args, **kwargs) - except Exception as ex: - result = str(ex) - if result is None: task_log.status = BackgroundTaskLog.StatusChoices.SUCCESSFULL - else: + task_log.log = str(result) + except Exception as e: task_log.status = BackgroundTaskLog.StatusChoices.FAILURE - # if task doesn't return error/expection result will be None - task_log.log = str(result) if result else None - task_log.end_datetime = timezone.now() - task_log.save() - return result - return wrapper + task_log.log = str(e) + finally: + task_log.end_datetime = timezone.now() + task_log.save() + return wrapper From 2df67f63053636efda62f623835def0803cc6ee8 Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Tue, 15 Apr 2025 21:01:03 +0530 Subject: [PATCH 6/9] add model changes --- backend/djangoindia/db/models/common.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/djangoindia/db/models/common.py b/backend/djangoindia/db/models/common.py index 9fc4811b..f5d2600c 100644 --- a/backend/djangoindia/db/models/common.py +++ b/backend/djangoindia/db/models/common.py @@ -7,13 +7,16 @@ class BackgroundTaskLog(BaseModel): model to hold the information about the background task that runs. """ class StatusChoices(models.TextChoices): - SUCCESSFULL = "successfully" - FAILURE = "FAILURE" + SUCCESSFULL = "successful" + FAILURE = "failure" + PENDING = "pending" + STARTED = "started" + name = models.CharField(max_length=256, null=False, blank=False) start_datetime = models.DateTimeField(null=False) end_datetime = models.DateTimeField(null=True, blank=True) args = models.JSONField(default=dict, null=True, blank=True, help_text="default arguments present in task") kwargs = models.JSONField(default=dict, null=True, blank=True, help_text="Additional arguments") - status = models.CharField(max_length=32, null=False, blank=False, choices=StatusChoices.choices) + status = models.CharField(max_length=32, default=StatusChoices.PENDING, null=False, blank=False, choices=StatusChoices.choices) log = models.CharField(max_length=512, null=True, blank=True) From a35b30b08e872c5e52014804bd7b23a033ea5c8f Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Sun, 20 Apr 2025 15:35:38 +0530 Subject: [PATCH 7/9] add started logger Co-authored-by: Bhuvnesh --- backend/djangoindia/utils/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/djangoindia/utils/common.py b/backend/djangoindia/utils/common.py index f1cc381d..e53dd03c 100644 --- a/backend/djangoindia/utils/common.py +++ b/backend/djangoindia/utils/common.py @@ -26,6 +26,7 @@ def wrapper(*args, **kwargs): kwargs=kwargs ) try: + task_log.status = BackgroundTaskLog.StatusChoices.STARTED result = func(*args, **kwargs) task_log.status = BackgroundTaskLog.StatusChoices.SUCCESSFULL task_log.log = str(result) From 80e9a35baf3f99e9115c7abbf517ce44ed1069cd Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Sun, 20 Apr 2025 15:45:27 +0530 Subject: [PATCH 8/9] chore: Register model in admin file to access log via admin panel --- backend/djangoindia/bg_tasks/common.py | 39 ++++++++++++++++++++++++++ backend/djangoindia/db/admin.py | 8 ++++++ 2 files changed, 47 insertions(+) create mode 100644 backend/djangoindia/bg_tasks/common.py diff --git a/backend/djangoindia/bg_tasks/common.py b/backend/djangoindia/bg_tasks/common.py new file mode 100644 index 00000000..e53dd03c --- /dev/null +++ b/backend/djangoindia/bg_tasks/common.py @@ -0,0 +1,39 @@ +from functools import wraps + +from django.utils import timezone +from djangoindia.db.models.common import BackgroundTaskLog + + +def log_bg_task(func): + """ + A decorator to log background task details. + + Usage: + ------- + >>> from djangoindia.utils.common import log_bg_task + >>> @log_bg_task + >>> def my_celery_task(*args, **kwargs): + >>> # Task logic here + >>> pass + """ + + @wraps(func) + def wrapper(*args, **kwargs): + task_log = BackgroundTaskLog.objects.create( + name=func.__name__, + start_datetime=timezone.now(), + args=list(args), + kwargs=kwargs + ) + try: + task_log.status = BackgroundTaskLog.StatusChoices.STARTED + result = func(*args, **kwargs) + task_log.status = BackgroundTaskLog.StatusChoices.SUCCESSFULL + task_log.log = str(result) + except Exception as e: + task_log.status = BackgroundTaskLog.StatusChoices.FAILURE + task_log.log = str(e) + finally: + task_log.end_datetime = timezone.now() + task_log.save() + return wrapper diff --git a/backend/djangoindia/db/admin.py b/backend/djangoindia/db/admin.py index ddbd5152..894ab105 100644 --- a/backend/djangoindia/db/admin.py +++ b/backend/djangoindia/db/admin.py @@ -33,6 +33,7 @@ Update, User, Volunteer, + BackgroundTaskLog ) from .forms import ( @@ -684,3 +685,10 @@ def recipient_count(self, obj): return obj.recipient.count() recipient_count.short_description = "Recipients" + + +@admin.register(BackgroundTaskLog) +class BackgroundTaskLogAdmin(admin.ModelAdmin): + list_display = ("name", "start_datetime", "end_datetime", "args", "kwargs", "status", "log") + list_filter = ("name", "status", "start_datetime", "end_datetime") + search_fields = ("name", "status") From 5e34ffbff1aa5eb9d5d2d810601009e6a6b06f99 Mon Sep 17 00:00:00 2001 From: Suraj Pisal Date: Sun, 20 Apr 2025 15:46:25 +0530 Subject: [PATCH 9/9] move task to bg_tasks --- backend/djangoindia/utils/common.py | 39 ----------------------------- 1 file changed, 39 deletions(-) delete mode 100644 backend/djangoindia/utils/common.py diff --git a/backend/djangoindia/utils/common.py b/backend/djangoindia/utils/common.py deleted file mode 100644 index e53dd03c..00000000 --- a/backend/djangoindia/utils/common.py +++ /dev/null @@ -1,39 +0,0 @@ -from functools import wraps - -from django.utils import timezone -from djangoindia.db.models.common import BackgroundTaskLog - - -def log_bg_task(func): - """ - A decorator to log background task details. - - Usage: - ------- - >>> from djangoindia.utils.common import log_bg_task - >>> @log_bg_task - >>> def my_celery_task(*args, **kwargs): - >>> # Task logic here - >>> pass - """ - - @wraps(func) - def wrapper(*args, **kwargs): - task_log = BackgroundTaskLog.objects.create( - name=func.__name__, - start_datetime=timezone.now(), - args=list(args), - kwargs=kwargs - ) - try: - task_log.status = BackgroundTaskLog.StatusChoices.STARTED - result = func(*args, **kwargs) - task_log.status = BackgroundTaskLog.StatusChoices.SUCCESSFULL - task_log.log = str(result) - except Exception as e: - task_log.status = BackgroundTaskLog.StatusChoices.FAILURE - task_log.log = str(e) - finally: - task_log.end_datetime = timezone.now() - task_log.save() - return wrapper