diff --git a/apiserver/plane/db/backfills/__init__.py b/apiserver/plane/db/backfills/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apiserver/plane/db/backfills/backfill_0070_page_versions.py b/apiserver/plane/db/backfills/backfill_0070_page_versions.py new file mode 100644 index 000000000..6afbd233d --- /dev/null +++ b/apiserver/plane/db/backfills/backfill_0070_page_versions.py @@ -0,0 +1,63 @@ +# Third party imports +from celery import shared_task + +# Django imports +from django.utils import timezone + +# Module imports +from plane.db.models import PageVersion, IssueType, Issue + + +@shared_task +def backfill_issue_type_task(projects): + # Create the issue types for all projects + IssueType.objects.bulk_create( + [ + IssueType( + name="Task", + description="A task that needs to be completed.", + project_id=project["id"], + workspace_id=project["workspace_id"], + ) + for project in projects + ], + batch_size=1000, + ) + + # Update the issue type for all existing issues + issue_types = { + str(issue_type["project_id"]): str(issue_type["id"]) + for issue_type in IssueType.objects.filter( + project_id__in=[project["id"] for project in projects] + ).values("id", "project_id") + } + # Update the issue type for all existing issues + bulk_issues = [] + for issue in Issue.objects.filter( + project_id__in=[project["id"] for project in projects] + ): + issue.type_id = issue_types[str(issue.project_id)] + bulk_issues.append(issue) + + # Update the issue type for all existing issues + Issue.objects.bulk_update(bulk_issues, ["type_id"], batch_size=1000) + + +@shared_task +def backfill_page_versions_task(pages): + # Create the page versions for all pages + PageVersion.objects.bulk_create( + [ + PageVersion( + page_id=page["id"], + workspace_id=page["workspace_id"], + last_saved_at=timezone.now(), + owned_by_id=page["owned_by_id"], + description_binary=page["description_binary"], + description_html=page["description_html"], + description_stripped=page["description_stripped"], + ) + for page in pages + ], + batch_size=1000, + ) diff --git a/apiserver/plane/db/migrations/0070_issuetype_issue_type.py b/apiserver/plane/db/migrations/0070_page_versions.py similarity index 82% rename from apiserver/plane/db/migrations/0070_issuetype_issue_type.py rename to apiserver/plane/db/migrations/0070_page_versions.py index df89ae700..f303d07f2 100644 --- a/apiserver/plane/db/migrations/0070_issuetype_issue_type.py +++ b/apiserver/plane/db/migrations/0070_page_versions.py @@ -4,38 +4,67 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion import uuid +from apiserver.plane.db.backfills.backfill_0070_page_versions import ( + backfill_issue_type_task, + backfill_page_versions_task, +) + +CHUNK_SIZE = 100 # Initial Delay in seconds +INITIAL_DELAY = 30 # Initial delay in seconds +INCREMENT_DELAY = 1 # Increment delay in seconds -def create_issue_types(apps, schema_editor): +def backfill_issue_types(apps, schema_editor): + start = 0 + end = CHUNK_SIZE + Project = apps.get_model("db", "Project") - Issue = apps.get_model("db", "Issue") - IssueType = apps.get_model("db", "IssueType") - # Create the issue types for all projects - IssueType.objects.bulk_create( - [ - IssueType( - name="Task", - description="A task that needs to be completed.", - project_id=project["id"], - workspace_id=project["workspace_id"], - ) - for project in Project.objects.values("id", "workspace_id") - ], - batch_size=1000, - ) - # Update the issue type for all existing issues - issue_types = { - str(issue_type["project_id"]): str(issue_type["id"]) - for issue_type in IssueType.objects.values("id", "project_id") - } - # Update the issue type for all existing issues - bulk_issues = [] - for issue in Issue.objects.all(): - issue.type_id = issue_types[str(issue.project_id)] - bulk_issues.append(issue) - # Update the issue type for all existing issues - Issue.objects.bulk_update(bulk_issues, ["type_id"], batch_size=1000) + total = Project.objects.count() + delay_increment = INITIAL_DELAY + + while start < total: + projects = list( + Project.objects.values("id", "workspace_id")[start:end] + ) + backfill_issue_type_task.apply_async( + (projects,), countdown=delay_increment + ) + delay_increment += ( + INCREMENT_DELAY # Increment delay for the next batch + ) + start += CHUNK_SIZE + end += CHUNK_SIZE + + +def backfill_page_versions(apps, schema_editor): + start = 0 + end = CHUNK_SIZE + + Page = apps.get_model("db", "Page") + + total = Page.objects.count() + delay_increment = INITIAL_DELAY + + while start < total: + pages = list( + Page.objects.values( + "id", + "workspace_id", + "owned_by_id", + "description_binary", + "description_html", + "description_stripped", + )[start:end] + ) + backfill_page_versions_task.apply_async( + (pages,), countdown=delay_increment + ) + delay_increment += ( + INCREMENT_DELAY # Increment delay for the next batch + ) + start += CHUNK_SIZE + end += CHUNK_SIZE class Migration(migrations.Migration): @@ -140,7 +169,6 @@ class Migration(migrations.Migration): name="is_service", field=models.BooleanField(default=False), ), - migrations.RunPython(create_issue_types), migrations.CreateModel( name="PageVersion", fields=[ @@ -180,6 +208,10 @@ class Migration(migrations.Migration): "description_stripped", models.TextField(blank=True, null=True), ), + ( + "description_json", + models.JSONField(blank=True, default=dict), + ), ( "created_by", models.ForeignKey( @@ -274,4 +306,6 @@ class Migration(migrations.Migration): name="target_date", field=models.DateTimeField(blank=True, null=True), ), + migrations.RunPython(backfill_issue_types), + migrations.RunPython(backfill_page_versions), ] diff --git a/apiserver/plane/db/models/page.py b/apiserver/plane/db/models/page.py index 71e9d3c25..bc6f17b34 100644 --- a/apiserver/plane/db/models/page.py +++ b/apiserver/plane/db/models/page.py @@ -281,6 +281,7 @@ class PageVersion(BaseModel): description_binary = models.BinaryField(null=True) description_html = models.TextField(blank=True, default="
") description_stripped = models.TextField(blank=True, null=True) + description_json = models.JSONField(default=dict, blank=True) class Meta: verbose_name = "Page Version" diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index 937ff0c2a..91cbac37e 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -276,6 +276,8 @@ CELERY_IMPORTS = ( "plane.bgtasks.api_logs_task", # management tasks "plane.bgtasks.dummy_data_task", + # backfill tasks + "plane.db.backfills.backfill_0070_page_versions", ) # Sentry Settings