[WEB-4373]: optimize backend query for workspace views and Project gantt view (#7267)

* feat: add IssueListDetailSerializer for detailed issue representation

- Introduced IssueListDetailSerializer to enhance issue data representation with expanded fields.
- Updated issue detail endpoint to utilize the new serializer for improved data handling.
- Added methods for retrieving related module, label, and assignee IDs, along with support for expanded relations.

* feat: add ViewIssueListSerializer and enhance issue ordering

- Introduced ViewIssueListSerializer for improved issue representation, including assignee, label, and module IDs.
- Updated WorkspaceViewIssuesViewSet to utilize the new serializer and optimized queryset with prefetching.
- Enhanced order_issue_queryset to maintain consistent ordering by created_at alongside other fields.
- Modified pagination logic to support total count retrieval for better performance.

* fix: optimize issue filtering and pagination logic

- Updated WorkspaceViewIssuesViewSet to apply filters more efficiently in the issue query.
- Refined pagination logic in OffsetPaginator to ensure consistent behavior using limit instead of cursor.value, improving overall pagination accuracy.

* fix: improve pagination logic in OffsetPaginator

- Updated the next_cursor calculation to use the length of results instead of cursor.value, ensuring accurate pagination behavior.
- Added a comment to clarify the purpose of checking for additional results after the current page.

* Move the common permission filters into a separate method

* fix: handle deleted related issues in serializers

- Updated IssueListDetailSerializer to skip null related issues when building relations.
- Enhanced ViewIssueListSerializer to safely access state.group, returning None if state is not present.
- Removed unused User import in base.py for cleaner code.

---------

Co-authored-by: Dheeraj Kumar Ketireddy <dheeru0198@gmail.com>
This commit is contained in:
Nikhil 2025-06-25 19:10:24 +05:30 committed by GitHub
parent 0e91feacc3
commit b8043f92b1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 316 additions and 227 deletions

View file

@ -39,7 +39,7 @@ from .project import (
ProjectMemberRoleSerializer,
)
from .state import StateSerializer, StateLiteSerializer
from .view import IssueViewSerializer
from .view import IssueViewSerializer, ViewIssueListSerializer
from .cycle import (
CycleSerializer,
CycleIssueSerializer,
@ -74,6 +74,7 @@ from .issue import (
IssueLinkLiteSerializer,
IssueVersionDetailSerializer,
IssueDescriptionVersionDetailSerializer,
IssueListDetailSerializer,
)
from .module import (

View file

@ -725,6 +725,110 @@ class IssueSerializer(DynamicBaseSerializer):
read_only_fields = fields
class IssueListDetailSerializer(serializers.Serializer):
def __init__(self, *args, **kwargs):
# Extract expand parameter and store it as instance variable
self.expand = kwargs.pop("expand", []) or []
# Extract fields parameter and store it as instance variable
self.fields = kwargs.pop("fields", []) or []
super().__init__(*args, **kwargs)
def get_module_ids(self, obj):
return [module.module_id for module in obj.issue_module.all()]
def get_label_ids(self, obj):
return [label.label_id for label in obj.label_issue.all()]
def get_assignee_ids(self, obj):
return [assignee.assignee_id for assignee in obj.issue_assignee.all()]
def to_representation(self, instance):
data = {
# Basic fields
"id": instance.id,
"name": instance.name,
"state_id": instance.state_id,
"sort_order": instance.sort_order,
"completed_at": instance.completed_at,
"estimate_point": instance.estimate_point_id,
"priority": instance.priority,
"start_date": instance.start_date,
"target_date": instance.target_date,
"sequence_id": instance.sequence_id,
"project_id": instance.project_id,
"parent_id": instance.parent_id,
"created_at": instance.created_at,
"updated_at": instance.updated_at,
"created_by": instance.created_by_id,
"updated_by": instance.updated_by_id,
"is_draft": instance.is_draft,
"archived_at": instance.archived_at,
# Computed fields
"cycle_id": instance.cycle_id,
"module_ids": self.get_module_ids(instance),
"label_ids": self.get_label_ids(instance),
"assignee_ids": self.get_assignee_ids(instance),
"sub_issues_count": instance.sub_issues_count,
"attachment_count": instance.attachment_count,
"link_count": instance.link_count,
}
# Handle expanded fields only when requested - using direct field access
if self.expand:
if "issue_relation" in self.expand:
relations = []
for relation in instance.issue_relation.all():
related_issue = relation.related_issue
# If the related issue is deleted, skip it
if not related_issue:
continue
# Add the related issue to the relations list
relations.append(
{
"id": related_issue.id,
"project_id": related_issue.project_id,
"sequence_id": related_issue.sequence_id,
"name": related_issue.name,
"relation_type": relation.relation_type,
"state_id": related_issue.state_id,
"priority": related_issue.priority,
"created_by": related_issue.created_by_id,
"created_at": related_issue.created_at,
"updated_at": related_issue.updated_at,
"updated_by": related_issue.updated_by_id,
}
)
data["issue_relation"] = relations
if "issue_related" in self.expand:
related = []
for relation in instance.issue_related.all():
issue = relation.issue
# If the related issue is deleted, skip it
if not issue:
continue
# Add the related issue to the related list
related.append(
{
"id": issue.id,
"project_id": issue.project_id,
"sequence_id": issue.sequence_id,
"name": issue.name,
"relation_type": relation.relation_type,
"state_id": issue.state_id,
"priority": issue.priority,
"created_by": issue.created_by_id,
"created_at": issue.created_at,
"updated_at": issue.updated_at,
"updated_by": issue.updated_by_id,
}
)
data["issue_related"] = related
return data
class IssueLiteSerializer(DynamicBaseSerializer):
class Meta:
model = Issue

View file

@ -7,6 +7,49 @@ from plane.db.models import IssueView
from plane.utils.issue_filters import issue_filters
class ViewIssueListSerializer(serializers.Serializer):
def get_assignee_ids(self, instance):
return [assignee.assignee_id for assignee in instance.issue_assignee.all()]
def get_label_ids(self, instance):
return [label.label_id for label in instance.label_issue.all()]
def get_module_ids(self, instance):
return [module.module_id for module in instance.issue_module.all()]
def to_representation(self, instance):
data = {
"id": instance.id,
"name": instance.name,
"state_id": instance.state_id,
"sort_order": instance.sort_order,
"completed_at": instance.completed_at,
"estimate_point": instance.estimate_point_id,
"priority": instance.priority,
"start_date": instance.start_date,
"target_date": instance.target_date,
"sequence_id": instance.sequence_id,
"project_id": instance.project_id,
"parent_id": instance.parent_id,
"cycle_id": instance.cycle_id,
"sub_issues_count": instance.sub_issues_count,
"created_at": instance.created_at,
"updated_at": instance.updated_at,
"created_by": instance.created_by_id,
"updated_by": instance.updated_by_id,
"attachment_count": instance.attachment_count,
"link_count": instance.link_count,
"is_draft": instance.is_draft,
"archived_at": instance.archived_at,
"state__group": instance.state.group if instance.state else None,
"assignee_ids": self.get_assignee_ids(instance),
"label_ids": self.get_label_ids(instance),
"module_ids": self.get_module_ids(instance),
}
return data
class IssueViewSerializer(DynamicBaseSerializer):
is_favorite = serializers.BooleanField(read_only=True)

View file

@ -32,6 +32,7 @@ from plane.app.serializers import (
IssueDetailSerializer,
IssueUserPropertySerializer,
IssueSerializer,
IssueListDetailSerializer,
)
from plane.bgtasks.issue_activities_task import issue_activity
from plane.db.models import (
@ -46,6 +47,9 @@ from plane.db.models import (
CycleIssue,
UserRecentVisit,
ModuleIssue,
IssueRelation,
IssueAssignee,
IssueLabel,
)
from plane.utils.grouper import (
issue_group_values,
@ -947,22 +951,22 @@ class IssueDetailEndpoint(BaseAPIView):
# check for the project member role, if the role is 5 then check for the guest_view_all_features
# if it is true then show all the issues else show only the issues created by the user
project_member_subquery = ProjectMember.objects.filter(
project_id=OuterRef("project_id"),
member=self.request.user,
is_active=True,
).filter(
Q(role__gt=ROLE.GUEST.value)
| Q(
role=ROLE.GUEST.value, project__guest_view_all_features=True
permission_subquery = (
Issue.issue_objects.filter(
workspace__slug=slug, project_id=project_id, id=OuterRef("id")
)
)
# Main issue query
issue = (
Issue.issue_objects.filter(workspace__slug=slug, project_id=project_id)
.filter(
Q(Exists(project_member_subquery))
Q(
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
project__project_projectmember__role__gt=ROLE.GUEST.value,
)
| Q(
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
project__project_projectmember__role=ROLE.GUEST.value,
project__guest_view_all_features=True,
)
| Q(
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
@ -971,7 +975,30 @@ class IssueDetailEndpoint(BaseAPIView):
created_by=self.request.user,
)
)
.prefetch_related("assignees", "labels", "issue_module__module")
.values("id")
)
# Main issue query
issue = (
Issue.issue_objects.filter(workspace__slug=slug, project_id=project_id)
.filter(Exists(permission_subquery))
.prefetch_related(
Prefetch(
"issue_assignee",
queryset=IssueAssignee.objects.all(),
)
)
.prefetch_related(
Prefetch(
"label_issue",
queryset=IssueLabel.objects.all(),
)
)
.prefetch_related(
Prefetch(
"issue_module",
queryset=ModuleIssue.objects.all(),
)
)
.annotate(
cycle_id=Subquery(
CycleIssue.objects.filter(
@ -979,43 +1006,6 @@ class IssueDetailEndpoint(BaseAPIView):
).values("cycle_id")[:1]
)
)
.annotate(
label_ids=Coalesce(
ArrayAgg(
"labels__id",
distinct=True,
filter=Q(
~Q(labels__id__isnull=True)
& Q(label_issue__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
assignee_ids=Coalesce(
ArrayAgg(
"assignees__id",
distinct=True,
filter=Q(
~Q(assignees__id__isnull=True)
& Q(assignees__member_project__is_active=True)
& Q(issue_assignee__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
module_ids=Coalesce(
ArrayAgg(
"issue_module__module_id",
distinct=True,
filter=Q(
~Q(issue_module__module_id__isnull=True)
& Q(issue_module__module__archived_at__isnull=True)
& Q(issue_module__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
)
.annotate(
link_count=IssueLink.objects.filter(issue=OuterRef("id"))
.order_by()
@ -1039,6 +1029,23 @@ class IssueDetailEndpoint(BaseAPIView):
)
)
# Add additional prefetch based on expand parameter
if self.expand:
if "issue_relation" in self.expand:
issue = issue.prefetch_related(
Prefetch(
"issue_relation",
queryset=IssueRelation.objects.select_related("related_issue"),
)
)
if "issue_related" in self.expand:
issue = issue.prefetch_related(
Prefetch(
"issue_related",
queryset=IssueRelation.objects.select_related("issue"),
)
)
issue = issue.filter(**filters)
order_by_param = request.GET.get("order_by", "-created_at")
# Issue queryset
@ -1049,7 +1056,7 @@ class IssueDetailEndpoint(BaseAPIView):
request=request,
order_by=order_by_param,
queryset=(issue),
on_results=lambda issue: IssueSerializer(
on_results=lambda issue: IssueListDetailSerializer(
issue, many=True, fields=self.fields, expand=self.expand
).data,
)

View file

@ -1,8 +1,13 @@
# Django imports
from django.contrib.postgres.aggregates import ArrayAgg
from django.contrib.postgres.fields import ArrayField
from django.db.models import Exists, F, Func, OuterRef, Q, UUIDField, Value, Subquery
from django.db.models.functions import Coalesce
from django.db.models import (
Exists,
F,
Func,
OuterRef,
Q,
Subquery,
Prefetch,
)
from django.utils.decorators import method_decorator
from django.views.decorators.gzip import gzip_page
from django.db import transaction
@ -13,7 +18,7 @@ from rest_framework.response import Response
# Module imports
from plane.app.permissions import allow_permission, ROLE
from plane.app.serializers import IssueViewSerializer
from plane.app.serializers import IssueViewSerializer, ViewIssueListSerializer
from plane.db.models import (
Issue,
FileAsset,
@ -25,15 +30,12 @@ from plane.db.models import (
Project,
CycleIssue,
UserRecentVisit,
)
from plane.utils.grouper import (
issue_group_values,
issue_on_results,
issue_queryset_grouper,
IssueAssignee,
IssueLabel,
ModuleIssue,
)
from plane.utils.issue_filters import issue_filters
from plane.utils.order_queryset import order_issue_queryset
from plane.utils.paginator import GroupedOffsetPaginator, SubGroupedOffsetPaginator
from plane.bgtasks.recent_visited_task import recent_visited_task
from .. import BaseViewSet
from plane.db.models import UserFavorite
@ -143,6 +145,28 @@ class WorkspaceViewViewSet(BaseViewSet):
class WorkspaceViewIssuesViewSet(BaseViewSet):
def _get_project_permission_filters(self):
"""
Get common project permission filters for guest users and role-based access control.
Returns Q object for filtering issues based on user role and project settings.
"""
return Q(
Q(
project__project_projectmember__role=5,
project__guest_view_all_features=True,
)
| Q(
project__project_projectmember__role=5,
project__guest_view_all_features=False,
created_by=self.request.user,
)
|
# For other roles (role > 5), show all issues
Q(project__project_projectmember__role__gt=5),
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
)
def get_queryset(self):
return (
Issue.issue_objects.annotate(
@ -152,12 +176,25 @@ class WorkspaceViewIssuesViewSet(BaseViewSet):
.values("count")
)
.filter(workspace__slug=self.kwargs.get("slug"))
.filter(
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
.select_related("state")
.prefetch_related(
Prefetch(
"issue_assignee",
queryset=IssueAssignee.objects.all(),
)
)
.prefetch_related(
Prefetch(
"label_issue",
queryset=IssueLabel.objects.all(),
)
)
.prefetch_related(
Prefetch(
"issue_module",
queryset=ModuleIssue.objects.all(),
)
)
.select_related("workspace", "project", "state", "parent")
.prefetch_related("assignees", "labels", "issue_module__module")
.annotate(
cycle_id=Subquery(
CycleIssue.objects.filter(
@ -186,43 +223,6 @@ class WorkspaceViewIssuesViewSet(BaseViewSet):
.annotate(count=Func(F("id"), function="Count"))
.values("count")
)
.annotate(
label_ids=Coalesce(
ArrayAgg(
"labels__id",
distinct=True,
filter=Q(
~Q(labels__id__isnull=True)
& Q(label_issue__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
assignee_ids=Coalesce(
ArrayAgg(
"assignees__id",
distinct=True,
filter=Q(
~Q(assignees__id__isnull=True)
& Q(assignees__member_project__is_active=True)
& Q(issue_assignee__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
module_ids=Coalesce(
ArrayAgg(
"issue_module__module_id",
distinct=True,
filter=Q(
~Q(issue_module__module_id__isnull=True)
& Q(issue_module__module__archived_at__isnull=True)
& Q(issue_module__deleted_at__isnull=True)
),
),
Value([], output_field=ArrayField(UUIDField())),
),
)
)
@method_decorator(gzip_page)
@ -233,126 +233,36 @@ class WorkspaceViewIssuesViewSet(BaseViewSet):
filters = issue_filters(request.query_params, "GET")
order_by_param = request.GET.get("order_by", "-created_at")
issue_queryset = (
self.get_queryset()
.filter(**filters)
.annotate(
cycle_id=Subquery(
CycleIssue.objects.filter(
issue=OuterRef("id"), deleted_at__isnull=True
).values("cycle_id")[:1]
)
)
issue_queryset = self.get_queryset().filter(**filters)
# Get common project permission filters
permission_filters = self._get_project_permission_filters()
# Base query for the counts
total_issue_count = (
Issue.issue_objects.filter(**filters)
.filter(workspace__slug=slug)
.filter(permission_filters)
.only("id")
)
# check for the project member role, if the role is 5 then check for the guest_view_all_features if it is true then show all the issues else show only the issues created by the user
issue_queryset = issue_queryset.filter(
Q(
project__project_projectmember__role=5,
project__guest_view_all_features=True,
)
| Q(
project__project_projectmember__role=5,
project__guest_view_all_features=False,
created_by=self.request.user,
)
|
# For other roles (role < 5), show all issues
Q(project__project_projectmember__role__gt=5),
project__project_projectmember__member=self.request.user,
project__project_projectmember__is_active=True,
)
# Apply project permission filters to the issue queryset
issue_queryset = issue_queryset.filter(permission_filters)
# Issue queryset
issue_queryset, order_by_param = order_issue_queryset(
issue_queryset=issue_queryset, order_by_param=order_by_param
)
# Group by
group_by = request.GET.get("group_by", False)
sub_group_by = request.GET.get("sub_group_by", False)
# issue queryset
issue_queryset = issue_queryset_grouper(
queryset=issue_queryset, group_by=group_by, sub_group_by=sub_group_by
# List Paginate
return self.paginate(
order_by=order_by_param,
request=request,
queryset=issue_queryset,
on_results=lambda issues: ViewIssueListSerializer(issues, many=True).data,
total_count_queryset=total_issue_count,
)
if group_by:
# Check group and sub group value paginate
if sub_group_by:
if group_by == sub_group_by:
return Response(
{
"error": "Group by and sub group by cannot have same parameters"
},
status=status.HTTP_400_BAD_REQUEST,
)
else:
# group and sub group pagination
return self.paginate(
request=request,
order_by=order_by_param,
queryset=issue_queryset,
on_results=lambda issues: issue_on_results(
group_by=group_by, issues=issues, sub_group_by=sub_group_by
),
paginator_cls=SubGroupedOffsetPaginator,
group_by_fields=issue_group_values(
field=group_by, slug=slug, project_id=None, filters=filters
),
sub_group_by_fields=issue_group_values(
field=sub_group_by,
slug=slug,
project_id=None,
filters=filters,
),
group_by_field_name=group_by,
sub_group_by_field_name=sub_group_by,
count_filter=Q(
Q(issue_intake__status=1)
| Q(issue_intake__status=-1)
| Q(issue_intake__status=2)
| Q(issue_intake__isnull=True),
archived_at__isnull=True,
is_draft=False,
),
)
# Group Paginate
else:
# Group paginate
return self.paginate(
request=request,
order_by=order_by_param,
queryset=issue_queryset,
on_results=lambda issues: issue_on_results(
group_by=group_by, issues=issues, sub_group_by=sub_group_by
),
paginator_cls=GroupedOffsetPaginator,
group_by_fields=issue_group_values(
field=group_by, slug=slug, project_id=None, filters=filters
),
group_by_field_name=group_by,
count_filter=Q(
Q(issue_intake__status=1)
| Q(issue_intake__status=-1)
| Q(issue_intake__status=2)
| Q(issue_intake__isnull=True),
archived_at__isnull=True,
is_draft=False,
),
)
else:
# List Paginate
return self.paginate(
order_by=order_by_param,
request=request,
queryset=issue_queryset,
on_results=lambda issues: issue_on_results(
group_by=group_by, issues=issues, sub_group_by=sub_group_by
),
)
class IssueViewViewSet(BaseViewSet):
serializer_class = IssueViewSerializer

View file

@ -16,7 +16,7 @@ def order_issue_queryset(issue_queryset, order_by_param="-created_at"):
],
output_field=CharField(),
)
).order_by("priority_order")
).order_by("priority_order", "-created_at")
order_by_param = (
"priority_order" if order_by_param.startswith("-") else "-priority_order"
)
@ -36,7 +36,7 @@ def order_issue_queryset(issue_queryset, order_by_param="-created_at"):
default=Value(len(state_order)),
output_field=CharField(),
)
).order_by("state_order")
).order_by("state_order", "-created_at")
order_by_param = (
"-state_order" if order_by_param.startswith("-") else "state_order"
)
@ -55,11 +55,18 @@ def order_issue_queryset(issue_queryset, order_by_param="-created_at"):
if order_by_param.startswith("-")
else order_by_param
)
).order_by("-min_values" if order_by_param.startswith("-") else "min_values")
).order_by(
"-min_values" if order_by_param.startswith("-") else "min_values",
"-created_at",
)
order_by_param = (
"-min_values" if order_by_param.startswith("-") else "min_values"
)
else:
issue_queryset = issue_queryset.order_by(order_by_param)
# If the order_by_param is created_at, then don't add the -created_at
if "created_at" in order_by_param:
issue_queryset = issue_queryset.order_by(order_by_param)
else:
issue_queryset = issue_queryset.order_by(order_by_param, "-created_at")
order_by_param = order_by_param
return issue_queryset, order_by_param

View file

@ -102,6 +102,7 @@ class OffsetPaginator:
max_limit=MAX_LIMIT,
max_offset=None,
on_results=None,
total_count_queryset=None,
):
# Key tuple and remove `-` if descending order by
self.key = (
@ -115,6 +116,7 @@ class OffsetPaginator:
self.max_limit = max_limit
self.max_offset = max_offset
self.on_results = on_results
self.total_count_queryset = total_count_queryset
def get_result(self, limit=1000, cursor=None):
# offset is page #
@ -138,9 +140,9 @@ class OffsetPaginator:
)
# The current page
page = cursor.offset
# The offset
offset = cursor.offset * cursor.value
stop = offset + (cursor.value or limit) + 1
# The offset - use limit instead of cursor.value for consistent pagination
offset = cursor.offset * limit
stop = offset + limit + 1
if self.max_offset is not None and offset >= self.max_offset:
raise BadPaginationError("Pagination offset too large")
@ -148,11 +150,21 @@ class OffsetPaginator:
raise BadPaginationError("Pagination offset cannot be negative")
results = queryset[offset:stop]
if cursor.value != limit:
# Only slice from the end if we're going backwards (previous page)
if cursor.value != limit and cursor.is_prev:
results = results[-(limit + 1) :]
total_count = (
self.total_count_queryset.count()
if self.total_count_queryset
else results.count()
)
# Check if there are more results available after the current page
# Adjust cursors based on the results for pagination
next_cursor = Cursor(limit, page + 1, False, results.count() > limit)
next_cursor = Cursor(limit, page + 1, False, len(results) > limit)
# If the page is greater than 0, then set the previous cursor
prev_cursor = Cursor(limit, page - 1, True, page > 0)
@ -164,7 +176,7 @@ class OffsetPaginator:
results = self.on_results(results)
# Count the queryset
count = queryset.count()
count = total_count
# Optionally, calculate the total count and max_hits if needed
max_hits = math.ceil(count / limit)
@ -196,6 +208,7 @@ class GroupedOffsetPaginator(OffsetPaginator):
group_by_field_name,
group_by_fields,
count_filter,
total_count_queryset=None,
*args,
**kwargs,
):
@ -404,6 +417,7 @@ class SubGroupedOffsetPaginator(OffsetPaginator):
group_by_fields,
sub_group_by_fields,
count_filter,
total_count_queryset=None,
*args,
**kwargs,
):
@ -694,6 +708,7 @@ class BasePaginator:
sub_group_by_field_name=None,
sub_group_by_fields=None,
count_filter=None,
total_count_queryset=None,
**paginator_kwargs,
):
"""Paginate the request"""
@ -719,6 +734,8 @@ class BasePaginator:
)
paginator_kwargs["sub_group_by_fields"] = sub_group_by_fields
paginator_kwargs["total_count_queryset"] = total_count_queryset
paginator = paginator_cls(**paginator_kwargs)
try: