bb-plane-fork/apiserver/plane/app/views/asset/v2.py
Nikhil 9b85306359
dev: move storage metadata collection to background job (#5818)
* fix: move storage metadata collection to background job

* fix: docker compose and env

* fix: archive endpoint
2024-10-16 13:55:49 +05:30

803 lines
26 KiB
Python

# Python imports
import uuid
# Django imports
from django.conf import settings
from django.http import HttpResponseRedirect
from django.utils import timezone
# Third party imports
from rest_framework import status
from rest_framework.response import Response
from rest_framework.permissions import AllowAny
# Module imports
from ..base import BaseAPIView
from plane.db.models import (
FileAsset,
Workspace,
Project,
User,
)
from plane.settings.storage import S3Storage
from plane.app.permissions import allow_permission, ROLE
from plane.utils.cache import invalidate_cache_directly
from plane.bgtasks.storage_metadata_task import get_asset_object_metadata
class UserAssetsV2Endpoint(BaseAPIView):
"""This endpoint is used to upload user profile images."""
def asset_delete(self, asset_id):
asset = FileAsset.objects.filter(id=asset_id).first()
if asset is None:
return
asset.is_deleted = True
asset.deleted_at = timezone.now()
asset.save()
return
def entity_asset_save(self, asset_id, entity_type, asset, request):
# User Avatar
if entity_type == FileAsset.EntityTypeContext.USER_AVATAR:
user = User.objects.get(id=asset.user_id)
user.avatar = ""
# Delete the previous avatar
if user.avatar_asset_id:
self.asset_delete(user.avatar_asset_id)
# Save the new avatar
user.avatar_asset_id = asset_id
user.save()
invalidate_cache_directly(
path="/api/users/me/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/settings/",
url_params=False,
user=True,
request=request,
)
return
# User Cover
if entity_type == FileAsset.EntityTypeContext.USER_COVER:
user = User.objects.get(id=asset.user_id)
user.cover_image = None
# Delete the previous cover image
if user.cover_image_asset_id:
self.asset_delete(user.cover_image_asset_id)
# Save the new cover image
user.cover_image_asset_id = asset_id
user.save()
invalidate_cache_directly(
path="/api/users/me/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/settings/",
url_params=False,
user=True,
request=request,
)
return
return
def entity_asset_delete(self, entity_type, asset, request):
# User Avatar
if entity_type == FileAsset.EntityTypeContext.USER_AVATAR:
user = User.objects.get(id=asset.user_id)
user.avatar_asset_id = None
user.save()
invalidate_cache_directly(
path="/api/users/me/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/settings/",
url_params=False,
user=True,
request=request,
)
return
# User Cover
if entity_type == FileAsset.EntityTypeContext.USER_COVER:
user = User.objects.get(id=asset.user_id)
user.cover_image_asset_id = None
user.save()
invalidate_cache_directly(
path="/api/users/me/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/settings/",
url_params=False,
user=True,
request=request,
)
return
return
def post(self, request):
# get the asset key
name = request.data.get("name")
type = request.data.get("type", "image/jpeg")
size = int(request.data.get("size", settings.FILE_SIZE_LIMIT))
entity_type = request.data.get("entity_type", False)
# Check if the file size is within the limit
size_limit = min(size, settings.FILE_SIZE_LIMIT)
# Check if the entity type is allowed
if not entity_type or entity_type not in ["USER_AVATAR", "USER_COVER"]:
return Response(
{
"error": "Invalid entity type.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Check if the file type is allowed
allowed_types = ["image/jpeg", "image/png", "image/webp", "image/jpg"]
if type not in allowed_types:
return Response(
{
"error": "Invalid file type. Only JPEG and PNG files are allowed.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# asset key
asset_key = f"{uuid.uuid4().hex}-{name}"
# Create a File Asset
asset = FileAsset.objects.create(
attributes={
"name": name,
"type": type,
"size": size_limit,
},
asset=asset_key,
size=size_limit,
user=request.user,
created_by=request.user,
entity_type=entity_type,
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
presigned_url = storage.generate_presigned_post(
object_name=asset_key,
file_type=type,
file_size=size_limit,
)
# Return the presigned URL
return Response(
{
"upload_data": presigned_url,
"asset_id": str(asset.id),
"asset_url": asset.asset_url,
},
status=status.HTTP_200_OK,
)
def patch(self, request, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id, user_id=request.user.id)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(asset_id))
# get the entity and save the asset id for the request field
self.entity_asset_save(
asset_id=asset_id,
entity_type=asset.entity_type,
asset=asset,
request=request,
)
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
def delete(self, request, asset_id):
asset = FileAsset.objects.get(id=asset_id, user_id=request.user.id)
asset.is_deleted = True
asset.deleted_at = timezone.now()
# get the entity and save the asset id for the request field
self.entity_asset_delete(
entity_type=asset.entity_type, asset=asset, request=request
)
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
class WorkspaceFileAssetEndpoint(BaseAPIView):
"""This endpoint is used to upload cover images/logos etc for workspace, projects and users."""
def get_entity_id_field(self, entity_type, entity_id):
# Workspace Logo
if entity_type == FileAsset.EntityTypeContext.WORKSPACE_LOGO:
return {
"workspace_id": entity_id,
}
# Project Cover
if entity_type == FileAsset.EntityTypeContext.PROJECT_COVER:
return {
"project_id": entity_id,
}
# User Avatar and Cover
if entity_type in [
FileAsset.EntityTypeContext.USER_AVATAR,
FileAsset.EntityTypeContext.USER_COVER,
]:
return {
"user_id": entity_id,
}
# Issue Attachment and Description
if entity_type in [
FileAsset.EntityTypeContext.ISSUE_ATTACHMENT,
FileAsset.EntityTypeContext.ISSUE_DESCRIPTION,
]:
return {
"issue_id": entity_id,
}
# Page Description
if entity_type == FileAsset.EntityTypeContext.PAGE_DESCRIPTION:
return {
"page_id": entity_id,
}
# Comment Description
if entity_type == FileAsset.EntityTypeContext.COMMENT_DESCRIPTION:
return {
"comment_id": entity_id,
}
return {}
def asset_delete(self, asset_id):
asset = FileAsset.objects.filter(id=asset_id).first()
# Check if the asset exists
if asset is None:
return
# Mark the asset as deleted
asset.is_deleted = True
asset.deleted_at = timezone.now()
asset.save()
return
def entity_asset_save(self, asset_id, entity_type, asset, request):
# Workspace Logo
if entity_type == FileAsset.EntityTypeContext.WORKSPACE_LOGO:
workspace = Workspace.objects.filter(id=asset.workspace_id).first()
if workspace is None:
return
# Delete the previous logo
if workspace.logo_asset_id:
self.asset_delete(workspace.logo_asset_id)
# Save the new logo
workspace.logo = ""
workspace.logo_asset_id = asset_id
workspace.save()
invalidate_cache_directly(
path="/api/workspaces/",
url_params=False,
user=False,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/workspaces/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/instances/",
url_params=False,
user=False,
request=request,
)
return
# Project Cover
elif entity_type == FileAsset.EntityTypeContext.PROJECT_COVER:
project = Project.objects.filter(id=asset.workspace_id).first()
if project is None:
return
# Delete the previous cover image
if project.cover_image_asset_id:
self.asset_delete(project.cover_image_asset_id)
# Save the new cover image
project.cover_image = ""
project.cover_image_asset_id = asset_id
project.save()
return
else:
return
def entity_asset_delete(self, entity_type, asset, request):
# Workspace Logo
if entity_type == FileAsset.EntityTypeContext.WORKSPACE_LOGO:
workspace = Workspace.objects.get(id=asset.workspace_id)
if workspace is None:
return
workspace.logo_asset_id = None
workspace.save()
invalidate_cache_directly(
path="/api/workspaces/",
url_params=False,
user=False,
request=request,
)
invalidate_cache_directly(
path="/api/users/me/workspaces/",
url_params=False,
user=True,
request=request,
)
invalidate_cache_directly(
path="/api/instances/",
url_params=False,
user=False,
request=request,
)
return
# Project Cover
elif entity_type == FileAsset.EntityTypeContext.PROJECT_COVER:
project = Project.objects.filter(id=asset.project_id).first()
if project is None:
return
project.cover_image_asset_id = None
project.save()
return
else:
return
def post(self, request, slug):
name = request.data.get("name")
type = request.data.get("type", "image/jpeg")
size = int(request.data.get("size", settings.FILE_SIZE_LIMIT))
entity_type = request.data.get("entity_type")
entity_identifier = request.data.get("entity_identifier", False)
# Check if the entity type is allowed
if entity_type not in FileAsset.EntityTypeContext.values:
return Response(
{
"error": "Invalid entity type.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Check if the file type is allowed
allowed_types = ["image/jpeg", "image/png", "image/webp", "image/jpg"]
if type not in allowed_types:
return Response(
{
"error": "Invalid file type. Only JPEG and PNG files are allowed.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Get the size limit
size_limit = min(settings.FILE_SIZE_LIMIT, size)
# Get the workspace
workspace = Workspace.objects.get(slug=slug)
# asset key
asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
# Create a File Asset
asset = FileAsset.objects.create(
attributes={
"name": name,
"type": type,
"size": size_limit,
},
asset=asset_key,
size=size_limit,
workspace=workspace,
created_by=request.user,
entity_type=entity_type,
**self.get_entity_id_field(
entity_type=entity_type, entity_id=entity_identifier
),
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
presigned_url = storage.generate_presigned_post(
object_name=asset_key,
file_type=type,
file_size=size_limit,
)
# Return the presigned URL
return Response(
{
"upload_data": presigned_url,
"asset_id": str(asset.id),
"asset_url": asset.asset_url,
},
status=status.HTTP_200_OK,
)
def patch(self, request, slug, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id, workspace__slug=slug)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(asset_id))
# get the entity and save the asset id for the request field
self.entity_asset_save(
asset_id=asset_id,
entity_type=asset.entity_type,
asset=asset,
request=request,
)
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
def delete(self, request, slug, asset_id):
asset = FileAsset.objects.get(id=asset_id, workspace__slug=slug)
asset.is_deleted = True
asset.deleted_at = timezone.now()
# get the entity and save the asset id for the request field
self.entity_asset_delete(
entity_type=asset.entity_type, asset=asset, request=request
)
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
def get(self, request, slug, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id, workspace__slug=slug)
# Check if the asset is uploaded
if not asset.is_uploaded:
return Response(
{
"error": "The requested asset could not be found.",
},
status=status.HTTP_404_NOT_FOUND,
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
signed_url = storage.generate_presigned_url(
object_name=asset.asset.name,
)
# Redirect to the signed URL
return HttpResponseRedirect(signed_url)
class StaticFileAssetEndpoint(BaseAPIView):
"""This endpoint is used to get the signed URL for a static asset."""
permission_classes = [
AllowAny,
]
def get(self, request, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id)
# Check if the asset is uploaded
if not asset.is_uploaded:
return Response(
{
"error": "The requested asset could not be found.",
},
status=status.HTTP_404_NOT_FOUND,
)
# Check if the entity type is allowed
if asset.entity_type not in [
FileAsset.EntityTypeContext.USER_AVATAR,
FileAsset.EntityTypeContext.USER_COVER,
FileAsset.EntityTypeContext.WORKSPACE_LOGO,
FileAsset.EntityTypeContext.PROJECT_COVER,
]:
return Response(
{
"error": "Invalid entity type.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
signed_url = storage.generate_presigned_url(
object_name=asset.asset.name,
)
# Redirect to the signed URL
return HttpResponseRedirect(signed_url)
class AssetRestoreEndpoint(BaseAPIView):
"""Endpoint to restore a deleted assets."""
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST], level="WORKSPACE")
def post(self, request, slug, asset_id):
asset = FileAsset.all_objects.get(id=asset_id, workspace__slug=slug)
asset.is_deleted = False
asset.deleted_at = None
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
class ProjectAssetEndpoint(BaseAPIView):
"""This endpoint is used to upload cover images/logos etc for workspace, projects and users."""
def get_entity_id_field(self, entity_type, entity_id):
if entity_type == FileAsset.EntityTypeContext.WORKSPACE_LOGO:
return {
"workspace_id": entity_id,
}
if entity_type == FileAsset.EntityTypeContext.PROJECT_COVER:
return {
"project_id": entity_id,
}
if entity_type in [
FileAsset.EntityTypeContext.USER_AVATAR,
FileAsset.EntityTypeContext.USER_COVER,
]:
return {
"user_id": entity_id,
}
if entity_type in [
FileAsset.EntityTypeContext.ISSUE_ATTACHMENT,
FileAsset.EntityTypeContext.ISSUE_DESCRIPTION,
]:
return {
"issue_id": entity_id,
}
if entity_type == FileAsset.EntityTypeContext.PAGE_DESCRIPTION:
return {
"page_id": entity_id,
}
if entity_type == FileAsset.EntityTypeContext.COMMENT_DESCRIPTION:
return {
"comment_id": entity_id,
}
if entity_type == FileAsset.EntityTypeContext.DRAFT_ISSUE_DESCRIPTION:
return {
"draft_issue_id": entity_id,
}
return {}
@allow_permission(
[ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST],
)
def post(self, request, slug, project_id):
name = request.data.get("name")
type = request.data.get("type", "image/jpeg")
size = int(request.data.get("size", settings.FILE_SIZE_LIMIT))
entity_type = request.data.get("entity_type", "")
entity_identifier = request.data.get("entity_identifier")
# Check if the entity type is allowed
if entity_type not in FileAsset.EntityTypeContext.values:
return Response(
{
"error": "Invalid entity type.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Check if the file type is allowed
allowed_types = ["image/jpeg", "image/png", "image/webp", "image/jpg"]
if type not in allowed_types:
return Response(
{
"error": "Invalid file type. Only JPEG and PNG files are allowed.",
"status": False,
},
status=status.HTTP_400_BAD_REQUEST,
)
# Get the size limit
size_limit = min(settings.FILE_SIZE_LIMIT, size)
# Get the workspace
workspace = Workspace.objects.get(slug=slug)
# asset key
asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
# Create a File Asset
asset = FileAsset.objects.create(
attributes={
"name": name,
"type": type,
"size": size_limit,
},
asset=asset_key,
size=size_limit,
workspace=workspace,
created_by=request.user,
entity_type=entity_type,
project_id=project_id,
**self.get_entity_id_field(entity_type, entity_identifier),
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
presigned_url = storage.generate_presigned_post(
object_name=asset_key,
file_type=type,
file_size=size_limit,
)
# Return the presigned URL
return Response(
{
"upload_data": presigned_url,
"asset_id": str(asset.id),
"asset_url": asset.asset_url,
},
status=status.HTTP_200_OK,
)
@allow_permission(
[ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST],
)
def patch(self, request, slug, project_id, pk):
# get the asset id
asset = FileAsset.objects.get(
id=pk,
)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(pk))
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST])
def delete(self, request, slug, project_id, pk):
# Get the asset
asset = FileAsset.objects.get(
id=pk,
workspace__slug=slug,
project_id=project_id,
)
# Check deleted assets
asset.is_deleted = True
asset.deleted_at = timezone.now()
# Save the asset
asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST])
def get(self, request, slug, project_id, pk):
# get the asset id
asset = FileAsset.objects.get(
workspace__slug=slug,
project_id=project_id,
pk=pk,
)
# Check if the asset is uploaded
if not asset.is_uploaded:
return Response(
{
"error": "The requested asset could not be found.",
},
status=status.HTTP_404_NOT_FOUND,
)
# Get the presigned URL
storage = S3Storage(request=request)
# Generate a presigned URL to share an S3 object
signed_url = storage.generate_presigned_url(
object_name=asset.asset.name,
)
# Redirect to the signed URL
return HttpResponseRedirect(signed_url)
class ProjectBulkAssetEndpoint(BaseAPIView):
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST])
def post(self, request, slug, project_id, entity_id):
asset_ids = request.data.get("asset_ids", [])
# Check if the asset ids are provided
if not asset_ids:
return Response(
{
"error": "No asset ids provided.",
},
status=status.HTTP_400_BAD_REQUEST,
)
# get the asset id
assets = FileAsset.objects.filter(
id__in=asset_ids,
workspace__slug=slug,
)
# Get the first asset
asset = assets.first()
if not asset:
return Response(
{
"error": "The requested asset could not be found.",
},
status=status.HTTP_404_NOT_FOUND,
)
# Check if the asset is uploaded
if asset.entity_type == FileAsset.EntityTypeContext.PROJECT_COVER:
assets.update(
project_id=project_id,
)
if asset.entity_type == FileAsset.EntityTypeContext.ISSUE_DESCRIPTION:
assets.update(
issue_id=entity_id,
)
if (
asset.entity_type
== FileAsset.EntityTypeContext.COMMENT_DESCRIPTION
):
assets.update(
comment_id=entity_id,
)
if asset.entity_type == FileAsset.EntityTypeContext.PAGE_DESCRIPTION:
assets.update(
page_id=entity_id,
)
if (
asset.entity_type
== FileAsset.EntityTypeContext.DRAFT_ISSUE_DESCRIPTION
):
assets.update(
draft_issue_id=entity_id,
)
return Response(status=status.HTTP_204_NO_CONTENT)