diff --git a/apiserver/plane/app/serializers/__init__.py b/apiserver/plane/app/serializers/__init__.py index 0d72f9192..07d22f423 100644 --- a/apiserver/plane/app/serializers/__init__.py +++ b/apiserver/plane/app/serializers/__init__.py @@ -7,6 +7,7 @@ from .user import ( UserAdminLiteSerializer, UserMeSerializer, UserMeSettingsSerializer, + UserAssetSerializer, ) from .workspace import ( WorkSpaceSerializer, diff --git a/apiserver/plane/app/serializers/base.py b/apiserver/plane/app/serializers/base.py index 20be382a8..880b19ab5 100644 --- a/apiserver/plane/app/serializers/base.py +++ b/apiserver/plane/app/serializers/base.py @@ -162,17 +162,10 @@ class DynamicBaseSerializer(BaseSerializer): class BaseFileSerializer(DynamicBaseSerializer): - download_url = serializers.SerializerMethodField() class Meta: abstract = True # Make this serializer abstract - def get_download_url(self, obj): - if hasattr(obj, "asset") and obj.asset: - storage = S3PrivateBucketStorage() - return storage.download_url(obj.asset.name) - return None - def to_representation(self, instance): """ Object instance -> Dict of primitive datatypes. diff --git a/apiserver/plane/app/serializers/user.py b/apiserver/plane/app/serializers/user.py index 8cd48827e..571560131 100644 --- a/apiserver/plane/app/serializers/user.py +++ b/apiserver/plane/app/serializers/user.py @@ -3,8 +3,7 @@ from rest_framework import serializers # Module import from .base import BaseSerializer -from plane.db.models import User, Workspace, WorkspaceMemberInvite -from plane.license.models import InstanceAdmin, Instance +from plane.db.models import User, Workspace, WorkspaceMemberInvite, UserAsset class UserSerializer(BaseSerializer): @@ -197,3 +196,12 @@ class ResetPasswordSerializer(serializers.Serializer): """ new_password = serializers.CharField(required=True, min_length=8) + + +class UserAssetSerializer(BaseSerializer): + class Meta: + model = UserAsset + fields = "__all__" + read_only_fields = [ + "user", + ] diff --git a/apiserver/plane/app/urls/asset.py b/apiserver/plane/app/urls/asset.py index 2d84b93e0..77461313c 100644 --- a/apiserver/plane/app/urls/asset.py +++ b/apiserver/plane/app/urls/asset.py @@ -19,16 +19,6 @@ urlpatterns = [ FileAssetEndpoint.as_view(), name="file-assets", ), - path( - "users/file-assets/", - UserAssetsEndpoint.as_view(), - name="user-file-assets", - ), - path( - "users/file-assets//", - UserAssetsEndpoint.as_view(), - name="user-file-assets", - ), path( "workspaces/file-assets///restore/", FileAssetViewSet.as_view( diff --git a/apiserver/plane/app/urls/user.py b/apiserver/plane/app/urls/user.py index 9dae7b5da..789b20188 100644 --- a/apiserver/plane/app/urls/user.py +++ b/apiserver/plane/app/urls/user.py @@ -8,6 +8,7 @@ from plane.app.views import ( UserActivityEndpoint, ChangePasswordEndpoint, SetUserPasswordEndpoint, + UserAssetsEndpoint, ## End User ## Workspaces UserWorkSpacesEndpoint, @@ -95,5 +96,15 @@ urlpatterns = [ SetUserPasswordEndpoint.as_view(), name="set-password", ), + path( + "users/assets/", + UserAssetsEndpoint.as_view(), + name="user-assets", + ), + path( + "users/assets//", + UserAssetsEndpoint.as_view(), + name="user-assets", + ), ## End User Graph ] diff --git a/apiserver/plane/app/views/__init__.py b/apiserver/plane/app/views/__init__.py index 0a959a667..6adffdb83 100644 --- a/apiserver/plane/app/views/__init__.py +++ b/apiserver/plane/app/views/__init__.py @@ -18,6 +18,7 @@ from .user import ( UpdateUserOnBoardedEndpoint, UpdateUserTourCompletedEndpoint, UserActivityEndpoint, + UserAssetsEndpoint, ) from .oauth import OauthEndpoint @@ -65,7 +66,7 @@ from .cycle import ( TransferCycleIssueEndpoint, CycleUserPropertiesEndpoint, ) -from .asset import FileAssetEndpoint, UserAssetsEndpoint, FileAssetViewSet +from .asset import FileAssetEndpoint, FileAssetViewSet from .issue import ( IssueViewSet, WorkSpaceIssuesEndpoint, @@ -180,7 +181,4 @@ from .webhook import ( WebhookSecretRegenerateEndpoint, ) -from .dashboard import ( - DashboardEndpoint, - WidgetsEndpoint -) \ No newline at end of file +from .dashboard import DashboardEndpoint, WidgetsEndpoint diff --git a/apiserver/plane/app/views/asset.py b/apiserver/plane/app/views/asset.py index fb5590610..a41bd2242 100644 --- a/apiserver/plane/app/views/asset.py +++ b/apiserver/plane/app/views/asset.py @@ -61,40 +61,3 @@ class FileAssetViewSet(BaseViewSet): file_asset.is_deleted = False file_asset.save() return Response(status=status.HTTP_204_NO_CONTENT) - - -class UserAssetsEndpoint(BaseAPIView): - parser_classes = (MultiPartParser, FormParser) - - def get(self, request, asset_key): - files = FileAsset.objects.filter( - asset=asset_key, created_by=request.user - ) - if files.exists(): - serializer = FileAssetSerializer( - files, context={"request": request} - ) - return Response( - {"data": serializer.data, "status": True}, - status=status.HTTP_200_OK, - ) - else: - return Response( - {"error": "Asset key does not exist", "status": False}, - status=status.HTTP_200_OK, - ) - - def post(self, request): - serializer = FileAssetSerializer(data=request.data) - if serializer.is_valid(): - serializer.save() - return Response(serializer.data, status=status.HTTP_201_CREATED) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - def delete(self, request, asset_key): - file_asset = FileAsset.objects.get( - asset=asset_key, created_by=request.user - ) - file_asset.is_deleted = True - file_asset.save() - return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/apiserver/plane/app/views/user.py b/apiserver/plane/app/views/user.py index 7764e3b97..bd0218ca6 100644 --- a/apiserver/plane/app/views/user.py +++ b/apiserver/plane/app/views/user.py @@ -1,7 +1,15 @@ +# Python imports +import requests + +# Django imports +from django.http import StreamingHttpResponse +from django.conf import settings + # Third party imports +import boto3 from rest_framework.response import Response from rest_framework import status - +from rest_framework.parsers import MultiPartParser, FormParser, JSONParser # Module imports from plane.app.serializers import ( @@ -9,13 +17,14 @@ from plane.app.serializers import ( IssueActivitySerializer, UserMeSerializer, UserMeSettingsSerializer, + UserAssetSerializer, ) from plane.app.views.base import BaseViewSet, BaseAPIView -from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember +from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember, UserAsset from plane.license.models import Instance, InstanceAdmin from plane.utils.paginator import BasePaginator - +from plane.utils.file_stream import get_file_streams from django.db.models import Q, F, Count, Case, When, IntegerField @@ -177,3 +186,27 @@ class UserActivityEndpoint(BaseAPIView, BasePaginator): issue_activities, many=True ).data, ) + + +class UserAssetsEndpoint(BaseAPIView): + parser_classes = (MultiPartParser, FormParser) + + def post(self, request): + serializer = UserAssetSerializer(data=request.data) + if serializer.is_valid(): + serializer.save(user=request.user) + return Response(serializer.data, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def delete(self, request, asset_key): + user_asset = UserAsset.objects.get( + asset=asset_key, created_by=request.user + ) + user_asset.is_deleted = True + user_asset.save() + return Response(status=status.HTTP_204_NO_CONTENT) + + + def get(self, request, key): + response = get_file_streams(key, key) + return response diff --git a/apiserver/plane/db/migrations/0059_auto_20240131_1334.py b/apiserver/plane/db/migrations/0059_auto_20240131_1334.py new file mode 100644 index 000000000..3058c1776 --- /dev/null +++ b/apiserver/plane/db/migrations/0059_auto_20240131_1334.py @@ -0,0 +1,175 @@ +# Generated by Django 4.2.7 on 2024-01-31 13:34 +import uuid +from django.db import migrations, models +from django.conf import settings +import django.db.models +import plane.db.models.asset + + +def update_urls(apps, schema_editor): + # Check if the app is using minio or s3 + if settings.USE_MINIO: + prefix = ( + f"{settings.AWS_S3_URL_PROTOCOL}//{settings.AWS_S3_CUSTOM_DOMAIN}/" + ) + prefix2 = prefix + else: + prefix = f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.{settings.AWS_REGION}.amazonaws.com/" + prefix2 = ( + f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/" + ) + + User = apps.get_model("db", "User") + UserAsset = apps.get_model("db", "UserAsset") + bulk_users = [] + bulk_user_assets = [] + for user in User.objects.all(): + if user.avatar and ( + user.avatar.startswith(prefix) or user.avatar.startswith(prefix2) + ): + avatar = user.avatar[len(prefix) :] + user.avatar = avatar + bulk_user_assets.append( + UserAsset( + user=user, + asset=avatar, + ) + ) + + if user.cover_image and ( + user.cover_image.startswith(prefix) + or user.cover_image.startswith(prefix2) + ): + cover_image = user.cover_image[len(prefix) :] + user.cover_image = cover_image + bulk_user_assets.append( + UserAsset( + user=user, + asset=cover_image, + ) + ) + User.objects.bulk_update( + bulk_users, ["avatar", "cover_image"], batch_size=100 + ) + UserAsset.objects.bulk_create(bulk_user_assets, batch_size=100) + + +class Migration(migrations.Migration): + dependencies = [ + ("db", "0058_alter_moduleissue_issue_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="UserAsset", + fields=[ + ( + "created_at", + models.DateTimeField( + auto_now_add=True, verbose_name="Created At" + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, verbose_name="Last Modified At" + ), + ), + ( + "id", + models.UUIDField( + db_index=True, + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ), + ), + ( + "asset", + models.FileField( + max_length=500, + storage=plane.settings.storage.S3PrivateBucketStorage(), + upload_to=plane.db.models.user.get_upload_path, + validators=[plane.db.models.user.file_size], + ), + ), + ("is_deleted", models.BooleanField(default=False)), + ("size", models.PositiveBigIntegerField(null=True)), + ("attributes", models.JSONField(default=dict)), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_created_by", + to=settings.AUTH_USER_MODEL, + verbose_name="Created By", + ), + ), + ( + "updated_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_updated_by", + to=settings.AUTH_USER_MODEL, + verbose_name="Last Modified By", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="assets", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "User Asset", + "verbose_name_plural": "User Assets", + "db_table": "user_assets", + "ordering": ("-created_at",), + }, + ), + migrations.AlterField( + model_name="fileasset", + name="asset", + field=models.FileField( + storage=plane.settings.storage.S3PrivateBucketStorage(), + upload_to=plane.db.models.asset.get_upload_path, + validators=[plane.db.models.asset.file_size], + ), + ), + migrations.RemoveField( + model_name="issueactivity", + name="attachments", + ), + migrations.RemoveField( + model_name="issuecomment", + name="attachments", + ), + migrations.AlterField( + model_name="integration", + name="avatar_url", + field=models.CharField(blank=True, null=True), + ), + migrations.AlterField( + model_name="project", + name="cover_image", + field=models.CharField(blank=True, max_length=800, null=True), + ), + migrations.AlterField( + model_name="user", + name="cover_image", + field=models.CharField(blank=True, max_length=800, null=True), + ), + migrations.AlterField( + model_name="workspace", + name="logo", + field=models.CharField(blank=True, null=True, verbose_name="Logo"), + ), + migrations.RunPython(update_urls), + ] diff --git a/apiserver/plane/db/models/__init__.py b/apiserver/plane/db/models/__init__.py index d9096bd01..9575702aa 100644 --- a/apiserver/plane/db/models/__init__.py +++ b/apiserver/plane/db/models/__init__.py @@ -1,6 +1,6 @@ from .base import BaseModel -from .user import User +from .user import User, UserAsset from .workspace import ( Workspace, diff --git a/apiserver/plane/db/models/integration/base.py b/apiserver/plane/db/models/integration/base.py index 0c68adfd2..355e14136 100644 --- a/apiserver/plane/db/models/integration/base.py +++ b/apiserver/plane/db/models/integration/base.py @@ -29,7 +29,7 @@ class Integration(AuditModel): redirect_url = models.TextField(blank=True) metadata = models.JSONField(default=dict) verified = models.BooleanField(default=False) - avatar_url = models.URLField(blank=True, null=True) + avatar_url = models.CharField(blank=True, null=True) def __str__(self): """Return provider of the integration""" diff --git a/apiserver/plane/db/models/issue.py b/apiserver/plane/db/models/issue.py index 05cb8d482..24bce3816 100644 --- a/apiserver/plane/db/models/issue.py +++ b/apiserver/plane/db/models/issue.py @@ -389,9 +389,6 @@ class IssueActivity(ProjectBaseModel): ) comment = models.TextField(verbose_name="Comment", blank=True) - attachments = ArrayField( - models.URLField(), size=10, blank=True, default=list - ) issue_comment = models.ForeignKey( "db.IssueComment", on_delete=models.SET_NULL, @@ -423,9 +420,6 @@ class IssueComment(ProjectBaseModel): comment_stripped = models.TextField(verbose_name="Comment", blank=True) comment_json = models.JSONField(blank=True, default=dict) comment_html = models.TextField(blank=True, default="

") - attachments = ArrayField( - models.URLField(), size=10, blank=True, default=list - ) issue = models.ForeignKey( Issue, on_delete=models.CASCADE, related_name="issue_comments" ) diff --git a/apiserver/plane/db/models/project.py b/apiserver/plane/db/models/project.py index b93174724..6fd3f2369 100644 --- a/apiserver/plane/db/models/project.py +++ b/apiserver/plane/db/models/project.py @@ -94,7 +94,7 @@ class Project(BaseModel): issue_views_view = models.BooleanField(default=True) page_view = models.BooleanField(default=True) inbox_view = models.BooleanField(default=False) - cover_image = models.URLField(blank=True, null=True, max_length=800) + cover_image = models.CharField(blank=True, null=True, max_length=800) estimate = models.ForeignKey( "db.Estimate", on_delete=models.SET_NULL, diff --git a/apiserver/plane/db/models/user.py b/apiserver/plane/db/models/user.py index 6f8a82e56..9ebafe0f9 100644 --- a/apiserver/plane/db/models/user.py +++ b/apiserver/plane/db/models/user.py @@ -15,12 +15,17 @@ from django.db.models.signals import post_save from django.conf import settings from django.dispatch import receiver from django.utils import timezone +from django.core.exceptions import ValidationError # Third party imports from sentry_sdk import capture_exception from slack_sdk import WebClient from slack_sdk.errors import SlackApiError +# Module imports +from . import BaseModel +from plane.settings.storage import S3PrivateBucketStorage + def get_default_onboarding(): return { @@ -49,7 +54,7 @@ class User(AbstractBaseUser, PermissionsMixin): first_name = models.CharField(max_length=255, blank=True) last_name = models.CharField(max_length=255, blank=True) avatar = models.CharField(max_length=255, blank=True) - cover_image = models.URLField(blank=True, null=True, max_length=800) + cover_image = models.CharField(blank=True, null=True, max_length=800) # tracking metrics date_joined = models.DateTimeField( @@ -144,6 +149,42 @@ class User(AbstractBaseUser, PermissionsMixin): super(User, self).save(*args, **kwargs) +def get_upload_path(instance, filename): + return f"user-{uuid.uuid4().hex}" + +def file_size(value): + if value.size > settings.FILE_SIZE_LIMIT: + raise ValidationError("File too large. Size should not exceed 5 MB.") + +class UserAsset(BaseModel): + + user = models.ForeignKey("db.User", on_delete=models.CASCADE, related_name="assets") + asset = models.FileField( + upload_to=get_upload_path, + validators=[ + file_size, + ], + storage=S3PrivateBucketStorage(), + max_length=500, + ) + is_deleted = models.BooleanField(default=False) + size = models.PositiveBigIntegerField(null=True) + attributes = models.JSONField(default=dict) + + def save(self, *args, **kwargs): + self.size = self.asset.size + super(UserAsset, self).save(*args, **kwargs) + + class Meta: + verbose_name = "User Asset" + verbose_name_plural = "User Assets" + db_table = "user_assets" + ordering = ("-created_at",) + + def __str__(self): + return str(self.asset) + + @receiver(post_save, sender=User) def send_welcome_slack(sender, instance, created, **kwargs): try: diff --git a/apiserver/plane/db/models/workspace.py b/apiserver/plane/db/models/workspace.py index 7e5d6d90b..f23203e55 100644 --- a/apiserver/plane/db/models/workspace.py +++ b/apiserver/plane/db/models/workspace.py @@ -131,7 +131,7 @@ def slug_validator(value): class Workspace(BaseModel): name = models.CharField(max_length=80, verbose_name="Workspace Name") - logo = models.URLField(verbose_name="Logo", blank=True, null=True) + logo = models.CharField(verbose_name="Logo", blank=True, null=True) owner = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.CASCADE, diff --git a/apiserver/plane/settings/storage.py b/apiserver/plane/settings/storage.py index 414495259..8993b8ac0 100644 --- a/apiserver/plane/settings/storage.py +++ b/apiserver/plane/settings/storage.py @@ -1,15 +1,9 @@ # Third party imports from storages.backends.s3boto3 import S3Boto3Storage -# Module imports -from plane.utils.presigned_url_generator import generate_download_presigned_url - class S3PrivateBucketStorage(S3Boto3Storage): def url(self, name): # Return an empty string or None, or implement custom logic here return name - - def download_url(self, name): - return generate_download_presigned_url(name) diff --git a/apiserver/plane/utils/file_stream.py b/apiserver/plane/utils/file_stream.py new file mode 100644 index 000000000..ab5f35fe7 --- /dev/null +++ b/apiserver/plane/utils/file_stream.py @@ -0,0 +1,59 @@ +# Python imports +import uuid +import requests + +# Django imports +from django.conf import settings +from django.http import StreamingHttpResponse +# Third party imports +import boto3 + + +def get_file_streams(key, filename=uuid.uuid4().hex): + + if settings.USE_MINIO: + s3 = boto3.client( + 's3', + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=boto3.session.Config(signature_version='s3v4'), + ) + else: + s3 = boto3.client( + 's3', + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=boto3.session.Config(signature_version='s3v4'), + ) + + presigned_url = s3.generate_presigned_url( + 'get_object', + Params={ + 'Bucket': settings.AWS_STORAGE_BUCKET_NAME, + 'Key': key, + }, + ExpiresIn=3600, + ) + + # Fetch the object metadata to get the content type + metadata = s3.head_object( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + Key=key, + ) + + + # Stream the file from the custom endpoint URL + def stream_file_from_url(url): + with requests.get(url, stream=True) as r: + r.raise_for_status() + for chunk in r.iter_content(chunk_size=8192): + if chunk: # filter out keep-alive new chunks + yield chunk + + + content_type = metadata['ContentType'] + response = StreamingHttpResponse(stream_file_from_url(presigned_url), content_type=content_type) + response['Content-Disposition'] = f'inline; filename={filename}' # Adjust filename as needed + + return response