dev: update user assets with backend streaming

apiserver/plane/app/serializers/__init__.py

@@ -7,6 +7,7 @@ from .user import (
     UserAdminLiteSerializer,
     UserMeSerializer,
     UserMeSettingsSerializer,
+    UserAssetSerializer,
 )
 from .workspace import (
     WorkSpaceSerializer,

apiserver/plane/app/serializers/base.py

@@ -162,17 +162,10 @@ class DynamicBaseSerializer(BaseSerializer):
 
 
 class BaseFileSerializer(DynamicBaseSerializer):
-    download_url = serializers.SerializerMethodField()
 
     class Meta:
         abstract = True  # Make this serializer abstract
 
-    def get_download_url(self, obj):
-        if hasattr(obj, "asset") and obj.asset:
-            storage = S3PrivateBucketStorage()
-            return storage.download_url(obj.asset.name)
-        return None
-
     def to_representation(self, instance):
         """
         Object instance -> Dict of primitive datatypes.

apiserver/plane/app/serializers/user.py

@@ -3,8 +3,7 @@ from rest_framework import serializers
 
 # Module import
 from .base import BaseSerializer
-from plane.db.models import User, Workspace, WorkspaceMemberInvite
+from plane.db.models import User, Workspace, WorkspaceMemberInvite, UserAsset
-from plane.license.models import InstanceAdmin, Instance
 
 
 class UserSerializer(BaseSerializer):
@@ -197,3 +196,12 @@ class ResetPasswordSerializer(serializers.Serializer):
     """
 
     new_password = serializers.CharField(required=True, min_length=8)
+
+
+class UserAssetSerializer(BaseSerializer):
+    class Meta:
+        model = UserAsset
+        fields = "__all__"
+        read_only_fields = [
+            "user",
+        ]

apiserver/plane/app/urls/asset.py

@@ -19,16 +19,6 @@ urlpatterns = [
         FileAssetEndpoint.as_view(),
         name="file-assets",
     ),
-    path(
-        "users/file-assets/",
-        UserAssetsEndpoint.as_view(),
-        name="user-file-assets",
-    ),
-    path(
-        "users/file-assets/<str:asset_key>/",
-        UserAssetsEndpoint.as_view(),
-        name="user-file-assets",
-    ),
     path(
         "workspaces/file-assets/<uuid:workspace_id>/<str:asset_key>/restore/",
         FileAssetViewSet.as_view(

apiserver/plane/app/urls/user.py

@@ -8,6 +8,7 @@ from plane.app.views import (
     UserActivityEndpoint,
     ChangePasswordEndpoint,
     SetUserPasswordEndpoint,
+    UserAssetsEndpoint,
     ## End User
     ## Workspaces
     UserWorkSpacesEndpoint,
@@ -95,5 +96,15 @@ urlpatterns = [
         SetUserPasswordEndpoint.as_view(),
         name="set-password",
     ),
+    path(
+        "users/assets/",
+        UserAssetsEndpoint.as_view(),
+        name="user-assets",
+    ),
+    path(
+        "users/assets/<str:key>/",
+        UserAssetsEndpoint.as_view(),
+        name="user-assets",
+    ),
     ## End User Graph
 ]

apiserver/plane/app/views/__init__.py

@@ -18,6 +18,7 @@ from .user import (
     UpdateUserOnBoardedEndpoint,
     UpdateUserTourCompletedEndpoint,
     UserActivityEndpoint,
+    UserAssetsEndpoint,
 )
 
 from .oauth import OauthEndpoint
@@ -65,7 +66,7 @@ from .cycle import (
     TransferCycleIssueEndpoint,
     CycleUserPropertiesEndpoint,
 )
-from .asset import FileAssetEndpoint, UserAssetsEndpoint, FileAssetViewSet
+from .asset import FileAssetEndpoint, FileAssetViewSet
 from .issue import (
     IssueViewSet,
     WorkSpaceIssuesEndpoint,
@@ -180,7 +181,4 @@ from .webhook import (
     WebhookSecretRegenerateEndpoint,
 )
 
-from .dashboard import (
+from .dashboard import DashboardEndpoint, WidgetsEndpoint
-    DashboardEndpoint,
-    WidgetsEndpoint
-)

apiserver/plane/app/views/asset.py

@@ -61,40 +61,3 @@ class FileAssetViewSet(BaseViewSet):
         file_asset.is_deleted = False
         file_asset.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class UserAssetsEndpoint(BaseAPIView):
-    parser_classes = (MultiPartParser, FormParser)
-
-    def get(self, request, asset_key):
-        files = FileAsset.objects.filter(
-            asset=asset_key, created_by=request.user
-        )
-        if files.exists():
-            serializer = FileAssetSerializer(
-                files, context={"request": request}
-            )
-            return Response(
-                {"data": serializer.data, "status": True},
-                status=status.HTTP_200_OK,
-            )
-        else:
-            return Response(
-                {"error": "Asset key does not exist", "status": False},
-                status=status.HTTP_200_OK,
-            )
-
-    def post(self, request):
-        serializer = FileAssetSerializer(data=request.data)
-        if serializer.is_valid():
-            serializer.save()
-            return Response(serializer.data, status=status.HTTP_201_CREATED)
-        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
-
-    def delete(self, request, asset_key):
-        file_asset = FileAsset.objects.get(
-            asset=asset_key, created_by=request.user
-        )
-        file_asset.is_deleted = True
-        file_asset.save()
-        return Response(status=status.HTTP_204_NO_CONTENT)

apiserver/plane/app/views/user.py

@@ -1,7 +1,15 @@
+# Python imports
+import requests
+
+# Django imports
+from django.http import StreamingHttpResponse
+from django.conf import settings
+
 # Third party imports
+import boto3
 from rest_framework.response import Response
 from rest_framework import status
-
+from rest_framework.parsers import MultiPartParser, FormParser, JSONParser
 
 # Module imports
 from plane.app.serializers import (
@@ -9,13 +17,14 @@ from plane.app.serializers import (
     IssueActivitySerializer,
     UserMeSerializer,
     UserMeSettingsSerializer,
+    UserAssetSerializer,
 )
 
 from plane.app.views.base import BaseViewSet, BaseAPIView
-from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember
+from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember, UserAsset
 from plane.license.models import Instance, InstanceAdmin
 from plane.utils.paginator import BasePaginator
-
+from plane.utils.file_stream import get_file_streams
 
 from django.db.models import Q, F, Count, Case, When, IntegerField
 
@@ -177,3 +186,27 @@ class UserActivityEndpoint(BaseAPIView, BasePaginator):
                 issue_activities, many=True
             ).data,
         )
+
+
+class UserAssetsEndpoint(BaseAPIView):
+    parser_classes = (MultiPartParser, FormParser)
+
+    def post(self, request):
+        serializer = UserAssetSerializer(data=request.data)
+        if serializer.is_valid():
+            serializer.save(user=request.user)
+            return Response(serializer.data, status=status.HTTP_201_CREATED)
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+    def delete(self, request, asset_key):
+        user_asset = UserAsset.objects.get(
+            asset=asset_key, created_by=request.user
+        )
+        user_asset.is_deleted = True
+        user_asset.save()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+    def get(self, request, key):
+        response = get_file_streams(key, key)
+        return response

apiserver/plane/db/migrations/0059_auto_20240131_1334.py

@@ -0,0 +1,175 @@
+# Generated by Django 4.2.7 on 2024-01-31 13:34
+import uuid
+from django.db import migrations, models
+from django.conf import settings
+import django.db.models
+import plane.db.models.asset
+
+
+def update_urls(apps, schema_editor):
+    # Check if the app is using minio or s3
+    if settings.USE_MINIO:
+        prefix = (
+            f"{settings.AWS_S3_URL_PROTOCOL}//{settings.AWS_S3_CUSTOM_DOMAIN}/"
+        )
+        prefix2 = prefix
+    else:
+        prefix = f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.{settings.AWS_REGION}.amazonaws.com/"
+        prefix2 = (
+            f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/"
+        )
+
+    User = apps.get_model("db", "User")
+    UserAsset = apps.get_model("db", "UserAsset")
+    bulk_users = []
+    bulk_user_assets = []
+    for user in User.objects.all():
+        if user.avatar and (
+            user.avatar.startswith(prefix) or user.avatar.startswith(prefix2)
+        ):
+            avatar = user.avatar[len(prefix) :]
+            user.avatar = avatar
+            bulk_user_assets.append(
+                UserAsset(
+                    user=user,
+                    asset=avatar,
+                )
+            )
+
+        if user.cover_image and (
+            user.cover_image.startswith(prefix)
+            or user.cover_image.startswith(prefix2)
+        ):
+            cover_image = user.cover_image[len(prefix) :]
+            user.cover_image = cover_image
+            bulk_user_assets.append(
+                UserAsset(
+                    user=user,
+                    asset=cover_image,
+                )
+            )
+    User.objects.bulk_update(
+        bulk_users, ["avatar", "cover_image"], batch_size=100
+    )
+    UserAsset.objects.bulk_create(bulk_user_assets, batch_size=100)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("db", "0058_alter_moduleissue_issue_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="UserAsset",
+            fields=[
+                (
+                    "created_at",
+                    models.DateTimeField(
+                        auto_now_add=True, verbose_name="Created At"
+                    ),
+                ),
+                (
+                    "updated_at",
+                    models.DateTimeField(
+                        auto_now=True, verbose_name="Last Modified At"
+                    ),
+                ),
+                (
+                    "id",
+                    models.UUIDField(
+                        db_index=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        unique=True,
+                    ),
+                ),
+                (
+                    "asset",
+                    models.FileField(
+                        max_length=500,
+                        storage=plane.settings.storage.S3PrivateBucketStorage(),
+                        upload_to=plane.db.models.user.get_upload_path,
+                        validators=[plane.db.models.user.file_size],
+                    ),
+                ),
+                ("is_deleted", models.BooleanField(default=False)),
+                ("size", models.PositiveBigIntegerField(null=True)),
+                ("attributes", models.JSONField(default=dict)),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="%(class)s_created_by",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="Created By",
+                    ),
+                ),
+                (
+                    "updated_by",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="%(class)s_updated_by",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="Last Modified By",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="assets",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "User Asset",
+                "verbose_name_plural": "User Assets",
+                "db_table": "user_assets",
+                "ordering": ("-created_at",),
+            },
+        ),
+        migrations.AlterField(
+            model_name="fileasset",
+            name="asset",
+            field=models.FileField(
+                storage=plane.settings.storage.S3PrivateBucketStorage(),
+                upload_to=plane.db.models.asset.get_upload_path,
+                validators=[plane.db.models.asset.file_size],
+            ),
+        ),
+        migrations.RemoveField(
+            model_name="issueactivity",
+            name="attachments",
+        ),
+        migrations.RemoveField(
+            model_name="issuecomment",
+            name="attachments",
+        ),
+        migrations.AlterField(
+            model_name="integration",
+            name="avatar_url",
+            field=models.CharField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="cover_image",
+            field=models.CharField(blank=True, max_length=800, null=True),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="cover_image",
+            field=models.CharField(blank=True, max_length=800, null=True),
+        ),
+        migrations.AlterField(
+            model_name="workspace",
+            name="logo",
+            field=models.CharField(blank=True, null=True, verbose_name="Logo"),
+        ),
+        migrations.RunPython(update_urls),
+    ]

apiserver/plane/db/models/__init__.py

@@ -1,6 +1,6 @@
 from .base import BaseModel
 
-from .user import User
+from .user import User, UserAsset
 
 from .workspace import (
     Workspace,

apiserver/plane/db/models/integration/base.py

@@ -29,7 +29,7 @@ class Integration(AuditModel):
     redirect_url = models.TextField(blank=True)
     metadata = models.JSONField(default=dict)
     verified = models.BooleanField(default=False)
-    avatar_url = models.URLField(blank=True, null=True)
+    avatar_url = models.CharField(blank=True, null=True)
 
     def __str__(self):
         """Return provider of the integration"""

apiserver/plane/db/models/issue.py

@@ -389,9 +389,6 @@ class IssueActivity(ProjectBaseModel):
     )
 
     comment = models.TextField(verbose_name="Comment", blank=True)
-    attachments = ArrayField(
-        models.URLField(), size=10, blank=True, default=list
-    )
     issue_comment = models.ForeignKey(
         "db.IssueComment",
         on_delete=models.SET_NULL,
@@ -423,9 +420,6 @@ class IssueComment(ProjectBaseModel):
     comment_stripped = models.TextField(verbose_name="Comment", blank=True)
     comment_json = models.JSONField(blank=True, default=dict)
     comment_html = models.TextField(blank=True, default="<p></p>")
-    attachments = ArrayField(
-        models.URLField(), size=10, blank=True, default=list
-    )
     issue = models.ForeignKey(
         Issue, on_delete=models.CASCADE, related_name="issue_comments"
     )

apiserver/plane/db/models/project.py

@@ -94,7 +94,7 @@ class Project(BaseModel):
     issue_views_view = models.BooleanField(default=True)
     page_view = models.BooleanField(default=True)
     inbox_view = models.BooleanField(default=False)
-    cover_image = models.URLField(blank=True, null=True, max_length=800)
+    cover_image = models.CharField(blank=True, null=True, max_length=800)
     estimate = models.ForeignKey(
         "db.Estimate",
         on_delete=models.SET_NULL,

apiserver/plane/db/models/user.py

@@ -15,12 +15,17 @@ from django.db.models.signals import post_save
 from django.conf import settings
 from django.dispatch import receiver
 from django.utils import timezone
+from django.core.exceptions import ValidationError
 
 # Third party imports
 from sentry_sdk import capture_exception
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 
+# Module imports
+from . import BaseModel
+from plane.settings.storage import S3PrivateBucketStorage
+
 
 def get_default_onboarding():
     return {
@@ -49,7 +54,7 @@ class User(AbstractBaseUser, PermissionsMixin):
     first_name = models.CharField(max_length=255, blank=True)
     last_name = models.CharField(max_length=255, blank=True)
     avatar = models.CharField(max_length=255, blank=True)
-    cover_image = models.URLField(blank=True, null=True, max_length=800)
+    cover_image = models.CharField(blank=True, null=True, max_length=800)
 
     # tracking metrics
     date_joined = models.DateTimeField(
@@ -144,6 +149,42 @@ class User(AbstractBaseUser, PermissionsMixin):
         super(User, self).save(*args, **kwargs)
 
 
+def get_upload_path(instance, filename):
+    return f"user-{uuid.uuid4().hex}"
+
+def file_size(value):
+    if value.size > settings.FILE_SIZE_LIMIT:
+        raise ValidationError("File too large. Size should not exceed 5 MB.")
+
+class UserAsset(BaseModel):
+
+    user = models.ForeignKey("db.User", on_delete=models.CASCADE, related_name="assets")
+    asset = models.FileField(
+        upload_to=get_upload_path,
+        validators=[
+            file_size,
+        ],
+        storage=S3PrivateBucketStorage(),
+        max_length=500,
+    )
+    is_deleted = models.BooleanField(default=False)
+    size = models.PositiveBigIntegerField(null=True)
+    attributes = models.JSONField(default=dict)
+
+    def save(self, *args, **kwargs):
+        self.size = self.asset.size
+        super(UserAsset, self).save(*args, **kwargs)
+
+    class Meta:
+        verbose_name = "User Asset"
+        verbose_name_plural = "User Assets"
+        db_table = "user_assets"
+        ordering = ("-created_at",)
+
+    def __str__(self):
+        return str(self.asset)
+
+
 @receiver(post_save, sender=User)
 def send_welcome_slack(sender, instance, created, **kwargs):
     try:

apiserver/plane/db/models/workspace.py

@@ -131,7 +131,7 @@ def slug_validator(value):
 
 class Workspace(BaseModel):
     name = models.CharField(max_length=80, verbose_name="Workspace Name")
-    logo = models.URLField(verbose_name="Logo", blank=True, null=True)
+    logo = models.CharField(verbose_name="Logo", blank=True, null=True)
     owner = models.ForeignKey(
         settings.AUTH_USER_MODEL,
         on_delete=models.CASCADE,

apiserver/plane/settings/storage.py

@@ -1,15 +1,9 @@
 # Third party imports
 from storages.backends.s3boto3 import S3Boto3Storage
 
-# Module imports
-from plane.utils.presigned_url_generator import generate_download_presigned_url
-
 
 class S3PrivateBucketStorage(S3Boto3Storage):
 
     def url(self, name):
         # Return an empty string or None, or implement custom logic here
         return name
-
-    def download_url(self, name):
-        return generate_download_presigned_url(name)

apiserver/plane/utils/file_stream.py

@@ -0,0 +1,59 @@
+# Python imports
+import uuid
+import requests
+
+# Django imports
+from django.conf import settings
+from django.http import StreamingHttpResponse
+# Third party imports
+import boto3
+
+
+def get_file_streams(key, filename=uuid.uuid4().hex):
+
+    if settings.USE_MINIO:
+        s3 = boto3.client(
+            's3',
+            endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+            aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+            config=boto3.session.Config(signature_version='s3v4'),
+        )
+    else:
+        s3 = boto3.client(
+            's3',
+            aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+            config=boto3.session.Config(signature_version='s3v4'),
+        )
+
+    presigned_url = s3.generate_presigned_url(
+        'get_object',
+        Params={
+            'Bucket': settings.AWS_STORAGE_BUCKET_NAME,
+            'Key': key,
+        },
+        ExpiresIn=3600,
+    )
+
+    # Fetch the object metadata to get the content type
+    metadata = s3.head_object(
+        Bucket=settings.AWS_STORAGE_BUCKET_NAME,
+        Key=key,
+    )
+
+
+    # Stream the file from the custom endpoint URL
+    def stream_file_from_url(url):
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+            for chunk in r.iter_content(chunk_size=8192):
+                if chunk:  # filter out keep-alive new chunks
+                    yield chunk
+
+
+    content_type = metadata['ContentType']
+    response = StreamingHttpResponse(stream_file_from_url(presigned_url), content_type=content_type)
+    response['Content-Disposition'] = f'inline; filename={filename}'  # Adjust filename as needed
+
+    return response