github/plane
0
0
Fork 1
mirror of https://github.com/makeplane/plane synced 2024-06-14 14:31:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

dev: update user assets with backend streaming

Browse Source
This commit is contained in:
pablohashescobar 2024-02-01 15:55:43 +05:30
parent 42f307421a
commit 94f445cc08
17 changed files with 341 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apiserver/plane/app/serializers/__init__.py
View File

@ -7,6 +7,7 @@ from .user import (
UserAdminLiteSerializer,
UserMeSerializer,
UserMeSettingsSerializer,
UserAssetSerializer,
)
from .workspace import (
WorkSpaceSerializer,

7
apiserver/plane/app/serializers/base.py
View File

@ -162,17 +162,10 @@ class DynamicBaseSerializer(BaseSerializer):
class BaseFileSerializer(DynamicBaseSerializer):
download_url = serializers.SerializerMethodField()
class Meta:
abstract = True # Make this serializer abstract
def get_download_url(self, obj):
if hasattr(obj, "asset") and obj.asset:
storage = S3PrivateBucketStorage()
return storage.download_url(obj.asset.name)
return None
def to_representation(self, instance):
"""
Object instance -> Dict of primitive datatypes.

12
apiserver/plane/app/serializers/user.py
View File

@ -3,8 +3,7 @@ from rest_framework import serializers
# Module import
from .base import BaseSerializer
from plane.db.models import User, Workspace, WorkspaceMemberInvite
from plane.license.models import InstanceAdmin, Instance
from plane.db.models import User, Workspace, WorkspaceMemberInvite, UserAsset
class UserSerializer(BaseSerializer):
@ -197,3 +196,12 @@ class ResetPasswordSerializer(serializers.Serializer):
"""
new_password = serializers.CharField(required=True, min_length=8)
class UserAssetSerializer(BaseSerializer):
class Meta:
model = UserAsset
fields = "__all__"
read_only_fields = [
"user",
]

10
apiserver/plane/app/urls/asset.py
View File

@ -19,16 +19,6 @@ urlpatterns = [
FileAssetEndpoint.as_view(),
name="file-assets",
),
path(
"users/file-assets/",
UserAssetsEndpoint.as_view(),
name="user-file-assets",
),
path(
"users/file-assets/<str:asset_key>/",
UserAssetsEndpoint.as_view(),
name="user-file-assets",
),
path(
"workspaces/file-assets/<uuid:workspace_id>/<str:asset_key>/restore/",
FileAssetViewSet.as_view(

11
apiserver/plane/app/urls/user.py
View File

@ -8,6 +8,7 @@ from plane.app.views import (
UserActivityEndpoint,
ChangePasswordEndpoint,
SetUserPasswordEndpoint,
UserAssetsEndpoint,
## End User
## Workspaces
UserWorkSpacesEndpoint,
@ -95,5 +96,15 @@ urlpatterns = [
SetUserPasswordEndpoint.as_view(),
name="set-password",
),
path(
"users/assets/",
UserAssetsEndpoint.as_view(),
name="user-assets",
),
path(
"users/assets/<str:key>/",
UserAssetsEndpoint.as_view(),
name="user-assets",
),
## End User Graph
]

8
apiserver/plane/app/views/__init__.py
View File

@ -18,6 +18,7 @@ from .user import (
UpdateUserOnBoardedEndpoint,
UpdateUserTourCompletedEndpoint,
UserActivityEndpoint,
UserAssetsEndpoint,
)
from .oauth import OauthEndpoint
@ -65,7 +66,7 @@ from .cycle import (
TransferCycleIssueEndpoint,
CycleUserPropertiesEndpoint,
)
from .asset import FileAssetEndpoint, UserAssetsEndpoint, FileAssetViewSet
from .asset import FileAssetEndpoint, FileAssetViewSet
from .issue import (
IssueViewSet,
WorkSpaceIssuesEndpoint,
@ -180,7 +181,4 @@ from .webhook import (
WebhookSecretRegenerateEndpoint,
)
from .dashboard import (
DashboardEndpoint,
WidgetsEndpoint
)
from .dashboard import DashboardEndpoint, WidgetsEndpoint

37
apiserver/plane/app/views/asset.py
View File

@ -61,40 +61,3 @@ class FileAssetViewSet(BaseViewSet):
file_asset.is_deleted = False
file_asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
class UserAssetsEndpoint(BaseAPIView):
parser_classes = (MultiPartParser, FormParser)
def get(self, request, asset_key):
files = FileAsset.objects.filter(
asset=asset_key, created_by=request.user
)
if files.exists():
serializer = FileAssetSerializer(
files, context={"request": request}
)
return Response(
{"data": serializer.data, "status": True},
status=status.HTTP_200_OK,
)
else:
return Response(
{"error": "Asset key does not exist", "status": False},
status=status.HTTP_200_OK,
)
def post(self, request):
serializer = FileAssetSerializer(data=request.data)
if serializer.is_valid():
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def delete(self, request, asset_key):
file_asset = FileAsset.objects.get(
asset=asset_key, created_by=request.user
)
file_asset.is_deleted = True
file_asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)

39
apiserver/plane/app/views/user.py
View File

@ -1,7 +1,15 @@
# Python imports
import requests
# Django imports
from django.http import StreamingHttpResponse
from django.conf import settings
# Third party imports
import boto3
from rest_framework.response import Response
from rest_framework import status
from rest_framework.parsers import MultiPartParser, FormParser, JSONParser
# Module imports
from plane.app.serializers import (
@ -9,13 +17,14 @@ from plane.app.serializers import (
IssueActivitySerializer,
UserMeSerializer,
UserMeSettingsSerializer,
UserAssetSerializer,
)
from plane.app.views.base import BaseViewSet, BaseAPIView
from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember
from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember, UserAsset
from plane.license.models import Instance, InstanceAdmin
from plane.utils.paginator import BasePaginator
from plane.utils.file_stream import get_file_streams
from django.db.models import Q, F, Count, Case, When, IntegerField
@ -177,3 +186,27 @@ class UserActivityEndpoint(BaseAPIView, BasePaginator):
issue_activities, many=True
).data,
)
class UserAssetsEndpoint(BaseAPIView):
parser_classes = (MultiPartParser, FormParser)
def post(self, request):
serializer = UserAssetSerializer(data=request.data)
if serializer.is_valid():
serializer.save(user=request.user)
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def delete(self, request, asset_key):
user_asset = UserAsset.objects.get(
asset=asset_key, created_by=request.user
)
user_asset.is_deleted = True
user_asset.save()
return Response(status=status.HTTP_204_NO_CONTENT)
def get(self, request, key):
response = get_file_streams(key, key)
return response

175
apiserver/plane/db/migrations/0059_auto_20240131_1334.py Normal file
View File

@ -0,0 +1,175 @@
# Generated by Django 4.2.7 on 2024-01-31 13:34
import uuid
from django.db import migrations, models
from django.conf import settings
import django.db.models
import plane.db.models.asset
def update_urls(apps, schema_editor):
# Check if the app is using minio or s3
if settings.USE_MINIO:
prefix = (
f"{settings.AWS_S3_URL_PROTOCOL}//{settings.AWS_S3_CUSTOM_DOMAIN}/"
)
prefix2 = prefix
else:
prefix = f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.{settings.AWS_REGION}.amazonaws.com/"
prefix2 = (
f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/"
)
User = apps.get_model("db", "User")
UserAsset = apps.get_model("db", "UserAsset")
bulk_users = []
bulk_user_assets = []
for user in User.objects.all():
if user.avatar and (
user.avatar.startswith(prefix) or user.avatar.startswith(prefix2)
):
avatar = user.avatar[len(prefix) :]
user.avatar = avatar
bulk_user_assets.append(
UserAsset(
user=user,
asset=avatar,
)
)
if user.cover_image and (
user.cover_image.startswith(prefix)
or user.cover_image.startswith(prefix2)
):
cover_image = user.cover_image[len(prefix) :]
user.cover_image = cover_image
bulk_user_assets.append(
UserAsset(
user=user,
asset=cover_image,
)
)
User.objects.bulk_update(
bulk_users, ["avatar", "cover_image"], batch_size=100
)
UserAsset.objects.bulk_create(bulk_user_assets, batch_size=100)
class Migration(migrations.Migration):
dependencies = [
("db", "0058_alter_moduleissue_issue_and_more"),
]
operations = [
migrations.CreateModel(
name="UserAsset",
fields=[
(
"created_at",
models.DateTimeField(
auto_now_add=True, verbose_name="Created At"
),
),
(
"updated_at",
models.DateTimeField(
auto_now=True, verbose_name="Last Modified At"
),
),
(
"id",
models.UUIDField(
db_index=True,
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
unique=True,
),
),
(
"asset",
models.FileField(
max_length=500,
storage=plane.settings.storage.S3PrivateBucketStorage(),
upload_to=plane.db.models.user.get_upload_path,
validators=[plane.db.models.user.file_size],
),
),
("is_deleted", models.BooleanField(default=False)),
("size", models.PositiveBigIntegerField(null=True)),
("attributes", models.JSONField(default=dict)),
(
"created_by",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="%(class)s_created_by",
to=settings.AUTH_USER_MODEL,
verbose_name="Created By",
),
),
(
"updated_by",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="%(class)s_updated_by",
to=settings.AUTH_USER_MODEL,
verbose_name="Last Modified By",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="assets",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"verbose_name": "User Asset",
"verbose_name_plural": "User Assets",
"db_table": "user_assets",
"ordering": ("-created_at",),
},
),
migrations.AlterField(
model_name="fileasset",
name="asset",
field=models.FileField(
storage=plane.settings.storage.S3PrivateBucketStorage(),
upload_to=plane.db.models.asset.get_upload_path,
validators=[plane.db.models.asset.file_size],
),
),
migrations.RemoveField(
model_name="issueactivity",
name="attachments",
),
migrations.RemoveField(
model_name="issuecomment",
name="attachments",
),
migrations.AlterField(
model_name="integration",
name="avatar_url",
field=models.CharField(blank=True, null=True),
),
migrations.AlterField(
model_name="project",
name="cover_image",
field=models.CharField(blank=True, max_length=800, null=True),
),
migrations.AlterField(
model_name="user",
name="cover_image",
field=models.CharField(blank=True, max_length=800, null=True),
),
migrations.AlterField(
model_name="workspace",
name="logo",
field=models.CharField(blank=True, null=True, verbose_name="Logo"),
),
migrations.RunPython(update_urls),
]

2
apiserver/plane/db/models/__init__.py
View File

@ -1,6 +1,6 @@
from .base import BaseModel
from .user import User
from .user import User, UserAsset
from .workspace import (
Workspace,

2
apiserver/plane/db/models/integration/base.py
View File

@ -29,7 +29,7 @@ class Integration(AuditModel):
redirect_url = models.TextField(blank=True)
metadata = models.JSONField(default=dict)
verified = models.BooleanField(default=False)
avatar_url = models.URLField(blank=True, null=True)
avatar_url = models.CharField(blank=True, null=True)
def __str__(self):
"""Return provider of the integration"""

6
apiserver/plane/db/models/issue.py
View File

@ -389,9 +389,6 @@ class IssueActivity(ProjectBaseModel):
)
comment = models.TextField(verbose_name="Comment", blank=True)
attachments = ArrayField(
models.URLField(), size=10, blank=True, default=list
)
issue_comment = models.ForeignKey(
"db.IssueComment",
on_delete=models.SET_NULL,
@ -423,9 +420,6 @@ class IssueComment(ProjectBaseModel):
comment_stripped = models.TextField(verbose_name="Comment", blank=True)
comment_json = models.JSONField(blank=True, default=dict)
comment_html = models.TextField(blank=True, default="<p></p>")
attachments = ArrayField(
models.URLField(), size=10, blank=True, default=list
)
issue = models.ForeignKey(
Issue, on_delete=models.CASCADE, related_name="issue_comments"
)

2
apiserver/plane/db/models/project.py
View File

@ -94,7 +94,7 @@ class Project(BaseModel):
issue_views_view = models.BooleanField(default=True)
page_view = models.BooleanField(default=True)
inbox_view = models.BooleanField(default=False)
cover_image = models.URLField(blank=True, null=True, max_length=800)
cover_image = models.CharField(blank=True, null=True, max_length=800)
estimate = models.ForeignKey(
"db.Estimate",
on_delete=models.SET_NULL,

43
apiserver/plane/db/models/user.py
View File

@ -15,12 +15,17 @@ from django.db.models.signals import post_save
from django.conf import settings
from django.dispatch import receiver
from django.utils import timezone
from django.core.exceptions import ValidationError
# Third party imports
from sentry_sdk import capture_exception
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
# Module imports
from . import BaseModel
from plane.settings.storage import S3PrivateBucketStorage
def get_default_onboarding():
return {
@ -49,7 +54,7 @@ class User(AbstractBaseUser, PermissionsMixin):
first_name = models.CharField(max_length=255, blank=True)
last_name = models.CharField(max_length=255, blank=True)
avatar = models.CharField(max_length=255, blank=True)
cover_image = models.URLField(blank=True, null=True, max_length=800)
cover_image = models.CharField(blank=True, null=True, max_length=800)
# tracking metrics
date_joined = models.DateTimeField(
@ -144,6 +149,42 @@ class User(AbstractBaseUser, PermissionsMixin):
super(User, self).save(*args, **kwargs)
def get_upload_path(instance, filename):
return f"user-{uuid.uuid4().hex}"
def file_size(value):
if value.size > settings.FILE_SIZE_LIMIT:
raise ValidationError("File too large. Size should not exceed 5 MB.")
class UserAsset(BaseModel):
user = models.ForeignKey("db.User", on_delete=models.CASCADE, related_name="assets")
asset = models.FileField(
upload_to=get_upload_path,
validators=[
file_size,
],
storage=S3PrivateBucketStorage(),
max_length=500,
)
is_deleted = models.BooleanField(default=False)
size = models.PositiveBigIntegerField(null=True)
attributes = models.JSONField(default=dict)
def save(self, *args, **kwargs):
self.size = self.asset.size
super(UserAsset, self).save(*args, **kwargs)
class Meta:
verbose_name = "User Asset"
verbose_name_plural = "User Assets"
db_table = "user_assets"
ordering = ("-created_at",)
def __str__(self):
return str(self.asset)
@receiver(post_save, sender=User)
def send_welcome_slack(sender, instance, created, **kwargs):
try:

2
apiserver/plane/db/models/workspace.py
View File

@ -131,7 +131,7 @@ def slug_validator(value):
class Workspace(BaseModel):
name = models.CharField(max_length=80, verbose_name="Workspace Name")
logo = models.URLField(verbose_name="Logo", blank=True, null=True)
logo = models.CharField(verbose_name="Logo", blank=True, null=True)
owner = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,

6
apiserver/plane/settings/storage.py
View File

@ -1,15 +1,9 @@
# Third party imports
from storages.backends.s3boto3 import S3Boto3Storage
# Module imports
from plane.utils.presigned_url_generator import generate_download_presigned_url
class S3PrivateBucketStorage(S3Boto3Storage):
def url(self, name):
# Return an empty string or None, or implement custom logic here
return name
def download_url(self, name):
return generate_download_presigned_url(name)

59
apiserver/plane/utils/file_stream.py Normal file
View File

@ -0,0 +1,59 @@
# Python imports
import uuid
import requests
# Django imports
from django.conf import settings
from django.http import StreamingHttpResponse
# Third party imports
import boto3
def get_file_streams(key, filename=uuid.uuid4().hex):
if settings.USE_MINIO:
s3 = boto3.client(
's3',
endpoint_url=settings.AWS_S3_ENDPOINT_URL,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
config=boto3.session.Config(signature_version='s3v4'),
)
else:
s3 = boto3.client(
's3',
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
config=boto3.session.Config(signature_version='s3v4'),
)
presigned_url = s3.generate_presigned_url(
'get_object',
Params={
'Bucket': settings.AWS_STORAGE_BUCKET_NAME,
'Key': key,
},
ExpiresIn=3600,
)
# Fetch the object metadata to get the content type
metadata = s3.head_object(
Bucket=settings.AWS_STORAGE_BUCKET_NAME,
Key=key,
)
# Stream the file from the custom endpoint URL
def stream_file_from_url(url):
with requests.get(url, stream=True) as r:
r.raise_for_status()
for chunk in r.iter_content(chunk_size=8192):
if chunk: # filter out keep-alive new chunks
yield chunk
content_type = metadata['ContentType']
response = StreamingHttpResponse(stream_file_from_url(presigned_url), content_type=content_type)
response['Content-Disposition'] = f'inline; filename={filename}' # Adjust filename as needed
return response