diff --git a/apiserver/plane/api/serializers/__init__.py b/apiserver/plane/api/serializers/__init__.py index bd49c9a6f..5855f0413 100644 --- a/apiserver/plane/api/serializers/__init__.py +++ b/apiserver/plane/api/serializers/__init__.py @@ -81,3 +81,5 @@ from .inbox import InboxSerializer, InboxIssueSerializer, IssueStateInboxSeriali from .analytic import AnalyticViewSerializer from .notification import NotificationSerializer + +from .exporter import ExporterHistorySerializer diff --git a/apiserver/plane/api/serializers/exporter.py b/apiserver/plane/api/serializers/exporter.py new file mode 100644 index 000000000..5c78cfa69 --- /dev/null +++ b/apiserver/plane/api/serializers/exporter.py @@ -0,0 +1,26 @@ +# Module imports +from .base import BaseSerializer +from plane.db.models import ExporterHistory +from .user import UserLiteSerializer + + +class ExporterHistorySerializer(BaseSerializer): + initiated_by_detail = UserLiteSerializer(source="initiated_by", read_only=True) + + class Meta: + model = ExporterHistory + fields = [ + "id", + "created_at", + "updated_at", + "project", + "provider", + "status", + "url", + "initiated_by", + "initiated_by_detail", + "token", + "created_by", + "updated_by", + ] + read_only_fields = fields diff --git a/apiserver/plane/api/views/__init__.py b/apiserver/plane/api/views/__init__.py index 39940bcb5..11223f90a 100644 --- a/apiserver/plane/api/views/__init__.py +++ b/apiserver/plane/api/views/__init__.py @@ -84,7 +84,6 @@ from .issue import ( IssueReactionPublicViewSet, CommentReactionPublicViewSet, IssueVotePublicViewSet, - ExportIssuesEndpoint ) from .auth_extended import ( @@ -162,4 +161,8 @@ from .analytic import ( DefaultAnalyticsEndpoint, ) -from .notification import NotificationViewSet, UnreadNotificationEndpoint \ No newline at end of file +from .notification import NotificationViewSet, UnreadNotificationEndpoint + +from .exporter import ( + ExportIssuesEndpoint, +) \ No newline at end of file diff --git a/apiserver/plane/api/views/exporter.py b/apiserver/plane/api/views/exporter.py new file mode 100644 index 000000000..f158f783d --- /dev/null +++ b/apiserver/plane/api/views/exporter.py @@ -0,0 +1,99 @@ +# Third Party imports +from rest_framework.response import Response +from rest_framework import status +from sentry_sdk import capture_exception + +# Module imports +from . import BaseAPIView +from plane.api.permissions import WorkSpaceAdminPermission +from plane.bgtasks.export_task import issue_export_task +from plane.db.models import Project, ExporterHistory, Workspace + +from plane.api.serializers import ExporterHistorySerializer + + +class ExportIssuesEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + model = ExporterHistory + serializer_class = ExporterHistorySerializer + + def post(self, request, slug): + try: + # Get the workspace + workspace = Workspace.objects.get(slug=slug) + + provider = request.data.get("provider", False) + multiple = request.data.get("multiple", False) + project_ids = request.data.get("project", []) + + if provider in ["csv", "xlsx", "json"]: + if not project_ids: + project_ids = Project.objects.filter( + workspace__slug=slug + ).values_list("id", flat=True) + project_ids = [str(project_id) for project_id in project_ids] + + exporter = ExporterHistory.objects.create( + workspace=workspace, + project=project_ids, + initiated_by=request.user, + provider=provider, + ) + + issue_export_task.delay( + provider=exporter.provider, + workspace_id=workspace.id, + project_ids=project_ids, + token_id=exporter.token, + multiple=multiple, + ) + return Response( + { + "message": f"Once the export is ready you will be able to download it" + }, + status=status.HTTP_200_OK, + ) + else: + return Response( + {"error": f"Provider '{provider}' not found."}, + status=status.HTTP_400_BAD_REQUEST, + ) + except Workspace.DoesNotExist: + return Response( + {"error": "Workspace does not exists"}, + status=status.HTTP_400_BAD_REQUEST, + ) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + def get(self, request, slug): + try: + exporter_history = ExporterHistory.objects.filter( + workspace__slug=slug + ).select_related("workspace","initiated_by") + + if request.GET.get("per_page", False) and request.GET.get("cursor", False): + return self.paginate( + request=request, + queryset=exporter_history, + on_results=lambda exporter_history: ExporterHistorySerializer( + exporter_history, many=True + ).data, + ) + else: + return Response( + {"error": "per_page and cursor are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/apiserver/plane/api/views/issue.py b/apiserver/plane/api/views/issue.py index 5ca14234a..77432e1e0 100644 --- a/apiserver/plane/api/views/issue.py +++ b/apiserver/plane/api/views/issue.py @@ -77,7 +77,6 @@ from plane.db.models import ( from plane.bgtasks.issue_activites_task import issue_activity from plane.utils.grouper import group_results from plane.utils.issue_filters import issue_filters -from plane.bgtasks.project_issue_export import issue_export_task class IssueViewSet(BaseViewSet): diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py new file mode 100644 index 000000000..de15bcdb8 --- /dev/null +++ b/apiserver/plane/bgtasks/export_task.py @@ -0,0 +1,357 @@ +# Python imports +import csv +import io +import json +import boto3 +import zipfile +from datetime import datetime, date, timedelta + +# Django imports +from django.conf import settings +from django.utils import timezone + +# Third party imports +from celery import shared_task +from sentry_sdk import capture_exception +from botocore.client import Config +from openpyxl import Workbook +from openpyxl.styles import NamedStyle +from openpyxl.utils.datetime import to_excel + +# Module imports +from plane.db.models import Issue, ExporterHistory, Project + + +class DateTimeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + return super().default(obj) + + +def create_csv_file(data): + csv_buffer = io.StringIO() + csv_writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) + + for row in data: + csv_writer.writerow(row) + + csv_buffer.seek(0) + return csv_buffer.getvalue() + + +def create_json_file(data): + return json.dumps(data, cls=DateTimeEncoder) + + +def create_xlsx_file(data): + workbook = Workbook() + sheet = workbook.active + + no_timezone_style = NamedStyle(name="no_timezone_style") + no_timezone_style.number_format = "yyyy-mm-dd hh:mm:ss" + + for row in data: + sheet.append(row) + + for column_cells in sheet.columns: + for cell in column_cells: + if isinstance(cell.value, datetime): + cell.style = no_timezone_style + cell.value = to_excel(cell.value.replace(tzinfo=None)) + + xlsx_buffer = io.BytesIO() + workbook.save(xlsx_buffer) + xlsx_buffer.seek(0) + return xlsx_buffer.getvalue() + + +def create_zip_file(files): + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf: + for filename, file_content in files: + zipf.writestr(filename, file_content) + + zip_buffer.seek(0) + return zip_buffer + + +def upload_to_s3(zip_file, workspace_id, token_id): + s3 = boto3.client( + "s3", + region_name="ap-south-1", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=Config(signature_version="s3v4"), + ) + file_name = f"{workspace_id}/issues-{datetime.now().date()}.zip" + + s3.upload_fileobj( + zip_file, + settings.AWS_S3_BUCKET_NAME, + file_name, + ExtraArgs={"ACL": "public-read", "ContentType": "application/zip"}, + ) + + expires_in = 7 * 24 * 60 * 60 + presigned_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": settings.AWS_S3_BUCKET_NAME, "Key": file_name}, + ExpiresIn=expires_in, + ) + + exporter_instance = ExporterHistory.objects.get(token=token_id) + + if presigned_url: + exporter_instance.url = presigned_url + exporter_instance.status = "completed" + exporter_instance.key = file_name + else: + exporter_instance.status = "failed" + + exporter_instance.save(update_fields=["status", "url","key"]) + + +def generate_table_row(issue): + return [ + f"""{issue["project__identifier"]}-{issue["sequence_id"]}""", + issue["project__name"], + issue["name"], + issue["description_stripped"], + issue["state__name"], + issue["priority"], + f"{issue['created_by__first_name']} {issue['created_by__last_name']}" + if issue["created_by__first_name"] and issue["created_by__last_name"] + else "", + f"{issue['assignees__first_name']} {issue['assignees__last_name']}" + if issue["assignees__first_name"] and issue["assignees__last_name"] + else "", + issue["labels__name"], + issue["issue_cycle__cycle__name"], + issue["issue_cycle__cycle__start_date"], + issue["issue_cycle__cycle__end_date"], + issue["issue_module__module__name"], + issue["issue_module__module__start_date"], + issue["issue_module__module__target_date"], + issue["created_at"], + issue["updated_at"], + issue["completed_at"], + issue["archived_at"], + ] + + +def generate_json_row(issue): + return { + "ID": f"""{issue["project__identifier"]}-{issue["sequence_id"]}""", + "Project": issue["project__name"], + "Name": issue["name"], + "Description": issue["description_stripped"], + "State": issue["state__name"], + "Priority": issue["priority"], + "Created By": f"{issue['created_by__first_name']} {issue['created_by__last_name']}" + if issue["created_by__first_name"] and issue["created_by__last_name"] + else "", + "Assignee": f"{issue['assignees__first_name']} {issue['assignees__last_name']}" + if issue["assignees__first_name"] and issue["assignees__last_name"] + else "", + "Labels": issue["labels__name"], + "Cycle Name": issue["issue_cycle__cycle__name"], + "Cycle Start Date": issue["issue_cycle__cycle__start_date"], + "Cycle End Date": issue["issue_cycle__cycle__end_date"], + "Module Name": issue["issue_module__module__name"], + "Module Start Date": issue["issue_module__module__start_date"], + "Module Target Date": issue["issue_module__module__target_date"], + "Created At": issue["created_at"], + "Updated At": issue["updated_at"], + "Completed At": issue["completed_at"], + "Archived At": issue["archived_at"], + } + + +def update_json_row(rows, row): + matched_index = next( + ( + index + for index, existing_row in enumerate(rows) + if existing_row["ID"] == row["ID"] + ), + None, + ) + + if matched_index is not None: + existing_assignees, existing_labels = ( + rows[matched_index]["Assignee"], + rows[matched_index]["Labels"], + ) + assignee, label = row["Assignee"], row["Labels"] + + if assignee is not None and assignee not in existing_assignees: + rows[matched_index]["Assignee"] += f", {assignee}" + if label is not None and label not in existing_labels: + rows[matched_index]["Labels"] += f", {label}" + else: + rows.append(row) + + +def update_table_row(rows, row): + matched_index = next( + (index for index, existing_row in enumerate(rows) if existing_row[0] == row[0]), + None, + ) + + if matched_index is not None: + existing_assignees, existing_labels = rows[matched_index][7:9] + assignee, label = row[7:9] + + if assignee is not None and assignee not in existing_assignees: + rows[matched_index][7] += f", {assignee}" + if label is not None and label not in existing_labels: + rows[matched_index][8] += f", {label}" + else: + rows.append(row) + + +def generate_csv(header, project_id, issues, files): + """ + Generate CSV export for all the passed issues. + """ + rows = [ + header, + ] + for issue in issues: + row = generate_table_row(issue) + update_table_row(rows, row) + csv_file = create_csv_file(rows) + files.append((f"{project_id}.csv", csv_file)) + + +def generate_json(header, project_id, issues, files): + rows = [] + for issue in issues: + row = generate_json_row(issue) + update_json_row(rows, row) + json_file = create_json_file(rows) + files.append((f"{project_id}.json", json_file)) + + +def generate_xlsx(header, project_id, issues, files): + rows = [header] + for issue in issues: + row = generate_table_row(issue) + update_table_row(rows, row) + xlsx_file = create_xlsx_file(rows) + files.append((f"{project_id}.xlsx", xlsx_file)) + + +@shared_task +def issue_export_task(provider, workspace_id, project_ids, token_id, multiple): + try: + exporter_instance = ExporterHistory.objects.get(token=token_id) + exporter_instance.status = "processing" + exporter_instance.save(update_fields=["status"]) + + workspace_issues = ( + ( + Issue.objects.filter( + workspace__id=workspace_id, project_id__in=project_ids + ) + .select_related("project", "workspace", "state", "parent", "created_by") + .prefetch_related( + "assignees", "labels", "issue_cycle__cycle", "issue_module__module" + ) + .values( + "id", + "project__identifier", + "project__name", + "project__id", + "sequence_id", + "name", + "description_stripped", + "priority", + "state__name", + "created_at", + "updated_at", + "completed_at", + "archived_at", + "issue_cycle__cycle__name", + "issue_cycle__cycle__start_date", + "issue_cycle__cycle__end_date", + "issue_module__module__name", + "issue_module__module__start_date", + "issue_module__module__target_date", + "created_by__first_name", + "created_by__last_name", + "assignees__first_name", + "assignees__last_name", + "labels__name", + ) + ) + .order_by("project__identifier","sequence_id") + .distinct() + ) + # CSV header + header = [ + "ID", + "Project", + "Name", + "Description", + "State", + "Priority", + "Created By", + "Assignee", + "Labels", + "Cycle Name", + "Cycle Start Date", + "Cycle End Date", + "Module Name", + "Module Start Date", + "Module Target Date", + "Created At", + "Updated At", + "Completed At", + "Archived At", + ] + + EXPORTER_MAPPER = { + "csv": generate_csv, + "json": generate_json, + "xlsx": generate_xlsx, + } + + files = [] + if multiple: + for project_id in project_ids: + issues = workspace_issues.filter(project__id=project_id) + exporter = EXPORTER_MAPPER.get(provider) + if exporter is not None: + exporter( + header, + project_id, + issues, + files, + ) + + else: + exporter = EXPORTER_MAPPER.get(provider) + if exporter is not None: + exporter( + header, + workspace_id, + workspace_issues, + files, + ) + + zip_buffer = create_zip_file(files) + upload_to_s3(zip_buffer, workspace_id, token_id) + + except Exception as e: + exporter_instance = ExporterHistory.objects.get(token=token_id) + exporter_instance.status = "failed" + exporter_instance.reason = str(e) + exporter_instance.save(update_fields=["status", "reason"]) + + # Print logs if in DEBUG mode + if settings.DEBUG: + print(e) + capture_exception(e) + return diff --git a/apiserver/plane/bgtasks/exporter_expired_task.py b/apiserver/plane/bgtasks/exporter_expired_task.py new file mode 100644 index 000000000..799904347 --- /dev/null +++ b/apiserver/plane/bgtasks/exporter_expired_task.py @@ -0,0 +1,38 @@ +# Python imports +import boto3 +from datetime import timedelta + +# Django imports +from django.conf import settings +from django.utils import timezone +from django.db.models import Q + +# Third party imports +from celery import shared_task +from botocore.client import Config + +# Module imports +from plane.db.models import ExporterHistory + + +@shared_task +def delete_old_s3_link(): + # Get a list of keys and IDs to process + expired_exporter_history = ExporterHistory.objects.filter( + Q(url__isnull=False) & Q(created_at__lte=timezone.now() - timedelta(days=8)) + ).values_list("key", "id") + + s3 = boto3.client( + "s3", + region_name="ap-south-1", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=Config(signature_version="s3v4"), + ) + + for file_name, exporter_id in expired_exporter_history: + # Delete object from S3 + if file_name: + s3.delete_object(Bucket=settings.AWS_S3_BUCKET_NAME, Key=file_name) + + ExporterHistory.objects.filter(id=exporter_id).update(url=None) diff --git a/apiserver/plane/bgtasks/project_issue_export.py b/apiserver/plane/bgtasks/project_issue_export.py deleted file mode 100644 index 75088be9d..000000000 --- a/apiserver/plane/bgtasks/project_issue_export.py +++ /dev/null @@ -1,191 +0,0 @@ -# Python imports -import csv -import io - -# Django imports -from django.core.mail import EmailMultiAlternatives -from django.template.loader import render_to_string -from django.utils.html import strip_tags -from django.conf import settings -from django.utils import timezone - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception - -# Module imports -from plane.db.models import Issue - -@shared_task -def issue_export_task(email, data, slug, exporter_name): - try: - - project_ids = data.get("project_id", []) - issues_filter = {"workspace__slug": slug} - - if project_ids: - issues_filter["project_id__in"] = project_ids - - issues = ( - Issue.objects.filter(**issues_filter) - .select_related("project", "workspace", "state", "parent", "created_by") - .prefetch_related( - "assignees", "labels", "issue_cycle__cycle", "issue_module__module" - ) - .values_list( - "project__identifier", - "sequence_id", - "name", - "description_stripped", - "priority", - "start_date", - "target_date", - "state__name", - "project__name", - "created_at", - "updated_at", - "completed_at", - "archived_at", - "issue_cycle__cycle__name", - "issue_cycle__cycle__start_date", - "issue_cycle__cycle__end_date", - "issue_module__module__name", - "issue_module__module__start_date", - "issue_module__module__target_date", - "created_by__first_name", - "created_by__last_name", - "assignees__first_name", - "assignees__last_name", - "labels__name", - ) - ) - - # CSV header - header = [ - "Issue ID", - "Project", - "Name", - "Description", - "State", - "Priority", - "Created By", - "Assignee", - "Labels", - "Cycle Name", - "Cycle Start Date", - "Cycle End Date", - "Module Name", - "Module Start Date", - "Module Target Date", - "Created At" - "Updated At" - "Completed At" - "Archived At" - ] - - # Prepare the CSV data - rows = [header] - - # Write data for each issue - for issue in issues: - ( - project_identifier, - sequence_id, - name, - description, - priority, - start_date, - target_date, - state_name, - project_name, - created_at, - updated_at, - completed_at, - archived_at, - cycle_name, - cycle_start_date, - cycle_end_date, - module_name, - module_start_date, - module_target_date, - created_by_first_name, - created_by_last_name, - assignees_first_names, - assignees_last_names, - labels_names, - ) = issue - - created_by_fullname = ( - f"{created_by_first_name} {created_by_last_name}" - if created_by_first_name and created_by_last_name - else "" - ) - - assignees_names = "" - if assignees_first_names and assignees_last_names: - assignees_names = ", ".join( - [ - f"{assignees_first_name} {assignees_last_name}" - for assignees_first_name, assignees_last_name in zip( - assignees_first_names, assignees_last_names - ) - ] - ) - - labels_names = ", ".join(labels_names) if labels_names else "" - - row = [ - f"{project_identifier}-{sequence_id}", - project_name, - name, - description, - state_name, - priority, - created_by_fullname, - assignees_names, - labels_names, - cycle_name, - cycle_start_date, - cycle_end_date, - module_name, - module_start_date, - module_target_date, - start_date, - target_date, - created_at, - updated_at, - completed_at, - archived_at, - ] - rows.append(row) - - # Create CSV file in-memory - csv_buffer = io.StringIO() - writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) - - # Write CSV data to the buffer - for row in rows: - writer.writerow(row) - - subject = "Your Issue Export is ready" - - context = { - "username": exporter_name, - } - - html_content = render_to_string("emails/exports/issues.html", context) - text_content = strip_tags(html_content) - - csv_buffer.seek(0) - msg = EmailMultiAlternatives( - subject, text_content, settings.EMAIL_FROM, [email] - ) - msg.attach(f"{slug}-issues-{timezone.now().date()}.csv", csv_buffer.read(), "text/csv") - msg.send(fail_silently=False) - - except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) - return diff --git a/apiserver/plane/celery.py b/apiserver/plane/celery.py index ed0dc419e..15fe8af52 100644 --- a/apiserver/plane/celery.py +++ b/apiserver/plane/celery.py @@ -20,6 +20,10 @@ app.conf.beat_schedule = { "task": "plane.bgtasks.issue_automation_task.archive_and_close_old_issues", "schedule": crontab(hour=0, minute=0), }, + "check-every-day-to-delete_exporter_history": { + "task": "plane.bgtasks.exporter_expired_task.delete_old_s3_link", + "schedule": crontab(hour=0, minute=0), + }, } # Load task modules from all registered Django app configs. diff --git a/apiserver/plane/db/models/__init__.py b/apiserver/plane/db/models/__init__.py index a1bd49ac5..659eea3eb 100644 --- a/apiserver/plane/db/models/__init__.py +++ b/apiserver/plane/db/models/__init__.py @@ -74,4 +74,6 @@ from .inbox import Inbox, InboxIssue from .analytic import AnalyticView -from .notification import Notification \ No newline at end of file +from .notification import Notification + +from .exporter import ExporterHistory \ No newline at end of file diff --git a/apiserver/plane/db/models/exporter.py b/apiserver/plane/db/models/exporter.py new file mode 100644 index 000000000..fce31c8e7 --- /dev/null +++ b/apiserver/plane/db/models/exporter.py @@ -0,0 +1,56 @@ +import uuid + +# Python imports +from uuid import uuid4 + +# Django imports +from django.db import models +from django.conf import settings +from django.contrib.postgres.fields import ArrayField + +# Module imports +from . import BaseModel + +def generate_token(): + return uuid4().hex + +class ExporterHistory(BaseModel): + workspace = models.ForeignKey( + "db.WorkSpace", on_delete=models.CASCADE, related_name="workspace_exporters" + ) + project = ArrayField(models.UUIDField(default=uuid.uuid4), blank=True, null=True) + provider = models.CharField( + max_length=50, + choices=( + ("json", "json"), + ("csv", "csv"), + ("xlsx", "xlsx"), + ), + ) + status = models.CharField( + max_length=50, + choices=( + ("queued", "Queued"), + ("processing", "Processing"), + ("completed", "Completed"), + ("failed", "Failed"), + ), + default="queued", + ) + reason = models.TextField(blank=True) + key = models.TextField(blank=True) + url = models.URLField(max_length=800, blank=True, null=True) + token = models.CharField(max_length=255, default=generate_token, unique=True) + initiated_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="workspace_exporters" + ) + + class Meta: + verbose_name = "Exporter" + verbose_name_plural = "Exporters" + db_table = "exporters" + ordering = ("-created_at",) + + def __str__(self): + """Return name of the service""" + return f"{self.provider} <{self.workspace.name}>" \ No newline at end of file diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index e3a918c18..59e0bd31b 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -214,4 +214,4 @@ SIMPLE_JWT = { CELERY_TIMEZONE = TIME_ZONE CELERY_TASK_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['application/json'] -CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task",) +CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task") diff --git a/apiserver/requirements/base.txt b/apiserver/requirements/base.txt index 76c3dace9..ca9d881ef 100644 --- a/apiserver/requirements/base.txt +++ b/apiserver/requirements/base.txt @@ -32,4 +32,5 @@ celery==5.3.1 django_celery_beat==2.5.0 psycopg-binary==3.1.9 psycopg-c==3.1.9 -scout-apm==2.26.1 \ No newline at end of file +scout-apm==2.26.1 +openpyxl==3.1.2 \ No newline at end of file