github/plane
0
0
Fork 1
mirror of https://github.com/makeplane/plane synced 2024-06-14 14:31:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: export issues in CSV, JSON and XLSX (#1794)

Browse Source 
* fix: file name change

* feat: added xml json and csv export

* chore: added openpyxl package

* fix: added initiated_by field

* fix: added initiated by details

* dev: refactoring

* fix: rendering assignee name and labels in sheet

* fix: handeled exception in label

* feat: implemented link expiration scheduler(8 days)

* fix: removed the expired field

---------

Co-authored-by: NarayanBavisetti <narayan311@gmail.com>
Co-authored-by: pablohashescobar <nikhilschacko@gmail.com>
This commit is contained in:
Bavisetti Narayan 2023-08-11 10:39:52 -04:00 committed by GitHub
parent feba1cc4d0
commit 1a9faa025a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 593 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apiserver/plane/api/serializers/__init__.py
View File

@ -81,3 +81,5 @@ from .inbox import InboxSerializer, InboxIssueSerializer, IssueStateInboxSeriali
from .analytic import AnalyticViewSerializer
from .notification import NotificationSerializer
from .exporter import ExporterHistorySerializer

26
apiserver/plane/api/serializers/exporter.py Normal file
View File

@ -0,0 +1,26 @@
# Module imports
from .base import BaseSerializer
from plane.db.models import ExporterHistory
from .user import UserLiteSerializer
class ExporterHistorySerializer(BaseSerializer):
initiated_by_detail = UserLiteSerializer(source="initiated_by", read_only=True)
class Meta:
model = ExporterHistory
fields = [
"id",
"created_at",
"updated_at",
"project",
"provider",
"status",
"url",
"initiated_by",
"initiated_by_detail",
"token",
"created_by",
"updated_by",
]
read_only_fields = fields

5
apiserver/plane/api/views/__init__.py
View File

@ -84,7 +84,6 @@ from .issue import (
IssueReactionPublicViewSet,
CommentReactionPublicViewSet,
IssueVotePublicViewSet,
ExportIssuesEndpoint
)
from .auth_extended import (
@ -163,3 +162,7 @@ from .analytic import (
)
from .notification import NotificationViewSet, UnreadNotificationEndpoint
from .exporter import (
ExportIssuesEndpoint,
)

99
apiserver/plane/api/views/exporter.py Normal file
View File

@ -0,0 +1,99 @@
# Third Party imports
from rest_framework.response import Response
from rest_framework import status
from sentry_sdk import capture_exception
# Module imports
from . import BaseAPIView
from plane.api.permissions import WorkSpaceAdminPermission
from plane.bgtasks.export_task import issue_export_task
from plane.db.models import Project, ExporterHistory, Workspace
from plane.api.serializers import ExporterHistorySerializer
class ExportIssuesEndpoint(BaseAPIView):
permission_classes = [
WorkSpaceAdminPermission,
]
model = ExporterHistory
serializer_class = ExporterHistorySerializer
def post(self, request, slug):
try:
# Get the workspace
workspace = Workspace.objects.get(slug=slug)
provider = request.data.get("provider", False)
multiple = request.data.get("multiple", False)
project_ids = request.data.get("project", [])
if provider in ["csv", "xlsx", "json"]:
if not project_ids:
project_ids = Project.objects.filter(
workspace__slug=slug
).values_list("id", flat=True)
project_ids = [str(project_id) for project_id in project_ids]
exporter = ExporterHistory.objects.create(
workspace=workspace,
project=project_ids,
initiated_by=request.user,
provider=provider,
)
issue_export_task.delay(
provider=exporter.provider,
workspace_id=workspace.id,
project_ids=project_ids,
token_id=exporter.token,
multiple=multiple,
)
return Response(
{
"message": f"Once the export is ready you will be able to download it"
},
status=status.HTTP_200_OK,
)
else:
return Response(
{"error": f"Provider '{provider}' not found."},
status=status.HTTP_400_BAD_REQUEST,
)
except Workspace.DoesNotExist:
return Response(
{"error": "Workspace does not exists"},
status=status.HTTP_400_BAD_REQUEST,
)
except Exception as e:
capture_exception(e)
return Response(
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)
def get(self, request, slug):
try:
exporter_history = ExporterHistory.objects.filter(
workspace__slug=slug
).select_related("workspace","initiated_by")
if request.GET.get("per_page", False) and request.GET.get("cursor", False):
return self.paginate(
request=request,
queryset=exporter_history,
on_results=lambda exporter_history: ExporterHistorySerializer(
exporter_history, many=True
).data,
)
else:
return Response(
{"error": "per_page and cursor are required"},
status=status.HTTP_400_BAD_REQUEST,
)
except Exception as e:
capture_exception(e)
return Response(
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)

1
apiserver/plane/api/views/issue.py
View File

@ -77,7 +77,6 @@ from plane.db.models import (
from plane.bgtasks.issue_activites_task import issue_activity
from plane.utils.grouper import group_results
from plane.utils.issue_filters import issue_filters
from plane.bgtasks.project_issue_export import issue_export_task
class IssueViewSet(BaseViewSet):

357
apiserver/plane/bgtasks/export_task.py Normal file
View File

@ -0,0 +1,357 @@
# Python imports
import csv
import io
import json
import boto3
import zipfile
from datetime import datetime, date, timedelta
# Django imports
from django.conf import settings
from django.utils import timezone
# Third party imports
from celery import shared_task
from sentry_sdk import capture_exception
from botocore.client import Config
from openpyxl import Workbook
from openpyxl.styles import NamedStyle
from openpyxl.utils.datetime import to_excel
# Module imports
from plane.db.models import Issue, ExporterHistory, Project
class DateTimeEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, (datetime, date)):
return obj.isoformat()
return super().default(obj)
def create_csv_file(data):
csv_buffer = io.StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL)
for row in data:
csv_writer.writerow(row)
csv_buffer.seek(0)
return csv_buffer.getvalue()
def create_json_file(data):
return json.dumps(data, cls=DateTimeEncoder)
def create_xlsx_file(data):
workbook = Workbook()
sheet = workbook.active
no_timezone_style = NamedStyle(name="no_timezone_style")
no_timezone_style.number_format = "yyyy-mm-dd hh:mm:ss"
for row in data:
sheet.append(row)
for column_cells in sheet.columns:
for cell in column_cells:
if isinstance(cell.value, datetime):
cell.style = no_timezone_style
cell.value = to_excel(cell.value.replace(tzinfo=None))
xlsx_buffer = io.BytesIO()
workbook.save(xlsx_buffer)
xlsx_buffer.seek(0)
return xlsx_buffer.getvalue()
def create_zip_file(files):
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf:
for filename, file_content in files:
zipf.writestr(filename, file_content)
zip_buffer.seek(0)
return zip_buffer
def upload_to_s3(zip_file, workspace_id, token_id):
s3 = boto3.client(
"s3",
region_name="ap-south-1",
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
config=Config(signature_version="s3v4"),
)
file_name = f"{workspace_id}/issues-{datetime.now().date()}.zip"
s3.upload_fileobj(
zip_file,
settings.AWS_S3_BUCKET_NAME,
file_name,
ExtraArgs={"ACL": "public-read", "ContentType": "application/zip"},
)
expires_in = 7 * 24 * 60 * 60
presigned_url = s3.generate_presigned_url(
"get_object",
Params={"Bucket": settings.AWS_S3_BUCKET_NAME, "Key": file_name},
ExpiresIn=expires_in,
)
exporter_instance = ExporterHistory.objects.get(token=token_id)
if presigned_url:
exporter_instance.url = presigned_url
exporter_instance.status = "completed"
exporter_instance.key = file_name
else:
exporter_instance.status = "failed"
exporter_instance.save(update_fields=["status", "url","key"])
def generate_table_row(issue):
return [
f"""{issue["project__identifier"]}-{issue["sequence_id"]}""",
issue["project__name"],
issue["name"],
issue["description_stripped"],
issue["state__name"],
issue["priority"],
f"{issue['created_by__first_name']} {issue['created_by__last_name']}"
if issue["created_by__first_name"] and issue["created_by__last_name"]
else "",
f"{issue['assignees__first_name']} {issue['assignees__last_name']}"
if issue["assignees__first_name"] and issue["assignees__last_name"]
else "",
issue["labels__name"],
issue["issue_cycle__cycle__name"],
issue["issue_cycle__cycle__start_date"],
issue["issue_cycle__cycle__end_date"],
issue["issue_module__module__name"],
issue["issue_module__module__start_date"],
issue["issue_module__module__target_date"],
issue["created_at"],
issue["updated_at"],
issue["completed_at"],
issue["archived_at"],
]
def generate_json_row(issue):
return {
"ID": f"""{issue["project__identifier"]}-{issue["sequence_id"]}""",
"Project": issue["project__name"],
"Name": issue["name"],
"Description": issue["description_stripped"],
"State": issue["state__name"],
"Priority": issue["priority"],
"Created By": f"{issue['created_by__first_name']} {issue['created_by__last_name']}"
if issue["created_by__first_name"] and issue["created_by__last_name"]
else "",
"Assignee": f"{issue['assignees__first_name']} {issue['assignees__last_name']}"
if issue["assignees__first_name"] and issue["assignees__last_name"]
else "",
"Labels": issue["labels__name"],
"Cycle Name": issue["issue_cycle__cycle__name"],
"Cycle Start Date": issue["issue_cycle__cycle__start_date"],
"Cycle End Date": issue["issue_cycle__cycle__end_date"],
"Module Name": issue["issue_module__module__name"],
"Module Start Date": issue["issue_module__module__start_date"],
"Module Target Date": issue["issue_module__module__target_date"],
"Created At": issue["created_at"],
"Updated At": issue["updated_at"],
"Completed At": issue["completed_at"],
"Archived At": issue["archived_at"],
}
def update_json_row(rows, row):
matched_index = next(
(
index
for index, existing_row in enumerate(rows)
if existing_row["ID"] == row["ID"]
),
None,
)
if matched_index is not None:
existing_assignees, existing_labels = (
rows[matched_index]["Assignee"],
rows[matched_index]["Labels"],
)
assignee, label = row["Assignee"], row["Labels"]
if assignee is not None and assignee not in existing_assignees:
rows[matched_index]["Assignee"] += f", {assignee}"
if label is not None and label not in existing_labels:
rows[matched_index]["Labels"] += f", {label}"
else:
rows.append(row)
def update_table_row(rows, row):
matched_index = next(
(index for index, existing_row in enumerate(rows) if existing_row[0] == row[0]),
None,
)
if matched_index is not None:
existing_assignees, existing_labels = rows[matched_index][7:9]
assignee, label = row[7:9]
if assignee is not None and assignee not in existing_assignees:
rows[matched_index][7] += f", {assignee}"
if label is not None and label not in existing_labels:
rows[matched_index][8] += f", {label}"
else:
rows.append(row)
def generate_csv(header, project_id, issues, files):
"""
Generate CSV export for all the passed issues.
"""
rows = [
header,
]
for issue in issues:
row = generate_table_row(issue)
update_table_row(rows, row)
csv_file = create_csv_file(rows)
files.append((f"{project_id}.csv", csv_file))
def generate_json(header, project_id, issues, files):
rows = []
for issue in issues:
row = generate_json_row(issue)
update_json_row(rows, row)
json_file = create_json_file(rows)
files.append((f"{project_id}.json", json_file))
def generate_xlsx(header, project_id, issues, files):
rows = [header]
for issue in issues:
row = generate_table_row(issue)
update_table_row(rows, row)
xlsx_file = create_xlsx_file(rows)
files.append((f"{project_id}.xlsx", xlsx_file))
@shared_task
def issue_export_task(provider, workspace_id, project_ids, token_id, multiple):
try:
exporter_instance = ExporterHistory.objects.get(token=token_id)
exporter_instance.status = "processing"
exporter_instance.save(update_fields=["status"])
workspace_issues = (
(
Issue.objects.filter(
workspace__id=workspace_id, project_id__in=project_ids
)
.select_related("project", "workspace", "state", "parent", "created_by")
.prefetch_related(
"assignees", "labels", "issue_cycle__cycle", "issue_module__module"
)
.values(
"id",
"project__identifier",
"project__name",
"project__id",
"sequence_id",
"name",
"description_stripped",
"priority",
"state__name",
"created_at",
"updated_at",
"completed_at",
"archived_at",
"issue_cycle__cycle__name",
"issue_cycle__cycle__start_date",
"issue_cycle__cycle__end_date",
"issue_module__module__name",
"issue_module__module__start_date",
"issue_module__module__target_date",
"created_by__first_name",
"created_by__last_name",
"assignees__first_name",
"assignees__last_name",
"labels__name",
)
)
.order_by("project__identifier","sequence_id")
.distinct()
)
# CSV header
header = [
"ID",
"Project",
"Name",
"Description",
"State",
"Priority",
"Created By",
"Assignee",
"Labels",
"Cycle Name",
"Cycle Start Date",
"Cycle End Date",
"Module Name",
"Module Start Date",
"Module Target Date",
"Created At",
"Updated At",
"Completed At",
"Archived At",
]
EXPORTER_MAPPER = {
"csv": generate_csv,
"json": generate_json,
"xlsx": generate_xlsx,
}
files = []
if multiple:
for project_id in project_ids:
issues = workspace_issues.filter(project__id=project_id)
exporter = EXPORTER_MAPPER.get(provider)
if exporter is not None:
exporter(
header,
project_id,
issues,
files,
)
else:
exporter = EXPORTER_MAPPER.get(provider)
if exporter is not None:
exporter(
header,
workspace_id,
workspace_issues,
files,
)
zip_buffer = create_zip_file(files)
upload_to_s3(zip_buffer, workspace_id, token_id)
except Exception as e:
exporter_instance = ExporterHistory.objects.get(token=token_id)
exporter_instance.status = "failed"
exporter_instance.reason = str(e)
exporter_instance.save(update_fields=["status", "reason"])
# Print logs if in DEBUG mode
if settings.DEBUG:
print(e)
capture_exception(e)
return

38
apiserver/plane/bgtasks/exporter_expired_task.py Normal file
View File

@ -0,0 +1,38 @@
# Python imports
import boto3
from datetime import timedelta
# Django imports
from django.conf import settings
from django.utils import timezone
from django.db.models import Q
# Third party imports
from celery import shared_task
from botocore.client import Config
# Module imports
from plane.db.models import ExporterHistory
@shared_task
def delete_old_s3_link():
# Get a list of keys and IDs to process
expired_exporter_history = ExporterHistory.objects.filter(
Q(url__isnull=False) & Q(created_at__lte=timezone.now() - timedelta(days=8))
).values_list("key", "id")
s3 = boto3.client(
"s3",
region_name="ap-south-1",
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
config=Config(signature_version="s3v4"),
)
for file_name, exporter_id in expired_exporter_history:
# Delete object from S3
if file_name:
s3.delete_object(Bucket=settings.AWS_S3_BUCKET_NAME, Key=file_name)
ExporterHistory.objects.filter(id=exporter_id).update(url=None)

191
apiserver/plane/bgtasks/project_issue_export.py
View File

@ -1,191 +0,0 @@
# Python imports
import csv
import io
# Django imports
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string
from django.utils.html import strip_tags
from django.conf import settings
from django.utils import timezone
# Third party imports
from celery import shared_task
from sentry_sdk import capture_exception
# Module imports
from plane.db.models import Issue
@shared_task
def issue_export_task(email, data, slug, exporter_name):
try:
project_ids = data.get("project_id", [])
issues_filter = {"workspace__slug": slug}
if project_ids:
issues_filter["project_id__in"] = project_ids
issues = (
Issue.objects.filter(**issues_filter)
.select_related("project", "workspace", "state", "parent", "created_by")
.prefetch_related(
"assignees", "labels", "issue_cycle__cycle", "issue_module__module"
)
.values_list(
"project__identifier",
"sequence_id",
"name",
"description_stripped",
"priority",
"start_date",
"target_date",
"state__name",
"project__name",
"created_at",
"updated_at",
"completed_at",
"archived_at",
"issue_cycle__cycle__name",
"issue_cycle__cycle__start_date",
"issue_cycle__cycle__end_date",
"issue_module__module__name",
"issue_module__module__start_date",
"issue_module__module__target_date",
"created_by__first_name",
"created_by__last_name",
"assignees__first_name",
"assignees__last_name",
"labels__name",
)
)
# CSV header
header = [
"Issue ID",
"Project",
"Name",
"Description",
"State",
"Priority",
"Created By",
"Assignee",
"Labels",
"Cycle Name",
"Cycle Start Date",
"Cycle End Date",
"Module Name",
"Module Start Date",
"Module Target Date",
"Created At"
"Updated At"
"Completed At"
"Archived At"
]
# Prepare the CSV data
rows = [header]
# Write data for each issue
for issue in issues:
(
project_identifier,
sequence_id,
name,
description,
priority,
start_date,
target_date,
state_name,
project_name,
created_at,
updated_at,
completed_at,
archived_at,
cycle_name,
cycle_start_date,
cycle_end_date,
module_name,
module_start_date,
module_target_date,
created_by_first_name,
created_by_last_name,
assignees_first_names,
assignees_last_names,
labels_names,
) = issue
created_by_fullname = (
f"{created_by_first_name} {created_by_last_name}"
if created_by_first_name and created_by_last_name
else ""
)
assignees_names = ""
if assignees_first_names and assignees_last_names:
assignees_names = ", ".join(
[
f"{assignees_first_name} {assignees_last_name}"
for assignees_first_name, assignees_last_name in zip(
assignees_first_names, assignees_last_names
)
]
)
labels_names = ", ".join(labels_names) if labels_names else ""
row = [
f"{project_identifier}-{sequence_id}",
project_name,
name,
description,
state_name,
priority,
created_by_fullname,
assignees_names,
labels_names,
cycle_name,
cycle_start_date,
cycle_end_date,
module_name,
module_start_date,
module_target_date,
start_date,
target_date,
created_at,
updated_at,
completed_at,
archived_at,
]
rows.append(row)
# Create CSV file in-memory
csv_buffer = io.StringIO()
writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL)
# Write CSV data to the buffer
for row in rows:
writer.writerow(row)
subject = "Your Issue Export is ready"
context = {
"username": exporter_name,
}
html_content = render_to_string("emails/exports/issues.html", context)
text_content = strip_tags(html_content)
csv_buffer.seek(0)
msg = EmailMultiAlternatives(
subject, text_content, settings.EMAIL_FROM, [email]
)
msg.attach(f"{slug}-issues-{timezone.now().date()}.csv", csv_buffer.read(), "text/csv")
msg.send(fail_silently=False)
except Exception as e:
# Print logs if in DEBUG mode
if settings.DEBUG:
print(e)
capture_exception(e)
return

4
apiserver/plane/celery.py
View File

@ -20,6 +20,10 @@ app.conf.beat_schedule = {
"task": "plane.bgtasks.issue_automation_task.archive_and_close_old_issues",
"schedule": crontab(hour=0, minute=0),
},
"check-every-day-to-delete_exporter_history": {
"task": "plane.bgtasks.exporter_expired_task.delete_old_s3_link",
"schedule": crontab(hour=0, minute=0),
},
}
# Load task modules from all registered Django app configs.

2
apiserver/plane/db/models/__init__.py
View File

@ -75,3 +75,5 @@ from .inbox import Inbox, InboxIssue
from .analytic import AnalyticView
from .notification import Notification
from .exporter import ExporterHistory

56
apiserver/plane/db/models/exporter.py Normal file
View File

@ -0,0 +1,56 @@
import uuid
# Python imports
from uuid import uuid4
# Django imports
from django.db import models
from django.conf import settings
from django.contrib.postgres.fields import ArrayField
# Module imports
from . import BaseModel
def generate_token():
return uuid4().hex
class ExporterHistory(BaseModel):
workspace = models.ForeignKey(
"db.WorkSpace", on_delete=models.CASCADE, related_name="workspace_exporters"
)
project = ArrayField(models.UUIDField(default=uuid.uuid4), blank=True, null=True)
provider = models.CharField(
max_length=50,
choices=(
("json", "json"),
("csv", "csv"),
("xlsx", "xlsx"),
),
)
status = models.CharField(
max_length=50,
choices=(
("queued", "Queued"),
("processing", "Processing"),
("completed", "Completed"),
("failed", "Failed"),
),
default="queued",
)
reason = models.TextField(blank=True)
key = models.TextField(blank=True)
url = models.URLField(max_length=800, blank=True, null=True)
token = models.CharField(max_length=255, default=generate_token, unique=True)
initiated_by = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="workspace_exporters"
)
class Meta:
verbose_name = "Exporter"
verbose_name_plural = "Exporters"
db_table = "exporters"
ordering = ("-created_at",)
def __str__(self):
"""Return name of the service"""
return f"{self.provider} <{self.workspace.name}>"

2
apiserver/plane/settings/common.py
View File

@ -214,4 +214,4 @@ SIMPLE_JWT = {
CELERY_TIMEZONE = TIME_ZONE
CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task",)
CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task")

1
apiserver/requirements/base.txt
View File

@ -33,3 +33,4 @@ django_celery_beat==2.5.0
psycopg-binary==3.1.9
psycopg-c==3.1.9
scout-apm==2.26.1
openpyxl==3.1.2