forked from github/plane
fix: export issues in CSV, JSON and XLSX (#1794)
* fix: file name change * feat: added xml json and csv export * chore: added openpyxl package * fix: added initiated_by field * fix: added initiated by details * dev: refactoring * fix: rendering assignee name and labels in sheet * fix: handeled exception in label * feat: implemented link expiration scheduler(8 days) * fix: removed the expired field --------- Co-authored-by: NarayanBavisetti <narayan311@gmail.com> Co-authored-by: pablohashescobar <nikhilschacko@gmail.com>
This commit is contained in:
parent
feba1cc4d0
commit
1a9faa025a
@ -81,3 +81,5 @@ from .inbox import InboxSerializer, InboxIssueSerializer, IssueStateInboxSeriali
|
|||||||
from .analytic import AnalyticViewSerializer
|
from .analytic import AnalyticViewSerializer
|
||||||
|
|
||||||
from .notification import NotificationSerializer
|
from .notification import NotificationSerializer
|
||||||
|
|
||||||
|
from .exporter import ExporterHistorySerializer
|
||||||
|
26
apiserver/plane/api/serializers/exporter.py
Normal file
26
apiserver/plane/api/serializers/exporter.py
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# Module imports
|
||||||
|
from .base import BaseSerializer
|
||||||
|
from plane.db.models import ExporterHistory
|
||||||
|
from .user import UserLiteSerializer
|
||||||
|
|
||||||
|
|
||||||
|
class ExporterHistorySerializer(BaseSerializer):
|
||||||
|
initiated_by_detail = UserLiteSerializer(source="initiated_by", read_only=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
model = ExporterHistory
|
||||||
|
fields = [
|
||||||
|
"id",
|
||||||
|
"created_at",
|
||||||
|
"updated_at",
|
||||||
|
"project",
|
||||||
|
"provider",
|
||||||
|
"status",
|
||||||
|
"url",
|
||||||
|
"initiated_by",
|
||||||
|
"initiated_by_detail",
|
||||||
|
"token",
|
||||||
|
"created_by",
|
||||||
|
"updated_by",
|
||||||
|
]
|
||||||
|
read_only_fields = fields
|
@ -84,7 +84,6 @@ from .issue import (
|
|||||||
IssueReactionPublicViewSet,
|
IssueReactionPublicViewSet,
|
||||||
CommentReactionPublicViewSet,
|
CommentReactionPublicViewSet,
|
||||||
IssueVotePublicViewSet,
|
IssueVotePublicViewSet,
|
||||||
ExportIssuesEndpoint
|
|
||||||
)
|
)
|
||||||
|
|
||||||
from .auth_extended import (
|
from .auth_extended import (
|
||||||
@ -162,4 +161,8 @@ from .analytic import (
|
|||||||
DefaultAnalyticsEndpoint,
|
DefaultAnalyticsEndpoint,
|
||||||
)
|
)
|
||||||
|
|
||||||
from .notification import NotificationViewSet, UnreadNotificationEndpoint
|
from .notification import NotificationViewSet, UnreadNotificationEndpoint
|
||||||
|
|
||||||
|
from .exporter import (
|
||||||
|
ExportIssuesEndpoint,
|
||||||
|
)
|
99
apiserver/plane/api/views/exporter.py
Normal file
99
apiserver/plane/api/views/exporter.py
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
# Third Party imports
|
||||||
|
from rest_framework.response import Response
|
||||||
|
from rest_framework import status
|
||||||
|
from sentry_sdk import capture_exception
|
||||||
|
|
||||||
|
# Module imports
|
||||||
|
from . import BaseAPIView
|
||||||
|
from plane.api.permissions import WorkSpaceAdminPermission
|
||||||
|
from plane.bgtasks.export_task import issue_export_task
|
||||||
|
from plane.db.models import Project, ExporterHistory, Workspace
|
||||||
|
|
||||||
|
from plane.api.serializers import ExporterHistorySerializer
|
||||||
|
|
||||||
|
|
||||||
|
class ExportIssuesEndpoint(BaseAPIView):
|
||||||
|
permission_classes = [
|
||||||
|
WorkSpaceAdminPermission,
|
||||||
|
]
|
||||||
|
model = ExporterHistory
|
||||||
|
serializer_class = ExporterHistorySerializer
|
||||||
|
|
||||||
|
def post(self, request, slug):
|
||||||
|
try:
|
||||||
|
# Get the workspace
|
||||||
|
workspace = Workspace.objects.get(slug=slug)
|
||||||
|
|
||||||
|
provider = request.data.get("provider", False)
|
||||||
|
multiple = request.data.get("multiple", False)
|
||||||
|
project_ids = request.data.get("project", [])
|
||||||
|
|
||||||
|
if provider in ["csv", "xlsx", "json"]:
|
||||||
|
if not project_ids:
|
||||||
|
project_ids = Project.objects.filter(
|
||||||
|
workspace__slug=slug
|
||||||
|
).values_list("id", flat=True)
|
||||||
|
project_ids = [str(project_id) for project_id in project_ids]
|
||||||
|
|
||||||
|
exporter = ExporterHistory.objects.create(
|
||||||
|
workspace=workspace,
|
||||||
|
project=project_ids,
|
||||||
|
initiated_by=request.user,
|
||||||
|
provider=provider,
|
||||||
|
)
|
||||||
|
|
||||||
|
issue_export_task.delay(
|
||||||
|
provider=exporter.provider,
|
||||||
|
workspace_id=workspace.id,
|
||||||
|
project_ids=project_ids,
|
||||||
|
token_id=exporter.token,
|
||||||
|
multiple=multiple,
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
{
|
||||||
|
"message": f"Once the export is ready you will be able to download it"
|
||||||
|
},
|
||||||
|
status=status.HTTP_200_OK,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return Response(
|
||||||
|
{"error": f"Provider '{provider}' not found."},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
except Workspace.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"error": "Workspace does not exists"},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
capture_exception(e)
|
||||||
|
return Response(
|
||||||
|
{"error": "Something went wrong please try again later"},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
|
||||||
|
def get(self, request, slug):
|
||||||
|
try:
|
||||||
|
exporter_history = ExporterHistory.objects.filter(
|
||||||
|
workspace__slug=slug
|
||||||
|
).select_related("workspace","initiated_by")
|
||||||
|
|
||||||
|
if request.GET.get("per_page", False) and request.GET.get("cursor", False):
|
||||||
|
return self.paginate(
|
||||||
|
request=request,
|
||||||
|
queryset=exporter_history,
|
||||||
|
on_results=lambda exporter_history: ExporterHistorySerializer(
|
||||||
|
exporter_history, many=True
|
||||||
|
).data,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return Response(
|
||||||
|
{"error": "per_page and cursor are required"},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
capture_exception(e)
|
||||||
|
return Response(
|
||||||
|
{"error": "Something went wrong please try again later"},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
@ -77,7 +77,6 @@ from plane.db.models import (
|
|||||||
from plane.bgtasks.issue_activites_task import issue_activity
|
from plane.bgtasks.issue_activites_task import issue_activity
|
||||||
from plane.utils.grouper import group_results
|
from plane.utils.grouper import group_results
|
||||||
from plane.utils.issue_filters import issue_filters
|
from plane.utils.issue_filters import issue_filters
|
||||||
from plane.bgtasks.project_issue_export import issue_export_task
|
|
||||||
|
|
||||||
|
|
||||||
class IssueViewSet(BaseViewSet):
|
class IssueViewSet(BaseViewSet):
|
||||||
|
357
apiserver/plane/bgtasks/export_task.py
Normal file
357
apiserver/plane/bgtasks/export_task.py
Normal file
@ -0,0 +1,357 @@
|
|||||||
|
# Python imports
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import boto3
|
||||||
|
import zipfile
|
||||||
|
from datetime import datetime, date, timedelta
|
||||||
|
|
||||||
|
# Django imports
|
||||||
|
from django.conf import settings
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
# Third party imports
|
||||||
|
from celery import shared_task
|
||||||
|
from sentry_sdk import capture_exception
|
||||||
|
from botocore.client import Config
|
||||||
|
from openpyxl import Workbook
|
||||||
|
from openpyxl.styles import NamedStyle
|
||||||
|
from openpyxl.utils.datetime import to_excel
|
||||||
|
|
||||||
|
# Module imports
|
||||||
|
from plane.db.models import Issue, ExporterHistory, Project
|
||||||
|
|
||||||
|
|
||||||
|
class DateTimeEncoder(json.JSONEncoder):
|
||||||
|
def default(self, obj):
|
||||||
|
if isinstance(obj, (datetime, date)):
|
||||||
|
return obj.isoformat()
|
||||||
|
return super().default(obj)
|
||||||
|
|
||||||
|
|
||||||
|
def create_csv_file(data):
|
||||||
|
csv_buffer = io.StringIO()
|
||||||
|
csv_writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL)
|
||||||
|
|
||||||
|
for row in data:
|
||||||
|
csv_writer.writerow(row)
|
||||||
|
|
||||||
|
csv_buffer.seek(0)
|
||||||
|
return csv_buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def create_json_file(data):
|
||||||
|
return json.dumps(data, cls=DateTimeEncoder)
|
||||||
|
|
||||||
|
|
||||||
|
def create_xlsx_file(data):
|
||||||
|
workbook = Workbook()
|
||||||
|
sheet = workbook.active
|
||||||
|
|
||||||
|
no_timezone_style = NamedStyle(name="no_timezone_style")
|
||||||
|
no_timezone_style.number_format = "yyyy-mm-dd hh:mm:ss"
|
||||||
|
|
||||||
|
for row in data:
|
||||||
|
sheet.append(row)
|
||||||
|
|
||||||
|
for column_cells in sheet.columns:
|
||||||
|
for cell in column_cells:
|
||||||
|
if isinstance(cell.value, datetime):
|
||||||
|
cell.style = no_timezone_style
|
||||||
|
cell.value = to_excel(cell.value.replace(tzinfo=None))
|
||||||
|
|
||||||
|
xlsx_buffer = io.BytesIO()
|
||||||
|
workbook.save(xlsx_buffer)
|
||||||
|
xlsx_buffer.seek(0)
|
||||||
|
return xlsx_buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def create_zip_file(files):
|
||||||
|
zip_buffer = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf:
|
||||||
|
for filename, file_content in files:
|
||||||
|
zipf.writestr(filename, file_content)
|
||||||
|
|
||||||
|
zip_buffer.seek(0)
|
||||||
|
return zip_buffer
|
||||||
|
|
||||||
|
|
||||||
|
def upload_to_s3(zip_file, workspace_id, token_id):
|
||||||
|
s3 = boto3.client(
|
||||||
|
"s3",
|
||||||
|
region_name="ap-south-1",
|
||||||
|
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
|
||||||
|
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
|
||||||
|
config=Config(signature_version="s3v4"),
|
||||||
|
)
|
||||||
|
file_name = f"{workspace_id}/issues-{datetime.now().date()}.zip"
|
||||||
|
|
||||||
|
s3.upload_fileobj(
|
||||||
|
zip_file,
|
||||||
|
settings.AWS_S3_BUCKET_NAME,
|
||||||
|
file_name,
|
||||||
|
ExtraArgs={"ACL": "public-read", "ContentType": "application/zip"},
|
||||||
|
)
|
||||||
|
|
||||||
|
expires_in = 7 * 24 * 60 * 60
|
||||||
|
presigned_url = s3.generate_presigned_url(
|
||||||
|
"get_object",
|
||||||
|
Params={"Bucket": settings.AWS_S3_BUCKET_NAME, "Key": file_name},
|
||||||
|
ExpiresIn=expires_in,
|
||||||
|
)
|
||||||
|
|
||||||
|
exporter_instance = ExporterHistory.objects.get(token=token_id)
|
||||||
|
|
||||||
|
if presigned_url:
|
||||||
|
exporter_instance.url = presigned_url
|
||||||
|
exporter_instance.status = "completed"
|
||||||
|
exporter_instance.key = file_name
|
||||||
|
else:
|
||||||
|
exporter_instance.status = "failed"
|
||||||
|
|
||||||
|
exporter_instance.save(update_fields=["status", "url","key"])
|
||||||
|
|
||||||
|
|
||||||
|
def generate_table_row(issue):
|
||||||
|
return [
|
||||||
|
f"""{issue["project__identifier"]}-{issue["sequence_id"]}""",
|
||||||
|
issue["project__name"],
|
||||||
|
issue["name"],
|
||||||
|
issue["description_stripped"],
|
||||||
|
issue["state__name"],
|
||||||
|
issue["priority"],
|
||||||
|
f"{issue['created_by__first_name']} {issue['created_by__last_name']}"
|
||||||
|
if issue["created_by__first_name"] and issue["created_by__last_name"]
|
||||||
|
else "",
|
||||||
|
f"{issue['assignees__first_name']} {issue['assignees__last_name']}"
|
||||||
|
if issue["assignees__first_name"] and issue["assignees__last_name"]
|
||||||
|
else "",
|
||||||
|
issue["labels__name"],
|
||||||
|
issue["issue_cycle__cycle__name"],
|
||||||
|
issue["issue_cycle__cycle__start_date"],
|
||||||
|
issue["issue_cycle__cycle__end_date"],
|
||||||
|
issue["issue_module__module__name"],
|
||||||
|
issue["issue_module__module__start_date"],
|
||||||
|
issue["issue_module__module__target_date"],
|
||||||
|
issue["created_at"],
|
||||||
|
issue["updated_at"],
|
||||||
|
issue["completed_at"],
|
||||||
|
issue["archived_at"],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_json_row(issue):
|
||||||
|
return {
|
||||||
|
"ID": f"""{issue["project__identifier"]}-{issue["sequence_id"]}""",
|
||||||
|
"Project": issue["project__name"],
|
||||||
|
"Name": issue["name"],
|
||||||
|
"Description": issue["description_stripped"],
|
||||||
|
"State": issue["state__name"],
|
||||||
|
"Priority": issue["priority"],
|
||||||
|
"Created By": f"{issue['created_by__first_name']} {issue['created_by__last_name']}"
|
||||||
|
if issue["created_by__first_name"] and issue["created_by__last_name"]
|
||||||
|
else "",
|
||||||
|
"Assignee": f"{issue['assignees__first_name']} {issue['assignees__last_name']}"
|
||||||
|
if issue["assignees__first_name"] and issue["assignees__last_name"]
|
||||||
|
else "",
|
||||||
|
"Labels": issue["labels__name"],
|
||||||
|
"Cycle Name": issue["issue_cycle__cycle__name"],
|
||||||
|
"Cycle Start Date": issue["issue_cycle__cycle__start_date"],
|
||||||
|
"Cycle End Date": issue["issue_cycle__cycle__end_date"],
|
||||||
|
"Module Name": issue["issue_module__module__name"],
|
||||||
|
"Module Start Date": issue["issue_module__module__start_date"],
|
||||||
|
"Module Target Date": issue["issue_module__module__target_date"],
|
||||||
|
"Created At": issue["created_at"],
|
||||||
|
"Updated At": issue["updated_at"],
|
||||||
|
"Completed At": issue["completed_at"],
|
||||||
|
"Archived At": issue["archived_at"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def update_json_row(rows, row):
|
||||||
|
matched_index = next(
|
||||||
|
(
|
||||||
|
index
|
||||||
|
for index, existing_row in enumerate(rows)
|
||||||
|
if existing_row["ID"] == row["ID"]
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if matched_index is not None:
|
||||||
|
existing_assignees, existing_labels = (
|
||||||
|
rows[matched_index]["Assignee"],
|
||||||
|
rows[matched_index]["Labels"],
|
||||||
|
)
|
||||||
|
assignee, label = row["Assignee"], row["Labels"]
|
||||||
|
|
||||||
|
if assignee is not None and assignee not in existing_assignees:
|
||||||
|
rows[matched_index]["Assignee"] += f", {assignee}"
|
||||||
|
if label is not None and label not in existing_labels:
|
||||||
|
rows[matched_index]["Labels"] += f", {label}"
|
||||||
|
else:
|
||||||
|
rows.append(row)
|
||||||
|
|
||||||
|
|
||||||
|
def update_table_row(rows, row):
|
||||||
|
matched_index = next(
|
||||||
|
(index for index, existing_row in enumerate(rows) if existing_row[0] == row[0]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if matched_index is not None:
|
||||||
|
existing_assignees, existing_labels = rows[matched_index][7:9]
|
||||||
|
assignee, label = row[7:9]
|
||||||
|
|
||||||
|
if assignee is not None and assignee not in existing_assignees:
|
||||||
|
rows[matched_index][7] += f", {assignee}"
|
||||||
|
if label is not None and label not in existing_labels:
|
||||||
|
rows[matched_index][8] += f", {label}"
|
||||||
|
else:
|
||||||
|
rows.append(row)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_csv(header, project_id, issues, files):
|
||||||
|
"""
|
||||||
|
Generate CSV export for all the passed issues.
|
||||||
|
"""
|
||||||
|
rows = [
|
||||||
|
header,
|
||||||
|
]
|
||||||
|
for issue in issues:
|
||||||
|
row = generate_table_row(issue)
|
||||||
|
update_table_row(rows, row)
|
||||||
|
csv_file = create_csv_file(rows)
|
||||||
|
files.append((f"{project_id}.csv", csv_file))
|
||||||
|
|
||||||
|
|
||||||
|
def generate_json(header, project_id, issues, files):
|
||||||
|
rows = []
|
||||||
|
for issue in issues:
|
||||||
|
row = generate_json_row(issue)
|
||||||
|
update_json_row(rows, row)
|
||||||
|
json_file = create_json_file(rows)
|
||||||
|
files.append((f"{project_id}.json", json_file))
|
||||||
|
|
||||||
|
|
||||||
|
def generate_xlsx(header, project_id, issues, files):
|
||||||
|
rows = [header]
|
||||||
|
for issue in issues:
|
||||||
|
row = generate_table_row(issue)
|
||||||
|
update_table_row(rows, row)
|
||||||
|
xlsx_file = create_xlsx_file(rows)
|
||||||
|
files.append((f"{project_id}.xlsx", xlsx_file))
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task
|
||||||
|
def issue_export_task(provider, workspace_id, project_ids, token_id, multiple):
|
||||||
|
try:
|
||||||
|
exporter_instance = ExporterHistory.objects.get(token=token_id)
|
||||||
|
exporter_instance.status = "processing"
|
||||||
|
exporter_instance.save(update_fields=["status"])
|
||||||
|
|
||||||
|
workspace_issues = (
|
||||||
|
(
|
||||||
|
Issue.objects.filter(
|
||||||
|
workspace__id=workspace_id, project_id__in=project_ids
|
||||||
|
)
|
||||||
|
.select_related("project", "workspace", "state", "parent", "created_by")
|
||||||
|
.prefetch_related(
|
||||||
|
"assignees", "labels", "issue_cycle__cycle", "issue_module__module"
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
"id",
|
||||||
|
"project__identifier",
|
||||||
|
"project__name",
|
||||||
|
"project__id",
|
||||||
|
"sequence_id",
|
||||||
|
"name",
|
||||||
|
"description_stripped",
|
||||||
|
"priority",
|
||||||
|
"state__name",
|
||||||
|
"created_at",
|
||||||
|
"updated_at",
|
||||||
|
"completed_at",
|
||||||
|
"archived_at",
|
||||||
|
"issue_cycle__cycle__name",
|
||||||
|
"issue_cycle__cycle__start_date",
|
||||||
|
"issue_cycle__cycle__end_date",
|
||||||
|
"issue_module__module__name",
|
||||||
|
"issue_module__module__start_date",
|
||||||
|
"issue_module__module__target_date",
|
||||||
|
"created_by__first_name",
|
||||||
|
"created_by__last_name",
|
||||||
|
"assignees__first_name",
|
||||||
|
"assignees__last_name",
|
||||||
|
"labels__name",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by("project__identifier","sequence_id")
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
# CSV header
|
||||||
|
header = [
|
||||||
|
"ID",
|
||||||
|
"Project",
|
||||||
|
"Name",
|
||||||
|
"Description",
|
||||||
|
"State",
|
||||||
|
"Priority",
|
||||||
|
"Created By",
|
||||||
|
"Assignee",
|
||||||
|
"Labels",
|
||||||
|
"Cycle Name",
|
||||||
|
"Cycle Start Date",
|
||||||
|
"Cycle End Date",
|
||||||
|
"Module Name",
|
||||||
|
"Module Start Date",
|
||||||
|
"Module Target Date",
|
||||||
|
"Created At",
|
||||||
|
"Updated At",
|
||||||
|
"Completed At",
|
||||||
|
"Archived At",
|
||||||
|
]
|
||||||
|
|
||||||
|
EXPORTER_MAPPER = {
|
||||||
|
"csv": generate_csv,
|
||||||
|
"json": generate_json,
|
||||||
|
"xlsx": generate_xlsx,
|
||||||
|
}
|
||||||
|
|
||||||
|
files = []
|
||||||
|
if multiple:
|
||||||
|
for project_id in project_ids:
|
||||||
|
issues = workspace_issues.filter(project__id=project_id)
|
||||||
|
exporter = EXPORTER_MAPPER.get(provider)
|
||||||
|
if exporter is not None:
|
||||||
|
exporter(
|
||||||
|
header,
|
||||||
|
project_id,
|
||||||
|
issues,
|
||||||
|
files,
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
|
exporter = EXPORTER_MAPPER.get(provider)
|
||||||
|
if exporter is not None:
|
||||||
|
exporter(
|
||||||
|
header,
|
||||||
|
workspace_id,
|
||||||
|
workspace_issues,
|
||||||
|
files,
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_buffer = create_zip_file(files)
|
||||||
|
upload_to_s3(zip_buffer, workspace_id, token_id)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
exporter_instance = ExporterHistory.objects.get(token=token_id)
|
||||||
|
exporter_instance.status = "failed"
|
||||||
|
exporter_instance.reason = str(e)
|
||||||
|
exporter_instance.save(update_fields=["status", "reason"])
|
||||||
|
|
||||||
|
# Print logs if in DEBUG mode
|
||||||
|
if settings.DEBUG:
|
||||||
|
print(e)
|
||||||
|
capture_exception(e)
|
||||||
|
return
|
38
apiserver/plane/bgtasks/exporter_expired_task.py
Normal file
38
apiserver/plane/bgtasks/exporter_expired_task.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
# Python imports
|
||||||
|
import boto3
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
# Django imports
|
||||||
|
from django.conf import settings
|
||||||
|
from django.utils import timezone
|
||||||
|
from django.db.models import Q
|
||||||
|
|
||||||
|
# Third party imports
|
||||||
|
from celery import shared_task
|
||||||
|
from botocore.client import Config
|
||||||
|
|
||||||
|
# Module imports
|
||||||
|
from plane.db.models import ExporterHistory
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task
|
||||||
|
def delete_old_s3_link():
|
||||||
|
# Get a list of keys and IDs to process
|
||||||
|
expired_exporter_history = ExporterHistory.objects.filter(
|
||||||
|
Q(url__isnull=False) & Q(created_at__lte=timezone.now() - timedelta(days=8))
|
||||||
|
).values_list("key", "id")
|
||||||
|
|
||||||
|
s3 = boto3.client(
|
||||||
|
"s3",
|
||||||
|
region_name="ap-south-1",
|
||||||
|
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
|
||||||
|
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
|
||||||
|
config=Config(signature_version="s3v4"),
|
||||||
|
)
|
||||||
|
|
||||||
|
for file_name, exporter_id in expired_exporter_history:
|
||||||
|
# Delete object from S3
|
||||||
|
if file_name:
|
||||||
|
s3.delete_object(Bucket=settings.AWS_S3_BUCKET_NAME, Key=file_name)
|
||||||
|
|
||||||
|
ExporterHistory.objects.filter(id=exporter_id).update(url=None)
|
@ -1,191 +0,0 @@
|
|||||||
# Python imports
|
|
||||||
import csv
|
|
||||||
import io
|
|
||||||
|
|
||||||
# Django imports
|
|
||||||
from django.core.mail import EmailMultiAlternatives
|
|
||||||
from django.template.loader import render_to_string
|
|
||||||
from django.utils.html import strip_tags
|
|
||||||
from django.conf import settings
|
|
||||||
from django.utils import timezone
|
|
||||||
|
|
||||||
# Third party imports
|
|
||||||
from celery import shared_task
|
|
||||||
from sentry_sdk import capture_exception
|
|
||||||
|
|
||||||
# Module imports
|
|
||||||
from plane.db.models import Issue
|
|
||||||
|
|
||||||
@shared_task
|
|
||||||
def issue_export_task(email, data, slug, exporter_name):
|
|
||||||
try:
|
|
||||||
|
|
||||||
project_ids = data.get("project_id", [])
|
|
||||||
issues_filter = {"workspace__slug": slug}
|
|
||||||
|
|
||||||
if project_ids:
|
|
||||||
issues_filter["project_id__in"] = project_ids
|
|
||||||
|
|
||||||
issues = (
|
|
||||||
Issue.objects.filter(**issues_filter)
|
|
||||||
.select_related("project", "workspace", "state", "parent", "created_by")
|
|
||||||
.prefetch_related(
|
|
||||||
"assignees", "labels", "issue_cycle__cycle", "issue_module__module"
|
|
||||||
)
|
|
||||||
.values_list(
|
|
||||||
"project__identifier",
|
|
||||||
"sequence_id",
|
|
||||||
"name",
|
|
||||||
"description_stripped",
|
|
||||||
"priority",
|
|
||||||
"start_date",
|
|
||||||
"target_date",
|
|
||||||
"state__name",
|
|
||||||
"project__name",
|
|
||||||
"created_at",
|
|
||||||
"updated_at",
|
|
||||||
"completed_at",
|
|
||||||
"archived_at",
|
|
||||||
"issue_cycle__cycle__name",
|
|
||||||
"issue_cycle__cycle__start_date",
|
|
||||||
"issue_cycle__cycle__end_date",
|
|
||||||
"issue_module__module__name",
|
|
||||||
"issue_module__module__start_date",
|
|
||||||
"issue_module__module__target_date",
|
|
||||||
"created_by__first_name",
|
|
||||||
"created_by__last_name",
|
|
||||||
"assignees__first_name",
|
|
||||||
"assignees__last_name",
|
|
||||||
"labels__name",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# CSV header
|
|
||||||
header = [
|
|
||||||
"Issue ID",
|
|
||||||
"Project",
|
|
||||||
"Name",
|
|
||||||
"Description",
|
|
||||||
"State",
|
|
||||||
"Priority",
|
|
||||||
"Created By",
|
|
||||||
"Assignee",
|
|
||||||
"Labels",
|
|
||||||
"Cycle Name",
|
|
||||||
"Cycle Start Date",
|
|
||||||
"Cycle End Date",
|
|
||||||
"Module Name",
|
|
||||||
"Module Start Date",
|
|
||||||
"Module Target Date",
|
|
||||||
"Created At"
|
|
||||||
"Updated At"
|
|
||||||
"Completed At"
|
|
||||||
"Archived At"
|
|
||||||
]
|
|
||||||
|
|
||||||
# Prepare the CSV data
|
|
||||||
rows = [header]
|
|
||||||
|
|
||||||
# Write data for each issue
|
|
||||||
for issue in issues:
|
|
||||||
(
|
|
||||||
project_identifier,
|
|
||||||
sequence_id,
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
priority,
|
|
||||||
start_date,
|
|
||||||
target_date,
|
|
||||||
state_name,
|
|
||||||
project_name,
|
|
||||||
created_at,
|
|
||||||
updated_at,
|
|
||||||
completed_at,
|
|
||||||
archived_at,
|
|
||||||
cycle_name,
|
|
||||||
cycle_start_date,
|
|
||||||
cycle_end_date,
|
|
||||||
module_name,
|
|
||||||
module_start_date,
|
|
||||||
module_target_date,
|
|
||||||
created_by_first_name,
|
|
||||||
created_by_last_name,
|
|
||||||
assignees_first_names,
|
|
||||||
assignees_last_names,
|
|
||||||
labels_names,
|
|
||||||
) = issue
|
|
||||||
|
|
||||||
created_by_fullname = (
|
|
||||||
f"{created_by_first_name} {created_by_last_name}"
|
|
||||||
if created_by_first_name and created_by_last_name
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
|
|
||||||
assignees_names = ""
|
|
||||||
if assignees_first_names and assignees_last_names:
|
|
||||||
assignees_names = ", ".join(
|
|
||||||
[
|
|
||||||
f"{assignees_first_name} {assignees_last_name}"
|
|
||||||
for assignees_first_name, assignees_last_name in zip(
|
|
||||||
assignees_first_names, assignees_last_names
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
labels_names = ", ".join(labels_names) if labels_names else ""
|
|
||||||
|
|
||||||
row = [
|
|
||||||
f"{project_identifier}-{sequence_id}",
|
|
||||||
project_name,
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
state_name,
|
|
||||||
priority,
|
|
||||||
created_by_fullname,
|
|
||||||
assignees_names,
|
|
||||||
labels_names,
|
|
||||||
cycle_name,
|
|
||||||
cycle_start_date,
|
|
||||||
cycle_end_date,
|
|
||||||
module_name,
|
|
||||||
module_start_date,
|
|
||||||
module_target_date,
|
|
||||||
start_date,
|
|
||||||
target_date,
|
|
||||||
created_at,
|
|
||||||
updated_at,
|
|
||||||
completed_at,
|
|
||||||
archived_at,
|
|
||||||
]
|
|
||||||
rows.append(row)
|
|
||||||
|
|
||||||
# Create CSV file in-memory
|
|
||||||
csv_buffer = io.StringIO()
|
|
||||||
writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL)
|
|
||||||
|
|
||||||
# Write CSV data to the buffer
|
|
||||||
for row in rows:
|
|
||||||
writer.writerow(row)
|
|
||||||
|
|
||||||
subject = "Your Issue Export is ready"
|
|
||||||
|
|
||||||
context = {
|
|
||||||
"username": exporter_name,
|
|
||||||
}
|
|
||||||
|
|
||||||
html_content = render_to_string("emails/exports/issues.html", context)
|
|
||||||
text_content = strip_tags(html_content)
|
|
||||||
|
|
||||||
csv_buffer.seek(0)
|
|
||||||
msg = EmailMultiAlternatives(
|
|
||||||
subject, text_content, settings.EMAIL_FROM, [email]
|
|
||||||
)
|
|
||||||
msg.attach(f"{slug}-issues-{timezone.now().date()}.csv", csv_buffer.read(), "text/csv")
|
|
||||||
msg.send(fail_silently=False)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
# Print logs if in DEBUG mode
|
|
||||||
if settings.DEBUG:
|
|
||||||
print(e)
|
|
||||||
capture_exception(e)
|
|
||||||
return
|
|
@ -20,6 +20,10 @@ app.conf.beat_schedule = {
|
|||||||
"task": "plane.bgtasks.issue_automation_task.archive_and_close_old_issues",
|
"task": "plane.bgtasks.issue_automation_task.archive_and_close_old_issues",
|
||||||
"schedule": crontab(hour=0, minute=0),
|
"schedule": crontab(hour=0, minute=0),
|
||||||
},
|
},
|
||||||
|
"check-every-day-to-delete_exporter_history": {
|
||||||
|
"task": "plane.bgtasks.exporter_expired_task.delete_old_s3_link",
|
||||||
|
"schedule": crontab(hour=0, minute=0),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Load task modules from all registered Django app configs.
|
# Load task modules from all registered Django app configs.
|
||||||
|
@ -74,4 +74,6 @@ from .inbox import Inbox, InboxIssue
|
|||||||
|
|
||||||
from .analytic import AnalyticView
|
from .analytic import AnalyticView
|
||||||
|
|
||||||
from .notification import Notification
|
from .notification import Notification
|
||||||
|
|
||||||
|
from .exporter import ExporterHistory
|
56
apiserver/plane/db/models/exporter.py
Normal file
56
apiserver/plane/db/models/exporter.py
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
import uuid
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
# Django imports
|
||||||
|
from django.db import models
|
||||||
|
from django.conf import settings
|
||||||
|
from django.contrib.postgres.fields import ArrayField
|
||||||
|
|
||||||
|
# Module imports
|
||||||
|
from . import BaseModel
|
||||||
|
|
||||||
|
def generate_token():
|
||||||
|
return uuid4().hex
|
||||||
|
|
||||||
|
class ExporterHistory(BaseModel):
|
||||||
|
workspace = models.ForeignKey(
|
||||||
|
"db.WorkSpace", on_delete=models.CASCADE, related_name="workspace_exporters"
|
||||||
|
)
|
||||||
|
project = ArrayField(models.UUIDField(default=uuid.uuid4), blank=True, null=True)
|
||||||
|
provider = models.CharField(
|
||||||
|
max_length=50,
|
||||||
|
choices=(
|
||||||
|
("json", "json"),
|
||||||
|
("csv", "csv"),
|
||||||
|
("xlsx", "xlsx"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
status = models.CharField(
|
||||||
|
max_length=50,
|
||||||
|
choices=(
|
||||||
|
("queued", "Queued"),
|
||||||
|
("processing", "Processing"),
|
||||||
|
("completed", "Completed"),
|
||||||
|
("failed", "Failed"),
|
||||||
|
),
|
||||||
|
default="queued",
|
||||||
|
)
|
||||||
|
reason = models.TextField(blank=True)
|
||||||
|
key = models.TextField(blank=True)
|
||||||
|
url = models.URLField(max_length=800, blank=True, null=True)
|
||||||
|
token = models.CharField(max_length=255, default=generate_token, unique=True)
|
||||||
|
initiated_by = models.ForeignKey(
|
||||||
|
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="workspace_exporters"
|
||||||
|
)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
verbose_name = "Exporter"
|
||||||
|
verbose_name_plural = "Exporters"
|
||||||
|
db_table = "exporters"
|
||||||
|
ordering = ("-created_at",)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
"""Return name of the service"""
|
||||||
|
return f"{self.provider} <{self.workspace.name}>"
|
@ -214,4 +214,4 @@ SIMPLE_JWT = {
|
|||||||
CELERY_TIMEZONE = TIME_ZONE
|
CELERY_TIMEZONE = TIME_ZONE
|
||||||
CELERY_TASK_SERIALIZER = 'json'
|
CELERY_TASK_SERIALIZER = 'json'
|
||||||
CELERY_ACCEPT_CONTENT = ['application/json']
|
CELERY_ACCEPT_CONTENT = ['application/json']
|
||||||
CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task",)
|
CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task")
|
||||||
|
@ -32,4 +32,5 @@ celery==5.3.1
|
|||||||
django_celery_beat==2.5.0
|
django_celery_beat==2.5.0
|
||||||
psycopg-binary==3.1.9
|
psycopg-binary==3.1.9
|
||||||
psycopg-c==3.1.9
|
psycopg-c==3.1.9
|
||||||
scout-apm==2.26.1
|
scout-apm==2.26.1
|
||||||
|
openpyxl==3.1.2
|
Loading…
Reference in New Issue
Block a user