diff --git a/apiserver/plane/api/serializers/__init__.py b/apiserver/plane/api/serializers/__init__.py index 79014c53d..505a9978d 100644 --- a/apiserver/plane/api/serializers/__init__.py +++ b/apiserver/plane/api/serializers/__init__.py @@ -70,3 +70,5 @@ from .importer import ImporterSerializer from .page import PageSerializer, PageBlockSerializer, PageFavoriteSerializer from .estimate import EstimateSerializer, EstimatePointSerializer, EstimateReadSerializer + +from .analytic import AnalyticViewSerializer diff --git a/apiserver/plane/api/serializers/analytic.py b/apiserver/plane/api/serializers/analytic.py new file mode 100644 index 000000000..5f35e1117 --- /dev/null +++ b/apiserver/plane/api/serializers/analytic.py @@ -0,0 +1,30 @@ +from .base import BaseSerializer +from plane.db.models import AnalyticView +from plane.utils.issue_filters import issue_filters + + +class AnalyticViewSerializer(BaseSerializer): + class Meta: + model = AnalyticView + fields = "__all__" + read_only_fields = [ + "workspace", + "query", + ] + + def create(self, validated_data): + query_params = validated_data.get("query_dict", {}) + if bool(query_params): + validated_data["query"] = issue_filters(query_params, "POST") + else: + validated_data["query"] = dict() + return AnalyticView.objects.create(**validated_data) + + def update(self, instance, validated_data): + query_params = validated_data.get("query_data", {}) + if bool(query_params): + validated_data["query"] = issue_filters(query_params, "POST") + else: + validated_data["query"] = dict() + validated_data["query"] = issue_filters(query_params, "PATCH") + return super().update(instance, validated_data) diff --git a/apiserver/plane/api/urls.py b/apiserver/plane/api/urls.py index a88744b4a..cf9ac92cf 100644 --- a/apiserver/plane/api/urls.py +++ b/apiserver/plane/api/urls.py @@ -148,6 +148,13 @@ from plane.api.views import ( # Release Notes ReleaseNotesEndpoint, ## End Release Notes + # Analytics + AnalyticsEndpoint, + AnalyticViewViewset, + SavedAnalyticEndpoint, + ExportAnalyticsEndpoint, + DefaultAnalyticsEndpoint, + ## End Analytics ) @@ -1285,4 +1292,38 @@ urlpatterns = [ name="release-notes", ), ## End Release Notes + # Analytics + path( + "workspaces//analytics/", + AnalyticsEndpoint.as_view(), + name="plane-analytics", + ), + path( + "workspaces//analytic-view/", + AnalyticViewViewset.as_view({"get": "list", "post": "create"}), + name="analytic-view", + ), + path( + "workspaces//analytic-view//", + AnalyticViewViewset.as_view( + {"get": "retrieve", "patch": "partial_update", "delete": "destroy"} + ), + name="analytic-view", + ), + path( + "workspaces//saved-analytic-view//", + SavedAnalyticEndpoint.as_view(), + name="saved-analytic-view", + ), + path( + "workspaces//export-analytics/", + ExportAnalyticsEndpoint.as_view(), + name="export-analytics", + ), + path( + "workspaces//default-analytics/", + DefaultAnalyticsEndpoint.as_view(), + name="default-analytics", + ), + ## End Analytics ] diff --git a/apiserver/plane/api/views/__init__.py b/apiserver/plane/api/views/__init__.py index 536fd83bf..65554f529 100644 --- a/apiserver/plane/api/views/__init__.py +++ b/apiserver/plane/api/views/__init__.py @@ -140,3 +140,11 @@ from .estimate import ( from .release import ReleaseNotesEndpoint + +from .analytic import ( + AnalyticsEndpoint, + AnalyticViewViewset, + SavedAnalyticEndpoint, + ExportAnalyticsEndpoint, + DefaultAnalyticsEndpoint, +) diff --git a/apiserver/plane/api/views/analytic.py b/apiserver/plane/api/views/analytic.py new file mode 100644 index 000000000..d22eb35e8 --- /dev/null +++ b/apiserver/plane/api/views/analytic.py @@ -0,0 +1,308 @@ +# Django imports +from django.db.models import ( + Q, + Count, + Sum, + Value, + Case, + When, + FloatField, + Subquery, + OuterRef, + F, + ExpressionWrapper, +) +from django.db.models.functions import ExtractMonth + +# Third party imports +from rest_framework import status +from rest_framework.response import Response +from sentry_sdk import capture_exception + +# Module imports +from plane.api.views import BaseAPIView, BaseViewSet +from plane.api.permissions import WorkSpaceAdminPermission +from plane.db.models import Issue, AnalyticView, Workspace, State, Label +from plane.api.serializers import AnalyticViewSerializer +from plane.utils.analytics_plot import build_graph_plot +from plane.bgtasks.analytic_plot_export import analytic_export_task + + +class AnalyticsEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + + def get(self, request, slug): + try: + x_axis = request.GET.get("x_axis", False) + y_axis = request.GET.get("y_axis", False) + + if not x_axis or not y_axis: + return Response( + {"error": "x-axis and y-axis dimensions are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + project_ids = request.GET.getlist("project") + cycle_ids = request.GET.getlist("cycle") + module_ids = request.GET.getlist("module") + + segment = request.GET.get("segment", False) + + queryset = Issue.objects.filter(workspace__slug=slug) + if project_ids: + queryset = queryset.filter(project_id__in=project_ids) + if cycle_ids: + queryset = queryset.filter(issue_cycle__cycle_id__in=cycle_ids) + if module_ids: + queryset = queryset.filter(issue_module__module_id__in=module_ids) + + total_issues = queryset.count() + distribution = build_graph_plot( + queryset=queryset, x_axis=x_axis, y_axis=y_axis, segment=segment + ) + + colors = dict() + if x_axis in ["state__name", "state__group"] or segment in [ + "state__name", + "state__group", + ]: + if x_axis in ["state__name", "state__group"]: + key = "name" if x_axis == "state__name" else "group" + else: + key = "name" if segment == "state__name" else "group" + + colors = ( + State.objects.filter( + workspace__slug=slug, project_id__in=project_ids + ).values(key, "color") + if project_ids + else State.objects.filter(workspace__slug=slug).values(key, "color") + ) + + if x_axis in ["labels__name"] or segment in ["labels__name"]: + colors = ( + Label.objects.filter( + workspace__slug=slug, project_id__in=project_ids + ).values("name", "color") + if project_ids + else Label.objects.filter(workspace__slug=slug).values( + "name", "color" + ) + ) + + return Response( + { + "total": total_issues, + "distribution": distribution, + "extras": {"colors": colors}, + }, + status=status.HTTP_200_OK, + ) + + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + +class AnalyticViewViewset(BaseViewSet): + permission_classes = [ + WorkSpaceAdminPermission, + ] + model = AnalyticView + serializer_class = AnalyticViewSerializer + + def perform_create(self, serializer): + workspace = Workspace.objects.get(slug=self.kwargs.get("slug")) + serializer.save(workspace_id=workspace.id) + + def get_queryset(self): + return self.filter_queryset( + super().get_queryset().filter(workspace__slug=self.kwargs.get("slug")) + ) + + +class SavedAnalyticEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + + def get(self, request, slug, analytic_id): + try: + analytic_view = AnalyticView.objects.get( + pk=analytic_id, workspace__slug=slug + ) + + filter = analytic_view.query + queryset = Issue.objects.filter(**filter) + + x_axis = analytic_view.query_dict.get("x_axis", False) + y_axis = analytic_view.query_dict.get("y_axis", False) + + if not x_axis or not y_axis: + return Response( + {"error": "x-axis and y-axis dimensions are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + segment = request.GET.get("segment", False) + distribution = build_graph_plot( + queryset=queryset, x_axis=x_axis, y_axis=y_axis, segment=segment + ) + total_issues = queryset.count() + return Response( + {"total": total_issues, "distribution": distribution}, + status=status.HTTP_200_OK, + ) + + except AnalyticView.DoesNotExist: + return Response( + {"error": "Analytic View Does not exist"}, + status=status.HTTP_400_BAD_REQUEST, + ) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + +class ExportAnalyticsEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + + def post(self, request, slug): + try: + x_axis = request.data.get("x_axis", False) + y_axis = request.data.get("y_axis", False) + + if not x_axis or not y_axis: + return Response( + {"error": "x-axis and y-axis dimensions are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + analytic_export_task.delay( + email=request.user.email, data=request.data, slug=slug + ) + + return Response( + { + "message": f"Once the export is ready it will be emailed to you at {str(request.user.email)}" + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + +class DefaultAnalyticsEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + + def get(self, request, slug): + try: + queryset = Issue.objects.filter(workspace__slug=slug) + + project_ids = request.GET.getlist("project") + cycle_ids = request.GET.getlist("cycle") + module_ids = request.GET.getlist("module") + + if project_ids: + queryset = queryset.filter(project_id__in=project_ids) + if cycle_ids: + queryset = queryset.filter(issue_cycle__cycle_id__in=cycle_ids) + if module_ids: + queryset = queryset.filter(issue_module__module_id__in=module_ids) + + total_issues = queryset.count() + + total_issues_classified = ( + queryset.annotate(state_group=F("state__group")) + .values("state_group") + .annotate(state_count=Count("state_group")) + .order_by("state_group") + ) + + open_issues = queryset.filter( + state__group__in=["backlog", "unstarted", "started"] + ).count() + + open_issues_classified = ( + queryset.filter(state__group__in=["backlog", "unstarted", "started"]) + .annotate(state_group=F("state__group")) + .values("state_group") + .annotate(state_count=Count("state_group")) + .order_by("state_group") + ) + + issue_completed_month_wise = ( + queryset.filter(completed_at__isnull=False) + .annotate(month=ExtractMonth("completed_at")) + .values("month") + .annotate(count=Count("*")) + .order_by("month") + ) + most_issue_created_user = ( + queryset.filter(created_by__isnull=False) + .values("assignees__email", "assignees__avatar") + .annotate(count=Count("id")) + .order_by("-count") + )[:5] + + most_issue_closed_user = ( + queryset.filter(completed_at__isnull=False, assignees__isnull=False) + .values("assignees__email", "assignees__avatar") + .annotate(count=Count("id")) + .order_by("-count") + )[:5] + + pending_issue_user = ( + queryset.filter(completed_at__isnull=True) + .values("assignees__email", "assignees__avatar") + .annotate(count=Count("id")) + .order_by("-count") + ) + + open_estimate_sum = ( + Issue.objects.filter( + state__group__in=["backlog", "unstarted", "started"] + ).aggregate(open_estimate_sum=Sum("estimate_point")) + )["open_estimate_sum"] + total_estimate_sum = Issue.objects.aggregate( + total_estimate_sum=Sum("estimate_point") + )["total_estimate_sum"] + + return Response( + { + "total_issues": total_issues, + "total_issues_classified": total_issues_classified, + "open_issues": open_issues, + "open_issues_classified": open_issues_classified, + "issue_completed_month_wise": issue_completed_month_wise, + "most_issue_created_user": most_issue_created_user, + "most_issue_closed_user": most_issue_closed_user, + "pending_issue_user": pending_issue_user, + "open_estimate_sum": open_estimate_sum, + "total_estimate_sum": total_estimate_sum, + }, + status=status.HTTP_200_OK, + ) + + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/apiserver/plane/api/views/view.py b/apiserver/plane/api/views/view.py index b4e300dcb..1b6fb42cc 100644 --- a/apiserver/plane/api/views/view.py +++ b/apiserver/plane/api/views/view.py @@ -18,10 +18,6 @@ from plane.api.permissions import ProjectEntityPermission from plane.db.models import ( IssueView, Issue, - IssueBlocker, - IssueLink, - CycleIssue, - ModuleIssue, IssueViewFavorite, ) from plane.utils.issue_filters import issue_filters diff --git a/apiserver/plane/bgtasks/analytic_plot_export.py b/apiserver/plane/bgtasks/analytic_plot_export.py new file mode 100644 index 000000000..f34f28c62 --- /dev/null +++ b/apiserver/plane/bgtasks/analytic_plot_export.py @@ -0,0 +1,138 @@ +# Python imports +import csv +import io + +# Django imports +from django.core.mail import EmailMultiAlternatives +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.conf import settings + +# Third party imports +from celery import shared_task +from sentry_sdk import capture_exception + +# Module imports +from plane.db.models import Issue +from plane.utils.analytics_plot import build_graph_plot +from plane.utils.issue_filters import issue_filters + +row_mapping = { + "state__name": "State", + "state__group": "State Group", + "labels__name": "Label", + "assignees__email": "Assignee Email", + "start_date": "Start Date", + "target_date": "Due Date", + "completed_at": "Completed At", + "created_at": "Created At", + "issue_count": "Issue Count", + "effort": "Effort", +} + + +@shared_task +def analytic_export_task(email, data, slug): + try: + filters = issue_filters(data, "POST") + queryset = Issue.objects.filter(**filters, workspace__slug=slug) + + x_axis = data.get("x_axis", False) + y_axis = data.get("y_axis", False) + segment = data.get("segment", False) + + distribution = build_graph_plot( + queryset=queryset, x_axis=x_axis, y_axis=y_axis, segment=segment + ) + + key = "count" if y_axis == "issue_count" else "effort" + + if segment: + row_zero = [ + row_mapping.get(x_axis, "X-Axis"), + ] + segment_zero = [] + for item in distribution: + current_dict = distribution.get(item) + for current in current_dict: + segment_zero.append(current.get("segment")) + + segment_zero = list(set(segment_zero)) + row_zero = row_zero + segment_zero + + rows = [] + for item in distribution: + generated_row = [] + data = distribution.get(item) + for segment in segment_zero[1:]: + value = [x for x in data if x.get("segment") == segment] + if len(value): + generated_row.append(value[0].get(key)) + else: + generated_row.append("") + + rows.append(tuple(generated_row)) + + rows = [tuple(row_zero)] + rows + csv_buffer = io.StringIO() + writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) + + # Write CSV data to the buffer + for row in rows: + writer.writerow(row) + + subject = "Your Export is ready" + + html_content = render_to_string("emails/exports/analytics.html", {}) + + text_content = strip_tags(html_content) + + msg = EmailMultiAlternatives( + subject, text_content, settings.EMAIL_FROM, [email] + ) + msg.attach(f"{slug}-analytics.csv", csv_buffer.read()) + msg.send(fail_silently=False) + + else: + row_zero = [ + row_mapping.get(x_axis, "X-Axis"), + row_mapping.get(y_axis, "Y-Axis"), + ] + rows = [] + for item in distribution: + rows.append( + tuple( + [ + item, + distribution.get(item)[0].get("count") + if y_axis == "issue_count" + else distribution.get(item)[0].get("effort"), + ] + ) + ) + + rows = [tuple(row_zero)] + rows + csv_buffer = io.StringIO() + writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) + + # Write CSV data to the buffer + for row in rows: + writer.writerow(row) + + subject = "Your Export is ready" + + html_content = render_to_string("emails/exports/analytics.html", {}) + + text_content = strip_tags(html_content) + + msg = EmailMultiAlternatives( + subject, text_content, settings.EMAIL_FROM, [email] + ) + msg.attach(f"{slug}-analytics.csv", csv_buffer.read()) + msg.send(fail_silently=False) + + + except Exception as e: + print(e) + capture_exception(e) + return diff --git a/apiserver/plane/bgtasks/issue_activites_task.py b/apiserver/plane/bgtasks/issue_activites_task.py index c749d9c15..417fe2324 100644 --- a/apiserver/plane/bgtasks/issue_activites_task.py +++ b/apiserver/plane/bgtasks/issue_activites_task.py @@ -136,7 +136,6 @@ def track_priority( comment=f"{actor.email} updated the priority to {requested_data.get('priority')}", ) ) - print(issue_activities) # Track chnages in state of the issue diff --git a/apiserver/plane/db/models/__init__.py b/apiserver/plane/db/models/__init__.py index e32d768e0..53b501716 100644 --- a/apiserver/plane/db/models/__init__.py +++ b/apiserver/plane/db/models/__init__.py @@ -67,3 +67,5 @@ from .importer import Importer from .page import Page, PageBlock, PageFavorite, PageLabel from .estimate import Estimate, EstimatePoint + +from .analytic import AnalyticView \ No newline at end of file diff --git a/apiserver/plane/db/models/analytic.py b/apiserver/plane/db/models/analytic.py new file mode 100644 index 000000000..d097051af --- /dev/null +++ b/apiserver/plane/db/models/analytic.py @@ -0,0 +1,25 @@ +# Django models +from django.db import models +from django.conf import settings + +from .base import BaseModel + + +class AnalyticView(BaseModel): + workspace = models.ForeignKey( + "db.Workspace", related_name="analytics", on_delete=models.CASCADE + ) + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + query = models.JSONField() + query_dict = models.JSONField(default=dict) + + class Meta: + verbose_name = "Analytic" + verbose_name_plural = "Analytics" + db_table = "analytic_views" + ordering = ("-created_at",) + + def __str__(self): + """Return name of the analytic view""" + return f"{self.name} <{self.workspace.name}>" diff --git a/apiserver/plane/utils/analytics_plot.py b/apiserver/plane/utils/analytics_plot.py new file mode 100644 index 000000000..408b41d4b --- /dev/null +++ b/apiserver/plane/utils/analytics_plot.py @@ -0,0 +1,57 @@ +# Python imports +from itertools import groupby + +# Django import +from django.db import models +from django.db.models import Count, DateField, F, Sum, Value, Case, When, Q +from django.db.models.functions import Cast, Concat, Coalesce + + +def build_graph_plot(queryset, x_axis, y_axis, segment=None): + if x_axis in ["created_at", "completed_at"]: + queryset = queryset.annotate(dimension=Cast(x_axis, DateField())) + x_axis = "dimension" + else: + queryset = queryset.annotate(dimension=F(x_axis)) + x_axis = "dimension" + + if x_axis in ["created_at", "start_date", "target_date", "completed_at"]: + queryset = queryset.exclude(x_axis__is_null=True) + + queryset = queryset.values(x_axis) + + # Group queryset by x_axis field + + if y_axis == "issue_count": + queryset = ( + queryset.annotate( + is_null=Case( + When(dimension__isnull=True, then=Value("None")), + default=Value("not_null"), + output_field=models.CharField(max_length=8), + ), + dimension_ex=Coalesce("dimension", Value("null")), + ) + .values("dimension") + ) + if segment: + queryset = queryset.annotate(segment=F(segment)).values("dimension", "segment") + else: + queryset = queryset.values("dimension") + + queryset = queryset.annotate(count=Count("*")).order_by("dimension") + + + if y_axis == "effort": + queryset = queryset.annotate(effort=Sum("estimate_point")).order_by(x_axis) + if segment: + queryset = queryset.annotate(segment=F(segment)).values("dimension", "segment", "effort") + else: + queryset = queryset.values("dimension", "effort") + + result_values = list(queryset) + grouped_data = {} + for date, items in groupby(result_values, key=lambda x: x[str("dimension")]): + grouped_data[str(date)] = list(items) + + return grouped_data diff --git a/apiserver/plane/utils/issue_filters.py b/apiserver/plane/utils/issue_filters.py index 8b62da722..c9328ecf1 100644 --- a/apiserver/plane/utils/issue_filters.py +++ b/apiserver/plane/utils/issue_filters.py @@ -198,6 +198,39 @@ def filter_issue_state_type(params, filter, method): return filter +def filter_project(params, filter, method): + if method == "GET": + projects = params.get("project").split(",") + if len(projects) and "" not in projects: + filter["project__in"] = projects + else: + if params.get("project", None) and len(params.get("project")): + filter["project__in"] = params.get("project") + return filter + + +def filter_cycle(params, filter, method): + if method == "GET": + cycles = params.get("cycle").split(",") + if len(cycles) and "" not in cycles: + filter["cycle__in"] = cycles + else: + if params.get("cycle", None) and len(params.get("cycle")): + filter["issue_cycle__cycle_id__in"] = params.get("cycle") + return filter + + +def filter_module(params, filter, method): + if method == "GET": + modules = params.get("module").split(",") + if len(modules) and "" not in modules: + filter["module__in"] = modules + else: + if params.get("module", None) and len(params.get("module")): + filter["issue_module__module_id__in"] = params.get("module") + return filter + + def issue_filters(query_params, method): filter = dict() @@ -216,6 +249,9 @@ def issue_filters(query_params, method): "target_date": filter_target_date, "completed_at": filter_completed_at, "type": filter_issue_state_type, + "project": filter_project, + "cycle": filter_cycle, + "module": filter_module, } for key, value in ISSUE_FILTER.items(): diff --git a/apiserver/templates/emails/exports/analytics.html b/apiserver/templates/emails/exports/analytics.html new file mode 100644 index 000000000..248c5513c --- /dev/null +++ b/apiserver/templates/emails/exports/analytics.html @@ -0,0 +1,4 @@ + + + Your Export is ready +