From ed8782757d2ee70779e36d18d2cfecbbd3cd4b69 Mon Sep 17 00:00:00 2001 From: Nikhil <118773738+pablohashescobar@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:39:50 +0530 Subject: [PATCH] [WEB - 471] dev: caching users and workspace apis (#3707) * dev: caching users and workspace apis * dev: cache user and config apis * dev: update caching function to use user_id instead of token * dev: update caching layer * dev: update caching logic * dev: format caching file * dev: refactor caching to include name space and user id as key * dev: cache project cover image endpoint --- apiserver/plane/app/views/config.py | 4 +- apiserver/plane/app/views/estimate.py | 5 +- apiserver/plane/app/views/issue.py | 39 +++++---- apiserver/plane/app/views/project.py | 4 +- apiserver/plane/app/views/state.py | 8 +- apiserver/plane/app/views/user.py | 21 ++++- apiserver/plane/app/views/workspace.py | 41 +++++++-- apiserver/plane/license/api/views/instance.py | 20 +++-- apiserver/plane/settings/local.py | 7 +- apiserver/plane/utils/cache.py | 84 +++++++++++++++++++ 10 files changed, 189 insertions(+), 44 deletions(-) create mode 100644 apiserver/plane/utils/cache.py diff --git a/apiserver/plane/app/views/config.py b/apiserver/plane/app/views/config.py index b2a27252c..354f0aebc 100644 --- a/apiserver/plane/app/views/config.py +++ b/apiserver/plane/app/views/config.py @@ -12,13 +12,14 @@ from rest_framework.response import Response # Module imports from .base import BaseAPIView from plane.license.utils.instance_value import get_configuration_value - +from plane.utils.cache import cache_response class ConfigurationEndpoint(BaseAPIView): permission_classes = [ AllowAny, ] + @cache_response(60 * 60 * 2, user=False) def get(self, request): # Get all the configuration ( @@ -136,6 +137,7 @@ class MobileConfigurationEndpoint(BaseAPIView): AllowAny, ] + @cache_response(60 * 60 * 2, user=False) def get(self, request): ( GOOGLE_CLIENT_ID, diff --git a/apiserver/plane/app/views/estimate.py b/apiserver/plane/app/views/estimate.py index 3402bb068..eae2e3351 100644 --- a/apiserver/plane/app/views/estimate.py +++ b/apiserver/plane/app/views/estimate.py @@ -11,7 +11,7 @@ from plane.app.serializers import ( EstimatePointSerializer, EstimateReadSerializer, ) - +from plane.utils.cache import invalidate_cache class ProjectEstimatePointEndpoint(BaseAPIView): permission_classes = [ @@ -49,6 +49,7 @@ class BulkEstimatePointEndpoint(BaseViewSet): serializer = EstimateReadSerializer(estimates, many=True) return Response(serializer.data, status=status.HTTP_200_OK) + @invalidate_cache(path="/api/workspaces/:slug/estimates/", url_params=True, user=False) def create(self, request, slug, project_id): if not request.data.get("estimate", False): return Response( @@ -114,6 +115,7 @@ class BulkEstimatePointEndpoint(BaseViewSet): status=status.HTTP_200_OK, ) + @invalidate_cache(path="/api/workspaces/:slug/estimates/", url_params=True, user=False) def partial_update(self, request, slug, project_id, estimate_id): if not request.data.get("estimate", False): return Response( @@ -182,6 +184,7 @@ class BulkEstimatePointEndpoint(BaseViewSet): status=status.HTTP_200_OK, ) + @invalidate_cache(path="/api/workspaces/:slug/estimates/", url_params=True, user=False) def destroy(self, request, slug, project_id, estimate_id): estimate = Estimate.objects.get( pk=estimate_id, workspace__slug=slug, project_id=project_id diff --git a/apiserver/plane/app/views/issue.py b/apiserver/plane/app/views/issue.py index 14e0b6a9a..4355f0ab5 100644 --- a/apiserver/plane/app/views/issue.py +++ b/apiserver/plane/app/views/issue.py @@ -78,6 +78,7 @@ from plane.bgtasks.issue_activites_task import issue_activity from plane.utils.grouper import group_results from plane.utils.issue_filters import issue_filters from collections import defaultdict +from plane.utils.cache import invalidate_cache class IssueListEndpoint(BaseAPIView): @@ -1001,6 +1002,21 @@ class LabelViewSet(BaseViewSet): ProjectMemberPermission, ] + def get_queryset(self): + return self.filter_queryset( + super() + .get_queryset() + .filter(workspace__slug=self.kwargs.get("slug")) + .filter(project_id=self.kwargs.get("project_id")) + .filter(project__project_projectmember__member=self.request.user) + .select_related("project") + .select_related("workspace") + .select_related("parent") + .distinct() + .order_by("sort_order") + ) + + @invalidate_cache(path="/api/workspaces/:slug/labels/", url_params=True, user=False) def create(self, request, slug, project_id): try: serializer = LabelSerializer(data=request.data) @@ -1020,22 +1036,13 @@ class LabelViewSet(BaseViewSet): status=status.HTTP_400_BAD_REQUEST, ) - def get_queryset(self): - return self.filter_queryset( - super() - .get_queryset() - .filter(workspace__slug=self.kwargs.get("slug")) - .filter(project_id=self.kwargs.get("project_id")) - .filter( - project__project_projectmember__member=self.request.user, - project__project_projectmember__is_active=True, - ) - .select_related("project") - .select_related("workspace") - .select_related("parent") - .distinct() - .order_by("sort_order") - ) + @invalidate_cache(path="/api/workspaces/:slug/labels/", url_params=True, user=False) + def partial_update(self, request, *args, **kwargs): + return super().partial_update(request, *args, **kwargs) + + @invalidate_cache(path="/api/workspaces/:slug/labels/", url_params=True, user=False) + def destroy(self, request, *args, **kwargs): + return super().destroy(request, *args, **kwargs) class BulkDeleteIssuesEndpoint(BaseAPIView): diff --git a/apiserver/plane/app/views/project.py b/apiserver/plane/app/views/project.py index 6f9b2618e..42b9c1f37 100644 --- a/apiserver/plane/app/views/project.py +++ b/apiserver/plane/app/views/project.py @@ -65,7 +65,7 @@ from plane.db.models import ( ) from plane.bgtasks.project_invitation_task import project_invitation - +from plane.utils.cache import cache_response class ProjectViewSet(WebhookMixin, BaseViewSet): serializer_class = ProjectListSerializer @@ -1045,6 +1045,8 @@ class ProjectPublicCoverImagesEndpoint(BaseAPIView): AllowAny, ] + # Cache the below api for 24 hours + @cache_response(60 * 60 * 24, user=False) def get(self, request): files = [] s3 = boto3.client( diff --git a/apiserver/plane/app/views/state.py b/apiserver/plane/app/views/state.py index 34b3d1dcc..6d4fd7782 100644 --- a/apiserver/plane/app/views/state.py +++ b/apiserver/plane/app/views/state.py @@ -9,14 +9,13 @@ from rest_framework.response import Response from rest_framework import status # Module imports -from . import BaseViewSet, BaseAPIView +from . import BaseViewSet from plane.app.serializers import StateSerializer from plane.app.permissions import ( ProjectEntityPermission, - WorkspaceEntityPermission, ) from plane.db.models import State, Issue - +from plane.utils.cache import invalidate_cache class StateViewSet(BaseViewSet): serializer_class = StateSerializer @@ -41,6 +40,7 @@ class StateViewSet(BaseViewSet): .distinct() ) + @invalidate_cache(path="workspaces/:slug/states/", url_params=True, user=False) def create(self, request, slug, project_id): serializer = StateSerializer(data=request.data) if serializer.is_valid(): @@ -61,6 +61,7 @@ class StateViewSet(BaseViewSet): return Response(state_dict, status=status.HTTP_200_OK) return Response(states, status=status.HTTP_200_OK) + @invalidate_cache(path="workspaces/:slug/states/", url_params=True, user=False) def mark_as_default(self, request, slug, project_id, pk): # Select all the states which are marked as default _ = State.objects.filter( @@ -71,6 +72,7 @@ class StateViewSet(BaseViewSet): ).update(default=True) return Response(status=status.HTTP_204_NO_CONTENT) + @invalidate_cache(path="workspaces/:slug/states/", url_params=True, user=False) def destroy(self, request, slug, project_id, pk): state = State.objects.get( ~Q(name="Triage"), diff --git a/apiserver/plane/app/views/user.py b/apiserver/plane/app/views/user.py index 7764e3b97..07049b8d5 100644 --- a/apiserver/plane/app/views/user.py +++ b/apiserver/plane/app/views/user.py @@ -1,8 +1,10 @@ +# Django imports +from django.db.models import Q, Count, Case, When, IntegerField + # Third party imports from rest_framework.response import Response from rest_framework import status - # Module imports from plane.app.serializers import ( UserSerializer, @@ -15,9 +17,7 @@ from plane.app.views.base import BaseViewSet, BaseAPIView from plane.db.models import User, IssueActivity, WorkspaceMember, ProjectMember from plane.license.models import Instance, InstanceAdmin from plane.utils.paginator import BasePaginator - - -from django.db.models import Q, F, Count, Case, When, IntegerField +from plane.utils.cache import cache_response, invalidate_cache class UserEndpoint(BaseViewSet): @@ -27,6 +27,7 @@ class UserEndpoint(BaseViewSet): def get_object(self): return self.request.user + @cache_response(60 * 60) def retrieve(self, request): serialized_data = UserMeSerializer(request.user).data return Response( @@ -34,10 +35,12 @@ class UserEndpoint(BaseViewSet): status=status.HTTP_200_OK, ) + @cache_response(60 * 60) def retrieve_user_settings(self, request): serialized_data = UserMeSettingsSerializer(request.user).data return Response(serialized_data, status=status.HTTP_200_OK) + @cache_response(60 * 60) def retrieve_instance_admin(self, request): instance = Instance.objects.first() is_admin = InstanceAdmin.objects.filter( @@ -47,6 +50,11 @@ class UserEndpoint(BaseViewSet): {"is_instance_admin": is_admin}, status=status.HTTP_200_OK ) + @invalidate_cache(path="/api/users/me/") + def partial_update(self, request, *args, **kwargs): + return super().partial_update(request, *args, **kwargs) + + @invalidate_cache(path="/api/users/me/") def deactivate(self, request): # Check all workspace user is active user = self.get_object() @@ -145,6 +153,8 @@ class UserEndpoint(BaseViewSet): class UpdateUserOnBoardedEndpoint(BaseAPIView): + + @invalidate_cache(path="/api/users/me/") def patch(self, request): user = User.objects.get(pk=request.user.id, is_active=True) user.is_onboarded = request.data.get("is_onboarded", False) @@ -155,6 +165,8 @@ class UpdateUserOnBoardedEndpoint(BaseAPIView): class UpdateUserTourCompletedEndpoint(BaseAPIView): + + @invalidate_cache(path="/api/users/me/") def patch(self, request): user = User.objects.get(pk=request.user.id, is_active=True) user.is_tour_completed = request.data.get("is_tour_completed", False) @@ -165,6 +177,7 @@ class UpdateUserTourCompletedEndpoint(BaseAPIView): class UserActivityEndpoint(BaseAPIView, BasePaginator): + def get(self, request): queryset = IssueActivity.objects.filter( actor=request.user diff --git a/apiserver/plane/app/views/workspace.py b/apiserver/plane/app/views/workspace.py index 7c4a5db8d..34765c3c7 100644 --- a/apiserver/plane/app/views/workspace.py +++ b/apiserver/plane/app/views/workspace.py @@ -57,6 +57,8 @@ from plane.app.serializers import ( WorkspaceEstimateSerializer, StateSerializer, LabelSerializer, + CycleSerializer, + ModuleSerializer, ) from plane.app.views.base import BaseAPIView from . import BaseViewSet @@ -77,7 +79,6 @@ from plane.db.models import ( Label, WorkspaceMember, CycleIssue, - IssueReaction, WorkspaceUserProperties, Estimate, EstimatePoint, @@ -91,17 +92,11 @@ from plane.app.permissions import ( WorkspaceEntityPermission, WorkspaceViewerPermission, WorkspaceUserPermission, - ProjectLitePermission, ) from plane.bgtasks.workspace_invitation_task import workspace_invitation from plane.utils.issue_filters import issue_filters from plane.bgtasks.event_tracking_task import workspace_invite_event -from plane.app.serializers.module import ( - ModuleSerializer, -) -from plane.app.serializers.cycle import ( - CycleSerializer, -) +from plane.utils.cache import cache_response, invalidate_cache class WorkSpaceViewSet(BaseViewSet): @@ -151,7 +146,8 @@ class WorkSpaceViewSet(BaseViewSet): .annotate(total_issues=issue_count) .select_related("owner") ) - + @invalidate_cache(path="/api/workspaces/", user=False) + @invalidate_cache(path="/api/users/me/workspaces/") def create(self, request): try: serializer = WorkSpaceSerializer(data=request.data) @@ -197,6 +193,20 @@ class WorkSpaceViewSet(BaseViewSet): status=status.HTTP_410_GONE, ) + @cache_response(60 * 60 * 2) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + @invalidate_cache(path="/api/workspaces/", user=False) + @invalidate_cache(path="/api/users/me/workspaces/") + def partial_update(self, request, *args, **kwargs): + return super().partial_update(request, *args, **kwargs) + + @invalidate_cache(path="/api/workspaces/", user=False) + @invalidate_cache(path="/api/users/me/workspaces/") + def destroy(self, request, *args, **kwargs): + return super().destroy(request, *args, **kwargs) + class UserWorkSpacesEndpoint(BaseAPIView): search_fields = [ @@ -206,6 +216,7 @@ class UserWorkSpacesEndpoint(BaseAPIView): "owner", ] + @cache_response(60 * 60 * 2) def get(self, request): fields = [ field @@ -403,6 +414,8 @@ class WorkspaceJoinEndpoint(BaseAPIView): ] """Invitation response endpoint the user can respond to the invitation""" + @invalidate_cache(path="/api/workspaces/", user=False) + @invalidate_cache(path="/api/users/me/workspaces/") def post(self, request, slug, pk): workspace_invite = WorkspaceMemberInvite.objects.get( pk=pk, workspace__slug=slug @@ -499,6 +512,9 @@ class UserWorkspaceInvitationsViewSet(BaseViewSet): .annotate(total_members=Count("workspace__workspace_member")) ) + @invalidate_cache(path="/api/workspaces/", user=False) + @invalidate_cache(path="/api/users/me/workspaces/") + @invalidate_cache(path="/api/workspaces/:slug/members/", url_params=True, user=False) def create(self, request): invitations = request.data.get("invitations", []) workspace_invitations = WorkspaceMemberInvite.objects.filter( @@ -569,6 +585,7 @@ class WorkSpaceMemberViewSet(BaseViewSet): .select_related("member") ) + @cache_response(60 * 60 * 2) def list(self, request, slug): workspace_member = WorkspaceMember.objects.get( member=request.user, @@ -593,6 +610,7 @@ class WorkSpaceMemberViewSet(BaseViewSet): ) return Response(serializer.data, status=status.HTTP_200_OK) + @invalidate_cache(path="/api/workspaces/:slug/members/", url_params=True, user=False) def partial_update(self, request, slug, pk): workspace_member = WorkspaceMember.objects.get( pk=pk, @@ -635,6 +653,7 @@ class WorkSpaceMemberViewSet(BaseViewSet): return Response(serializer.data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + @invalidate_cache(path="/api/workspaces/:slug/members/", url_params=True, user=False) def destroy(self, request, slug, pk): # Check the user role who is deleting the user workspace_member = WorkspaceMember.objects.get( @@ -699,6 +718,7 @@ class WorkSpaceMemberViewSet(BaseViewSet): workspace_member.save() return Response(status=status.HTTP_204_NO_CONTENT) + @invalidate_cache(path="/api/workspaces/:slug/members/", url_params=True, user=False) def leave(self, request, slug): workspace_member = WorkspaceMember.objects.get( workspace__slug=slug, @@ -1550,6 +1570,7 @@ class WorkspaceLabelsEndpoint(BaseAPIView): WorkspaceViewerPermission, ] + @cache_response(60 * 60 * 2) def get(self, request, slug): labels = Label.objects.filter( workspace__slug=slug, @@ -1565,6 +1586,7 @@ class WorkspaceStatesEndpoint(BaseAPIView): WorkspaceEntityPermission, ] + @cache_response(60 * 60 * 2) def get(self, request, slug): states = State.objects.filter( workspace__slug=slug, @@ -1580,6 +1602,7 @@ class WorkspaceEstimatesEndpoint(BaseAPIView): WorkspaceEntityPermission, ] + @cache_response(60 * 60 * 2) def get(self, request, slug): estimate_ids = Project.objects.filter( workspace__slug=slug, estimate__isnull=False diff --git a/apiserver/plane/license/api/views/instance.py b/apiserver/plane/license/api/views/instance.py index 112c68bc8..c8608cbe5 100644 --- a/apiserver/plane/license/api/views/instance.py +++ b/apiserver/plane/license/api/views/instance.py @@ -1,17 +1,11 @@ # Python imports -import json -import os -import requests import uuid -import random -import string # Django imports from django.utils import timezone from django.contrib.auth.hashers import make_password from django.core.validators import validate_email from django.core.exceptions import ValidationError -from django.conf import settings # Third party imports from rest_framework import status @@ -30,9 +24,9 @@ from plane.license.api.serializers import ( from plane.license.api.permissions import ( InstanceAdminPermission, ) -from plane.db.models import User, WorkspaceMember, ProjectMember +from plane.db.models import User from plane.license.utils.encryption import encrypt_data - +from plane.utils.cache import cache_response, invalidate_cache class InstanceEndpoint(BaseAPIView): def get_permissions(self): @@ -44,6 +38,7 @@ class InstanceEndpoint(BaseAPIView): AllowAny(), ] + @cache_response(60 * 60 * 2, user=False) def get(self, request): instance = Instance.objects.first() # get the instance @@ -58,6 +53,7 @@ class InstanceEndpoint(BaseAPIView): data["is_activated"] = True return Response(data, status=status.HTTP_200_OK) + @invalidate_cache(path="/api/instances/", user=False) def patch(self, request): # Get the instance instance = Instance.objects.first() @@ -75,6 +71,7 @@ class InstanceAdminEndpoint(BaseAPIView): InstanceAdminPermission, ] + @invalidate_cache(path="/api/instances/", user=False) # Create an instance admin def post(self, request): email = request.data.get("email", False) @@ -104,6 +101,7 @@ class InstanceAdminEndpoint(BaseAPIView): serializer = InstanceAdminSerializer(instance_admin) return Response(serializer.data, status=status.HTTP_201_CREATED) + @cache_response(60 * 60 * 2) def get(self, request): instance = Instance.objects.first() if instance is None: @@ -115,6 +113,7 @@ class InstanceAdminEndpoint(BaseAPIView): serializer = InstanceAdminSerializer(instance_admins, many=True) return Response(serializer.data, status=status.HTTP_200_OK) + @invalidate_cache(path="/api/instances/", user=False) def delete(self, request, pk): instance = Instance.objects.first() instance_admin = InstanceAdmin.objects.filter( @@ -128,6 +127,7 @@ class InstanceConfigurationEndpoint(BaseAPIView): InstanceAdminPermission, ] + @cache_response(60 * 60 * 2, user=False) def get(self, request): instance_configurations = InstanceConfiguration.objects.all() serializer = InstanceConfigurationSerializer( @@ -135,6 +135,8 @@ class InstanceConfigurationEndpoint(BaseAPIView): ) return Response(serializer.data, status=status.HTTP_200_OK) + @invalidate_cache(path="/api/configs/", user=False) + @invalidate_cache(path="/api/mobile-configs/", user=False) def patch(self, request): configurations = InstanceConfiguration.objects.filter( key__in=request.data.keys() @@ -170,6 +172,7 @@ class InstanceAdminSignInEndpoint(BaseAPIView): AllowAny, ] + @invalidate_cache(path="/api/instances/", user=False) def post(self, request): # Check instance first instance = Instance.objects.first() @@ -260,6 +263,7 @@ class SignUpScreenVisitedEndpoint(BaseAPIView): AllowAny, ] + @invalidate_cache(path="/api/instances/", user=False) def post(self, request): instance = Instance.objects.first() if instance is None: diff --git a/apiserver/plane/settings/local.py b/apiserver/plane/settings/local.py index 8f27d4234..4dc998e55 100644 --- a/apiserver/plane/settings/local.py +++ b/apiserver/plane/settings/local.py @@ -1,4 +1,5 @@ """Development settings""" + from .common import * # noqa DEBUG = True @@ -14,7 +15,11 @@ EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" CACHES = { "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": REDIS_URL, + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, } } diff --git a/apiserver/plane/utils/cache.py b/apiserver/plane/utils/cache.py new file mode 100644 index 000000000..dba89c4a6 --- /dev/null +++ b/apiserver/plane/utils/cache.py @@ -0,0 +1,84 @@ +from django.core.cache import cache +# from django.utils.encoding import force_bytes +# import hashlib +from functools import wraps +from rest_framework.response import Response + + +def generate_cache_key(custom_path, auth_header=None): + """Generate a cache key with the given params""" + if auth_header: + key_data = f"{custom_path}:{auth_header}" + else: + key_data = custom_path + return key_data + + +def cache_response(timeout=60 * 60, path=None, user=True): + """decorator to create cache per user""" + + def decorator(view_func): + @wraps(view_func) + def _wrapped_view(instance, request, *args, **kwargs): + # Function to generate cache key + auth_header = ( + None if request.user.is_anonymous else str(request.user.id) if user else None + ) + custom_path = path if path is not None else request.get_full_path() + key = generate_cache_key(custom_path, auth_header) + cached_result = cache.get(key) + if cached_result is not None: + print("Cache Hit") + return Response( + cached_result["data"], status=cached_result["status"] + ) + + print("Cache Miss") + response = view_func(instance, request, *args, **kwargs) + + if response.status_code == 200: + cache.set( + key, + {"data": response.data, "status": response.status_code}, + timeout, + ) + + return response + + return _wrapped_view + + return decorator + + +def invalidate_cache(path=None, url_params=False, user=True): + """invalidate cache per user""" + + def decorator(view_func): + @wraps(view_func) + def _wrapped_view(instance, request, *args, **kwargs): + # Invalidate cache before executing the view function + if url_params: + path_with_values = path + for key, value in kwargs.items(): + path_with_values = path_with_values.replace( + f":{key}", str(value) + ) + + custom_path = path_with_values + else: + custom_path = ( + path if path is not None else request.get_full_path() + ) + + auth_header = ( + None if request.user.is_anonymous else str(request.user.id) if user else None + ) + key = generate_cache_key(custom_path, auth_header) + cache.delete(key) + print("Invalidating cache") + # Execute the view function + return view_func(instance, request, *args, **kwargs) + + return _wrapped_view + + return decorator