From 82ba9833f2ac2e213cf6b7af5188d745fe6f3593 Mon Sep 17 00:00:00 2001 From: Nikhil <118773738+pablohashescobar@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:27:02 +0530 Subject: [PATCH] dev: enable api logging (#2538) * dev: enable api logging and control worker count through env * dev: enable logger instead of printing * dev: remove worker counts * dev: enable global level log settings * dev: add rotating logger * fix: logging configuration * dev: api logging and moving the capture exception to utils for logging and then capturing * fix: information leaking through print logs * dev: linting fix * dev: logging configuration for django * fix: linting errors * dev: add logs for migrator * dev: logging cofiguration * dev: add permision for captain user in Plane * dev: add log paths in compose * dev: create directory for logs * dev: fix linting errors --- .gitignore | 1 + apiserver/.env.example | 1 - apiserver/Dockerfile.api | 2 + apiserver/plane/api/views/base.py | 26 +++--- apiserver/plane/app/views/base.py | 28 +++---- apiserver/plane/app/views/cycle/base.py | 8 -- apiserver/plane/app/views/dashboard/base.py | 1 - apiserver/plane/app/views/issue/base.py | 30 +------ apiserver/plane/app/views/issue/draft.py | 81 +++++++++---------- apiserver/plane/app/views/project/base.py | 6 +- .../plane/bgtasks/analytic_plot_export.py | 38 +++++---- .../plane/bgtasks/email_notification_task.py | 30 ++++--- .../plane/bgtasks/event_tracking_task.py | 10 ++- apiserver/plane/bgtasks/export_task.py | 20 +++-- .../plane/bgtasks/forgot_password_task.py | 18 ++--- .../plane/bgtasks/issue_activites_task.py | 37 +++++---- .../plane/bgtasks/issue_automation_task.py | 21 ++--- .../plane/bgtasks/magic_link_code_task.py | 17 ++-- .../plane/bgtasks/project_invitation_task.py | 20 +++-- apiserver/plane/bgtasks/webhook_task.py | 71 ++++++++-------- .../bgtasks/workspace_invitation_task.py | 22 +++-- apiserver/plane/db/models/user.py | 15 ++-- apiserver/plane/settings/common.py | 11 +-- apiserver/plane/settings/local.py | 43 +++++++++- apiserver/plane/settings/production.py | 64 ++++++++++++++- apiserver/plane/settings/test.py | 2 +- apiserver/plane/space/views/base.py | 27 +++---- apiserver/plane/utils/exception_logger.py | 15 ++++ apiserver/plane/utils/logging.py | 46 +++++++++++ apiserver/requirements/base.txt | 1 + deploy/selfhost/docker-compose.yml | 12 +++ 31 files changed, 421 insertions(+), 303 deletions(-) create mode 100644 apiserver/plane/utils/exception_logger.py create mode 100644 apiserver/plane/utils/logging.py diff --git a/.gitignore b/.gitignore index 0b655bd0e..3989f4356 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ staticfiles mediafiles .env .DS_Store +logs/ node_modules/ assets/dist/ diff --git a/apiserver/.env.example b/apiserver/.env.example index 97dc4dda8..d8554f400 100644 --- a/apiserver/.env.example +++ b/apiserver/.env.example @@ -44,4 +44,3 @@ WEB_URL="http://localhost" # Gunicorn Workers GUNICORN_WORKERS=2 - diff --git a/apiserver/Dockerfile.api b/apiserver/Dockerfile.api index 0e4e0ac50..c5113e059 100644 --- a/apiserver/Dockerfile.api +++ b/apiserver/Dockerfile.api @@ -48,8 +48,10 @@ USER root RUN apk --no-cache add "bash~=5.2" COPY ./bin ./bin/ +RUN mkdir /code/plane/logs RUN chmod +x ./bin/takeoff ./bin/worker ./bin/beat RUN chmod -R 777 /code +RUN chown -R captain:plane /code USER captain diff --git a/apiserver/plane/api/views/base.py b/apiserver/plane/api/views/base.py index 146f61f48..0cf5e8731 100644 --- a/apiserver/plane/api/views/base.py +++ b/apiserver/plane/api/views/base.py @@ -1,26 +1,26 @@ # Python imports -import zoneinfo from urllib.parse import urlparse +import zoneinfo # Django imports from django.conf import settings -from django.db import IntegrityError from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.db import IntegrityError from django.utils import timezone +from rest_framework import status +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response # Third party imports from rest_framework.views import APIView -from rest_framework.response import Response -from rest_framework.permissions import IsAuthenticated -from rest_framework import status -from sentry_sdk import capture_exception # Module imports from plane.api.middleware.api_authentication import APIKeyAuthentication from plane.api.rate_limit import ApiKeyRateThrottle -from plane.utils.paginator import BasePaginator from plane.bgtasks.webhook_task import send_webhook +from plane.utils.exception_logger import log_exception +from plane.utils.paginator import BasePaginator class TimezoneMixin: @@ -106,27 +106,23 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator): if isinstance(e, ValidationError): return Response( - { - "error": "The provided payload is not valid please try with a valid payload" - }, + {"error": "Please provide valid detail"}, status=status.HTTP_400_BAD_REQUEST, ) if isinstance(e, ObjectDoesNotExist): return Response( - {"error": "The required object does not exist."}, + {"error": "The requested resource does not exist."}, status=status.HTTP_404_NOT_FOUND, ) if isinstance(e, KeyError): return Response( - {"error": " The required key does not exist."}, + {"error": "The required key does not exist."}, status=status.HTTP_400_BAD_REQUEST, ) - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return Response( {"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/apiserver/plane/app/views/base.py b/apiserver/plane/app/views/base.py index cdba62350..1908cfdc9 100644 --- a/apiserver/plane/app/views/base.py +++ b/apiserver/plane/app/views/base.py @@ -1,27 +1,27 @@ # Python imports import zoneinfo +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.db import IntegrityError # Django imports from django.urls import resolve -from django.conf import settings from django.utils import timezone -from django.db import IntegrityError -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django_filters.rest_framework import DjangoFilterBackend # Third part imports from rest_framework import status -from rest_framework.viewsets import ModelViewSet -from rest_framework.response import Response from rest_framework.exceptions import APIException -from rest_framework.views import APIView from rest_framework.filters import SearchFilter from rest_framework.permissions import IsAuthenticated -from sentry_sdk import capture_exception -from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.viewsets import ModelViewSet # Module imports -from plane.utils.paginator import BasePaginator from plane.bgtasks.webhook_task import send_webhook +from plane.utils.exception_logger import log_exception +from plane.utils.paginator import BasePaginator class TimezoneMixin: @@ -87,7 +87,7 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): try: return self.model.objects.all() except Exception as e: - capture_exception(e) + log_exception(e) raise APIException( "Please check the view", status.HTTP_400_BAD_REQUEST ) @@ -121,13 +121,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): ) if isinstance(e, KeyError): - capture_exception(e) + log_exception(e) return Response( {"error": "The required key does not exist."}, status=status.HTTP_400_BAD_REQUEST, ) - capture_exception(e) + log_exception(e) return Response( {"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -233,9 +233,7 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator): status=status.HTTP_400_BAD_REQUEST, ) - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return Response( {"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/apiserver/plane/app/views/cycle/base.py b/apiserver/plane/app/views/cycle/base.py index e777a93a6..5a57ebef2 100644 --- a/apiserver/plane/app/views/cycle/base.py +++ b/apiserver/plane/app/views/cycle/base.py @@ -15,10 +15,7 @@ from django.db.models import ( Value, CharField, ) -from django.core import serializers from django.utils import timezone -from django.utils.decorators import method_decorator -from django.views.decorators.gzip import gzip_page from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField from django.db.models import UUIDField @@ -32,9 +29,7 @@ from rest_framework import status from .. import BaseViewSet, BaseAPIView, WebhookMixin from plane.app.serializers import ( CycleSerializer, - CycleIssueSerializer, CycleFavoriteSerializer, - IssueSerializer, CycleWriteSerializer, CycleUserPropertiesSerializer, ) @@ -48,13 +43,10 @@ from plane.db.models import ( CycleIssue, Issue, CycleFavorite, - IssueLink, - IssueAttachment, Label, CycleUserProperties, ) from plane.bgtasks.issue_activites_task import issue_activity -from plane.utils.issue_filters import issue_filters from plane.utils.analytics_plot import burndown_plot diff --git a/apiserver/plane/app/views/dashboard/base.py b/apiserver/plane/app/views/dashboard/base.py index 27e45f59c..e6757faf9 100644 --- a/apiserver/plane/app/views/dashboard/base.py +++ b/apiserver/plane/app/views/dashboard/base.py @@ -38,7 +38,6 @@ from plane.db.models import ( IssueLink, IssueAttachment, IssueRelation, - IssueAssignee, User, ) from plane.app.serializers import ( diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 63d4358b0..f1b4c7627 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -1,7 +1,5 @@ # Python imports import json -import random -from itertools import chain # Django imports from django.utils import timezone @@ -21,64 +19,38 @@ from django.db.models import ( from django.core.serializers.json import DjangoJSONEncoder from django.utils.decorators import method_decorator from django.views.decorators.gzip import gzip_page -from django.db import IntegrityError from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField -from django.db.models import Value, UUIDField +from django.db.models import UUIDField from django.db.models.functions import Coalesce # Third Party imports from rest_framework.response import Response from rest_framework import status -from rest_framework.parsers import MultiPartParser, FormParser # Module imports from .. import BaseViewSet, BaseAPIView, WebhookMixin from plane.app.serializers import ( - IssueActivitySerializer, - IssueCommentSerializer, IssuePropertySerializer, IssueSerializer, IssueCreateSerializer, - LabelSerializer, - IssueFlatSerializer, - IssueLinkSerializer, - IssueLiteSerializer, - IssueAttachmentSerializer, - IssueSubscriberSerializer, - ProjectMemberLiteSerializer, - IssueReactionSerializer, - CommentReactionSerializer, - IssueRelationSerializer, - RelatedIssueSerializer, IssueDetailSerializer, ) from plane.app.permissions import ( ProjectEntityPermission, - WorkSpaceAdminPermission, - ProjectMemberPermission, ProjectLitePermission, ) from plane.db.models import ( Project, Issue, - IssueActivity, - IssueComment, IssueProperty, - Label, IssueLink, IssueAttachment, IssueSubscriber, - ProjectMember, IssueReaction, - CommentReaction, - IssueRelation, ) from plane.bgtasks.issue_activites_task import issue_activity -from plane.utils.grouper import group_results from plane.utils.issue_filters import issue_filters -from collections import defaultdict -from plane.utils.cache import invalidate_cache class IssueListEndpoint(BaseAPIView): diff --git a/apiserver/plane/app/views/issue/draft.py b/apiserver/plane/app/views/issue/draft.py index 08032934b..db6b5b9fb 100644 --- a/apiserver/plane/app/views/issue/draft.py +++ b/apiserver/plane/app/views/issue/draft.py @@ -1,52 +1,54 @@ # Python imports import json -# Django imports -from django.utils import timezone -from django.db.models import ( - Prefetch, - OuterRef, - Func, - F, - Q, - Case, - Value, - CharField, - When, - Exists, - Max, - UUIDField, -) -from django.core.serializers.json import DjangoJSONEncoder -from django.utils.decorators import method_decorator -from django.views.decorators.gzip import gzip_page from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField +from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import ( + Case, + CharField, + Exists, + F, + Func, + Max, + OuterRef, + Prefetch, + Q, + UUIDField, + Value, + When, +) from django.db.models.functions import Coalesce +# Django imports +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views.decorators.gzip import gzip_page +from rest_framework import status + # Third Party imports from rest_framework.response import Response -from rest_framework import status + +from plane.app.permissions import ProjectEntityPermission +from plane.app.serializers import ( + IssueCreateSerializer, + IssueDetailSerializer, + IssueFlatSerializer, + IssueSerializer, +) +from plane.bgtasks.issue_activites_task import issue_activity +from plane.db.models import ( + Issue, + IssueAttachment, + IssueLink, + IssueReaction, + IssueSubscriber, + Project, +) +from plane.utils.issue_filters import issue_filters # Module imports from .. import BaseViewSet -from plane.app.serializers import ( - IssueSerializer, - IssueCreateSerializer, - IssueFlatSerializer, - IssueDetailSerializer, -) -from plane.app.permissions import ProjectEntityPermission -from plane.db.models import ( - Project, - Issue, - IssueLink, - IssueAttachment, - IssueSubscriber, - IssueReaction, -) -from plane.bgtasks.issue_activites_task import issue_activity -from plane.utils.issue_filters import issue_filters class IssueDraftViewSet(BaseViewSet): @@ -117,11 +119,6 @@ class IssueDraftViewSet(BaseViewSet): @method_decorator(gzip_page) def list(self, request, slug, project_id): filters = issue_filters(request.query_params, "GET") - fields = [ - field - for field in request.GET.get("fields", "").split(",") - if field - ] # Custom ordering for priority and state priority_order = ["urgent", "high", "medium", "low", "none"] diff --git a/apiserver/plane/app/views/project/base.py b/apiserver/plane/app/views/project/base.py index 74d4e3466..5dbc99a27 100644 --- a/apiserver/plane/app/views/project/base.py +++ b/apiserver/plane/app/views/project/base.py @@ -346,12 +346,12 @@ class ProjectViewSet(WebhookMixin, BaseViewSet): {"name": "The project name is already taken"}, status=status.HTTP_410_GONE, ) - except Workspace.DoesNotExist as e: + except Workspace.DoesNotExist: return Response( {"error": "Workspace does not exist"}, status=status.HTTP_404_NOT_FOUND, ) - except serializers.ValidationError as e: + except serializers.ValidationError: return Response( {"identifier": "The project identifier is already taken"}, status=status.HTTP_410_GONE, @@ -410,7 +410,7 @@ class ProjectViewSet(WebhookMixin, BaseViewSet): {"error": "Project does not exist"}, status=status.HTTP_404_NOT_FOUND, ) - except serializers.ValidationError as e: + except serializers.ValidationError: return Response( {"identifier": "The project identifier is already taken"}, status=status.HTTP_410_GONE, diff --git a/apiserver/plane/bgtasks/analytic_plot_export.py b/apiserver/plane/bgtasks/analytic_plot_export.py index 62620ab9d..c797f68a5 100644 --- a/apiserver/plane/bgtasks/analytic_plot_export.py +++ b/apiserver/plane/bgtasks/analytic_plot_export.py @@ -1,22 +1,22 @@ # Python imports import csv import io +import logging + +# Third party imports +from celery import shared_task # Django imports from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception # Module imports from plane.db.models import Issue -from plane.utils.analytics_plot import build_graph_plot -from plane.utils.issue_filters import issue_filters from plane.license.utils.instance_value import get_email_configuration +from plane.utils.analytics_plot import build_graph_plot +from plane.utils.exception_logger import log_exception +from plane.utils.issue_filters import issue_filters row_mapping = { "state__name": "State", @@ -210,9 +210,9 @@ def generate_segmented_rows( None, ) if assignee: - generated_row[ - 0 - ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + generated_row[0] = ( + f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + ) if x_axis == LABEL_ID: label = next( @@ -279,9 +279,9 @@ def generate_segmented_rows( None, ) if assignee: - row_zero[ - index + 2 - ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + row_zero[index + 2] = ( + f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + ) if segmented == LABEL_ID: for index, segm in enumerate(row_zero[2:]): @@ -366,9 +366,9 @@ def generate_non_segmented_rows( None, ) if assignee: - row[ - 0 - ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + row[0] = ( + f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}" + ) if x_axis == LABEL_ID: label = next( @@ -504,10 +504,8 @@ def analytic_export_task(email, data, slug): csv_buffer = generate_csv_from_rows(rows) send_export_email(email, slug, csv_buffer, rows) + logging.getLogger("plane").info("Email sent succesfully.") return except Exception as e: - print(e) - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/email_notification_task.py b/apiserver/plane/bgtasks/email_notification_task.py index c3e6e214a..26c3f6b8f 100644 --- a/apiserver/plane/bgtasks/email_notification_task.py +++ b/apiserver/plane/bgtasks/email_notification_task.py @@ -1,21 +1,22 @@ +import logging from datetime import datetime + from bs4 import BeautifulSoup # Third party imports from celery import shared_task -from sentry_sdk import capture_exception +from django.core.mail import EmailMultiAlternatives, get_connection +from django.template.loader import render_to_string # Django imports from django.utils import timezone -from django.core.mail import EmailMultiAlternatives, get_connection -from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings # Module imports -from plane.db.models import EmailNotificationLog, User, Issue +from plane.db.models import EmailNotificationLog, Issue, User from plane.license.utils.instance_value import get_email_configuration from plane.settings.redis import redis_instance +from plane.utils.exception_logger import log_exception # acquire and delete redis lock @@ -69,7 +70,9 @@ def stack_email_notification(): receiver_notification.get("entity_identifier"), {} ).setdefault( str(receiver_notification.get("triggered_by_id")), [] - ).append(receiver_notification.get("data")) + ).append( + receiver_notification.get("data") + ) # append processed notifications processed_notifications.append(receiver_notification.get("id")) email_notification_ids.append(receiver_notification.get("id")) @@ -296,7 +299,9 @@ def send_email_notification( ) msg.attach_alternative(html_content, "text/html") msg.send() + logging.getLogger("plane").info("Email Sent Successfully") + # Update the logs EmailNotificationLog.objects.filter( pk__in=email_notification_ids ).update(sent_at=timezone.now()) @@ -305,15 +310,20 @@ def send_email_notification( release_lock(lock_id=lock_id) return except Exception as e: - capture_exception(e) + log_exception(e) # release the lock release_lock(lock_id=lock_id) return else: - print("Duplicate task recived. Skipping...") + logging.getLogger("plane").info( + "Duplicate email received skipping" + ) return except (Issue.DoesNotExist, User.DoesNotExist) as e: - if settings.DEBUG: - print(e) + log_exception(e) + release_lock(lock_id=lock_id) + return + except Exception as e: + log_exception(e) release_lock(lock_id=lock_id) return diff --git a/apiserver/plane/bgtasks/event_tracking_task.py b/apiserver/plane/bgtasks/event_tracking_task.py index 82a8281a9..135ae1dd1 100644 --- a/apiserver/plane/bgtasks/event_tracking_task.py +++ b/apiserver/plane/bgtasks/event_tracking_task.py @@ -1,13 +1,13 @@ -import uuid import os +import uuid # third party imports from celery import shared_task -from sentry_sdk import capture_exception from posthog import Posthog # module imports from plane.license.utils.instance_value import get_configuration_value +from plane.utils.exception_logger import log_exception def posthogConfiguration(): @@ -51,7 +51,8 @@ def auth_events(user, email, user_agent, ip, event_name, medium, first_time): }, ) except Exception as e: - capture_exception(e) + log_exception(e) + return @shared_task @@ -77,4 +78,5 @@ def workspace_invite_event( }, ) except Exception as e: - capture_exception(e) + log_exception(e) + return diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index f99e54215..2e0d88994 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -2,21 +2,22 @@ import csv import io import json -import boto3 import zipfile +import boto3 +from botocore.client import Config + +# Third party imports +from celery import shared_task + # Django imports from django.conf import settings from django.utils import timezone - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception -from botocore.client import Config from openpyxl import Workbook # Module imports -from plane.db.models import Issue, ExporterHistory +from plane.db.models import ExporterHistory, Issue +from plane.utils.exception_logger import log_exception def dateTimeConverter(time): @@ -403,8 +404,5 @@ def issue_export_task( exporter_instance.status = "failed" exporter_instance.reason = str(e) exporter_instance.save(update_fields=["status", "reason"]) - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/forgot_password_task.py b/apiserver/plane/bgtasks/forgot_password_task.py index 1d3b68477..13dd505df 100644 --- a/apiserver/plane/bgtasks/forgot_password_task.py +++ b/apiserver/plane/bgtasks/forgot_password_task.py @@ -1,17 +1,17 @@ -# Python import +# Python imports +import logging + +# Third party imports +from celery import shared_task # Django imports from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception # Module imports from plane.license.utils.instance_value import get_email_configuration +from plane.utils.exception_logger import log_exception @shared_task @@ -60,10 +60,8 @@ def forgot_password(first_name, email, uidb64, token, current_site): ) msg.attach_alternative(html_content, "text/html") msg.send() + logging.getLogger("plane").info("Email sent successfully") return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/issue_activites_task.py b/apiserver/plane/bgtasks/issue_activites_task.py index 6aa6b6695..9a4e57a49 100644 --- a/apiserver/plane/bgtasks/issue_activites_task.py +++ b/apiserver/plane/bgtasks/issue_activites_task.py @@ -1,34 +1,36 @@ # Python imports import json + import requests +# Third Party imports +from celery import shared_task + # Django imports from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder from django.utils import timezone -# Third Party imports -from celery import shared_task -from sentry_sdk import capture_exception +from plane.app.serializers import IssueActivitySerializer +from plane.bgtasks.notification_task import notifications # Module imports from plane.db.models import ( - User, - Issue, - Project, - Label, - IssueActivity, - State, - Cycle, - Module, - IssueReaction, CommentReaction, + Cycle, + Issue, + IssueActivity, IssueComment, + IssueReaction, IssueSubscriber, + Label, + Module, + Project, + State, + User, ) -from plane.app.serializers import IssueActivitySerializer -from plane.bgtasks.notification_task import notifications from plane.settings.redis import redis_instance +from plane.utils.exception_logger import log_exception # Track Changes in name @@ -1647,7 +1649,7 @@ def issue_activity( headers=headers, ) except Exception as e: - capture_exception(e) + log_exception(e) if notification: notifications.delay( @@ -1668,8 +1670,5 @@ def issue_activity( return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/issue_automation_task.py b/apiserver/plane/bgtasks/issue_automation_task.py index 08c07b7b3..cdcdcd174 100644 --- a/apiserver/plane/bgtasks/issue_automation_task.py +++ b/apiserver/plane/bgtasks/issue_automation_task.py @@ -2,18 +2,17 @@ import json from datetime import timedelta -# Django imports -from django.utils import timezone -from django.db.models import Q -from django.conf import settings - # Third party imports from celery import shared_task -from sentry_sdk import capture_exception +from django.db.models import Q + +# Django imports +from django.utils import timezone # Module imports -from plane.db.models import Issue, Project, State from plane.bgtasks.issue_activites_task import issue_activity +from plane.db.models import Issue, Project, State +from plane.utils.exception_logger import log_exception @shared_task @@ -96,9 +95,7 @@ def archive_old_issues(): ] return except Exception as e: - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return @@ -179,7 +176,5 @@ def close_old_issues(): ] return except Exception as e: - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/magic_link_code_task.py b/apiserver/plane/bgtasks/magic_link_code_task.py index 019f5b13c..8698bebe6 100644 --- a/apiserver/plane/bgtasks/magic_link_code_task.py +++ b/apiserver/plane/bgtasks/magic_link_code_task.py @@ -1,17 +1,17 @@ # Python imports +import logging + +# Third party imports +from celery import shared_task # Django imports from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception # Module imports from plane.license.utils.instance_value import get_email_configuration +from plane.utils.exception_logger import log_exception @shared_task @@ -52,11 +52,8 @@ def magic_link(email, key, token, current_site): ) msg.attach_alternative(html_content, "text/html") msg.send() + logging.getLogger("plane").info("Email sent successfully.") return except Exception as e: - print(e) - capture_exception(e) - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/project_invitation_task.py b/apiserver/plane/bgtasks/project_invitation_task.py index d24db5ae9..cbb440130 100644 --- a/apiserver/plane/bgtasks/project_invitation_task.py +++ b/apiserver/plane/bgtasks/project_invitation_task.py @@ -1,18 +1,18 @@ -# Python import +# Python imports +import logging + +# Third party imports +from celery import shared_task # Django imports from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception # Module imports -from plane.db.models import Project, User, ProjectMemberInvite +from plane.db.models import Project, ProjectMemberInvite, User from plane.license.utils.instance_value import get_email_configuration +from plane.utils.exception_logger import log_exception @shared_task @@ -73,12 +73,10 @@ def project_invitation(email, project_id, token, current_site, invitor): msg.attach_alternative(html_content, "text/html") msg.send() + logging.getLogger("plane").info("Email sent successfully.") return except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist): return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/webhook_task.py b/apiserver/plane/bgtasks/webhook_task.py index 358fd7a85..fe59ce939 100644 --- a/apiserver/plane/bgtasks/webhook_task.py +++ b/apiserver/plane/bgtasks/webhook_task.py @@ -1,44 +1,45 @@ -import requests -import uuid import hashlib -import json import hmac +import json +import logging +import uuid -# Django imports -from django.conf import settings -from django.core.serializers.json import DjangoJSONEncoder -from django.core.mail import EmailMultiAlternatives, get_connection -from django.template.loader import render_to_string -from django.utils.html import strip_tags +import requests # Third party imports from celery import shared_task -from sentry_sdk import capture_exception -from plane.db.models import ( - Webhook, - WebhookLog, - Project, - Issue, - Cycle, - Module, - ModuleIssue, - CycleIssue, - IssueComment, - User, -) -from plane.api.serializers import ( - ProjectSerializer, - CycleSerializer, - ModuleSerializer, - CycleIssueSerializer, - ModuleIssueSerializer, - IssueCommentSerializer, - IssueExpandSerializer, -) +# Django imports +from django.conf import settings +from django.core.mail import EmailMultiAlternatives, get_connection +from django.core.serializers.json import DjangoJSONEncoder +from django.template.loader import render_to_string +from django.utils.html import strip_tags # Module imports +from plane.api.serializers import ( + CycleIssueSerializer, + CycleSerializer, + IssueCommentSerializer, + IssueExpandSerializer, + ModuleIssueSerializer, + ModuleSerializer, + ProjectSerializer, +) +from plane.db.models import ( + Cycle, + CycleIssue, + Issue, + IssueComment, + Module, + ModuleIssue, + Project, + User, + Webhook, + WebhookLog, +) from plane.license.utils.instance_value import get_email_configuration +from plane.utils.exception_logger import log_exception SERIALIZER_MAPPER = { "project": ProjectSerializer, @@ -174,7 +175,7 @@ def webhook_task(self, webhook, slug, event, event_data, action, current_site): except Exception as e: if settings.DEBUG: print(e) - capture_exception(e) + log_exception(e) return @@ -241,7 +242,7 @@ def send_webhook(event, payload, kw, action, slug, bulk, current_site): except Exception as e: if settings.DEBUG: print(e) - capture_exception(e) + log_exception(e) return @@ -295,8 +296,8 @@ def send_webhook_deactivation_email( ) msg.attach_alternative(html_content, "text/html") msg.send() - + logging.getLogger("plane").info("Email sent successfully.") return except Exception as e: - print(e) + log_exception(e) return diff --git a/apiserver/plane/bgtasks/workspace_invitation_task.py b/apiserver/plane/bgtasks/workspace_invitation_task.py index cc3000bbb..46837731a 100644 --- a/apiserver/plane/bgtasks/workspace_invitation_task.py +++ b/apiserver/plane/bgtasks/workspace_invitation_task.py @@ -1,18 +1,18 @@ # Python imports +import logging + +# Third party imports +from celery import shared_task # Django imports from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import render_to_string from django.utils.html import strip_tags -from django.conf import settings - -# Third party imports -from celery import shared_task -from sentry_sdk import capture_exception # Module imports -from plane.db.models import Workspace, WorkspaceMemberInvite, User +from plane.db.models import User, Workspace, WorkspaceMemberInvite from plane.license.utils.instance_value import get_email_configuration +from plane.utils.exception_logger import log_exception @shared_task @@ -76,14 +76,12 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor): ) msg.attach_alternative(html_content, "text/html") msg.send() + logging.getLogger("plane").info("Email sent succesfully") return - except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist): - print("Workspace or WorkspaceMember Invite Does not exists") + except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e: + log_exception(e) return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return diff --git a/apiserver/plane/db/models/user.py b/apiserver/plane/db/models/user.py index c9a8b4cb6..5f932d2ea 100644 --- a/apiserver/plane/db/models/user.py +++ b/apiserver/plane/db/models/user.py @@ -1,16 +1,17 @@ # Python imports -import uuid -import string import random +import string +import uuid + import pytz +from django.contrib.auth.models import ( + AbstractBaseUser, + PermissionsMixin, + UserManager, +) # Django imports from django.db import models -from django.contrib.auth.models import ( - AbstractBaseUser, - UserManager, - PermissionsMixin, -) from django.db.models.signals import post_save from django.dispatch import receiver from django.utils import timezone diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index 5c8947e73..886ad4cb4 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -3,19 +3,20 @@ # Python imports import os import ssl -import certifi from datetime import timedelta from urllib.parse import urlparse -# Django imports -from django.core.management.utils import get_random_secret_key +import certifi # Third party imports import dj_database_url import sentry_sdk + +# Django imports +from django.core.management.utils import get_random_secret_key +from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -from sentry_sdk.integrations.celery import CeleryIntegration BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -23,7 +24,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = os.environ.get("SECRET_KEY", get_random_secret_key()) # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = int(os.environ.get("DEBUG", "0")) # Allowed Hosts ALLOWED_HOSTS = ["*"] diff --git a/apiserver/plane/settings/local.py b/apiserver/plane/settings/local.py index a09a55ccf..b00684eae 100644 --- a/apiserver/plane/settings/local.py +++ b/apiserver/plane/settings/local.py @@ -7,8 +7,8 @@ from .common import * # noqa DEBUG = True # Debug Toolbar settings -INSTALLED_APPS += ("debug_toolbar",) -MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",) +INSTALLED_APPS += ("debug_toolbar",) # noqa +MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",) # noqa DEBUG_TOOLBAR_PATCH_SETTINGS = False @@ -18,7 +18,7 @@ EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": REDIS_URL, + "LOCATION": REDIS_URL, # noqa "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", }, @@ -28,7 +28,7 @@ CACHES = { INTERNAL_IPS = ("127.0.0.1",) MEDIA_URL = "/uploads/" -MEDIA_ROOT = os.path.join(BASE_DIR, "uploads") +MEDIA_ROOT = os.path.join(BASE_DIR, "uploads") # noqa CORS_ALLOWED_ORIGINS = [ "http://localhost:3000", @@ -36,3 +36,38 @@ CORS_ALLOWED_ORIGINS = [ "http://localhost:4000", "http://127.0.0.1:4000", ] + +LOG_DIR = os.path.join(BASE_DIR, "logs") # noqa + +if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}", + "style": "{", + }, + }, + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + }, + "loggers": { + "django.request": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": False, + }, + "plane": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": False, + }, + }, +} diff --git a/apiserver/plane/settings/production.py b/apiserver/plane/settings/production.py index 5a9c3413d..caf6804a3 100644 --- a/apiserver/plane/settings/production.py +++ b/apiserver/plane/settings/production.py @@ -1,15 +1,16 @@ """Production settings""" import os + from .common import * # noqa # SECURITY WARNING: don't run with debug turned on in production! DEBUG = int(os.environ.get("DEBUG", 0)) == 1 - +DEBUG = True # Honor the 'X-Forwarded-Proto' header for request.is_secure() SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -INSTALLED_APPS += ("scout_apm.django",) +INSTALLED_APPS += ("scout_apm.django",) # noqa # Honor the 'X-Forwarded-Proto' header for request.is_secure() SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") @@ -18,3 +19,62 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") SCOUT_MONITOR = os.environ.get("SCOUT_MONITOR", False) SCOUT_KEY = os.environ.get("SCOUT_KEY", "") SCOUT_NAME = "Plane" + +LOG_DIR = os.path.join(BASE_DIR, "logs") # noqa + +if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}", + "style": "{", + }, + "json": { + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "verbose", + "level": "INFO", + }, + "file": { + "class": "plane.utils.logging.SizedTimedRotatingFileHandler", + "filename": ( + os.path.join(BASE_DIR, "logs", "plane-debug.log") # noqa + if DEBUG + else os.path.join(BASE_DIR, "logs", "plane-error.log") # noqa + ), + "when": "s", + "maxBytes": 1024 * 1024 * 1, + "interval": 1, + "backupCount": 5, + "formatter": "json", + "level": "DEBUG" if DEBUG else "ERROR", + }, + }, + "loggers": { + "django": { + "handlers": ["console", "file"], + "level": "INFO", + "propagate": True, + }, + "django.request": { + "handlers": ["console", "file"], + "level": "INFO", + "propagate": False, + }, + "plane": { + "level": "DEBUG" if DEBUG else "ERROR", + "handlers": ["console", "file"], + "propagate": False, + }, + }, +} diff --git a/apiserver/plane/settings/test.py b/apiserver/plane/settings/test.py index 84153d37a..a86b044a3 100644 --- a/apiserver/plane/settings/test.py +++ b/apiserver/plane/settings/test.py @@ -7,6 +7,6 @@ DEBUG = True # Send it in a dummy outbox EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend" -INSTALLED_APPS.append( +INSTALLED_APPS.append( # noqa "plane.tests", ) diff --git a/apiserver/plane/space/views/base.py b/apiserver/plane/space/views/base.py index 54dac080c..023f27bbc 100644 --- a/apiserver/plane/space/views/base.py +++ b/apiserver/plane/space/views/base.py @@ -1,25 +1,25 @@ # Python imports import zoneinfo +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.db import IntegrityError # Django imports from django.urls import resolve -from django.conf import settings from django.utils import timezone -from django.db import IntegrityError -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django_filters.rest_framework import DjangoFilterBackend # Third part imports from rest_framework import status -from rest_framework.viewsets import ModelViewSet -from rest_framework.response import Response from rest_framework.exceptions import APIException -from rest_framework.views import APIView from rest_framework.filters import SearchFilter from rest_framework.permissions import IsAuthenticated -from sentry_sdk import capture_exception -from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.viewsets import ModelViewSet # Module imports +from plane.utils.exception_logger import log_exception from plane.utils.paginator import BasePaginator @@ -57,7 +57,7 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): try: return self.model.objects.all() except Exception as e: - capture_exception(e) + log_exception(e) raise APIException( "Please check the view", status.HTTP_400_BAD_REQUEST ) @@ -90,14 +90,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): ) if isinstance(e, KeyError): - capture_exception(e) + log_exception(e) return Response( {"error": "The required key does not exist."}, status=status.HTTP_400_BAD_REQUEST, ) - print(e) if settings.DEBUG else print("Server Error") - capture_exception(e) + log_exception(e) return Response( {"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -185,9 +184,7 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator): status=status.HTTP_400_BAD_REQUEST, ) - if settings.DEBUG: - print(e) - capture_exception(e) + log_exception(e) return Response( {"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/apiserver/plane/utils/exception_logger.py b/apiserver/plane/utils/exception_logger.py new file mode 100644 index 000000000..f7bb50de2 --- /dev/null +++ b/apiserver/plane/utils/exception_logger.py @@ -0,0 +1,15 @@ +# Python imports +import logging + +# Third party imports +from sentry_sdk import capture_exception + + +def log_exception(e): + # Log the error + logger = logging.getLogger("plane") + logger.error(e) + + # Capture in sentry if configured + capture_exception(e) + return diff --git a/apiserver/plane/utils/logging.py b/apiserver/plane/utils/logging.py new file mode 100644 index 000000000..8021689e9 --- /dev/null +++ b/apiserver/plane/utils/logging.py @@ -0,0 +1,46 @@ +import logging.handlers as handlers +import time + + +class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler): + """ + Handler for logging to a set of files, which switches from one file + to the next when the current file reaches a certain size, or at certain + timed intervals + """ + + def __init__( + self, + filename, + maxBytes=0, + backupCount=0, + encoding=None, + delay=0, + when="h", + interval=1, + utc=False, + ): + handlers.TimedRotatingFileHandler.__init__( + self, filename, when, interval, backupCount, encoding, delay, utc + ) + self.maxBytes = maxBytes + + def shouldRollover(self, record): + """ + Determine if rollover should occur. + + Basically, see if the supplied record would cause the file to exceed + the size limit we have. + """ + if self.stream is None: # delay was set... + self.stream = self._open() + if self.maxBytes > 0: # are we rolling over? + msg = "%s\n" % self.format(record) + # due to non-posix-compliant Windows feature + self.stream.seek(0, 2) + if self.stream.tell() + len(msg) >= self.maxBytes: + return 1 + t = int(time.time()) + if t >= self.rolloverAt: + return 1 + return 0 diff --git a/apiserver/requirements/base.txt b/apiserver/requirements/base.txt index eb0f54201..28b45adbe 100644 --- a/apiserver/requirements/base.txt +++ b/apiserver/requirements/base.txt @@ -27,6 +27,7 @@ psycopg-binary==3.1.12 psycopg-c==3.1.12 scout-apm==2.26.1 openpyxl==3.1.2 +python-json-logger==2.0.7 beautifulsoup4==4.12.2 dj-database-url==2.1.0 posthog==3.0.2 diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index 3af4b8449..b8d50177c 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -70,6 +70,8 @@ services: command: ./bin/takeoff deploy: replicas: ${API_REPLICAS:-1} + volumes: + - logs_api:/code/plane/logs depends_on: - plane-db - plane-redis @@ -80,6 +82,8 @@ services: pull_policy: ${PULL_POLICY:-always} restart: unless-stopped command: ./bin/worker + volumes: + - logs_worker:/code/plane/logs depends_on: - api - plane-db @@ -91,6 +95,8 @@ services: pull_policy: ${PULL_POLICY:-always} restart: unless-stopped command: ./bin/beat + volumes: + - logs_beat-worker:/code/plane/logs depends_on: - api - plane-db @@ -104,6 +110,8 @@ services: command: > sh -c "python manage.py wait_for_db && python manage.py migrate" + volumes: + - logs_migrator:/code/plane/logs depends_on: - plane-db - plane-redis @@ -149,3 +157,7 @@ volumes: pgdata: redisdata: uploads: + logs_api: + logs_worker: + logs_beat-worker: + logs_migrator: