dev: enable api logging (#2538)

* dev: enable api logging and control worker count through env

* dev: enable logger instead of printing

* dev: remove worker counts

* dev: enable global level log settings

* dev: add rotating logger

* fix: logging configuration

* dev: api logging and moving the capture exception to utils for logging and then capturing

* fix: information leaking through print logs

* dev: linting fix

* dev: logging configuration for django

* fix: linting errors

* dev: add logs for migrator

* dev: logging cofiguration

* dev: add permision for captain user in Plane

* dev: add log paths in compose

* dev: create directory for logs

* dev: fix linting errors

.gitignore

@@ -51,6 +51,7 @@ staticfiles
 mediafiles
 .env
 .DS_Store
+logs/
 
 node_modules/
 assets/dist/

apiserver/.env.example

@@ -44,4 +44,3 @@ WEB_URL="http://localhost"
 
 # Gunicorn Workers
 GUNICORN_WORKERS=2
-

apiserver/Dockerfile.api

@@ -48,8 +48,10 @@ USER root
 RUN apk --no-cache add "bash~=5.2"
 COPY ./bin ./bin/
 
+RUN mkdir /code/plane/logs
 RUN chmod +x ./bin/takeoff ./bin/worker ./bin/beat
 RUN chmod -R 777 /code
+RUN chown -R captain:plane /code
 
 USER captain
 

apiserver/plane/api/views/base.py

@@ -1,26 +1,26 @@
 # Python imports
-import zoneinfo
 from urllib.parse import urlparse
 
+import zoneinfo
 
 # Django imports
 from django.conf import settings
-from django.db import IntegrityError
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.db import IntegrityError
 from django.utils import timezone
+from rest_framework import status
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
 
 # Third party imports
 from rest_framework.views import APIView
-from rest_framework.response import Response
-from rest_framework.permissions import IsAuthenticated
-from rest_framework import status
-from sentry_sdk import capture_exception
 
 # Module imports
 from plane.api.middleware.api_authentication import APIKeyAuthentication
 from plane.api.rate_limit import ApiKeyRateThrottle
-from plane.utils.paginator import BasePaginator
 from plane.bgtasks.webhook_task import send_webhook
+from plane.utils.exception_logger import log_exception
+from plane.utils.paginator import BasePaginator
 
 
 class TimezoneMixin:
@@ -106,27 +106,23 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
 
             if isinstance(e, ValidationError):
                 return Response(
-                    {
+                    {"error": "Please provide valid detail"},
-                        "error": "The provided payload is not valid please try with a valid payload"
-                    },
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
             if isinstance(e, ObjectDoesNotExist):
                 return Response(
-                    {"error": "The required object does not exist."},
+                    {"error": "The requested resource does not exist."},
                     status=status.HTTP_404_NOT_FOUND,
                 )
 
             if isinstance(e, KeyError):
                 return Response(
-                    {"error": " The required key does not exist."},
+                    {"error": "The required key does not exist."},
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
-            if settings.DEBUG:
+            log_exception(e)
-                print(e)
-            capture_exception(e)
             return Response(
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,

apiserver/plane/app/views/base.py

@@ -1,27 +1,27 @@
 # Python imports
 import zoneinfo
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.db import IntegrityError
 
 # Django imports
 from django.urls import resolve
-from django.conf import settings
 from django.utils import timezone
-from django.db import IntegrityError
+from django_filters.rest_framework import DjangoFilterBackend
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
 
 # Third part imports
 from rest_framework import status
-from rest_framework.viewsets import ModelViewSet
-from rest_framework.response import Response
 from rest_framework.exceptions import APIException
-from rest_framework.views import APIView
 from rest_framework.filters import SearchFilter
 from rest_framework.permissions import IsAuthenticated
-from sentry_sdk import capture_exception
+from rest_framework.response import Response
-from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework.views import APIView
+from rest_framework.viewsets import ModelViewSet
 
 # Module imports
-from plane.utils.paginator import BasePaginator
 from plane.bgtasks.webhook_task import send_webhook
+from plane.utils.exception_logger import log_exception
+from plane.utils.paginator import BasePaginator
 
 
 class TimezoneMixin:
@@ -87,7 +87,7 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
         try:
             return self.model.objects.all()
         except Exception as e:
-            capture_exception(e)
+            log_exception(e)
             raise APIException(
                 "Please check the view", status.HTTP_400_BAD_REQUEST
             )
@@ -121,13 +121,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
                 )
 
             if isinstance(e, KeyError):
-                capture_exception(e)
+                log_exception(e)
                 return Response(
                     {"error": "The required key does not exist."},
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
-            capture_exception(e)
+            log_exception(e)
             return Response(
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -233,9 +233,7 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
-            if settings.DEBUG:
+            log_exception(e)
-                print(e)
-            capture_exception(e)
             return Response(
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,

apiserver/plane/app/views/cycle/base.py

@@ -15,10 +15,7 @@ from django.db.models import (
     Value,
     CharField,
 )
-from django.core import serializers
 from django.utils import timezone
-from django.utils.decorators import method_decorator
-from django.views.decorators.gzip import gzip_page
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import ArrayField
 from django.db.models import UUIDField
@@ -32,9 +29,7 @@ from rest_framework import status
 from .. import BaseViewSet, BaseAPIView, WebhookMixin
 from plane.app.serializers import (
     CycleSerializer,
-    CycleIssueSerializer,
     CycleFavoriteSerializer,
-    IssueSerializer,
     CycleWriteSerializer,
     CycleUserPropertiesSerializer,
 )
@@ -48,13 +43,10 @@ from plane.db.models import (
     CycleIssue,
     Issue,
     CycleFavorite,
-    IssueLink,
-    IssueAttachment,
     Label,
     CycleUserProperties,
 )
 from plane.bgtasks.issue_activites_task import issue_activity
-from plane.utils.issue_filters import issue_filters
 from plane.utils.analytics_plot import burndown_plot
 
 

apiserver/plane/app/views/dashboard/base.py

@@ -38,7 +38,6 @@ from plane.db.models import (
     IssueLink,
     IssueAttachment,
     IssueRelation,
-    IssueAssignee,
     User,
 )
 from plane.app.serializers import (

apiserver/plane/app/views/issue/base.py

@@ -1,7 +1,5 @@
 # Python imports
 import json
-import random
-from itertools import chain
 
 # Django imports
 from django.utils import timezone
@@ -21,64 +19,38 @@ from django.db.models import (
 from django.core.serializers.json import DjangoJSONEncoder
 from django.utils.decorators import method_decorator
 from django.views.decorators.gzip import gzip_page
-from django.db import IntegrityError
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import ArrayField
-from django.db.models import Value, UUIDField
+from django.db.models import UUIDField
 from django.db.models.functions import Coalesce
 
 # Third Party imports
 from rest_framework.response import Response
 from rest_framework import status
-from rest_framework.parsers import MultiPartParser, FormParser
 
 # Module imports
 from .. import BaseViewSet, BaseAPIView, WebhookMixin
 from plane.app.serializers import (
-    IssueActivitySerializer,
-    IssueCommentSerializer,
     IssuePropertySerializer,
     IssueSerializer,
     IssueCreateSerializer,
-    LabelSerializer,
-    IssueFlatSerializer,
-    IssueLinkSerializer,
-    IssueLiteSerializer,
-    IssueAttachmentSerializer,
-    IssueSubscriberSerializer,
-    ProjectMemberLiteSerializer,
-    IssueReactionSerializer,
-    CommentReactionSerializer,
-    IssueRelationSerializer,
-    RelatedIssueSerializer,
     IssueDetailSerializer,
 )
 from plane.app.permissions import (
     ProjectEntityPermission,
-    WorkSpaceAdminPermission,
-    ProjectMemberPermission,
     ProjectLitePermission,
 )
 from plane.db.models import (
     Project,
     Issue,
-    IssueActivity,
-    IssueComment,
     IssueProperty,
-    Label,
     IssueLink,
     IssueAttachment,
     IssueSubscriber,
-    ProjectMember,
     IssueReaction,
-    CommentReaction,
-    IssueRelation,
 )
 from plane.bgtasks.issue_activites_task import issue_activity
-from plane.utils.grouper import group_results
 from plane.utils.issue_filters import issue_filters
-from collections import defaultdict
-from plane.utils.cache import invalidate_cache
 
 class IssueListEndpoint(BaseAPIView):
 

apiserver/plane/app/views/issue/draft.py

@@ -1,52 +1,54 @@
 # Python imports
 import json
 
-# Django imports
-from django.utils import timezone
-from django.db.models import (
-    Prefetch,
-    OuterRef,
-    Func,
-    F,
-    Q,
-    Case,
-    Value,
-    CharField,
-    When,
-    Exists,
-    Max,
-    UUIDField,
-)
-from django.core.serializers.json import DjangoJSONEncoder
-from django.utils.decorators import method_decorator
-from django.views.decorators.gzip import gzip_page
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import ArrayField
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models import (
+    Case,
+    CharField,
+    Exists,
+    F,
+    Func,
+    Max,
+    OuterRef,
+    Prefetch,
+    Q,
+    UUIDField,
+    Value,
+    When,
+)
 from django.db.models.functions import Coalesce
 
+# Django imports
+from django.utils import timezone
+from django.utils.decorators import method_decorator
+from django.views.decorators.gzip import gzip_page
+from rest_framework import status
+
 # Third Party imports
 from rest_framework.response import Response
-from rest_framework import status
+
+from plane.app.permissions import ProjectEntityPermission
+from plane.app.serializers import (
+    IssueCreateSerializer,
+    IssueDetailSerializer,
+    IssueFlatSerializer,
+    IssueSerializer,
+)
+from plane.bgtasks.issue_activites_task import issue_activity
+from plane.db.models import (
+    Issue,
+    IssueAttachment,
+    IssueLink,
+    IssueReaction,
+    IssueSubscriber,
+    Project,
+)
+from plane.utils.issue_filters import issue_filters
 
 # Module imports
 from .. import BaseViewSet
-from plane.app.serializers import (
-    IssueSerializer,
-    IssueCreateSerializer,
-    IssueFlatSerializer,
-    IssueDetailSerializer,
-)
-from plane.app.permissions import ProjectEntityPermission
-from plane.db.models import (
-    Project,
-    Issue,
-    IssueLink,
-    IssueAttachment,
-    IssueSubscriber,
-    IssueReaction,
-)
-from plane.bgtasks.issue_activites_task import issue_activity
-from plane.utils.issue_filters import issue_filters
 
 
 class IssueDraftViewSet(BaseViewSet):
@@ -117,11 +119,6 @@ class IssueDraftViewSet(BaseViewSet):
     @method_decorator(gzip_page)
     def list(self, request, slug, project_id):
         filters = issue_filters(request.query_params, "GET")
-        fields = [
-            field
-            for field in request.GET.get("fields", "").split(",")
-            if field
-        ]
 
         # Custom ordering for priority and state
         priority_order = ["urgent", "high", "medium", "low", "none"]

apiserver/plane/app/views/project/base.py

@@ -346,12 +346,12 @@ class ProjectViewSet(WebhookMixin, BaseViewSet):
                     {"name": "The project name is already taken"},
                     status=status.HTTP_410_GONE,
                 )
-        except Workspace.DoesNotExist as e:
+        except Workspace.DoesNotExist:
             return Response(
                 {"error": "Workspace does not exist"},
                 status=status.HTTP_404_NOT_FOUND,
             )
-        except serializers.ValidationError as e:
+        except serializers.ValidationError:
             return Response(
                 {"identifier": "The project identifier is already taken"},
                 status=status.HTTP_410_GONE,
@@ -410,7 +410,7 @@ class ProjectViewSet(WebhookMixin, BaseViewSet):
                 {"error": "Project does not exist"},
                 status=status.HTTP_404_NOT_FOUND,
             )
-        except serializers.ValidationError as e:
+        except serializers.ValidationError:
             return Response(
                 {"identifier": "The project identifier is already taken"},
                 status=status.HTTP_410_GONE,

apiserver/plane/bgtasks/analytic_plot_export.py

@@ -1,22 +1,22 @@
 # Python imports
 import csv
 import io
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
 
 # Module imports
 from plane.db.models import Issue
-from plane.utils.analytics_plot import build_graph_plot
-from plane.utils.issue_filters import issue_filters
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.analytics_plot import build_graph_plot
+from plane.utils.exception_logger import log_exception
+from plane.utils.issue_filters import issue_filters
 
 row_mapping = {
     "state__name": "State",
@@ -210,9 +210,9 @@ def generate_segmented_rows(
                 None,
             )
             if assignee:
-                generated_row[
+                generated_row[0] = (
-                    0
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
         if x_axis == LABEL_ID:
             label = next(
@@ -279,9 +279,9 @@ def generate_segmented_rows(
                 None,
             )
             if assignee:
-                row_zero[
+                row_zero[index + 2] = (
-                    index + 2
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
     if segmented == LABEL_ID:
         for index, segm in enumerate(row_zero[2:]):
@@ -366,9 +366,9 @@ def generate_non_segmented_rows(
                 None,
             )
             if assignee:
-                row[
+                row[0] = (
-                    0
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
         if x_axis == LABEL_ID:
             label = next(
@@ -504,10 +504,8 @@ def analytic_export_task(email, data, slug):
 
         csv_buffer = generate_csv_from_rows(rows)
         send_export_email(email, slug, csv_buffer, rows)
+        logging.getLogger("plane").info("Email sent succesfully.")
         return
     except Exception as e:
-        print(e)
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/email_notification_task.py

@@ -1,21 +1,22 @@
+import logging
 from datetime import datetime
+
 from bs4 import BeautifulSoup
 
 # Third party imports
 from celery import shared_task
-from sentry_sdk import capture_exception
+from django.core.mail import EmailMultiAlternatives, get_connection
+from django.template.loader import render_to_string
 
 # Django imports
 from django.utils import timezone
-from django.core.mail import EmailMultiAlternatives, get_connection
-from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
 
 # Module imports
-from plane.db.models import EmailNotificationLog, User, Issue
+from plane.db.models import EmailNotificationLog, Issue, User
 from plane.license.utils.instance_value import get_email_configuration
 from plane.settings.redis import redis_instance
+from plane.utils.exception_logger import log_exception
 
 
 # acquire and delete redis lock
@@ -69,7 +70,9 @@ def stack_email_notification():
                 receiver_notification.get("entity_identifier"), {}
             ).setdefault(
                 str(receiver_notification.get("triggered_by_id")), []
-            ).append(receiver_notification.get("data"))
+            ).append(
+                receiver_notification.get("data")
+            )
             # append processed notifications
             processed_notifications.append(receiver_notification.get("id"))
             email_notification_ids.append(receiver_notification.get("id"))
@@ -296,7 +299,9 @@ def send_email_notification(
                 )
                 msg.attach_alternative(html_content, "text/html")
                 msg.send()
+                logging.getLogger("plane").info("Email Sent Successfully")
 
+                # Update the logs
                 EmailNotificationLog.objects.filter(
                     pk__in=email_notification_ids
                 ).update(sent_at=timezone.now())
@@ -305,15 +310,20 @@ def send_email_notification(
                 release_lock(lock_id=lock_id)
                 return
             except Exception as e:
-                capture_exception(e)
+                log_exception(e)
                 # release the lock
                 release_lock(lock_id=lock_id)
                 return
         else:
-            print("Duplicate task recived. Skipping...")
+            logging.getLogger("plane").info(
+                "Duplicate email received skipping"
+            )
             return
     except (Issue.DoesNotExist, User.DoesNotExist) as e:
-        if settings.DEBUG:
+        log_exception(e)
-            print(e)
+        release_lock(lock_id=lock_id)
+        return
+    except Exception as e:
+        log_exception(e)
         release_lock(lock_id=lock_id)
         return

apiserver/plane/bgtasks/event_tracking_task.py

@@ -1,13 +1,13 @@
-import uuid
 import os
+import uuid
 
 # third party imports
 from celery import shared_task
-from sentry_sdk import capture_exception
 from posthog import Posthog
 
 # module imports
 from plane.license.utils.instance_value import get_configuration_value
+from plane.utils.exception_logger import log_exception
 
 
 def posthogConfiguration():
@@ -51,7 +51,8 @@ def auth_events(user, email, user_agent, ip, event_name, medium, first_time):
                 },
             )
     except Exception as e:
-        capture_exception(e)
+        log_exception(e)
+        return
 
 
 @shared_task
@@ -77,4 +78,5 @@ def workspace_invite_event(
                 },
             )
     except Exception as e:
-        capture_exception(e)
+        log_exception(e)
+        return

apiserver/plane/bgtasks/export_task.py

@@ -2,21 +2,22 @@
 import csv
 import io
 import json
-import boto3
 import zipfile
 
+import boto3
+from botocore.client import Config
+
+# Third party imports
+from celery import shared_task
+
 # Django imports
 from django.conf import settings
 from django.utils import timezone
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
-from botocore.client import Config
 from openpyxl import Workbook
 
 # Module imports
-from plane.db.models import Issue, ExporterHistory
+from plane.db.models import ExporterHistory, Issue
+from plane.utils.exception_logger import log_exception
 
 
 def dateTimeConverter(time):
@@ -403,8 +404,5 @@ def issue_export_task(
         exporter_instance.status = "failed"
         exporter_instance.reason = str(e)
         exporter_instance.save(update_fields=["status", "reason"])
-        # Print logs if in DEBUG mode
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/forgot_password_task.py

@@ -1,17 +1,17 @@
-# Python import
+# Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
 
 # Module imports
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.exception_logger import log_exception
 
 
 @shared_task
@@ -60,10 +60,8 @@ def forgot_password(first_name, email, uidb64, token, current_site):
         )
         msg.attach_alternative(html_content, "text/html")
         msg.send()
+        logging.getLogger("plane").info("Email sent successfully")
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/issue_activites_task.py

@@ -1,34 +1,36 @@
 # Python imports
 import json
+
 import requests
 
+# Third Party imports
+from celery import shared_task
+
 # Django imports
 from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 from django.utils import timezone
 
-# Third Party imports
+from plane.app.serializers import IssueActivitySerializer
-from celery import shared_task
+from plane.bgtasks.notification_task import notifications
-from sentry_sdk import capture_exception
 
 # Module imports
 from plane.db.models import (
-    User,
-    Issue,
-    Project,
-    Label,
-    IssueActivity,
-    State,
-    Cycle,
-    Module,
-    IssueReaction,
     CommentReaction,
+    Cycle,
+    Issue,
+    IssueActivity,
     IssueComment,
+    IssueReaction,
     IssueSubscriber,
+    Label,
+    Module,
+    Project,
+    State,
+    User,
 )
-from plane.app.serializers import IssueActivitySerializer
-from plane.bgtasks.notification_task import notifications
 from plane.settings.redis import redis_instance
+from plane.utils.exception_logger import log_exception
 
 
 # Track Changes in name
@@ -1647,7 +1649,7 @@ def issue_activity(
                             headers=headers,
                         )
             except Exception as e:
-                capture_exception(e)
+                log_exception(e)
 
         if notification:
             notifications.delay(
@@ -1668,8 +1670,5 @@ def issue_activity(
 
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/issue_automation_task.py

@@ -2,18 +2,17 @@
 import json
 from datetime import timedelta
 
-# Django imports
-from django.utils import timezone
-from django.db.models import Q
-from django.conf import settings
-
 # Third party imports
 from celery import shared_task
-from sentry_sdk import capture_exception
+from django.db.models import Q
+
+# Django imports
+from django.utils import timezone
 
 # Module imports
-from plane.db.models import Issue, Project, State
 from plane.bgtasks.issue_activites_task import issue_activity
+from plane.db.models import Issue, Project, State
+from plane.utils.exception_logger import log_exception
 
 
 @shared_task
@@ -96,9 +95,7 @@ def archive_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
+        log_exception(e)
-            print(e)
-        capture_exception(e)
         return
 
 
@@ -179,7 +176,5 @@ def close_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
+        log_exception(e)
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/magic_link_code_task.py

@@ -1,17 +1,17 @@
 # Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
 
 # Module imports
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.exception_logger import log_exception
 
 
 @shared_task
@@ -52,11 +52,8 @@ def magic_link(email, key, token, current_site):
         )
         msg.attach_alternative(html_content, "text/html")
         msg.send()
+        logging.getLogger("plane").info("Email sent successfully.")
         return
     except Exception as e:
-        print(e)
+        log_exception(e)
-        capture_exception(e)
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
         return

apiserver/plane/bgtasks/project_invitation_task.py

@@ -1,18 +1,18 @@
-# Python import
+# Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
 
 # Module imports
-from plane.db.models import Project, User, ProjectMemberInvite
+from plane.db.models import Project, ProjectMemberInvite, User
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.exception_logger import log_exception
 
 
 @shared_task
@@ -73,12 +73,10 @@ def project_invitation(email, project_id, token, current_site, invitor):
 
         msg.attach_alternative(html_content, "text/html")
         msg.send()
+        logging.getLogger("plane").info("Email sent successfully.")
         return
     except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist):
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/bgtasks/webhook_task.py

@@ -1,44 +1,45 @@
-import requests
-import uuid
 import hashlib
-import json
 import hmac
+import json
+import logging
+import uuid
 
-# Django imports
+import requests
-from django.conf import settings
-from django.core.serializers.json import DjangoJSONEncoder
-from django.core.mail import EmailMultiAlternatives, get_connection
-from django.template.loader import render_to_string
-from django.utils.html import strip_tags
 
 # Third party imports
 from celery import shared_task
-from sentry_sdk import capture_exception
 
-from plane.db.models import (
+# Django imports
-    Webhook,
+from django.conf import settings
-    WebhookLog,
+from django.core.mail import EmailMultiAlternatives, get_connection
-    Project,
+from django.core.serializers.json import DjangoJSONEncoder
-    Issue,
+from django.template.loader import render_to_string
-    Cycle,
+from django.utils.html import strip_tags
-    Module,
-    ModuleIssue,
-    CycleIssue,
-    IssueComment,
-    User,
-)
-from plane.api.serializers import (
-    ProjectSerializer,
-    CycleSerializer,
-    ModuleSerializer,
-    CycleIssueSerializer,
-    ModuleIssueSerializer,
-    IssueCommentSerializer,
-    IssueExpandSerializer,
-)
 
 # Module imports
+from plane.api.serializers import (
+    CycleIssueSerializer,
+    CycleSerializer,
+    IssueCommentSerializer,
+    IssueExpandSerializer,
+    ModuleIssueSerializer,
+    ModuleSerializer,
+    ProjectSerializer,
+)
+from plane.db.models import (
+    Cycle,
+    CycleIssue,
+    Issue,
+    IssueComment,
+    Module,
+    ModuleIssue,
+    Project,
+    User,
+    Webhook,
+    WebhookLog,
+)
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.exception_logger import log_exception
 
 SERIALIZER_MAPPER = {
     "project": ProjectSerializer,
@@ -174,7 +175,7 @@ def webhook_task(self, webhook, slug, event, event_data, action, current_site):
     except Exception as e:
         if settings.DEBUG:
             print(e)
-        capture_exception(e)
+        log_exception(e)
         return
 
 
@@ -241,7 +242,7 @@ def send_webhook(event, payload, kw, action, slug, bulk, current_site):
     except Exception as e:
         if settings.DEBUG:
             print(e)
-        capture_exception(e)
+        log_exception(e)
         return
 
 
@@ -295,8 +296,8 @@ def send_webhook_deactivation_email(
         )
         msg.attach_alternative(html_content, "text/html")
         msg.send()
-
+        logging.getLogger("plane").info("Email sent successfully.")
         return
     except Exception as e:
-        print(e)
+        log_exception(e)
         return

apiserver/plane/bgtasks/workspace_invitation_task.py

@@ -1,18 +1,18 @@
 # Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
-from sentry_sdk import capture_exception
 
 # Module imports
-from plane.db.models import Workspace, WorkspaceMemberInvite, User
+from plane.db.models import User, Workspace, WorkspaceMemberInvite
 from plane.license.utils.instance_value import get_email_configuration
+from plane.utils.exception_logger import log_exception
 
 
 @shared_task
@@ -76,14 +76,12 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor):
         )
         msg.attach_alternative(html_content, "text/html")
         msg.send()
+        logging.getLogger("plane").info("Email sent succesfully")
 
         return
-    except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist):
+    except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e:
-        print("Workspace or WorkspaceMember Invite Does not exists")
+        log_exception(e)
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        log_exception(e)
-        if settings.DEBUG:
-            print(e)
-        capture_exception(e)
         return

apiserver/plane/db/models/user.py

@@ -1,16 +1,17 @@
 # Python imports
-import uuid
-import string
 import random
+import string
+import uuid
+
 import pytz
+from django.contrib.auth.models import (
+    AbstractBaseUser,
+    PermissionsMixin,
+    UserManager,
+)
 
 # Django imports
 from django.db import models
-from django.contrib.auth.models import (
-    AbstractBaseUser,
-    UserManager,
-    PermissionsMixin,
-)
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils import timezone

apiserver/plane/settings/common.py

@@ -3,19 +3,20 @@
 # Python imports
 import os
 import ssl
-import certifi
 from datetime import timedelta
 from urllib.parse import urlparse
 
-# Django imports
+import certifi
-from django.core.management.utils import get_random_secret_key
 
 # Third party imports
 import dj_database_url
 import sentry_sdk
+
+# Django imports
+from django.core.management.utils import get_random_secret_key
+from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
-from sentry_sdk.integrations.celery import CeleryIntegration
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -23,7 +24,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SECRET_KEY = os.environ.get("SECRET_KEY", get_random_secret_key())
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = int(os.environ.get("DEBUG", "0"))
 
 # Allowed Hosts
 ALLOWED_HOSTS = ["*"]

apiserver/plane/settings/local.py

@@ -7,8 +7,8 @@ from .common import *  # noqa
 DEBUG = True
 
 # Debug Toolbar settings
-INSTALLED_APPS += ("debug_toolbar",)
+INSTALLED_APPS += ("debug_toolbar",)  # noqa
-MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
+MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)  # noqa
 
 DEBUG_TOOLBAR_PATCH_SETTINGS = False
 
@@ -18,7 +18,7 @@ EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": REDIS_URL,
+        "LOCATION": REDIS_URL,  # noqa
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
         },
@@ -28,7 +28,7 @@ CACHES = {
 INTERNAL_IPS = ("127.0.0.1",)
 
 MEDIA_URL = "/uploads/"
-MEDIA_ROOT = os.path.join(BASE_DIR, "uploads")
+MEDIA_ROOT = os.path.join(BASE_DIR, "uploads")  # noqa
 
 CORS_ALLOWED_ORIGINS = [
     "http://localhost:3000",
@@ -36,3 +36,38 @@ CORS_ALLOWED_ORIGINS = [
     "http://localhost:4000",
     "http://127.0.0.1:4000",
 ]
+
+LOG_DIR = os.path.join(BASE_DIR, "logs")  # noqa
+
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
+            "style": "{",
+        },
+    },
+    "handlers": {
+        "console": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+        },
+    },
+    "loggers": {
+        "django.request": {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": False,
+        },
+        "plane": {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": False,
+        },
+    },
+}

apiserver/plane/settings/production.py

@@ -1,15 +1,16 @@
 """Production settings"""
 
 import os
+
 from .common import *  # noqa
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = int(os.environ.get("DEBUG", 0)) == 1
-
+DEBUG = True
 # Honor the 'X-Forwarded-Proto' header for request.is_secure()
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 
-INSTALLED_APPS += ("scout_apm.django",)
+INSTALLED_APPS += ("scout_apm.django",)  # noqa
 
 # Honor the 'X-Forwarded-Proto' header for request.is_secure()
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
@@ -18,3 +19,62 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 SCOUT_MONITOR = os.environ.get("SCOUT_MONITOR", False)
 SCOUT_KEY = os.environ.get("SCOUT_KEY", "")
 SCOUT_NAME = "Plane"
+
+LOG_DIR = os.path.join(BASE_DIR, "logs")  # noqa
+
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
+
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
+            "style": "{",
+        },
+        "json": {
+            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",
+            "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+            "level": "INFO",
+        },
+        "file": {
+            "class": "plane.utils.logging.SizedTimedRotatingFileHandler",
+            "filename": (
+                os.path.join(BASE_DIR, "logs", "plane-debug.log")  # noqa
+                if DEBUG
+                else os.path.join(BASE_DIR, "logs", "plane-error.log")  # noqa
+            ),
+            "when": "s",
+            "maxBytes": 1024 * 1024 * 1,
+            "interval": 1,
+            "backupCount": 5,
+            "formatter": "json",
+            "level": "DEBUG" if DEBUG else "ERROR",
+        },
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["console", "file"],
+            "level": "INFO",
+            "propagate": True,
+        },
+        "django.request": {
+            "handlers": ["console", "file"],
+            "level": "INFO",
+            "propagate": False,
+        },
+        "plane": {
+            "level": "DEBUG" if DEBUG else "ERROR",
+            "handlers": ["console", "file"],
+            "propagate": False,
+        },
+    },
+}

apiserver/plane/settings/test.py

@@ -7,6 +7,6 @@ DEBUG = True
 # Send it in a dummy outbox
 EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
 
-INSTALLED_APPS.append(
+INSTALLED_APPS.append(  # noqa
     "plane.tests",
 )

apiserver/plane/space/views/base.py

@@ -1,25 +1,25 @@
 # Python imports
 import zoneinfo
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.db import IntegrityError
 
 # Django imports
 from django.urls import resolve
-from django.conf import settings
 from django.utils import timezone
-from django.db import IntegrityError
+from django_filters.rest_framework import DjangoFilterBackend
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
 
 # Third part imports
 from rest_framework import status
-from rest_framework.viewsets import ModelViewSet
-from rest_framework.response import Response
 from rest_framework.exceptions import APIException
-from rest_framework.views import APIView
 from rest_framework.filters import SearchFilter
 from rest_framework.permissions import IsAuthenticated
-from sentry_sdk import capture_exception
+from rest_framework.response import Response
-from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework.views import APIView
+from rest_framework.viewsets import ModelViewSet
 
 # Module imports
+from plane.utils.exception_logger import log_exception
 from plane.utils.paginator import BasePaginator
 
 
@@ -57,7 +57,7 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
         try:
             return self.model.objects.all()
         except Exception as e:
-            capture_exception(e)
+            log_exception(e)
             raise APIException(
                 "Please check the view", status.HTTP_400_BAD_REQUEST
             )
@@ -90,14 +90,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
                 )
 
             if isinstance(e, KeyError):
-                capture_exception(e)
+                log_exception(e)
                 return Response(
                     {"error": "The required key does not exist."},
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
-            print(e) if settings.DEBUG else print("Server Error")
+            log_exception(e)
-            capture_exception(e)
             return Response(
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -185,9 +184,7 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
-            if settings.DEBUG:
+            log_exception(e)
-                print(e)
-            capture_exception(e)
             return Response(
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,

apiserver/plane/utils/exception_logger.py

@@ -0,0 +1,15 @@
+# Python imports
+import logging
+
+# Third party imports
+from sentry_sdk import capture_exception
+
+
+def log_exception(e):
+    # Log the error
+    logger = logging.getLogger("plane")
+    logger.error(e)
+
+    # Capture in sentry if configured
+    capture_exception(e)
+    return

apiserver/plane/utils/logging.py

@@ -0,0 +1,46 @@
+import logging.handlers as handlers
+import time
+
+
+class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler):
+    """
+    Handler for logging to a set of files, which switches from one file
+    to the next when the current file reaches a certain size, or at certain
+    timed intervals
+    """
+
+    def __init__(
+        self,
+        filename,
+        maxBytes=0,
+        backupCount=0,
+        encoding=None,
+        delay=0,
+        when="h",
+        interval=1,
+        utc=False,
+    ):
+        handlers.TimedRotatingFileHandler.__init__(
+            self, filename, when, interval, backupCount, encoding, delay, utc
+        )
+        self.maxBytes = maxBytes
+
+    def shouldRollover(self, record):
+        """
+        Determine if rollover should occur.
+
+        Basically, see if the supplied record would cause the file to exceed
+        the size limit we have.
+        """
+        if self.stream is None:  # delay was set...
+            self.stream = self._open()
+        if self.maxBytes > 0:  # are we rolling over?
+            msg = "%s\n" % self.format(record)
+            # due to non-posix-compliant Windows feature
+            self.stream.seek(0, 2)
+            if self.stream.tell() + len(msg) >= self.maxBytes:
+                return 1
+        t = int(time.time())
+        if t >= self.rolloverAt:
+            return 1
+        return 0

apiserver/requirements/base.txt

@@ -27,6 +27,7 @@ psycopg-binary==3.1.12
 psycopg-c==3.1.12
 scout-apm==2.26.1
 openpyxl==3.1.2
+python-json-logger==2.0.7
 beautifulsoup4==4.12.2
 dj-database-url==2.1.0
 posthog==3.0.2

deploy/selfhost/docker-compose.yml

@@ -70,6 +70,8 @@ services:
     command: ./bin/takeoff
     deploy:
       replicas: ${API_REPLICAS:-1}
+    volumes:
+      - logs_api:/code/plane/logs
     depends_on:
       - plane-db
       - plane-redis
@@ -80,6 +82,8 @@ services:
     pull_policy: ${PULL_POLICY:-always}
     restart: unless-stopped
     command: ./bin/worker
+    volumes:
+      - logs_worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -91,6 +95,8 @@ services:
     pull_policy: ${PULL_POLICY:-always}
     restart: unless-stopped
     command: ./bin/beat
+    volumes:
+      - logs_beat-worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -104,6 +110,8 @@ services:
     command: >
       sh -c "python manage.py wait_for_db &&
            python manage.py migrate"
+    volumes:
+      - logs_migrator:/code/plane/logs
     depends_on:
       - plane-db
       - plane-redis
@@ -149,3 +157,7 @@ volumes:
   pgdata:
   redisdata:
   uploads:
+  logs_api:
+  logs_worker:
+  logs_beat-worker:
+  logs_migrator: