Merge branch 'dev/api_logging' of github.com:makeplane/plane into dev/api_logging

.gitignore

@@ -51,6 +51,7 @@ staticfiles
 mediafiles
 .env
 .DS_Store
+logs/
 
 node_modules/
 assets/dist/

apiserver/.env.example

@@ -63,4 +63,3 @@ WEB_URL="http://localhost"
 
 # Gunicorn Workers
 GUNICORN_WORKERS=2
-

apiserver/plane/api/views/base.py

@@ -1,26 +1,27 @@
 # Python imports
-import zoneinfo
+import logging
 from urllib.parse import urlparse
 
+import zoneinfo
 
 # Django imports
 from django.conf import settings
-from django.db import IntegrityError
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.db import IntegrityError
 from django.utils import timezone
+from rest_framework import status
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
 
 # Third party imports
 from rest_framework.views import APIView
-from rest_framework.response import Response
-from rest_framework.permissions import IsAuthenticated
-from rest_framework import status
 from sentry_sdk import capture_exception
 
 # Module imports
 from plane.api.middleware.api_authentication import APIKeyAuthentication
 from plane.api.rate_limit import ApiKeyRateThrottle
-from plane.utils.paginator import BasePaginator
 from plane.bgtasks.webhook_task import send_webhook
+from plane.utils.paginator import BasePaginator
 
 
 class TimezoneMixin:
@@ -104,6 +105,35 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
                     status=status.HTTP_400_BAD_REQUEST,
                 )
 
+            if isinstance(e, ValidationError):
+                return Response(
+                    {"error": "Please provide valid detail"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            if isinstance(e, ObjectDoesNotExist):
+                model_name = str(exc).split(" matching query does not exist.")[
+                    0
+                ]
+                return Response(
+                    {"error": f"{model_name} does not exist."},
+                    status=status.HTTP_404_NOT_FOUND,
+                )
+
+            if isinstance(e, KeyError):
+                return Response(
+                    {"error": f"key {e} does not exist"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            logger = logging.getLogger("plane")
+            logger.error(e)
+            capture_exception(e)
+            return Response(
+                {"error": "Something went wrong please try again later"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
             if isinstance(e, ValidationError):
                 return Response(
                     {

apiserver/plane/bgtasks/analytic_plot_export.py

@@ -1,22 +1,23 @@
 # Python imports
 import csv
 import io
+import logging
+
+# Third party imports
+from celery import shared_task
+from django.conf import settings
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
 from sentry_sdk import capture_exception
 
 # Module imports
 from plane.db.models import Issue
+from plane.license.utils.instance_value import get_email_configuration
 from plane.utils.analytics_plot import build_graph_plot
 from plane.utils.issue_filters import issue_filters
-from plane.license.utils.instance_value import get_email_configuration
 
 row_mapping = {
     "state__name": "State",
@@ -210,9 +211,9 @@ def generate_segmented_rows(
                 None,
             )
             if assignee:
-                generated_row[
+                generated_row[0] = (
-                    0
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
         if x_axis == LABEL_ID:
             label = next(
@@ -279,9 +280,9 @@ def generate_segmented_rows(
                 None,
             )
             if assignee:
-                row_zero[
+                row_zero[index + 2] = (
-                    index + 2
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
     if segmented == LABEL_ID:
         for index, segm in enumerate(row_zero[2:]):
@@ -366,9 +367,9 @@ def generate_non_segmented_rows(
                 None,
             )
             if assignee:
-                row[
+                row[0] = (
-                    0
+                    f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
-                ] = f"{assignee['assignees__first_name']} {assignee['assignees__last_name']}"
+                )
 
         if x_axis == LABEL_ID:
             label = next(
@@ -506,8 +507,7 @@ def analytic_export_task(email, data, slug):
         send_export_email(email, slug, csv_buffer, rows)
         return
     except Exception as e:
-        print(e)
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/export_task.py

@@ -4,6 +4,8 @@ import io
 import json
 import boto3
 import zipfile
+import logging
+from urllib.parse import urlparse, urlunparse
 
 # Django imports
 from django.conf import settings
@@ -403,8 +405,7 @@ def issue_export_task(
         exporter_instance.status = "failed"
         exporter_instance.reason = str(e)
         exporter_instance.save(update_fields=["status", "reason"])
-        # Print logs if in DEBUG mode
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/forgot_password_task.py

@@ -1,13 +1,13 @@
-# Python import
+# Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
 from sentry_sdk import capture_exception
 
 # Module imports
@@ -62,8 +62,7 @@ def forgot_password(first_name, email, uidb64, token, current_site):
         msg.send()
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/issue_activites_task.py

@@ -1,6 +1,7 @@
 # Python imports
 import json
 import requests
+import logging
 
 # Django imports
 from django.conf import settings
@@ -1668,8 +1669,7 @@ def issue_activity(
 
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/issue_automation_task.py

@@ -1,6 +1,7 @@
 # Python imports
 import json
 from datetime import timedelta
+import logging
 
 # Django imports
 from django.utils import timezone
@@ -96,8 +97,8 @@ def archive_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
+        logger = logging.getLogger("plane")
-            print(e)
+        logger.error(e)
         capture_exception(e)
         return
 
@@ -179,7 +180,7 @@ def close_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
+        logger = logging.getLogger("plane")
-            print(e)
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/magic_link_code_task.py

@@ -1,13 +1,14 @@
 # Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
+from django.conf import settings
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
 from sentry_sdk import capture_exception
 
 # Module imports
@@ -54,9 +55,7 @@ def magic_link(email, key, token, current_site):
         msg.send()
         return
     except Exception as e:
-        print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
         return

apiserver/plane/bgtasks/project_invitation_task.py

@@ -1,17 +1,17 @@
-# Python import
+# Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
 from sentry_sdk import capture_exception
 
 # Module imports
-from plane.db.models import Project, User, ProjectMemberInvite
+from plane.db.models import Project, ProjectMemberInvite, User
 from plane.license.utils.instance_value import get_email_configuration
 
 
@@ -77,8 +77,7 @@ def project_invitation(email, project_id, token, current_site, invitor):
     except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist):
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/workspace_invitation_task.py

@@ -1,17 +1,17 @@
 # Python imports
+import logging
+
+# Third party imports
+from celery import shared_task
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives, get_connection
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
-from django.conf import settings
-
-# Third party imports
-from celery import shared_task
 from sentry_sdk import capture_exception
 
 # Module imports
-from plane.db.models import Workspace, WorkspaceMemberInvite, User
+from plane.db.models import User, Workspace, WorkspaceMemberInvite
 from plane.license.utils.instance_value import get_email_configuration
 
 
@@ -82,8 +82,7 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor):
         print("Workspace or WorkspaceMember Invite Does not exists")
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
+        logger = logging.getLogger("plane")
-        if settings.DEBUG:
+        logger.error(e)
-            print(e)
         capture_exception(e)
         return

apiserver/plane/db/models/user.py

@@ -1,16 +1,17 @@
 # Python imports
-import uuid
-import string
 import random
+import string
+import uuid
+
 import pytz
+from django.contrib.auth.models import (
+    AbstractBaseUser,
+    PermissionsMixin,
+    UserManager,
+)
 
 # Django imports
 from django.db import models
-from django.contrib.auth.models import (
-    AbstractBaseUser,
-    UserManager,
-    PermissionsMixin,
-)
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils import timezone

apiserver/plane/settings/common.py

@@ -3,19 +3,20 @@
 # Python imports
 import os
 import ssl
-import certifi
 from datetime import timedelta
 from urllib.parse import urlparse
 
-# Django imports
+import certifi
-from django.core.management.utils import get_random_secret_key
 
 # Third party imports
 import dj_database_url
 import sentry_sdk
+
+# Django imports
+from django.core.management.utils import get_random_secret_key
+from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
-from sentry_sdk.integrations.celery import CeleryIntegration
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -345,3 +346,49 @@ INSTANCE_KEY = os.environ.get(
 SKIP_ENV_VAR = os.environ.get("SKIP_ENV_VAR", "1") == "1"
 
 DATA_UPLOAD_MAX_MEMORY_SIZE = int(os.environ.get("FILE_SIZE_LIMIT", 5242880))
+
+LOG_DIR = os.path.join(BASE_DIR, "logs")
+
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
+
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
+            "style": "{",
+        },
+        "json": {
+            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",
+            "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+        },
+        "file": {
+            "class": "logging.handlers.TimedRotatingFileHandler",
+            "filename": (
+                os.path.join(BASE_DIR, "logs", "debug.log")
+                if DEBUG
+                else os.path.join(BASE_DIR, "logs", "error.log")
+            ),
+            "when": "midnight",
+            "interval": 1,  # One day
+            "backupCount": 5,  # Keep last 5 days of logs,
+            "formatter": "json",
+        },
+    },
+    "loggers": {
+        "plane": {
+            "level": "DEBUG" if DEBUG else "ERROR",
+            "handlers": ["console", "file"],
+            "propagate": False,
+        },
+    },
+}

deploy/selfhost/docker-compose.yml

@@ -90,6 +90,8 @@ services:
     command: ./bin/takeoff
     deploy:
       replicas: ${API_REPLICAS:-1}
+    volumes:
+      - logs/api:/code/plane/logs
     depends_on:
       - plane-db
       - plane-redis
@@ -100,6 +102,8 @@ services:
     pull_policy: ${PULL_POLICY:-always}
     restart: unless-stopped
     command: ./bin/worker
+    volumes:
+      - logs/worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -111,6 +115,8 @@ services:
     pull_policy: ${PULL_POLICY:-always}
     restart: unless-stopped
     command: ./bin/beat
+    volumes:
+      - logs/beat-worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -169,3 +175,4 @@ volumes:
   pgdata:
   redisdata:
   uploads:
+  logs: