Merge branch 'develop' into dev/api_logging

.gitignore

@@ -51,6 +51,7 @@ staticfiles
 mediafiles
 .env
 .DS_Store
+logs/
 
 node_modules/
 assets/dist/

apiserver/.env.example

@@ -70,6 +70,5 @@ ENABLE_MAGIC_LINK_LOGIN="0"
 # Email redirections and minio domain settings
 WEB_URL="http://localhost"
 
-
 # Gunicorn Workers
 GUNICORN_WORKERS=2

apiserver/plane/api/views/base.py

@@ -1,5 +1,6 @@
 # Python imports
 import zoneinfo
+import logging
 
 # Django imports
 from django.urls import resolve
@@ -58,6 +59,8 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
         try:
             return self.model.objects.all()
         except Exception as e:
+            logger = logging.getLogger("plane")
+            logger.error(e)
             capture_exception(e)
             raise APIException("Please check the view", status.HTTP_400_BAD_REQUEST)
         
@@ -81,10 +84,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
                 return Response({"error": f"{model_name} does not exist."}, status=status.HTTP_404_NOT_FOUND)
             
             if isinstance(e, KeyError):
+                logger = logging.getLogger("plane")
+                logger.error(e)
                 capture_exception(e)
                 return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST)
-            
-            print(e) if settings.DEBUG else print("Server Error")
+
+            logger = logging.getLogger("plane")
+            logger.error(e)
             capture_exception(e)
             return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
@@ -161,8 +167,9 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
             
             if isinstance(e, KeyError):
                 return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST)
-            
-            print(e) if settings.DEBUG else print("Server Error")
+
+            logger = logging.getLogger("plane")
+            logger.error(e)
             capture_exception(e)
             return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 

apiserver/plane/api/views/oauth.py

@@ -2,6 +2,7 @@
 import uuid
 import requests
 import os
+import logging
 
 # Django imports
 from django.utils import timezone
@@ -48,6 +49,8 @@ def validate_google_token(token, client_id):
         }
         return data
     except Exception as e:
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         raise exceptions.AuthenticationFailed("Error with Google connection.")
 

apiserver/plane/bgtasks/analytic_plot_export.py

@@ -1,6 +1,7 @@
 # Python imports
 import csv
 import io
+import logging
 
 # Django imports
 from django.core.mail import EmailMultiAlternatives
@@ -419,6 +420,6 @@ def analytic_export_task(email, data, slug):
         csv_buffer = generate_csv_from_rows(rows)
         send_export_email(email, slug, csv_buffer)
     except Exception as e:
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)

apiserver/plane/bgtasks/email_verification_task.py

@@ -1,3 +1,6 @@
+# Python imports
+import logging
+
 # Django imports
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
@@ -39,8 +42,7 @@ def email_verification(first_name, email, token, current_site):
         msg.send()
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/export_task.py

@@ -4,6 +4,8 @@ import io
 import json
 import boto3
 import zipfile
+import logging
+from urllib.parse import urlparse, urlunparse
 
 # Django imports
 from django.conf import settings
@@ -367,8 +369,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s
         exporter_instance.status = "failed"
         exporter_instance.reason = str(e)
         exporter_instance.save(update_fields=["status", "reason"])
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/forgot_password_task.py

@@ -1,3 +1,6 @@
+# Python imports
+import logging
+
 # Django imports
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
@@ -35,8 +38,7 @@ def forgot_password(first_name, email, uidb64, token, current_site):
         msg.send()
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/importer_task.py

@@ -2,6 +2,10 @@
 import json
 import requests
 import uuid
+import jwt
+import logging
+from datetime import datetime
+
 
 # Django imports
 from django.conf import settings
@@ -187,8 +191,7 @@ def service_importer(service, importer_id):
         importer = Importer.objects.get(pk=importer_id)
         importer.status = "failed"
         importer.save()
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/issue_activites_task.py

@@ -1,6 +1,7 @@
 # Python imports
 import json
 import requests
+import logging
 
 # Django imports
 from django.conf import settings
@@ -1568,8 +1569,7 @@ def issue_activity(
 
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/issue_automation_task.py

@@ -1,6 +1,7 @@
 # Python imports
 import json
 from datetime import timedelta
+import logging
 
 # Django imports
 from django.utils import timezone
@@ -86,8 +87,8 @@ def archive_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return
 
@@ -158,7 +159,7 @@ def close_old_issues():
                     ]
         return
     except Exception as e:
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/magic_link_code_task.py

@@ -1,3 +1,6 @@
+# Python imports
+import logging
+
 # Django imports
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
@@ -30,8 +33,7 @@ def magic_link(email, key, token, current_site):
         msg.send()
         return
     except Exception as e:
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
         return

apiserver/plane/bgtasks/project_invitation_task.py

@@ -1,3 +1,6 @@
+# Python imports 
+import logging
+
 # Django imports
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
@@ -50,8 +53,7 @@ def project_invitation(email, project_id, token, current_site):
     except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist) as e:
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/user_welcome_task.py

@@ -1,3 +1,6 @@
+# Python imports
+import logging
+
 # Django imports
 from django.conf import settings
 
@@ -29,8 +32,7 @@ def send_welcome_slack(user_id, created, message):
                     print(f"Got an error: {e.response['error']}")
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/bgtasks/workspace_invitation_task.py

@@ -1,3 +1,6 @@
+# Python imports
+import logging
+
 # Django imports
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
@@ -66,8 +69,7 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor):
     except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e:
         return
     except Exception as e:
-        # Print logs if in DEBUG mode
-        if settings.DEBUG:
-            print(e)
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/db/models/user.py

@@ -3,6 +3,7 @@ import uuid
 import string
 import random
 import pytz
+import logging
 
 # Django imports
 from django.db import models
@@ -139,5 +140,7 @@ def send_welcome_slack(sender, instance, created, **kwargs):
                     print(f"Got an error: {e.response['error']}")
         return
     except Exception as e:
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return

apiserver/plane/settings/common.py

@@ -215,3 +215,46 @@ CELERY_TIMEZONE = TIME_ZONE
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_ACCEPT_CONTENT = ['application/json']
 CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task")
+
+LOG_DIR = os.path.join(BASE_DIR, 'logs')
+
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
+
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
+            "style": "{",
+        },
+        "json": {
+            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",
+            "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+        },
+        "file": {
+            "class": "logging.handlers.TimedRotatingFileHandler",
+            "filename": os.path.join(BASE_DIR, 'logs', 'debug.log') if DEBUG else os.path.join(BASE_DIR, 'logs', 'error.log'),
+            "when": "midnight",
+            "interval": 1,  # One day
+            "backupCount": 5,  # Keep last 5 days of logs,
+            "formatter": "json",
+        },
+    },
+    "loggers": {
+        "plane": {
+            "level": "DEBUG" if DEBUG else "ERROR",
+            "handlers": ["console", "file"],
+            "propagate": False,
+        },
+    },
+}
+

apiserver/plane/settings/selfhosted.py

@@ -70,9 +70,7 @@ AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "secret-key")
 # The name of the bucket to store files in.
 AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_S3_BUCKET_NAME", "uploads")
 # The full URL to the S3 endpoint. Leave blank to use the default region URL.
-AWS_S3_ENDPOINT_URL = os.environ.get(
-    "AWS_S3_ENDPOINT_URL", "http://plane-minio:9000"
-)
+AWS_S3_ENDPOINT_URL = os.environ.get("AWS_S3_ENDPOINT_URL", "http://plane-minio:9000")
 # Default permissions
 AWS_DEFAULT_ACL = "public-read"
 AWS_QUERYSTRING_AUTH = False
@@ -98,7 +96,7 @@ CSRF_COOKIE_SECURE = True
 # Redis URL
 REDIS_URL = os.environ.get("REDIS_URL")
 
-# Caches 
+# Caches
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",

apiserver/plane/utils/importers/jira.py

@@ -1,5 +1,9 @@
+# Python imports
 import requests
 from requests.auth import HTTPBasicAuth
+import logging
+
+# Third party imports
 from sentry_sdk import capture_exception
 
 
@@ -49,5 +53,7 @@ def jira_project_issue_summary(email, api_token, project_key, hostname):
             ),
         }
     except Exception as e:
+        logger = logging.getLogger("plane")
+        logger.error(e)
         capture_exception(e)
         return {"error": "Something went wrong could not fetch information from jira"}

apiserver/requirements/base.txt

@@ -34,4 +34,5 @@ psycopg-binary==3.1.10
 psycopg-c==3.1.10
 scout-apm==2.26.1
 openpyxl==3.1.2
-beautifulsoup4==4.12.2
+python-json-logger==2.0.7
+beautifulsoup4==4.12.2

deploy/selfhost/docker-compose.yml

@@ -89,6 +89,8 @@ services:
     command: ./bin/takeoff
     deploy:
       replicas: ${API_REPLICAS:-1}
+    volumes:
+      - logs/api:/code/plane/logs
     depends_on:
       - plane-db
       - plane-redis
@@ -100,6 +102,8 @@ services:
     image: makeplane/plane-backend:${APP_RELEASE:-latest}
     restart: unless-stopped
     command: ./bin/worker
+    volumes:
+      - logs/worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -112,6 +116,8 @@ services:
     image: makeplane/plane-backend:${APP_RELEASE:-latest}
     restart: unless-stopped
     command: ./bin/beat
+    volumes:
+      - logs/beat-worker:/code/plane/logs
     depends_on:
       - api
       - plane-db
@@ -168,3 +174,4 @@ volumes:
   pgdata:
   redisdata:
   uploads:
+  logs: