From 93cfa13955238cd0134de641c498ebbe77a2e60c Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Mon, 30 Oct 2023 14:41:44 +0530 Subject: [PATCH 1/3] dev: enable logger instead of printing --- apiserver/plane/api/views/base.py | 15 +++++++--- apiserver/plane/api/views/oauth.py | 3 ++ .../plane/bgtasks/analytic_plot_export.py | 5 ++-- .../plane/bgtasks/email_verification_task.py | 8 ++++-- apiserver/plane/bgtasks/export_task.py | 6 ++-- .../plane/bgtasks/forgot_password_task.py | 8 ++++-- apiserver/plane/bgtasks/importer_task.py | 6 ++-- .../plane/bgtasks/issue_activites_task.py | 6 ++-- .../plane/bgtasks/issue_automation_task.py | 9 +++--- .../plane/bgtasks/magic_link_code_task.py | 8 ++++-- .../plane/bgtasks/project_invitation_task.py | 8 ++++-- apiserver/plane/bgtasks/user_welcome_task.py | 8 ++++-- .../bgtasks/workspace_invitation_task.py | 8 ++++-- apiserver/plane/db/models/user.py | 3 ++ apiserver/plane/settings/selfhosted.py | 28 ++++++++----------- apiserver/plane/utils/importers/jira.py | 6 ++++ 16 files changed, 82 insertions(+), 53 deletions(-) diff --git a/apiserver/plane/api/views/base.py b/apiserver/plane/api/views/base.py index 7ab660e81..72ced3398 100644 --- a/apiserver/plane/api/views/base.py +++ b/apiserver/plane/api/views/base.py @@ -1,5 +1,6 @@ # Python imports import zoneinfo +import logging # Django imports from django.urls import resolve @@ -58,6 +59,8 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): try: return self.model.objects.all() except Exception as e: + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) raise APIException("Please check the view", status.HTTP_400_BAD_REQUEST) @@ -81,10 +84,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator): return Response({"error": f"{model_name} does not exist."}, status=status.HTTP_404_NOT_FOUND) if isinstance(e, KeyError): + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST) - - print(e) if settings.DEBUG else print("Server Error") + + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -161,8 +167,9 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator): if isinstance(e, KeyError): return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST) - - print(e) if settings.DEBUG else print("Server Error") + + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/apiserver/plane/api/views/oauth.py b/apiserver/plane/api/views/oauth.py index 04c83813a..a7740b49a 100644 --- a/apiserver/plane/api/views/oauth.py +++ b/apiserver/plane/api/views/oauth.py @@ -2,6 +2,7 @@ import uuid import requests import os +import logging # Django imports from django.utils import timezone @@ -49,6 +50,8 @@ def validate_google_token(token, client_id): } return data except Exception as e: + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) raise exceptions.AuthenticationFailed("Error with Google connection.") diff --git a/apiserver/plane/bgtasks/analytic_plot_export.py b/apiserver/plane/bgtasks/analytic_plot_export.py index a041fd169..bf507a810 100644 --- a/apiserver/plane/bgtasks/analytic_plot_export.py +++ b/apiserver/plane/bgtasks/analytic_plot_export.py @@ -1,6 +1,7 @@ # Python imports import csv import io +import logging # Django imports from django.core.mail import EmailMultiAlternatives @@ -420,6 +421,6 @@ def analytic_export_task(email, data, slug): csv_buffer = generate_csv_from_rows(rows) send_export_email(email, slug, csv_buffer) except Exception as e: - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) diff --git a/apiserver/plane/bgtasks/email_verification_task.py b/apiserver/plane/bgtasks/email_verification_task.py index 93b15c425..76c2441b3 100644 --- a/apiserver/plane/bgtasks/email_verification_task.py +++ b/apiserver/plane/bgtasks/email_verification_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -39,8 +42,7 @@ def email_verification(first_name, email, token, current_site): msg.send() return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index a45120eb5..959c7e418 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -4,6 +4,7 @@ import io import json import boto3 import zipfile +import logging from urllib.parse import urlparse, urlunparse # Django imports @@ -368,8 +369,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s exporter_instance.status = "failed" exporter_instance.reason = str(e) exporter_instance.save(update_fields=["status", "reason"]) - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/forgot_password_task.py b/apiserver/plane/bgtasks/forgot_password_task.py index 95828765c..c1883f343 100644 --- a/apiserver/plane/bgtasks/forgot_password_task.py +++ b/apiserver/plane/bgtasks/forgot_password_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -37,8 +40,7 @@ def forgot_password(first_name, email, uidb64, token, current_site): msg.send() return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/importer_task.py b/apiserver/plane/bgtasks/importer_task.py index 20dc65e51..fe6a08a51 100644 --- a/apiserver/plane/bgtasks/importer_task.py +++ b/apiserver/plane/bgtasks/importer_task.py @@ -3,6 +3,7 @@ import json import requests import uuid import jwt +import logging from datetime import datetime # Django imports @@ -190,8 +191,7 @@ def service_importer(service, importer_id): importer = Importer.objects.get(pk=importer_id) importer.status = "failed" importer.save() - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/issue_activites_task.py b/apiserver/plane/bgtasks/issue_activites_task.py index 1dbaf8bc9..9a6d95b49 100644 --- a/apiserver/plane/bgtasks/issue_activites_task.py +++ b/apiserver/plane/bgtasks/issue_activites_task.py @@ -1,6 +1,7 @@ # Python imports import json import requests +import logging # Django imports from django.conf import settings @@ -1538,8 +1539,7 @@ def issue_activity( return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/issue_automation_task.py b/apiserver/plane/bgtasks/issue_automation_task.py index 68c64403a..f0e8c19e7 100644 --- a/apiserver/plane/bgtasks/issue_automation_task.py +++ b/apiserver/plane/bgtasks/issue_automation_task.py @@ -1,6 +1,7 @@ # Python imports import json from datetime import timedelta +import logging # Django imports from django.utils import timezone @@ -86,8 +87,8 @@ def archive_old_issues(): ] return except Exception as e: - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return @@ -158,7 +159,7 @@ def close_old_issues(): ] return except Exception as e: - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/magic_link_code_task.py b/apiserver/plane/bgtasks/magic_link_code_task.py index 91cc461bb..6bdfca2d4 100644 --- a/apiserver/plane/bgtasks/magic_link_code_task.py +++ b/apiserver/plane/bgtasks/magic_link_code_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -30,8 +33,7 @@ def magic_link(email, key, token, current_site): msg.send() return except Exception as e: + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) return diff --git a/apiserver/plane/bgtasks/project_invitation_task.py b/apiserver/plane/bgtasks/project_invitation_task.py index 8b8ef6e48..7d91ad886 100644 --- a/apiserver/plane/bgtasks/project_invitation_task.py +++ b/apiserver/plane/bgtasks/project_invitation_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -50,8 +53,7 @@ def project_invitation(email, project_id, token, current_site): except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist) as e: return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/user_welcome_task.py b/apiserver/plane/bgtasks/user_welcome_task.py index 33f4b5686..6a7cbae56 100644 --- a/apiserver/plane/bgtasks/user_welcome_task.py +++ b/apiserver/plane/bgtasks/user_welcome_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.conf import settings @@ -29,8 +32,7 @@ def send_welcome_slack(user_id, created, message): print(f"Got an error: {e.response['error']}") return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/bgtasks/workspace_invitation_task.py b/apiserver/plane/bgtasks/workspace_invitation_task.py index d84a0b414..d4fa4da82 100644 --- a/apiserver/plane/bgtasks/workspace_invitation_task.py +++ b/apiserver/plane/bgtasks/workspace_invitation_task.py @@ -1,3 +1,6 @@ +# Python imports +import logging + # Django imports from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -66,8 +69,7 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor): except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e: return except Exception as e: - # Print logs if in DEBUG mode - if settings.DEBUG: - print(e) + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/db/models/user.py b/apiserver/plane/db/models/user.py index e90e19c5e..5b0dd9c7f 100644 --- a/apiserver/plane/db/models/user.py +++ b/apiserver/plane/db/models/user.py @@ -3,6 +3,7 @@ import uuid import string import random import pytz +import logging # Django imports from django.db import models @@ -139,5 +140,7 @@ def send_welcome_slack(sender, instance, created, **kwargs): print(f"Got an error: {e.response['error']}") return except Exception as e: + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return diff --git a/apiserver/plane/settings/selfhosted.py b/apiserver/plane/settings/selfhosted.py index ad6296ba9..43c4dcb26 100644 --- a/apiserver/plane/settings/selfhosted.py +++ b/apiserver/plane/settings/selfhosted.py @@ -70,9 +70,7 @@ AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "secret-key") # The name of the bucket to store files in. AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_S3_BUCKET_NAME", "uploads") # The full URL to the S3 endpoint. Leave blank to use the default region URL. -AWS_S3_ENDPOINT_URL = os.environ.get( - "AWS_S3_ENDPOINT_URL", "http://plane-minio:9000" -) +AWS_S3_ENDPOINT_URL = os.environ.get("AWS_S3_ENDPOINT_URL", "http://plane-minio:9000") # Default permissions AWS_DEFAULT_ACL = "public-read" AWS_QUERYSTRING_AUTH = False @@ -98,7 +96,7 @@ CSRF_COOKIE_SECURE = True # Redis URL REDIS_URL = os.environ.get("REDIS_URL") -# Caches +# Caches CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", @@ -137,31 +135,29 @@ LOGGING = { "style": "{", }, "json": { - '()': 'pythonjsonlogger.jsonlogger.JsonFormatter', - 'fmt': '%(levelname)s %(asctime)s %(name)s %(message)s' + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s", }, }, "handlers": { "console": { - "level": "DEBUG", "class": "logging.StreamHandler", "formatter": "verbose", }, "file": { - "level": "DEBUG", - 'class': 'logging.handlers.TimedRotatingFileHandler', - 'filename': "debug.log", - 'when': 'midnight', - 'interval': 1, # One day - 'backupCount': 5, # Keep last 5 days of logs, - "formatter": "json" + "class": "logging.handlers.TimedRotatingFileHandler", + "filename": "debug.log", + "when": "midnight", + "interval": 1, # One day + "backupCount": 5, # Keep last 5 days of logs, + "formatter": "json", }, }, "loggers": { "plane": { + "level": "ERROR", "handlers": ["console", "file"], - "propagate": True, + "propagate": False, }, }, } - diff --git a/apiserver/plane/utils/importers/jira.py b/apiserver/plane/utils/importers/jira.py index b427ba14f..151c08e14 100644 --- a/apiserver/plane/utils/importers/jira.py +++ b/apiserver/plane/utils/importers/jira.py @@ -1,5 +1,9 @@ +# Python imports import requests from requests.auth import HTTPBasicAuth +import logging + +# Third party imports from sentry_sdk import capture_exception @@ -49,5 +53,7 @@ def jira_project_issue_summary(email, api_token, project_key, hostname): ), } except Exception as e: + logger = logging.getLogger("plane") + logger.error(e) capture_exception(e) return {"error": "Something went wrong could not fetch information from jira"} From 099cd5955cec688f44ae8eb830f643bf132addcf Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Mon, 30 Oct 2023 14:42:25 +0530 Subject: [PATCH 2/3] dev: remove worker counts --- apiserver/.env.example | 3 --- apiserver/bin/takeoff | 2 +- deploy/selfhost/docker-compose.yml | 1 - deploy/selfhost/variables.env | 1 - 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/apiserver/.env.example b/apiserver/.env.example index 2c22a2a38..404ddde69 100644 --- a/apiserver/.env.example +++ b/apiserver/.env.example @@ -69,6 +69,3 @@ ENABLE_MAGIC_LINK_LOGIN="0" # Email redirections and minio domain settings WEB_URL="http://localhost" - -# Gunicorn workers -WORKERS=2 \ No newline at end of file diff --git a/apiserver/bin/takeoff b/apiserver/bin/takeoff index a1ab20507..dc25a14e2 100755 --- a/apiserver/bin/takeoff +++ b/apiserver/bin/takeoff @@ -6,4 +6,4 @@ python manage.py migrate # Create a Default User python bin/user_script.py -exec gunicorn -w $WORKERS -k uvicorn.workers.UvicornWorker plane.asgi:application --bind 0.0.0.0:8000 --max-requests 1200 --max-requests-jitter 1000 --access-logfile - +exec gunicorn -w 8 -k uvicorn.workers.UvicornWorker plane.asgi:application --bind 0.0.0.0:8000 --max-requests 1200 --max-requests-jitter 1000 --access-logfile - diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index 4a5fe550d..33a0f6673 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -10,7 +10,6 @@ x-app-env : &app-env - SENTRY_DSN=${SENTRY_DSN:-""} - GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET:-""} - DOCKERIZED=${DOCKERIZED:-1} - - WORKERS=${WORKERS:-2} #DB SETTINGS - PGHOST=${PGHOST:-plane-db} - PGDATABASE=${PGDATABASE:-plane} diff --git a/deploy/selfhost/variables.env b/deploy/selfhost/variables.env index b80c4c808..1e507a54b 100644 --- a/deploy/selfhost/variables.env +++ b/deploy/selfhost/variables.env @@ -12,7 +12,6 @@ NEXT_PUBLIC_DEPLOY_URL=http://localhost/spaces SENTRY_DSN="" GITHUB_CLIENT_SECRET="" DOCKERIZED=1 -WORKERS=2 #DB SETTINGS PGHOST=plane-db From 8d4e4a7485a5fdf56be20d50258b9c6d7a773011 Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Tue, 31 Oct 2023 12:38:52 +0530 Subject: [PATCH 3/3] dev: enable global level log settings --- .gitignore | 1 + apiserver/plane/settings/common.py | 43 ++++++++++++++++++++++++++ apiserver/plane/settings/selfhosted.py | 36 --------------------- deploy/selfhost/docker-compose.yml | 7 +++++ 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 7568602d3..f48505350 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ staticfiles mediafiles .env .DS_Store +logs/ node_modules/ assets/dist/ diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index 27da44d9c..6f233a06c 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -215,3 +215,46 @@ CELERY_TIMEZONE = TIME_ZONE CELERY_TASK_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task") + +LOG_DIR = os.path.join(BASE_DIR, 'logs') + +if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}", + "style": "{", + }, + "json": { + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + "file": { + "class": "logging.handlers.TimedRotatingFileHandler", + "filename": os.path.join(BASE_DIR, 'logs', 'debug.log') if DEBUG else os.path.join(BASE_DIR, 'logs', 'error.log'), + "when": "midnight", + "interval": 1, # One day + "backupCount": 5, # Keep last 5 days of logs, + "formatter": "json", + }, + }, + "loggers": { + "plane": { + "level": "DEBUG" if DEBUG else "ERROR", + "handlers": ["console", "file"], + "propagate": False, + }, + }, +} + diff --git a/apiserver/plane/settings/selfhosted.py b/apiserver/plane/settings/selfhosted.py index 43c4dcb26..799d2b5e7 100644 --- a/apiserver/plane/settings/selfhosted.py +++ b/apiserver/plane/settings/selfhosted.py @@ -125,39 +125,3 @@ OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1") OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", False) GPT_ENGINE = os.environ.get("GPT_ENGINE", "gpt-3.5-turbo") - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "verbose": { - "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}", - "style": "{", - }, - "json": { - "()": "pythonjsonlogger.jsonlogger.JsonFormatter", - "fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s", - }, - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "verbose", - }, - "file": { - "class": "logging.handlers.TimedRotatingFileHandler", - "filename": "debug.log", - "when": "midnight", - "interval": 1, # One day - "backupCount": 5, # Keep last 5 days of logs, - "formatter": "json", - }, - }, - "loggers": { - "plane": { - "level": "ERROR", - "handlers": ["console", "file"], - "propagate": False, - }, - }, -} diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index 33a0f6673..9a369a65f 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -87,6 +87,8 @@ services: command: ./bin/takeoff deploy: replicas: ${API_REPLICAS:-1} + volumes: + - logs/api:/code/plane/logs depends_on: - plane-db - plane-redis @@ -98,6 +100,8 @@ services: image: makeplane/plane-backend:${APP_RELEASE:-latest} restart: unless-stopped command: ./bin/worker + volumes: + - logs/worker:/code/plane/logs depends_on: - api - plane-db @@ -110,6 +114,8 @@ services: image: makeplane/plane-backend:${APP_RELEASE:-latest} restart: unless-stopped command: ./bin/beat + volumes: + - logs/beat-worker:/code/plane/logs depends_on: - api - plane-db @@ -166,3 +172,4 @@ volumes: pgdata: redisdata: uploads: + logs: