Compare commits

...

7 Commits

Author SHA1 Message Date
Nikhil
4fb9ab3998
Merge branch 'develop' into dev/api_logging 2023-11-12 00:13:45 +05:30
pablohashescobar
8d4e4a7485 dev: enable global level log settings 2023-10-31 12:38:52 +05:30
pablohashescobar
edb4280ec1 Merge branch 'develop' of github.com:makeplane/plane into dev/api_logging 2023-10-31 11:49:54 +05:30
pablohashescobar
099cd5955c dev: remove worker counts 2023-10-30 14:42:25 +05:30
pablohashescobar
93cfa13955 dev: enable logger instead of printing 2023-10-30 14:41:44 +05:30
pablohashescobar
f231ac0a79 Merge branch 'develop' of github.com:makeplane/plane into dev/api_logging 2023-10-30 11:35:38 +05:30
pablohashescobar
00757a6704 dev: enable api logging and control worker count through env 2023-10-25 19:25:14 +05:30
21 changed files with 129 additions and 43 deletions

1
.gitignore vendored
View File

@ -51,6 +51,7 @@ staticfiles
mediafiles mediafiles
.env .env
.DS_Store .DS_Store
logs/
node_modules/ node_modules/
assets/dist/ assets/dist/

View File

@ -70,6 +70,5 @@ ENABLE_MAGIC_LINK_LOGIN="0"
# Email redirections and minio domain settings # Email redirections and minio domain settings
WEB_URL="http://localhost" WEB_URL="http://localhost"
# Gunicorn Workers # Gunicorn Workers
GUNICORN_WORKERS=2 GUNICORN_WORKERS=2

View File

@ -1,5 +1,6 @@
# Python imports # Python imports
import zoneinfo import zoneinfo
import logging
# Django imports # Django imports
from django.urls import resolve from django.urls import resolve
@ -58,6 +59,8 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
try: try:
return self.model.objects.all() return self.model.objects.all()
except Exception as e: except Exception as e:
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
raise APIException("Please check the view", status.HTTP_400_BAD_REQUEST) raise APIException("Please check the view", status.HTTP_400_BAD_REQUEST)
@ -81,10 +84,13 @@ class BaseViewSet(TimezoneMixin, ModelViewSet, BasePaginator):
return Response({"error": f"{model_name} does not exist."}, status=status.HTTP_404_NOT_FOUND) return Response({"error": f"{model_name} does not exist."}, status=status.HTTP_404_NOT_FOUND)
if isinstance(e, KeyError): if isinstance(e, KeyError):
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST) return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST)
print(e) if settings.DEBUG else print("Server Error") logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@ -161,8 +167,9 @@ class BaseAPIView(TimezoneMixin, APIView, BasePaginator):
if isinstance(e, KeyError): if isinstance(e, KeyError):
return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST) return Response({"error": f"key {e} does not exist"}, status=status.HTTP_400_BAD_REQUEST)
print(e) if settings.DEBUG else print("Server Error") logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response({"error": "Something went wrong please try again later"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

View File

@ -2,6 +2,7 @@
import uuid import uuid
import requests import requests
import os import os
import logging
# Django imports # Django imports
from django.utils import timezone from django.utils import timezone
@ -48,6 +49,8 @@ def validate_google_token(token, client_id):
} }
return data return data
except Exception as e: except Exception as e:
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
raise exceptions.AuthenticationFailed("Error with Google connection.") raise exceptions.AuthenticationFailed("Error with Google connection.")

View File

@ -1,6 +1,7 @@
# Python imports # Python imports
import csv import csv
import io import io
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
@ -419,6 +420,6 @@ def analytic_export_task(email, data, slug):
csv_buffer = generate_csv_from_rows(rows) csv_buffer = generate_csv_from_rows(rows)
send_export_email(email, slug, csv_buffer) send_export_email(email, slug, csv_buffer)
except Exception as e: except Exception as e:
if settings.DEBUG: logger = logging.getLogger("plane")
print(e) logger.error(e)
capture_exception(e) capture_exception(e)

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string from django.template.loader import render_to_string
@ -39,8 +42,7 @@ def email_verification(first_name, email, token, current_site):
msg.send() msg.send()
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -4,6 +4,8 @@ import io
import json import json
import boto3 import boto3
import zipfile import zipfile
import logging
from urllib.parse import urlparse, urlunparse
# Django imports # Django imports
from django.conf import settings from django.conf import settings
@ -367,8 +369,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s
exporter_instance.status = "failed" exporter_instance.status = "failed"
exporter_instance.reason = str(e) exporter_instance.reason = str(e)
exporter_instance.save(update_fields=["status", "reason"]) exporter_instance.save(update_fields=["status", "reason"])
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string from django.template.loader import render_to_string
@ -35,8 +38,7 @@ def forgot_password(first_name, email, uidb64, token, current_site):
msg.send() msg.send()
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -2,6 +2,10 @@
import json import json
import requests import requests
import uuid import uuid
import jwt
import logging
from datetime import datetime
# Django imports # Django imports
from django.conf import settings from django.conf import settings
@ -187,8 +191,7 @@ def service_importer(service, importer_id):
importer = Importer.objects.get(pk=importer_id) importer = Importer.objects.get(pk=importer_id)
importer.status = "failed" importer.status = "failed"
importer.save() importer.save()
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,6 +1,7 @@
# Python imports # Python imports
import json import json
import requests import requests
import logging
# Django imports # Django imports
from django.conf import settings from django.conf import settings
@ -1568,8 +1569,7 @@ def issue_activity(
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,6 +1,7 @@
# Python imports # Python imports
import json import json
from datetime import timedelta from datetime import timedelta
import logging
# Django imports # Django imports
from django.utils import timezone from django.utils import timezone
@ -86,8 +87,8 @@ def archive_old_issues():
] ]
return return
except Exception as e: except Exception as e:
if settings.DEBUG: logger = logging.getLogger("plane")
print(e) logger.error(e)
capture_exception(e) capture_exception(e)
return return
@ -158,7 +159,7 @@ def close_old_issues():
] ]
return return
except Exception as e: except Exception as e:
if settings.DEBUG: logger = logging.getLogger("plane")
print(e) logger.error(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string from django.template.loader import render_to_string
@ -30,8 +33,7 @@ def magic_link(email, key, token, current_site):
msg.send() msg.send()
return return
except Exception as e: except Exception as e:
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
# Print logs if in DEBUG mode
if settings.DEBUG:
print(e)
return return

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string from django.template.loader import render_to_string
@ -50,8 +53,7 @@ def project_invitation(email, project_id, token, current_site):
except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist) as e: except (Project.DoesNotExist, ProjectMemberInvite.DoesNotExist) as e:
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.conf import settings from django.conf import settings
@ -29,8 +32,7 @@ def send_welcome_slack(user_id, created, message):
print(f"Got an error: {e.response['error']}") print(f"Got an error: {e.response['error']}")
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -1,3 +1,6 @@
# Python imports
import logging
# Django imports # Django imports
from django.core.mail import EmailMultiAlternatives from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string from django.template.loader import render_to_string
@ -66,8 +69,7 @@ def workspace_invitation(email, workspace_id, token, current_site, invitor):
except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e: except (Workspace.DoesNotExist, WorkspaceMemberInvite.DoesNotExist) as e:
return return
except Exception as e: except Exception as e:
# Print logs if in DEBUG mode logger = logging.getLogger("plane")
if settings.DEBUG: logger.error(e)
print(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -3,6 +3,7 @@ import uuid
import string import string
import random import random
import pytz import pytz
import logging
# Django imports # Django imports
from django.db import models from django.db import models
@ -139,5 +140,7 @@ def send_welcome_slack(sender, instance, created, **kwargs):
print(f"Got an error: {e.response['error']}") print(f"Got an error: {e.response['error']}")
return return
except Exception as e: except Exception as e:
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
return return

View File

@ -215,3 +215,46 @@ CELERY_TIMEZONE = TIME_ZONE
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task") CELERY_IMPORTS = ("plane.bgtasks.issue_automation_task","plane.bgtasks.exporter_expired_task")
LOG_DIR = os.path.join(BASE_DIR, 'logs')
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"verbose": {
"format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
"style": "{",
},
"json": {
"()": "pythonjsonlogger.jsonlogger.JsonFormatter",
"fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s",
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "verbose",
},
"file": {
"class": "logging.handlers.TimedRotatingFileHandler",
"filename": os.path.join(BASE_DIR, 'logs', 'debug.log') if DEBUG else os.path.join(BASE_DIR, 'logs', 'error.log'),
"when": "midnight",
"interval": 1, # One day
"backupCount": 5, # Keep last 5 days of logs,
"formatter": "json",
},
},
"loggers": {
"plane": {
"level": "DEBUG" if DEBUG else "ERROR",
"handlers": ["console", "file"],
"propagate": False,
},
},
}

View File

@ -70,9 +70,7 @@ AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "secret-key")
# The name of the bucket to store files in. # The name of the bucket to store files in.
AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_S3_BUCKET_NAME", "uploads") AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_S3_BUCKET_NAME", "uploads")
# The full URL to the S3 endpoint. Leave blank to use the default region URL. # The full URL to the S3 endpoint. Leave blank to use the default region URL.
AWS_S3_ENDPOINT_URL = os.environ.get( AWS_S3_ENDPOINT_URL = os.environ.get("AWS_S3_ENDPOINT_URL", "http://plane-minio:9000")
"AWS_S3_ENDPOINT_URL", "http://plane-minio:9000"
)
# Default permissions # Default permissions
AWS_DEFAULT_ACL = "public-read" AWS_DEFAULT_ACL = "public-read"
AWS_QUERYSTRING_AUTH = False AWS_QUERYSTRING_AUTH = False
@ -98,7 +96,7 @@ CSRF_COOKIE_SECURE = True
# Redis URL # Redis URL
REDIS_URL = os.environ.get("REDIS_URL") REDIS_URL = os.environ.get("REDIS_URL")
# Caches # Caches
CACHES = { CACHES = {
"default": { "default": {
"BACKEND": "django_redis.cache.RedisCache", "BACKEND": "django_redis.cache.RedisCache",

View File

@ -1,5 +1,9 @@
# Python imports
import requests import requests
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
import logging
# Third party imports
from sentry_sdk import capture_exception from sentry_sdk import capture_exception
@ -49,5 +53,7 @@ def jira_project_issue_summary(email, api_token, project_key, hostname):
), ),
} }
except Exception as e: except Exception as e:
logger = logging.getLogger("plane")
logger.error(e)
capture_exception(e) capture_exception(e)
return {"error": "Something went wrong could not fetch information from jira"} return {"error": "Something went wrong could not fetch information from jira"}

View File

@ -34,4 +34,5 @@ psycopg-binary==3.1.10
psycopg-c==3.1.10 psycopg-c==3.1.10
scout-apm==2.26.1 scout-apm==2.26.1
openpyxl==3.1.2 openpyxl==3.1.2
beautifulsoup4==4.12.2 python-json-logger==2.0.7
beautifulsoup4==4.12.2

View File

@ -89,6 +89,8 @@ services:
command: ./bin/takeoff command: ./bin/takeoff
deploy: deploy:
replicas: ${API_REPLICAS:-1} replicas: ${API_REPLICAS:-1}
volumes:
- logs/api:/code/plane/logs
depends_on: depends_on:
- plane-db - plane-db
- plane-redis - plane-redis
@ -100,6 +102,8 @@ services:
image: makeplane/plane-backend:${APP_RELEASE:-latest} image: makeplane/plane-backend:${APP_RELEASE:-latest}
restart: unless-stopped restart: unless-stopped
command: ./bin/worker command: ./bin/worker
volumes:
- logs/worker:/code/plane/logs
depends_on: depends_on:
- api - api
- plane-db - plane-db
@ -112,6 +116,8 @@ services:
image: makeplane/plane-backend:${APP_RELEASE:-latest} image: makeplane/plane-backend:${APP_RELEASE:-latest}
restart: unless-stopped restart: unless-stopped
command: ./bin/beat command: ./bin/beat
volumes:
- logs/beat-worker:/code/plane/logs
depends_on: depends_on:
- api - api
- plane-db - plane-db
@ -168,3 +174,4 @@ volumes:
pgdata: pgdata:
redisdata: redisdata:
uploads: uploads:
logs: