[WEB-3927] chore: add logging to support json logging (#6955)

* chore: update logging to json based logging

* chore: add logging to file

apiserver/plane/celery.py

@@ -1,8 +1,16 @@
+# Python imports
 import os
+import logging
+
+# Third party imports
 from celery import Celery
-from plane.settings.redis import redis_instance
+from pythonjsonlogger.jsonlogger import JsonFormatter
+from celery.signals import after_setup_logger, after_setup_task_logger
 from celery.schedules import crontab
 
+# Module imports
+from plane.settings.redis import redis_instance
+
 # Set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
 
@@ -47,6 +55,28 @@ app.conf.beat_schedule = {
     },
 }
 
+
+# Setup logging
+@after_setup_logger.connect
+def setup_loggers(logger, *args, **kwargs):
+    formatter = JsonFormatter(
+        '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s'
+    )
+    handler = logging.StreamHandler()
+    handler.setFormatter(fmt=formatter)
+    logger.addHandler(handler)
+
+
+@after_setup_task_logger.connect
+def setup_task_loggers(logger, *args, **kwargs):
+    formatter = JsonFormatter(
+        '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s'
+    )
+    handler = logging.StreamHandler()
+    handler.setFormatter(fmt=formatter)
+    logger.addHandler(handler)
+
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 

apiserver/plane/middleware/api_log_middleware.py

@@ -1,40 +0,0 @@
-# Module imports
-from plane.db.models import APIActivityLog
-from plane.utils.ip_address import get_client_ip
-
-class APITokenLogMiddleware:
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        request_body = request.body
-        response = self.get_response(request)
-        self.process_request(request, response, request_body)
-        return response
-
-    def process_request(self, request, response, request_body):
-        api_key_header = "X-Api-Key"
-        api_key = request.headers.get(api_key_header)
-        # If the API key is present, log the request
-        if api_key:
-            try:
-                APIActivityLog.objects.create(
-                    token_identifier=api_key,
-                    path=request.path,
-                    method=request.method,
-                    query_params=request.META.get("QUERY_STRING", ""),
-                    headers=str(request.headers),
-                    body=(request_body.decode("utf-8") if request_body else None),
-                    response_body=(
-                        response.content.decode("utf-8") if response.content else None
-                    ),
-                    response_code=response.status_code,
-                    ip_address=get_client_ip(request=request),
-                    user_agent=request.META.get("HTTP_USER_AGENT", None),
-                )
-
-            except Exception as e:
-                print(e)
-                # If the token does not exist, you can decide whether to log this as an invalid attempt
-
-        return None

apiserver/plane/middleware/logger.py

@@ -10,8 +10,10 @@ from rest_framework.request import Request
 
 # Module imports
 from plane.utils.ip_address import get_client_ip
+from plane.db.models import APIActivityLog
 
-api_logger = logging.getLogger("plane.api")
+
+api_logger = logging.getLogger("plane.api.request")
 
 
 class RequestLoggerMiddleware:
@@ -69,3 +71,41 @@ class RequestLoggerMiddleware:
 
         # return the response
         return response
+
+
+class APITokenLogMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        request_body = request.body
+        response = self.get_response(request)
+        self.process_request(request, response, request_body)
+        return response
+
+    def process_request(self, request, response, request_body):
+        api_key_header = "X-Api-Key"
+        api_key = request.headers.get(api_key_header)
+        # If the API key is present, log the request
+        if api_key:
+            try:
+                APIActivityLog.objects.create(
+                    token_identifier=api_key,
+                    path=request.path,
+                    method=request.method,
+                    query_params=request.META.get("QUERY_STRING", ""),
+                    headers=str(request.headers),
+                    body=(request_body.decode("utf-8") if request_body else None),
+                    response_body=(
+                        response.content.decode("utf-8") if response.content else None
+                    ),
+                    response_code=response.status_code,
+                    ip_address=get_client_ip(request=request),
+                    user_agent=request.META.get("HTTP_USER_AGENT", None),
+                )
+
+            except Exception as e:
+                api_logger.exception(e)
+                # If the token does not exist, you can decide whether to log this as an invalid attempt
+
+        return None

apiserver/plane/settings/common.py

@@ -58,7 +58,7 @@ MIDDLEWARE = [
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "crum.CurrentRequestUserMiddleware",
     "django.middleware.gzip.GZipMiddleware",
-    "plane.middleware.api_log_middleware.APITokenLogMiddleware",
+    "plane.middleware.logger.APITokenLogMiddleware",
     "plane.middleware.logger.RequestLoggerMiddleware",
 ]
 

apiserver/plane/settings/local.py

@@ -56,6 +56,11 @@ LOGGING = {
         }
     },
     "loggers": {
+        "plane.api.request": {
+            "level": "INFO",
+            "handlers": ["console"],
+            "propagate": False,
+        },
         "plane.api": {"level": "INFO", "handlers": ["console"], "propagate": False},
         "plane.worker": {"level": "INFO", "handlers": ["console"], "propagate": False},
         "plane.exception": {
@@ -63,5 +68,10 @@ LOGGING = {
             "handlers": ["console"],
             "propagate": False,
         },
+        "plane.external": {
+            "level": "INFO",
+            "handlers": ["console"],
+            "propagate": False,
+        },
     },
 }

apiserver/plane/settings/production.py

@@ -58,6 +58,11 @@ LOGGING = {
         },
     },
     "loggers": {
+        "plane.api.request": {
+            "level": "DEBUG" if DEBUG else "INFO",
+            "handlers": ["console"],
+            "propagate": False,
+        },
         "plane.api": {
             "level": "DEBUG" if DEBUG else "INFO",
             "handlers": ["console"],
@@ -70,6 +75,11 @@ LOGGING = {
         },
         "plane.exception": {
             "level": "DEBUG" if DEBUG else "ERROR",
+            "handlers": ["console", "file"],
+            "propagate": False,
+        },
+        "plane.external": {
+            "level": "INFO",
             "handlers": ["console"],
             "propagate": False,
         },

apiserver/plane/utils/exception_logger.py

@@ -9,7 +9,7 @@ from django.conf import settings
 def log_exception(e):
     # Log the error
     logger = logging.getLogger("plane.exception")
-    logger.error(str(e))
+    logger.exception(e)
 
     if settings.DEBUG:
         # Print the traceback if in debug mode