refactor: move all background task from rqworker to celery (#668)

* refactor: move all background task from rqworker to celery

* dev: update background job to take input in parameters rather than a single dict

* dev: update procfile for new worker

* dev: docker updates for new celery worker

apiserver/plane/settings/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

apiserver/plane/settings/celery.py

@@ -0,0 +1,15 @@
+import os
+from celery import Celery
+from django.conf import settings
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
+
+app = Celery("plane")
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

apiserver/plane/settings/common.py

@@ -35,7 +35,6 @@ INSTALLED_APPS = [
     "rest_framework_simplejwt.token_blacklist",
     "corsheaders",
     "taggit",
-    "django_rq",
 ]
 
 MIDDLEWARE = [

apiserver/plane/settings/local.py

@@ -59,16 +59,8 @@ if os.environ.get("SENTRY_DSN", False):
 
 REDIS_HOST = "localhost"
 REDIS_PORT = 6379
-REDIS_URL = False
+REDIS_URL = os.environ.get("REDIS_URL")
 
-RQ_QUEUES = {
-    "default": {
-        "HOST": "localhost",
-        "PORT": 6379,
-        "DB": 0,
-        "DEFAULT_TIMEOUT": 360,
-    },
-}
 
 MEDIA_URL = "/uploads/"
 MEDIA_ROOT = os.path.join(BASE_DIR, "uploads")
@@ -88,3 +80,6 @@ GPT_ENGINE = os.environ.get("GPT_ENGINE", "text-davinci-003")
 SLACK_BOT_TOKEN = os.environ.get("SLACK_BOT_TOKEN", False)
 
 LOGGER_BASE_URL = os.environ.get("LOGGER_BASE_URL", False)
+
+CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL")
+CELERY_BROKER_URL = os.environ.get("REDIS_URL")

apiserver/plane/settings/production.py

@@ -236,3 +236,6 @@ GPT_ENGINE = os.environ.get("GPT_ENGINE", "text-davinci-003")
 SLACK_BOT_TOKEN = os.environ.get("SLACK_BOT_TOKEN", False)
 
 LOGGER_BASE_URL = os.environ.get("LOGGER_BASE_URL", False)
+
+CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL")
+CELERY_BROKER_URL = os.environ.get("REDIS_URL")

apiserver/plane/settings/redis.py

@@ -1,23 +1,23 @@
+import os
 import redis
 from django.conf import settings
 from urllib.parse import urlparse
 
+
 def redis_instance():
-    # Run in local redis url is false
-    if not settings.REDIS_URL:
-        ri = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0)
+    # connect to redis
+    if settings.DOCKERIZED or os.environ.get(
+        "DJANGO_SETTINGS_MODULE", "plane.settings.local"
+    ):
+        ri = redis.Redis.from_url(settings.REDIS_URL, db=0)
     else:
-    # Run in prod redis url is true check with dockerized value
-        if settings.DOCKERIZED:
-            ri = redis.from_url(settings.REDIS_URL, db=0)
-        else:
-            url = urlparse(settings.REDIS_URL)
-            ri = redis.Redis(
-                host=url.hostname,
-                port=url.port,
-                password=url.password,
-                ssl=True,
-                ssl_cert_reqs=None,
-            )
-    
-    return ri
+        url = urlparse(settings.REDIS_URL)
+        ri = redis.Redis(
+            host=url.hostname,
+            port=url.port,
+            password=url.password,
+            ssl=True,
+            ssl_cert_reqs=None,
+        )
+
+    return ri

apiserver/plane/settings/staging.py

@@ -197,3 +197,6 @@ GPT_ENGINE = os.environ.get("GPT_ENGINE", "text-davinci-003")
 SLACK_BOT_TOKEN = os.environ.get("SLACK_BOT_TOKEN", False)
 
 LOGGER_BASE_URL = os.environ.get("LOGGER_BASE_URL", False)
+
+CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL")
+CELERY_BROKER_URL = os.environ.get("REDIS_URL")