Add RabbitMQ Service to Docker Compose Configuration (#5439)

* fix: celery broker setup

* fix: docker compose update

* fixed rabbitmq vhost issue

* fix: env fixes

* fix-envs-issue in selfhost docker compose

* volume name fix

* added depends on for rabbitmq service

* Add: AMQP_URL for remote rabbitmq urls

* added amqp url im docker compose

* changed default user to guest

* fix: changes the Rabbit mq password var name

---------

Co-authored-by: sriram veeraghanta <veeraghanta.sriram@gmail.com>

.env.example

@@ -8,6 +8,13 @@ PGDATA="/var/lib/postgresql/data"
 REDIS_HOST="plane-redis"
 REDIS_PORT="6379"
 
+# RabbitMQ Settings
+RABBITMQ_HOST="plane-mq"
+RABBITMQ_PORT="5672"
+RABBITMQ_USER="plane"
+RABBITMQ_PASSWORD="plane"
+RABBITMQ_VHOST="plane"
+
 # AWS Settings
 AWS_REGION=""
 AWS_ACCESS_KEY_ID="access-key"

apiserver/.env.example

@@ -15,12 +15,18 @@ POSTGRES_DB="plane"
 POSTGRES_PORT=5432
 DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
 
-
 # Redis Settings
 REDIS_HOST="plane-redis"
 REDIS_PORT="6379"
 REDIS_URL="redis://${REDIS_HOST}:6379/"
 
+# RabbitMQ Settings
+RABBITMQ_HOST="plane-mq"
+RABBITMQ_PORT="5672"
+RABBITMQ_USER="plane"
+RABBITMQ_PASSWORD="plane"
+RABBITMQ_VHOST="plane"
+
 # AWS Settings
 AWS_REGION=""
 AWS_ACCESS_KEY_ID="access-key"

apiserver/plane/settings/common.py

@@ -254,18 +254,25 @@ if AWS_S3_ENDPOINT_URL and USE_MINIO:
     AWS_S3_CUSTOM_DOMAIN = f"{parsed_url.netloc}/{AWS_STORAGE_BUCKET_NAME}"
     AWS_S3_URL_PROTOCOL = f"{parsed_url.scheme}:"
 
+# RabbitMQ connection settings
+RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST", "localhost")
+RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT", "5672")
+RABBITMQ_USER = os.environ.get("RABBITMQ_USER", "guest")
+RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD", "guest")
+RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "/")
+AMQP_URL = os.environ.get("AMQP_URL")
 
 # Celery Configuration
+if AMQP_URL:
+    CELERY_BROKER_URL = AMQP_URL
+else:
+    CELERY_BROKER_URL = f"amqp://{RABBITMQ_USER}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}"
+ 
 CELERY_TIMEZONE = TIME_ZONE
 CELERY_TASK_SERIALIZER = "json"
+CELERY_RESULT_SERIALIZER = "json"
 CELERY_ACCEPT_CONTENT = ["application/json"]
 
-if REDIS_SSL:
-    redis_url = os.environ.get("REDIS_URL")
-    broker_url = f"{redis_url}?ssl_cert_reqs={ssl.CERT_NONE.name}&ssl_ca_certs={certifi.where()}"
-    CELERY_BROKER_URL = broker_url
-else:
-    CELERY_BROKER_URL = REDIS_URL
 
 CELERY_IMPORTS = (
     # scheduled tasks

apiserver/requirements/base.txt

@@ -17,6 +17,7 @@ django-cors-headers==4.3.1
 # celery
 celery==5.4.0
 django_celery_beat==2.6.0
+django-celery-results==2.5.1
 # file serve
 whitenoise==6.6.0
 # fake data

deploy/selfhost/docker-compose.yml

@@ -21,6 +21,15 @@ x-app-env: &app-env
     - REDIS_HOST=${REDIS_HOST:-plane-redis}
     - REDIS_PORT=${REDIS_PORT:-6379}
     - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
+
+    # RabbitMQ Settings
+    - RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq}
+    - RABBITMQ_PORT=${RABBITMQ_PORT:-5672}
+    - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane}
+    - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane}
+    - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane}
+    - RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane}
+    - AMQP_URL=${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane}
     # Application secret
     - SECRET_KEY=${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5}
     # DATA STORE SETTINGS
@@ -94,6 +103,8 @@ services:
     depends_on:
       - plane-db
       - plane-redis
+      - plane-mq
+
 
   worker:
     <<: *app-env
@@ -108,6 +119,7 @@ services:
       - api
       - plane-db
       - plane-redis
+      - plane-mq
 
   beat-worker:
     <<: *app-env
@@ -122,6 +134,7 @@ services:
       - api
       - plane-db
       - plane-redis
+      - plane-mq
 
   migrator:
     <<: *app-env
@@ -153,6 +166,13 @@ services:
     volumes:
       - redisdata:/data
 
+  plane-mq:
+    <<: *app-env
+    image: rabbitmq:3.13.6-management-alpine
+    restart: always
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+
   plane-minio:
     <<: *app-env
     image: minio/minio:latest
@@ -179,8 +199,10 @@ services:
 volumes:
   pgdata:
   redisdata:
+  
   uploads:
   logs_api:
   logs_worker:
   logs_beat-worker:
   logs_migrator:
+  rabbitmq_data:

deploy/selfhost/variables.env

@@ -28,6 +28,14 @@ REDIS_HOST=plane-redis
 REDIS_PORT=6379
 REDIS_URL=
 
+# RabbitMQ Settings
+RABBITMQ_HOST="plane-mq"
+RABBITMQ_PORT="5672"
+RABBITMQ_USER="plane"
+RABBITMQ_PASSWORD="plane"
+RABBITMQ_VHOST="plane"
+AMQP_URL=
+
 # Secret Key
 SECRET_KEY=60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5
 

docker-compose-local.yml

@@ -1,12 +1,3 @@
-networks:
-  dev_env:
-    driver: bridge
-
-volumes:
-  redisdata:
-  uploads:
-  pgdata:
-
 services:
   plane-redis:
     image: valkey/valkey:7.2.5-alpine
@@ -16,6 +7,20 @@ services:
     volumes:
       - redisdata:/data
 
+  plane-mq:
+    image: rabbitmq:3.13.6-management-alpine
+    restart: unless-stopped
+    networks:
+      - dev_env
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+    env_file:
+      - .env
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
+      RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST}
+
   plane-minio:
     image: minio/minio
     restart: unless-stopped
@@ -179,3 +184,13 @@ services:
       - api
       - space
       - admin
+
+volumes:
+  redisdata:
+  uploads:
+  pgdata:
+  rabbitmq_data:
+
+networks:
+  dev_env:
+    driver: bridge

docker-compose.yml

@@ -121,6 +121,19 @@ services:
     volumes:
       - redisdata:/data
 
+  plane-mq:
+    container_name: plane-mq
+    image: rabbitmq:3.13.6-management-alpine
+    restart: always
+    env_file:
+      - .env
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
+      RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST}
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+
   plane-minio:
     container_name: plane-minio
     image: minio/minio
@@ -154,3 +167,4 @@ volumes:
   pgdata:
   redisdata:
   uploads:
+  rabbitmq_data: