From 2c950713a71c0723036152065bb9385d116ffeab Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 2 Sep 2024 17:40:17 +0530 Subject: [PATCH] Add RabbitMQ Service to Docker Compose Configuration (#5439) * fix: celery broker setup * fix: docker compose update * fixed rabbitmq vhost issue * fix: env fixes * fix-envs-issue in selfhost docker compose * volume name fix * added depends on for rabbitmq service * Add: AMQP_URL for remote rabbitmq urls * added amqp url im docker compose * changed default user to guest * fix: changes the Rabbit mq password var name --------- Co-authored-by: sriram veeraghanta --- .env.example | 7 +++++++ apiserver/.env.example | 8 +++++++- apiserver/plane/settings/common.py | 19 +++++++++++------ apiserver/requirements/base.txt | 1 + deploy/selfhost/docker-compose.yml | 22 ++++++++++++++++++++ deploy/selfhost/variables.env | 8 ++++++++ docker-compose-local.yml | 33 ++++++++++++++++++++++-------- docker-compose.yml | 14 +++++++++++++ 8 files changed, 96 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 71a9074a6..8b11217b5 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,13 @@ PGDATA="/var/lib/postgresql/data" REDIS_HOST="plane-redis" REDIS_PORT="6379" +# RabbitMQ Settings +RABBITMQ_HOST="plane-mq" +RABBITMQ_PORT="5672" +RABBITMQ_USER="plane" +RABBITMQ_PASSWORD="plane" +RABBITMQ_VHOST="plane" + # AWS Settings AWS_REGION="" AWS_ACCESS_KEY_ID="access-key" diff --git a/apiserver/.env.example b/apiserver/.env.example index ad2140a5d..733e448d6 100644 --- a/apiserver/.env.example +++ b/apiserver/.env.example @@ -15,12 +15,18 @@ POSTGRES_DB="plane" POSTGRES_PORT=5432 DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - # Redis Settings REDIS_HOST="plane-redis" REDIS_PORT="6379" REDIS_URL="redis://${REDIS_HOST}:6379/" +# RabbitMQ Settings +RABBITMQ_HOST="plane-mq" +RABBITMQ_PORT="5672" +RABBITMQ_USER="plane" +RABBITMQ_PASSWORD="plane" +RABBITMQ_VHOST="plane" + # AWS Settings AWS_REGION="" AWS_ACCESS_KEY_ID="access-key" diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index 9fed15516..af6560ca1 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -254,18 +254,25 @@ if AWS_S3_ENDPOINT_URL and USE_MINIO: AWS_S3_CUSTOM_DOMAIN = f"{parsed_url.netloc}/{AWS_STORAGE_BUCKET_NAME}" AWS_S3_URL_PROTOCOL = f"{parsed_url.scheme}:" +# RabbitMQ connection settings +RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST", "localhost") +RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT", "5672") +RABBITMQ_USER = os.environ.get("RABBITMQ_USER", "guest") +RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD", "guest") +RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "/") +AMQP_URL = os.environ.get("AMQP_URL") # Celery Configuration +if AMQP_URL: + CELERY_BROKER_URL = AMQP_URL +else: + CELERY_BROKER_URL = f"amqp://{RABBITMQ_USER}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}" + CELERY_TIMEZONE = TIME_ZONE CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" CELERY_ACCEPT_CONTENT = ["application/json"] -if REDIS_SSL: - redis_url = os.environ.get("REDIS_URL") - broker_url = f"{redis_url}?ssl_cert_reqs={ssl.CERT_NONE.name}&ssl_ca_certs={certifi.where()}" - CELERY_BROKER_URL = broker_url -else: - CELERY_BROKER_URL = REDIS_URL CELERY_IMPORTS = ( # scheduled tasks diff --git a/apiserver/requirements/base.txt b/apiserver/requirements/base.txt index 9667346a8..c24904a08 100644 --- a/apiserver/requirements/base.txt +++ b/apiserver/requirements/base.txt @@ -17,6 +17,7 @@ django-cors-headers==4.3.1 # celery celery==5.4.0 django_celery_beat==2.6.0 +django-celery-results==2.5.1 # file serve whitenoise==6.6.0 # fake data diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index c574c1915..6df1eeebe 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -21,6 +21,15 @@ x-app-env: &app-env - REDIS_HOST=${REDIS_HOST:-plane-redis} - REDIS_PORT=${REDIS_PORT:-6379} - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/} + + # RabbitMQ Settings + - RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq} + - RABBITMQ_PORT=${RABBITMQ_PORT:-5672} + - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane} + - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane} + - RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane} + - AMQP_URL=${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane} # Application secret - SECRET_KEY=${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5} # DATA STORE SETTINGS @@ -94,6 +103,8 @@ services: depends_on: - plane-db - plane-redis + - plane-mq + worker: <<: *app-env @@ -108,6 +119,7 @@ services: - api - plane-db - plane-redis + - plane-mq beat-worker: <<: *app-env @@ -122,6 +134,7 @@ services: - api - plane-db - plane-redis + - plane-mq migrator: <<: *app-env @@ -153,6 +166,13 @@ services: volumes: - redisdata:/data + plane-mq: + <<: *app-env + image: rabbitmq:3.13.6-management-alpine + restart: always + volumes: + - rabbitmq_data:/var/lib/rabbitmq + plane-minio: <<: *app-env image: minio/minio:latest @@ -179,8 +199,10 @@ services: volumes: pgdata: redisdata: + uploads: logs_api: logs_worker: logs_beat-worker: logs_migrator: + rabbitmq_data: diff --git a/deploy/selfhost/variables.env b/deploy/selfhost/variables.env index 4120ee502..fa9cc57ad 100644 --- a/deploy/selfhost/variables.env +++ b/deploy/selfhost/variables.env @@ -28,6 +28,14 @@ REDIS_HOST=plane-redis REDIS_PORT=6379 REDIS_URL= +# RabbitMQ Settings +RABBITMQ_HOST="plane-mq" +RABBITMQ_PORT="5672" +RABBITMQ_USER="plane" +RABBITMQ_PASSWORD="plane" +RABBITMQ_VHOST="plane" +AMQP_URL= + # Secret Key SECRET_KEY=60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5 diff --git a/docker-compose-local.yml b/docker-compose-local.yml index ee7345b2e..ca5c6d0ec 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -1,12 +1,3 @@ -networks: - dev_env: - driver: bridge - -volumes: - redisdata: - uploads: - pgdata: - services: plane-redis: image: valkey/valkey:7.2.5-alpine @@ -16,6 +7,20 @@ services: volumes: - redisdata:/data + plane-mq: + image: rabbitmq:3.13.6-management-alpine + restart: unless-stopped + networks: + - dev_env + volumes: + - rabbitmq_data:/var/lib/rabbitmq + env_file: + - .env + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} + RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST} + plane-minio: image: minio/minio restart: unless-stopped @@ -179,3 +184,13 @@ services: - api - space - admin + +volumes: + redisdata: + uploads: + pgdata: + rabbitmq_data: + +networks: + dev_env: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index e833b9e76..93dd25a16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,19 @@ services: volumes: - redisdata:/data + plane-mq: + container_name: plane-mq + image: rabbitmq:3.13.6-management-alpine + restart: always + env_file: + - .env + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} + RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST} + volumes: + - rabbitmq_data:/var/lib/rabbitmq + plane-minio: container_name: plane-minio image: minio/minio @@ -154,3 +167,4 @@ volumes: pgdata: redisdata: uploads: + rabbitmq_data: