From bcb470db9bec4dd7191cdbb1eff96f54b23aaaa2 Mon Sep 17 00:00:00 2001 From: Daniel O'Connell Date: Sat, 1 Nov 2025 15:55:59 +0100 Subject: [PATCH] use redis for celery backend --- docker-compose.yaml | 29 +++++++++++++------------- requirements/requirements-common.txt | 2 +- src/memory/common/settings.py | 31 ++++++++++++++++++---------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 2bc0945..883a637 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,17 +19,17 @@ secrets: volumes: db_data: {} # Postgres qdrant_data: {} # Qdrant - rabbitmq_data: {} # RabbitMQ + redis_data: {} # Redis # ------------------------------ X-templates ---------------------------- x-common-env: &env - RABBITMQ_USER: kb - RABBITMQ_HOST: rabbitmq + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 CELERY_BROKER_PASSWORD: ${CELERY_BROKER_PASSWORD} QDRANT_HOST: qdrant DB_HOST: postgres DB_PORT: 5432 - RABBITMQ_PORT: 5672 FILE_STORAGE_DIR: /app/memory_files TZ: "Etc/UTC" @@ -40,7 +40,7 @@ x-worker-base: &worker-base restart: unless-stopped networks: [ kbnet ] security_opt: [ "no-new-privileges=true" ] - depends_on: [ postgres, rabbitmq, qdrant ] + depends_on: [ postgres, redis, qdrant ] env_file: [ .env ] environment: &worker-env <<: *env @@ -103,22 +103,21 @@ services: volumes: - ./db:/app/db:ro - rabbitmq: - image: rabbitmq:3.13-management + redis: + image: redis:7.2-alpine restart: unless-stopped networks: [ kbnet ] - environment: - <<: *env - RABBITMQ_DEFAULT_USER: "kb" - RABBITMQ_DEFAULT_PASS: "${CELERY_BROKER_PASSWORD}" + command: ["redis-server", "--save", "", "--appendonly", "no", "--requirepass", "${CELERY_BROKER_PASSWORD}"] volumes: - - rabbitmq_data:/var/lib/rabbitmq:rw + - redis_data:/data:rw healthcheck: - test: [ "CMD", "rabbitmq-diagnostics", "ping" ] + test: [ "CMD", "redis-cli", "--pass", "${CELERY_BROKER_PASSWORD}", "ping" ] interval: 15s timeout: 5s retries: 5 security_opt: [ "no-new-privileges=true" ] + cap_drop: [ ALL ] + user: redis qdrant: image: qdrant/qdrant:v1.14.0 @@ -148,7 +147,7 @@ services: SESSION_COOKIE_NAME: "${SESSION_COOKIE_NAME:-session_id}" restart: unless-stopped networks: [kbnet] - depends_on: [postgres, rabbitmq, qdrant] + depends_on: [postgres, redis, qdrant] environment: <<: *env POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password @@ -186,7 +185,7 @@ services: environment: <<: *worker-env DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN} - DISCORD_NOTIFICATIONS_ENABLED: true + DISCORD_NOTIFICATIONS_ENABLED: ${DISCORD_NOTIFICATIONS_ENABLED:-true} DISCORD_COLLECTOR_ENABLED: true volumes: - ./memory_files:/app/memory_files:rw diff --git a/requirements/requirements-common.txt b/requirements/requirements-common.txt index a00f38e..3683f32 100644 --- a/requirements/requirements-common.txt +++ b/requirements/requirements-common.txt @@ -9,4 +9,4 @@ anthropic==0.69.0 openai==2.3.0 # Pin the httpx version, as newer versions break the anthropic client httpx==0.27.0 -celery[sqs]==5.3.6 +celery[redis,sqs]==5.3.6 diff --git a/src/memory/common/settings.py b/src/memory/common/settings.py index c5b61a9..842a32c 100644 --- a/src/memory/common/settings.py +++ b/src/memory/common/settings.py @@ -34,15 +34,26 @@ DB_URL = os.getenv("DATABASE_URL", make_db_url()) # Broker settings CELERY_QUEUE_PREFIX = os.getenv("CELERY_QUEUE_PREFIX", "memory") -CELERY_BROKER_TYPE = os.getenv("CELERY_BROKER_TYPE", "amqp").lower() # amqp or sqs -CELERY_BROKER_USER = os.getenv("CELERY_BROKER_USER", "kb") -CELERY_BROKER_PASSWORD = os.getenv("CELERY_BROKER_PASSWORD", "kb") +CELERY_BROKER_TYPE = os.getenv("CELERY_BROKER_TYPE", "redis").lower() +REDIS_HOST = os.getenv("REDIS_HOST", "redis") +REDIS_PORT = os.getenv("REDIS_PORT", "6379") +REDIS_DB = os.getenv("REDIS_DB", "0") +CELERY_BROKER_USER = os.getenv( + "CELERY_BROKER_USER", "kb" if CELERY_BROKER_TYPE == "amqp" else "" +) +CELERY_BROKER_PASSWORD = os.getenv( + "CELERY_BROKER_PASSWORD", "" if CELERY_BROKER_TYPE == "redis" else "kb" +) + CELERY_BROKER_HOST = os.getenv("CELERY_BROKER_HOST", "") -if not CELERY_BROKER_HOST and CELERY_BROKER_TYPE == "amqp": - RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq") - RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", "5672") - CELERY_BROKER_HOST = f"{RABBITMQ_HOST}:{RABBITMQ_PORT}//" +if not CELERY_BROKER_HOST: + if CELERY_BROKER_TYPE == "amqp": + RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq") + RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", "5672") + CELERY_BROKER_HOST = f"{RABBITMQ_HOST}:{RABBITMQ_PORT}//" + elif CELERY_BROKER_TYPE == "redis": + CELERY_BROKER_HOST = f"{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", f"db+{DB_URL}") @@ -161,7 +172,7 @@ STATIC_DIR = pathlib.Path( ) # Discord notification settings -DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN", "") +DISCORD_BOT_ID = int(os.getenv("DISCORD_BOT_ID", "0")) DISCORD_ERROR_CHANNEL = os.getenv("DISCORD_ERROR_CHANNEL", "memory-errors") DISCORD_ACTIVITY_CHANNEL = os.getenv("DISCORD_ACTIVITY_CHANNEL", "memory-activity") DISCORD_DISCOVERY_CHANNEL = os.getenv("DISCORD_DISCOVERY_CHANNEL", "memory-discoveries") @@ -169,9 +180,7 @@ DISCORD_CHAT_CHANNEL = os.getenv("DISCORD_CHAT_CHANNEL", "memory-chat") # Enable Discord notifications if bot token is set -DISCORD_NOTIFICATIONS_ENABLED = bool( - boolean_env("DISCORD_NOTIFICATIONS_ENABLED", True) and DISCORD_BOT_TOKEN -) +DISCORD_NOTIFICATIONS_ENABLED = boolean_env("DISCORD_NOTIFICATIONS_ENABLED", True) DISCORD_PROCESS_MESSAGES = boolean_env("DISCORD_PROCESS_MESSAGES", True) DISCORD_MODEL = os.getenv("DISCORD_MODEL", "anthropic/claude-sonnet-4-5") DISCORD_MAX_TOOL_CALLS = int(os.getenv("DISCORD_MAX_TOOL_CALLS", 10))