use redis for celery backend

docker-compose.yaml

@@ -19,17 +19,17 @@ secrets:
 volumes:
   db_data: {} # Postgres
   qdrant_data: {} # Qdrant
-  rabbitmq_data: {} # RabbitMQ
+  redis_data: {} # Redis
 
 # ------------------------------ X-templates ----------------------------
 x-common-env: &env
-  RABBITMQ_USER: kb
+  REDIS_HOST: redis
-  RABBITMQ_HOST: rabbitmq
+  REDIS_PORT: 6379
+  REDIS_DB: 0
   CELERY_BROKER_PASSWORD: ${CELERY_BROKER_PASSWORD}
   QDRANT_HOST: qdrant
   DB_HOST: postgres
   DB_PORT: 5432
-  RABBITMQ_PORT: 5672
   FILE_STORAGE_DIR: /app/memory_files
   TZ: "Etc/UTC"
 
@@ -40,7 +40,7 @@ x-worker-base: &worker-base
   restart: unless-stopped
   networks: [ kbnet ]
   security_opt: [ "no-new-privileges=true" ]
-  depends_on: [ postgres, rabbitmq, qdrant ]
+  depends_on: [ postgres, redis, qdrant ]
   env_file: [ .env ]
   environment: &worker-env
     <<: *env
@@ -103,22 +103,21 @@ services:
     volumes:
       - ./db:/app/db:ro
 
-  rabbitmq:
+  redis:
-    image: rabbitmq:3.13-management
+    image: redis:7.2-alpine
     restart: unless-stopped
     networks: [ kbnet ]
-    environment:
+    command: ["redis-server", "--save", "", "--appendonly", "no", "--requirepass", "${CELERY_BROKER_PASSWORD}"]
-      <<: *env
-      RABBITMQ_DEFAULT_USER: "kb"
-      RABBITMQ_DEFAULT_PASS: "${CELERY_BROKER_PASSWORD}"
     volumes:
-      - rabbitmq_data:/var/lib/rabbitmq:rw
+      - redis_data:/data:rw
     healthcheck:
-      test: [ "CMD", "rabbitmq-diagnostics", "ping" ]
+      test: [ "CMD", "redis-cli", "--pass", "${CELERY_BROKER_PASSWORD}", "ping" ]
       interval: 15s
       timeout: 5s
       retries: 5
     security_opt: [ "no-new-privileges=true" ]
+    cap_drop: [ ALL ]
+    user: redis
 
   qdrant:
     image: qdrant/qdrant:v1.14.0
@@ -148,7 +147,7 @@ services:
         SESSION_COOKIE_NAME: "${SESSION_COOKIE_NAME:-session_id}"
     restart: unless-stopped
     networks: [kbnet]
-    depends_on: [postgres, rabbitmq, qdrant]
+    depends_on: [postgres, redis, qdrant]
     environment:
       <<: *env
       POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
@@ -186,7 +185,7 @@ services:
     environment:
       <<: *worker-env
       DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN}
-      DISCORD_NOTIFICATIONS_ENABLED: true
+      DISCORD_NOTIFICATIONS_ENABLED: ${DISCORD_NOTIFICATIONS_ENABLED:-true}
       DISCORD_COLLECTOR_ENABLED: true
     volumes:
       - ./memory_files:/app/memory_files:rw

requirements/requirements-common.txt

@@ -9,4 +9,4 @@ anthropic==0.69.0
 openai==2.3.0
 # Pin the httpx version, as newer versions break the anthropic client
 httpx==0.27.0
-celery[sqs]==5.3.6
+celery[redis,sqs]==5.3.6

src/memory/common/settings.py

@@ -34,15 +34,26 @@ DB_URL = os.getenv("DATABASE_URL", make_db_url())
 
 # Broker settings
 CELERY_QUEUE_PREFIX = os.getenv("CELERY_QUEUE_PREFIX", "memory")
-CELERY_BROKER_TYPE = os.getenv("CELERY_BROKER_TYPE", "amqp").lower()  # amqp or sqs
+CELERY_BROKER_TYPE = os.getenv("CELERY_BROKER_TYPE", "redis").lower()
-CELERY_BROKER_USER = os.getenv("CELERY_BROKER_USER", "kb")
+REDIS_HOST = os.getenv("REDIS_HOST", "redis")
-CELERY_BROKER_PASSWORD = os.getenv("CELERY_BROKER_PASSWORD", "kb")
+REDIS_PORT = os.getenv("REDIS_PORT", "6379")
+REDIS_DB = os.getenv("REDIS_DB", "0")
+CELERY_BROKER_USER = os.getenv(
+    "CELERY_BROKER_USER", "kb" if CELERY_BROKER_TYPE == "amqp" else ""
+)
+CELERY_BROKER_PASSWORD = os.getenv(
+    "CELERY_BROKER_PASSWORD", "" if CELERY_BROKER_TYPE == "redis" else "kb"
+)
+
 
 CELERY_BROKER_HOST = os.getenv("CELERY_BROKER_HOST", "")
-if not CELERY_BROKER_HOST and CELERY_BROKER_TYPE == "amqp":
+if not CELERY_BROKER_HOST:
+    if CELERY_BROKER_TYPE == "amqp":
         RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq")
         RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", "5672")
         CELERY_BROKER_HOST = f"{RABBITMQ_HOST}:{RABBITMQ_PORT}//"
+    elif CELERY_BROKER_TYPE == "redis":
+        CELERY_BROKER_HOST = f"{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
 
 CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", f"db+{DB_URL}")
 
@@ -161,7 +172,7 @@ STATIC_DIR = pathlib.Path(
 )
 
 # Discord notification settings
-DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN", "")
+DISCORD_BOT_ID = int(os.getenv("DISCORD_BOT_ID", "0"))
 DISCORD_ERROR_CHANNEL = os.getenv("DISCORD_ERROR_CHANNEL", "memory-errors")
 DISCORD_ACTIVITY_CHANNEL = os.getenv("DISCORD_ACTIVITY_CHANNEL", "memory-activity")
 DISCORD_DISCOVERY_CHANNEL = os.getenv("DISCORD_DISCOVERY_CHANNEL", "memory-discoveries")
@@ -169,9 +180,7 @@ DISCORD_CHAT_CHANNEL = os.getenv("DISCORD_CHAT_CHANNEL", "memory-chat")
 
 
 # Enable Discord notifications if bot token is set
-DISCORD_NOTIFICATIONS_ENABLED = bool(
+DISCORD_NOTIFICATIONS_ENABLED = boolean_env("DISCORD_NOTIFICATIONS_ENABLED", True)
-    boolean_env("DISCORD_NOTIFICATIONS_ENABLED", True) and DISCORD_BOT_TOKEN
-)
 DISCORD_PROCESS_MESSAGES = boolean_env("DISCORD_PROCESS_MESSAGES", True)
 DISCORD_MODEL = os.getenv("DISCORD_MODEL", "anthropic/claude-sonnet-4-5")
 DISCORD_MAX_TOOL_CALLS = int(os.getenv("DISCORD_MAX_TOOL_CALLS", 10))