diff --git a/docker-compose.yaml b/docker-compose.yaml index 833af1d..31782c2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,9 @@ secrets: postgres_password: { file: ./secrets/postgres_password.txt } openai_key: { file: ./secrets/openai_key.txt } anthropic_key: { file: ./secrets/anthropic_key.txt } + ssh_private_key: { file: ./secrets/ssh_private_key } + ssh_public_key: { file: ./secrets/ssh_public_key } + ssh_known_hosts: { file: ./secrets/ssh_known_hosts } # --------------------------------------------------------------------- volumes volumes: @@ -46,9 +49,12 @@ x-worker-base: &worker-base QDRANT_URL: http://qdrant:6333 OPENAI_API_KEY_FILE: /run/secrets/openai_key ANTHROPIC_API_KEY_FILE: /run/secrets/anthropic_key - secrets: [ postgres_password, openai_key, anthropic_key ] + secrets: [ postgres_password, openai_key, anthropic_key, ssh_private_key, ssh_public_key, ssh_known_hosts ] read_only: true - tmpfs: [ /tmp, /var/tmp ] + tmpfs: + - /tmp + - /var/tmp + - /home/kb/.ssh:uid=1000,gid=1000,mode=700 cap_drop: [ ALL ] volumes: - ./memory_files:/app/memory_files:rw diff --git a/docker/workers/Dockerfile b/docker/workers/Dockerfile index 72068af..1ff8ad5 100644 --- a/docker/workers/Dockerfile +++ b/docker/workers/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.11-slim WORKDIR /app RUN apt-get update && apt-get install -y \ - libpq-dev gcc pandoc \ + libpq-dev gcc pandoc git openssh-client \ texlive-xetex texlive-fonts-recommended texlive-plain-generic \ texlive-lang-greek texlive-lang-cyrillic texlive-lang-european \ texlive-luatex texlive-latex-extra texlive-latex-recommended \ @@ -31,12 +31,18 @@ RUN mkdir -p /app/memory_files COPY docker/workers/unnest-table.lua ./unnest-table.lua # Create user and set permissions -RUN useradd -m kb +RUN useradd -m -u 1000 kb RUN mkdir -p /var/cache/fontconfig /home/kb/.cache/fontconfig && \ chown -R kb:kb /var/cache/fontconfig /home/kb/.cache/fontconfig /app USER kb +# Git config will be set via environment variables +ENV GIT_USER_EMAIL=${GIT_USER_EMAIL:-me@some.domain} +ENV GIT_USER_NAME=${GIT_USER_NAME:-memory} +RUN git config --global user.email "${GIT_USER_EMAIL}" && \ + git config --global user.name "${GIT_USER_NAME}" + # Default queues to process ENV QUEUES="ebooks,email,comic,blogs,forums,photo_embed,maintenance" ENV PYTHONPATH="/app" diff --git a/docker/workers/entry.sh b/docker/workers/entry.sh index e936234..c3fe9bb 100644 --- a/docker/workers/entry.sh +++ b/docker/workers/entry.sh @@ -1,6 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +# SSH Setup for git operations +if [ -f /run/secrets/ssh_private_key ]; then + echo "Setting up SSH keys for git operations..." + mkdir -p ~/.ssh + cp /run/secrets/ssh_private_key ~/.ssh/id_rsa + cp /run/secrets/ssh_public_key ~/.ssh/id_rsa.pub + cp /run/secrets/ssh_known_hosts ~/.ssh/known_hosts + chmod 700 ~/.ssh + chmod 600 ~/.ssh/id_rsa + chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/known_hosts + echo "SSH keys configured successfully" +fi + QUEUE_PREFIX=${QUEUE_PREFIX:-memory} QUEUES=${QUEUES:-default} QUEUES=$(IFS=,; echo "${QUEUES}" | tr ',' '\n' | sed "s/^/${QUEUE_PREFIX}-/" | paste -sd, -) diff --git a/src/memory/workers/tasks/notes.py b/src/memory/workers/tasks/notes.py index 8941ca4..402e461 100644 --- a/src/memory/workers/tasks/notes.py +++ b/src/memory/workers/tasks/notes.py @@ -1,6 +1,10 @@ import logging import pathlib +import contextlib +import subprocess +import shlex +from memory.common import settings from memory.common.db.connection import make_session from memory.common.db.models import Note from memory.common.celery_app import app, SYNC_NOTE, SYNC_NOTES @@ -15,6 +19,42 @@ from memory.workers.tasks.content_processing import ( logger = logging.getLogger(__name__) +def git_command(repo_root: pathlib.Path, *args: str): + if not (repo_root / ".git").exists(): + return + + # Properly escape arguments for shell execution + escaped_args = [shlex.quote(arg) for arg in args] + cmd = f"git -C {shlex.quote(repo_root.as_posix())} {' '.join(escaped_args)}" + + logger.info(f"Running git command: {cmd}") + res = subprocess.run( + cmd, + shell=True, + text=True, + capture_output=True, # Capture both stdout and stderr + ) + if res.returncode != 0: + logger.error(f"Git command failed: {res.returncode}") + logger.error(f"stderr: {res.stderr}") + if res.stdout: + logger.error(f"stdout: {res.stdout}") + return res + + +@contextlib.contextmanager +def git_tracking(repo_root: pathlib.Path, commit_message: str = "Sync note"): + git_command(repo_root, "fetch") + git_command(repo_root, "reset", "--hard", "origin/master") + git_command(repo_root, "clean", "-fd") + + yield + + git_command(repo_root, "add", ".") + git_command(repo_root, "commit", "-m", commit_message) + git_command(repo_root, "push") + + @app.task(name=SYNC_NOTE) @safe_task_execution def sync_note( @@ -62,7 +102,10 @@ def sync_note( note.tags = tags # type: ignore note.update_confidences(confidences) - note.save_to_file() + with git_tracking( + settings.NOTES_STORAGE_DIR, f"Sync note {filename}: {subject}" + ): + note.save_to_file() return process_content_item(note, session)