add git pushing for notes

docker-compose.yaml

@@ -11,6 +11,9 @@ secrets:
   postgres_password: { file: ./secrets/postgres_password.txt }
   openai_key: { file: ./secrets/openai_key.txt }
   anthropic_key: { file: ./secrets/anthropic_key.txt }
+  ssh_private_key: { file: ./secrets/ssh_private_key }
+  ssh_public_key: { file: ./secrets/ssh_public_key }
+  ssh_known_hosts: { file: ./secrets/ssh_known_hosts }
 
 # --------------------------------------------------------------------- volumes
 volumes:
@@ -46,9 +49,12 @@ x-worker-base: &worker-base
     QDRANT_URL: http://qdrant:6333
     OPENAI_API_KEY_FILE: /run/secrets/openai_key
     ANTHROPIC_API_KEY_FILE: /run/secrets/anthropic_key
-  secrets: [ postgres_password, openai_key, anthropic_key ]
+  secrets: [ postgres_password, openai_key, anthropic_key, ssh_private_key, ssh_public_key, ssh_known_hosts ]
   read_only: true
-  tmpfs: [ /tmp, /var/tmp ]
+  tmpfs: 
+    - /tmp
+    - /var/tmp
+    - /home/kb/.ssh:uid=1000,gid=1000,mode=700
   cap_drop: [ ALL ]
   volumes:
     - ./memory_files:/app/memory_files:rw

docker/workers/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.11-slim
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y \
-    libpq-dev gcc pandoc \
+    libpq-dev gcc pandoc git openssh-client \
     texlive-xetex texlive-fonts-recommended texlive-plain-generic \
     texlive-lang-greek texlive-lang-cyrillic texlive-lang-european \
     texlive-luatex texlive-latex-extra texlive-latex-recommended \
@@ -31,12 +31,18 @@ RUN mkdir -p /app/memory_files
 COPY docker/workers/unnest-table.lua ./unnest-table.lua
 
 # Create user and set permissions
-RUN useradd -m kb
+RUN useradd -m -u 1000 kb
 RUN mkdir -p /var/cache/fontconfig /home/kb/.cache/fontconfig && \
     chown -R kb:kb /var/cache/fontconfig /home/kb/.cache/fontconfig /app
 
 USER kb
 
+# Git config will be set via environment variables
+ENV GIT_USER_EMAIL=${GIT_USER_EMAIL:-me@some.domain}
+ENV GIT_USER_NAME=${GIT_USER_NAME:-memory}
+RUN git config --global user.email "${GIT_USER_EMAIL}" && \
+    git config --global user.name "${GIT_USER_NAME}"
+
 # Default queues to process
 ENV QUEUES="ebooks,email,comic,blogs,forums,photo_embed,maintenance"
 ENV PYTHONPATH="/app"

docker/workers/entry.sh

@@ -1,6 +1,19 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# SSH Setup for git operations
+if [ -f /run/secrets/ssh_private_key ]; then
+    echo "Setting up SSH keys for git operations..."
+    mkdir -p ~/.ssh
+    cp /run/secrets/ssh_private_key ~/.ssh/id_rsa
+    cp /run/secrets/ssh_public_key ~/.ssh/id_rsa.pub
+    cp /run/secrets/ssh_known_hosts ~/.ssh/known_hosts
+    chmod 700 ~/.ssh
+    chmod 600 ~/.ssh/id_rsa
+    chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/known_hosts
+    echo "SSH keys configured successfully"
+fi
+
 QUEUE_PREFIX=${QUEUE_PREFIX:-memory}
 QUEUES=${QUEUES:-default}
 QUEUES=$(IFS=,; echo "${QUEUES}" | tr ',' '\n' | sed "s/^/${QUEUE_PREFIX}-/" | paste -sd, -)

src/memory/workers/tasks/notes.py

@@ -1,6 +1,10 @@
 import logging
 import pathlib
+import contextlib
+import subprocess
+import shlex
 
+from memory.common import settings
 from memory.common.db.connection import make_session
 from memory.common.db.models import Note
 from memory.common.celery_app import app, SYNC_NOTE, SYNC_NOTES
@@ -15,6 +19,42 @@ from memory.workers.tasks.content_processing import (
 logger = logging.getLogger(__name__)
 
 
+def git_command(repo_root: pathlib.Path, *args: str):
+    if not (repo_root / ".git").exists():
+        return
+
+    # Properly escape arguments for shell execution
+    escaped_args = [shlex.quote(arg) for arg in args]
+    cmd = f"git -C {shlex.quote(repo_root.as_posix())} {' '.join(escaped_args)}"
+
+    logger.info(f"Running git command: {cmd}")
+    res = subprocess.run(
+        cmd,
+        shell=True,
+        text=True,
+        capture_output=True,  # Capture both stdout and stderr
+    )
+    if res.returncode != 0:
+        logger.error(f"Git command failed: {res.returncode}")
+        logger.error(f"stderr: {res.stderr}")
+        if res.stdout:
+            logger.error(f"stdout: {res.stdout}")
+    return res
+
+
+@contextlib.contextmanager
+def git_tracking(repo_root: pathlib.Path, commit_message: str = "Sync note"):
+    git_command(repo_root, "fetch")
+    git_command(repo_root, "reset", "--hard", "origin/master")
+    git_command(repo_root, "clean", "-fd")
+
+    yield
+
+    git_command(repo_root, "add", ".")
+    git_command(repo_root, "commit", "-m", commit_message)
+    git_command(repo_root, "push")
+
+
 @app.task(name=SYNC_NOTE)
 @safe_task_execution
 def sync_note(
@@ -62,7 +102,10 @@ def sync_note(
             note.tags = tags  # type: ignore
 
         note.update_confidences(confidences)
-        note.save_to_file()
+        with git_tracking(
+            settings.NOTES_STORAGE_DIR, f"Sync note {filename}: {subject}"
+        ):
+            note.save_to_file()
         return process_content_item(note, session)