mirror of
https://github.com/mruwnik/memory.git
synced 2026-01-02 09:12:58 +01:00
google docs + frontend
This commit is contained in:
Daniel O'Connell
parent
a238ca6329
commit
849f03877a
187
db/migrations/versions/20251229_120000_add_google_drive.py
Normal file
187
db/migrations/versions/20251229_120000_add_google_drive.py
Normal file
@ -0,0 +1,187 @@
|
||||
"""Add Google Drive integration tables
|
||||
|
||||
Revision ID: e1f2a3b4c5d6
|
||||
Revises: d0e1f2a3b4c5
|
||||
Create Date: 2025-12-29 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "e1f2a3b4c5d6"
|
||||
down_revision: Union[str, None] = "d0e1f2a3b4c5"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Create google_oauth_config table (for storing OAuth credentials)
|
||||
op.create_table(
|
||||
"google_oauth_config",
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("name", sa.Text(), nullable=False),
|
||||
sa.Column("client_id", sa.Text(), nullable=False),
|
||||
sa.Column("client_secret", sa.Text(), nullable=False),
|
||||
sa.Column("project_id", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"auth_uri",
|
||||
sa.Text(),
|
||||
server_default="https://accounts.google.com/o/oauth2/auth",
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"token_uri",
|
||||
sa.Text(),
|
||||
server_default="https://oauth2.googleapis.com/token",
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"redirect_uris", sa.ARRAY(sa.Text()), server_default="{}", nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"javascript_origins", sa.ARRAY(sa.Text()), server_default="{}", nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("name"),
|
||||
)
|
||||
|
||||
# Create google_accounts table
|
||||
op.create_table(
|
||||
"google_accounts",
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("name", sa.Text(), nullable=False),
|
||||
sa.Column("email", sa.Text(), nullable=False),
|
||||
sa.Column("access_token", sa.Text(), nullable=True),
|
||||
sa.Column("refresh_token", sa.Text(), nullable=True),
|
||||
sa.Column("token_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("scopes", sa.ARRAY(sa.Text()), server_default="{}", nullable=False),
|
||||
sa.Column("active", sa.Boolean(), server_default="true", nullable=False),
|
||||
sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("sync_error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("email"),
|
||||
)
|
||||
op.create_index(
|
||||
"google_accounts_active_idx", "google_accounts", ["active", "last_sync_at"]
|
||||
)
|
||||
op.create_index("google_accounts_email_idx", "google_accounts", ["email"])
|
||||
|
||||
# Create google_folders table
|
||||
op.create_table(
|
||||
"google_folders",
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("account_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("folder_id", sa.Text(), nullable=False),
|
||||
sa.Column("folder_name", sa.Text(), nullable=False),
|
||||
sa.Column("folder_path", sa.Text(), nullable=True),
|
||||
sa.Column("recursive", sa.Boolean(), server_default="true", nullable=False),
|
||||
sa.Column(
|
||||
"include_shared", sa.Boolean(), server_default="false", nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"mime_type_filter", sa.ARRAY(sa.Text()), server_default="{}", nullable=False
|
||||
),
|
||||
sa.Column("tags", sa.ARRAY(sa.Text()), server_default="{}", nullable=False),
|
||||
sa.Column("check_interval", sa.Integer(), server_default="60", nullable=False),
|
||||
sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("active", sa.Boolean(), server_default="true", nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"], ["google_accounts.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"account_id", "folder_id", name="unique_folder_per_account"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"google_folders_active_idx", "google_folders", ["active", "last_sync_at"]
|
||||
)
|
||||
|
||||
# Create google_doc table (inherits from source_item)
|
||||
op.create_table(
|
||||
"google_doc",
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("google_file_id", sa.Text(), nullable=False),
|
||||
sa.Column("google_modified_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("title", sa.Text(), nullable=False),
|
||||
sa.Column("original_mime_type", sa.Text(), nullable=True),
|
||||
sa.Column("folder_id", sa.BigInteger(), nullable=True),
|
||||
sa.Column("folder_path", sa.Text(), nullable=True),
|
||||
sa.Column("owner", sa.Text(), nullable=True),
|
||||
sa.Column("last_modified_by", sa.Text(), nullable=True),
|
||||
sa.Column("word_count", sa.Integer(), nullable=True),
|
||||
sa.Column("content_hash", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(["id"], ["source_item.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["folder_id"], ["google_folders.id"], ondelete="SET NULL"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"google_doc_file_id_idx", "google_doc", ["google_file_id"], unique=True
|
||||
)
|
||||
op.create_index("google_doc_folder_idx", "google_doc", ["folder_id"])
|
||||
op.create_index("google_doc_modified_idx", "google_doc", ["google_modified_at"])
|
||||
op.create_index("google_doc_title_idx", "google_doc", ["title"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Drop google_doc table
|
||||
op.drop_index("google_doc_title_idx", table_name="google_doc")
|
||||
op.drop_index("google_doc_modified_idx", table_name="google_doc")
|
||||
op.drop_index("google_doc_folder_idx", table_name="google_doc")
|
||||
op.drop_index("google_doc_file_id_idx", table_name="google_doc")
|
||||
op.drop_table("google_doc")
|
||||
|
||||
# Drop google_folders table
|
||||
op.drop_index("google_folders_active_idx", table_name="google_folders")
|
||||
op.drop_table("google_folders")
|
||||
|
||||
# Drop google_accounts table
|
||||
op.drop_index("google_accounts_email_idx", table_name="google_accounts")
|
||||
op.drop_index("google_accounts_active_idx", table_name="google_accounts")
|
||||
op.drop_table("google_accounts")
|
||||
|
||||
# Drop google_oauth_config table
|
||||
op.drop_table("google_oauth_config")
|
||||
@ -959,4 +959,980 @@ body {
|
||||
align-items: stretch;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* ============================================
|
||||
Sources Management Styles
|
||||
============================================ */
|
||||
|
||||
.sources-view {
|
||||
min-height: 100vh;
|
||||
background: #f8fafc;
|
||||
padding: 2rem;
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.sources-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
margin-bottom: 2rem;
|
||||
padding-bottom: 1rem;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.sources-header h2 {
|
||||
color: #2d3748;
|
||||
font-size: 1.8rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.sources-tabs {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 2rem;
|
||||
border-bottom: 2px solid #e2e8f0;
|
||||
padding-bottom: 0;
|
||||
}
|
||||
|
||||
.sources-tabs .tab {
|
||||
background: none;
|
||||
border: none;
|
||||
padding: 0.75rem 1.5rem;
|
||||
font-size: 1rem;
|
||||
color: #718096;
|
||||
cursor: pointer;
|
||||
border-bottom: 2px solid transparent;
|
||||
margin-bottom: -2px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.sources-tabs .tab:hover {
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.sources-tabs .tab.active {
|
||||
color: #667eea;
|
||||
border-bottom-color: #667eea;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Source Panel */
|
||||
.source-panel {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
padding: 1.5rem;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.panel-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.panel-header h3 {
|
||||
color: #2d3748;
|
||||
font-size: 1.2rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Source List */
|
||||
.source-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
/* Source Card */
|
||||
.source-card {
|
||||
background: #f8fafc;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 8px;
|
||||
padding: 1rem;
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
|
||||
.source-card:hover {
|
||||
border-color: #cbd5e0;
|
||||
}
|
||||
|
||||
.source-card.inactive {
|
||||
opacity: 0.7;
|
||||
background: #f1f5f9;
|
||||
}
|
||||
|
||||
.source-card-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-start;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.source-card-info h4 {
|
||||
color: #2d3748;
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
|
||||
.source-subtitle {
|
||||
color: #718096;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.source-details {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.75rem;
|
||||
margin: 0.5rem 0;
|
||||
font-size: 0.85rem;
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.source-description {
|
||||
color: #4a5568;
|
||||
font-size: 0.9rem;
|
||||
margin: 0.5rem 0;
|
||||
}
|
||||
|
||||
.source-card-actions {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
margin-top: 1rem;
|
||||
padding-top: 0.75rem;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.source-card-actions-inline {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
/* Status Badge */
|
||||
.status-badge {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.status-badge.active {
|
||||
background: #c6f6d5;
|
||||
color: #276749;
|
||||
}
|
||||
|
||||
.status-badge.inactive {
|
||||
background: #fed7d7;
|
||||
color: #c53030;
|
||||
}
|
||||
|
||||
/* Sync Status */
|
||||
.sync-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.8rem;
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
.sync-error {
|
||||
color: #c53030;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Buttons */
|
||||
.add-btn {
|
||||
background: #667eea;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 6px;
|
||||
font-size: 0.9rem;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.add-btn:hover {
|
||||
background: #5a67d8;
|
||||
}
|
||||
|
||||
.add-btn.small {
|
||||
padding: 0.25rem 0.75rem;
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.sync-btn {
|
||||
background: #48bb78;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.375rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.sync-btn:hover:not(:disabled) {
|
||||
background: #38a169;
|
||||
}
|
||||
|
||||
.sync-btn:disabled {
|
||||
background: #a0aec0;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.edit-btn {
|
||||
background: #edf2f7;
|
||||
color: #4a5568;
|
||||
border: 1px solid #e2e8f0;
|
||||
padding: 0.375rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.edit-btn:hover {
|
||||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
.delete-btn {
|
||||
background: #fff5f5;
|
||||
color: #c53030;
|
||||
border: 1px solid #fed7d7;
|
||||
padding: 0.375rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.delete-btn:hover {
|
||||
background: #fed7d7;
|
||||
}
|
||||
|
||||
.delete-btn.small {
|
||||
padding: 0.25rem 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.toggle-btn {
|
||||
background: #edf2f7;
|
||||
color: #4a5568;
|
||||
border: 1px solid #e2e8f0;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.toggle-btn:hover {
|
||||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
/* Empty State */
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: 3rem;
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
.empty-state p {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
/* Loading State */
|
||||
.loading-state {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 3rem;
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
.loading-state .loading-spinner {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
/* Error State */
|
||||
.error-state {
|
||||
text-align: center;
|
||||
padding: 2rem;
|
||||
color: #c53030;
|
||||
background: #fff5f5;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.retry-btn {
|
||||
background: #667eea;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 6px;
|
||||
margin-top: 1rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Modal */
|
||||
.modal-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 1000;
|
||||
}
|
||||
|
||||
.modal {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
width: 90%;
|
||||
max-width: 500px;
|
||||
max-height: 90vh;
|
||||
overflow: auto;
|
||||
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
|
||||
.modal-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 1rem 1.5rem;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.modal-header h3 {
|
||||
color: #2d3748;
|
||||
font-size: 1.1rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.modal-close {
|
||||
background: none;
|
||||
border: none;
|
||||
font-size: 1.5rem;
|
||||
color: #718096;
|
||||
cursor: pointer;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.modal-close:hover {
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.modal-content {
|
||||
padding: 1.5rem;
|
||||
}
|
||||
|
||||
/* Form Styles */
|
||||
.source-form {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.form-group label {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.form-group input,
|
||||
.form-group select,
|
||||
.form-group textarea {
|
||||
padding: 0.5rem 0.75rem;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 6px;
|
||||
font-size: 0.9rem;
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
|
||||
.form-group input:focus,
|
||||
.form-group select:focus,
|
||||
.form-group textarea:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
box-shadow: 0 0 0 2px rgba(102, 126, 234, 0.1);
|
||||
}
|
||||
|
||||
.form-group.checkbox label {
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.form-group.checkboxes {
|
||||
flex-direction: row;
|
||||
flex-wrap: wrap;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.checkbox-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
font-size: 0.9rem;
|
||||
color: #4a5568;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.form-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.form-hint {
|
||||
font-size: 0.75rem;
|
||||
color: #718096;
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
|
||||
.form-error {
|
||||
background: #fff5f5;
|
||||
color: #c53030;
|
||||
padding: 0.75rem;
|
||||
border-radius: 6px;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.form-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 0.75rem;
|
||||
margin-top: 0.5rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.cancel-btn {
|
||||
background: #edf2f7;
|
||||
color: #4a5568;
|
||||
border: 1px solid #e2e8f0;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 6px;
|
||||
font-size: 0.9rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.cancel-btn:hover {
|
||||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
.submit-btn {
|
||||
background: #667eea;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1.5rem;
|
||||
border-radius: 6px;
|
||||
font-size: 0.9rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.submit-btn:hover:not(:disabled) {
|
||||
background: #5a67d8;
|
||||
}
|
||||
|
||||
.submit-btn:disabled {
|
||||
background: #a0aec0;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* Tags Input */
|
||||
.tags-input {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.tags-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.tags-input .tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
background: #edf2f7;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.tags-input .tag button {
|
||||
background: none;
|
||||
border: none;
|
||||
color: #718096;
|
||||
cursor: pointer;
|
||||
font-size: 1rem;
|
||||
line-height: 1;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.tags-input .tag button:hover {
|
||||
color: #c53030;
|
||||
}
|
||||
|
||||
.tag-input {
|
||||
padding: 0.375rem 0.5rem;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 4px;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
/* Interval Input */
|
||||
.interval-input {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.interval-input label {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.interval-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.interval-controls input {
|
||||
width: 60px;
|
||||
padding: 0.375rem 0.5rem;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 4px;
|
||||
font-size: 0.9rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.interval-controls span {
|
||||
color: #718096;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
/* Confirm Dialog */
|
||||
.confirm-dialog-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 1001;
|
||||
}
|
||||
|
||||
.confirm-dialog {
|
||||
background: white;
|
||||
padding: 1.5rem;
|
||||
border-radius: 8px;
|
||||
max-width: 400px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.confirm-dialog p {
|
||||
margin-bottom: 1rem;
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.confirm-dialog-buttons {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.confirm-btn {
|
||||
background: #e53e3e;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.confirm-btn:hover {
|
||||
background: #c53030;
|
||||
}
|
||||
|
||||
/* GitHub Account Card */
|
||||
.github-account-card,
|
||||
.google-account-card {
|
||||
background: #f8fafc;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 8px;
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.repos-section,
|
||||
.folders-section {
|
||||
margin-top: 1rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.repos-header,
|
||||
.folders-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.repos-header h5,
|
||||
.folders-header h5 {
|
||||
font-size: 0.9rem;
|
||||
color: #4a5568;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.repos-list,
|
||||
.folders-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.repo-card,
|
||||
.folder-card {
|
||||
background: white;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 6px;
|
||||
padding: 0.75rem;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.75rem;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.repo-card.inactive,
|
||||
.folder-card.inactive {
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.repo-info,
|
||||
.folder-info {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.repo-path {
|
||||
font-weight: 500;
|
||||
color: #2d3748;
|
||||
}
|
||||
|
||||
.folder-name {
|
||||
font-weight: 500;
|
||||
color: #2d3748;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
a.folder-name:hover {
|
||||
color: #667eea;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.folder-path {
|
||||
font-size: 0.8rem;
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
.repo-tracking,
|
||||
.folder-settings {
|
||||
display: flex;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.tracking-badge,
|
||||
.setting-badge {
|
||||
background: #e2e8f0;
|
||||
color: #4a5568;
|
||||
padding: 0.125rem 0.375rem;
|
||||
border-radius: 3px;
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
|
||||
.repo-actions,
|
||||
.folder-actions {
|
||||
display: flex;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.no-repos,
|
||||
.no-folders {
|
||||
color: #718096;
|
||||
font-size: 0.9rem;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.sync-error-banner {
|
||||
background: #fff5f5;
|
||||
color: #c53030;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.85rem;
|
||||
margin: 0.5rem 0;
|
||||
}
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
.sources-view {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.sources-header {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.sources-tabs {
|
||||
overflow-x: auto;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
}
|
||||
|
||||
.sources-tabs .tab {
|
||||
padding: 0.5rem 1rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.panel-header {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.source-card-header {
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.source-card-actions {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.form-row {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.repo-card,
|
||||
.folder-card {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.repo-actions,
|
||||
.folder-actions {
|
||||
width: 100%;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
}
|
||||
|
||||
/* === Folder Browser === */
|
||||
|
||||
.folder-browser {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 400px;
|
||||
max-height: 60vh;
|
||||
}
|
||||
|
||||
.folder-breadcrumb {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
padding: 0.75rem;
|
||||
background: #f8fafc;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.breadcrumb-sep {
|
||||
color: #a0aec0;
|
||||
margin: 0 0.25rem;
|
||||
}
|
||||
|
||||
.breadcrumb-item {
|
||||
background: none;
|
||||
border: none;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
color: #667eea;
|
||||
cursor: pointer;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.breadcrumb-item:hover:not(:disabled) {
|
||||
background: #edf2f7;
|
||||
}
|
||||
|
||||
.breadcrumb-item.current {
|
||||
color: #2d3748;
|
||||
font-weight: 500;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.folder-loading,
|
||||
.folder-empty {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex: 1;
|
||||
color: #718096;
|
||||
font-size: 0.875rem;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.folder-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.folder-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border-radius: 6px;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.folder-item:hover {
|
||||
background: #f7fafc;
|
||||
}
|
||||
|
||||
.folder-item.selected {
|
||||
background: #ebf8ff;
|
||||
}
|
||||
|
||||
.folder-item-checkbox {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.folder-item-checkbox input {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.folder-item-icon {
|
||||
font-size: 1.25rem;
|
||||
width: 24px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.folder-item-name {
|
||||
flex: 1;
|
||||
font-size: 0.875rem;
|
||||
color: #2d3748;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
a.folder-item-name {
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
a.folder-item-name:hover {
|
||||
color: #667eea;
|
||||
text-decoration: underline;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.folder-item-recursive {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
font-size: 0.75rem;
|
||||
color: #718096;
|
||||
margin-left: auto;
|
||||
padding: 0.25rem 0.5rem;
|
||||
background: #f7fafc;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.folder-item-recursive input {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
|
||||
.folder-item-enter {
|
||||
background: none;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 4px;
|
||||
padding: 0.25rem 0.5rem;
|
||||
color: #667eea;
|
||||
cursor: pointer;
|
||||
font-size: 0.875rem;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.folder-item-enter:hover {
|
||||
background: #667eea;
|
||||
color: white;
|
||||
border-color: #667eea;
|
||||
}
|
||||
|
||||
.folder-browser-footer {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.75rem 1rem;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
background: #f8fafc;
|
||||
}
|
||||
|
||||
.selected-count {
|
||||
font-size: 0.875rem;
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
.folder-browser-actions {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.folders-actions {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.add-btn.secondary {
|
||||
background: #f7fafc;
|
||||
color: #4a5568;
|
||||
border: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.add-btn.secondary:hover {
|
||||
background: #edf2f7;
|
||||
}
|
||||
@ -4,7 +4,7 @@ import './App.css'
|
||||
|
||||
import { useAuth } from '@/hooks/useAuth'
|
||||
import { useOAuth } from '@/hooks/useOAuth'
|
||||
import { Loading, LoginPrompt, AuthError, Dashboard, Search } from '@/components'
|
||||
import { Loading, LoginPrompt, AuthError, Dashboard, Search, Sources } from '@/components'
|
||||
|
||||
// AuthWrapper handles redirects based on auth state
|
||||
const AuthWrapper = () => {
|
||||
@ -94,6 +94,14 @@ const AuthWrapper = () => {
|
||||
)
|
||||
} />
|
||||
|
||||
<Route path="/ui/sources" element={
|
||||
isAuthenticated ? (
|
||||
<Sources />
|
||||
) : (
|
||||
<Navigate to="/ui/login" replace />
|
||||
)
|
||||
} />
|
||||
|
||||
{/* Default redirect */}
|
||||
<Route path="/" element={
|
||||
<Navigate to={isAuthenticated ? "/ui/dashboard" : "/ui/login"} replace />
|
||||
|
||||
@ -26,6 +26,11 @@ const Dashboard = ({ onLogout }) => {
|
||||
<p>Search through your knowledge base</p>
|
||||
</Link>
|
||||
|
||||
<Link to="/ui/sources" className="feature-card">
|
||||
<h3>Sources</h3>
|
||||
<p>Manage email, GitHub, RSS feeds, and Google Drive</p>
|
||||
</Link>
|
||||
|
||||
<div className="feature-card" onClick={async () => console.log(await listNotes())}>
|
||||
<h3>📝 Notes</h3>
|
||||
<p>Create and manage your notes</p>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
export { default as Loading } from './Loading'
|
||||
export { default as Dashboard } from './Dashboard'
|
||||
export { default as Search } from './search'
|
||||
export { default as Sources } from './sources'
|
||||
export { default as LoginPrompt } from './auth/LoginPrompt'
|
||||
export { default as AuthError } from './auth/AuthError'
|
||||
1528
frontend/src/components/sources/Sources.tsx
Normal file
1528
frontend/src/components/sources/Sources.tsx
Normal file
File diff suppressed because it is too large
Load Diff
1
frontend/src/components/sources/index.js
Normal file
1
frontend/src/components/sources/index.js
Normal file
@ -0,0 +1 @@
|
||||
export { default } from './Sources'
|
||||
310
frontend/src/components/sources/shared.tsx
Normal file
310
frontend/src/components/sources/shared.tsx
Normal file
@ -0,0 +1,310 @@
|
||||
import { useState, useCallback } from 'react'
|
||||
|
||||
// Format relative time
|
||||
export const formatRelativeTime = (dateString: string | null): string => {
|
||||
if (!dateString) return 'Never'
|
||||
const date = new Date(dateString)
|
||||
const now = new Date()
|
||||
const diffMs = now.getTime() - date.getTime()
|
||||
const diffMins = Math.floor(diffMs / 60000)
|
||||
const diffHours = Math.floor(diffMins / 60)
|
||||
const diffDays = Math.floor(diffHours / 24)
|
||||
|
||||
if (diffMins < 1) return 'Just now'
|
||||
if (diffMins < 60) return `${diffMins}m ago`
|
||||
if (diffHours < 24) return `${diffHours}h ago`
|
||||
if (diffDays < 7) return `${diffDays}d ago`
|
||||
return date.toLocaleDateString()
|
||||
}
|
||||
|
||||
// Status Badge
|
||||
interface StatusBadgeProps {
|
||||
active: boolean
|
||||
onClick?: () => void
|
||||
}
|
||||
|
||||
export const StatusBadge = ({ active, onClick }: StatusBadgeProps) => (
|
||||
<span
|
||||
className={`status-badge ${active ? 'active' : 'inactive'}`}
|
||||
onClick={onClick}
|
||||
style={{ cursor: onClick ? 'pointer' : 'default' }}
|
||||
>
|
||||
{active ? 'Active' : 'Inactive'}
|
||||
</span>
|
||||
)
|
||||
|
||||
// Sync Status
|
||||
interface SyncStatusProps {
|
||||
lastSyncAt: string | null
|
||||
syncError?: string | null
|
||||
}
|
||||
|
||||
export const SyncStatus = ({ lastSyncAt, syncError }: SyncStatusProps) => (
|
||||
<div className="sync-status">
|
||||
<span className="sync-time">
|
||||
Last sync: {formatRelativeTime(lastSyncAt)}
|
||||
</span>
|
||||
{syncError && <span className="sync-error" title={syncError}>Error</span>}
|
||||
</div>
|
||||
)
|
||||
|
||||
// Sync Button
|
||||
interface SyncButtonProps {
|
||||
onSync: () => Promise<void>
|
||||
disabled?: boolean
|
||||
label?: string
|
||||
}
|
||||
|
||||
export const SyncButton = ({ onSync, disabled, label = 'Sync' }: SyncButtonProps) => {
|
||||
const [syncing, setSyncing] = useState(false)
|
||||
|
||||
const handleSync = async () => {
|
||||
setSyncing(true)
|
||||
try {
|
||||
await onSync()
|
||||
} finally {
|
||||
setSyncing(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<button
|
||||
className="sync-btn"
|
||||
onClick={handleSync}
|
||||
disabled={disabled || syncing}
|
||||
>
|
||||
{syncing ? 'Syncing...' : label}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
// Tags Input
|
||||
interface TagsInputProps {
|
||||
tags: string[]
|
||||
onChange: (tags: string[]) => void
|
||||
placeholder?: string
|
||||
}
|
||||
|
||||
export const TagsInput = ({ tags, onChange, placeholder = 'Add tag...' }: TagsInputProps) => {
|
||||
const [input, setInput] = useState('')
|
||||
|
||||
const addTag = () => {
|
||||
const trimmed = input.trim()
|
||||
if (trimmed && !tags.includes(trimmed)) {
|
||||
onChange([...tags, trimmed])
|
||||
setInput('')
|
||||
}
|
||||
}
|
||||
|
||||
const removeTag = (tag: string) => {
|
||||
onChange(tags.filter(t => t !== tag))
|
||||
}
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault()
|
||||
addTag()
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="tags-input">
|
||||
<div className="tags-list">
|
||||
{tags.map(tag => (
|
||||
<span key={tag} className="tag">
|
||||
{tag}
|
||||
<button type="button" onClick={() => removeTag(tag)}>×</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
<input
|
||||
type="text"
|
||||
value={input}
|
||||
onChange={e => setInput(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
onBlur={addTag}
|
||||
placeholder={placeholder}
|
||||
className="tag-input"
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Interval Input (minutes)
|
||||
interface IntervalInputProps {
|
||||
value: number
|
||||
onChange: (value: number) => void
|
||||
label?: string
|
||||
}
|
||||
|
||||
export const IntervalInput = ({ value, onChange, label = 'Check interval' }: IntervalInputProps) => {
|
||||
const hours = Math.floor(value / 60)
|
||||
const minutes = value % 60
|
||||
|
||||
const setHours = (h: number) => onChange(h * 60 + minutes)
|
||||
const setMinutes = (m: number) => onChange(hours * 60 + m)
|
||||
|
||||
return (
|
||||
<div className="interval-input">
|
||||
<label>{label}</label>
|
||||
<div className="interval-controls">
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
value={hours}
|
||||
onChange={e => setHours(parseInt(e.target.value) || 0)}
|
||||
/>
|
||||
<span>h</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="59"
|
||||
value={minutes}
|
||||
onChange={e => setMinutes(parseInt(e.target.value) || 0)}
|
||||
/>
|
||||
<span>m</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Confirm Dialog
|
||||
interface ConfirmDialogProps {
|
||||
message: string
|
||||
onConfirm: () => void
|
||||
onCancel: () => void
|
||||
}
|
||||
|
||||
export const ConfirmDialog = ({ message, onConfirm, onCancel }: ConfirmDialogProps) => (
|
||||
<div className="confirm-dialog-overlay">
|
||||
<div className="confirm-dialog">
|
||||
<p>{message}</p>
|
||||
<div className="confirm-dialog-buttons">
|
||||
<button className="cancel-btn" onClick={onCancel}>Cancel</button>
|
||||
<button className="confirm-btn" onClick={onConfirm}>Confirm</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
// Modal wrapper
|
||||
interface ModalProps {
|
||||
title: string
|
||||
onClose: () => void
|
||||
children: React.ReactNode
|
||||
}
|
||||
|
||||
export const Modal = ({ title, onClose, children }: ModalProps) => (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<div className="modal" onClick={e => e.stopPropagation()}>
|
||||
<div className="modal-header">
|
||||
<h3>{title}</h3>
|
||||
<button className="modal-close" onClick={onClose}>×</button>
|
||||
</div>
|
||||
<div className="modal-content">
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
// Source Card wrapper
|
||||
interface SourceCardProps {
|
||||
title: string
|
||||
subtitle?: string
|
||||
active: boolean
|
||||
lastSyncAt: string | null
|
||||
syncError?: string | null
|
||||
onToggleActive?: () => void
|
||||
onEdit?: () => void
|
||||
onDelete?: () => void
|
||||
onSync?: () => Promise<void>
|
||||
children?: React.ReactNode
|
||||
}
|
||||
|
||||
export const SourceCard = ({
|
||||
title,
|
||||
subtitle,
|
||||
active,
|
||||
lastSyncAt,
|
||||
syncError,
|
||||
onToggleActive,
|
||||
onEdit,
|
||||
onDelete,
|
||||
onSync,
|
||||
children,
|
||||
}: SourceCardProps) => {
|
||||
const [confirmDelete, setConfirmDelete] = useState(false)
|
||||
|
||||
return (
|
||||
<div className={`source-card ${active ? '' : 'inactive'}`}>
|
||||
<div className="source-card-header">
|
||||
<div className="source-card-info">
|
||||
<h4>{title}</h4>
|
||||
{subtitle && <p className="source-subtitle">{subtitle}</p>}
|
||||
</div>
|
||||
<StatusBadge active={active} onClick={onToggleActive} />
|
||||
</div>
|
||||
|
||||
<SyncStatus lastSyncAt={lastSyncAt} syncError={syncError} />
|
||||
|
||||
{children}
|
||||
|
||||
<div className="source-card-actions">
|
||||
{onSync && <SyncButton onSync={onSync} disabled={!active} />}
|
||||
{onEdit && <button className="edit-btn" onClick={onEdit}>Edit</button>}
|
||||
{onDelete && (
|
||||
<button className="delete-btn" onClick={() => setConfirmDelete(true)}>Delete</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{confirmDelete && (
|
||||
<ConfirmDialog
|
||||
message={`Are you sure you want to delete "${title}"?`}
|
||||
onConfirm={() => {
|
||||
onDelete?.()
|
||||
setConfirmDelete(false)
|
||||
}}
|
||||
onCancel={() => setConfirmDelete(false)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Empty state
|
||||
interface EmptyStateProps {
|
||||
message: string
|
||||
actionLabel?: string
|
||||
onAction?: () => void
|
||||
}
|
||||
|
||||
export const EmptyState = ({ message, actionLabel, onAction }: EmptyStateProps) => (
|
||||
<div className="empty-state">
|
||||
<p>{message}</p>
|
||||
{actionLabel && onAction && (
|
||||
<button className="add-btn" onClick={onAction}>{actionLabel}</button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
|
||||
// Loading state
|
||||
export const LoadingState = () => (
|
||||
<div className="loading-state">
|
||||
<div className="loading-spinner"></div>
|
||||
<p>Loading...</p>
|
||||
</div>
|
||||
)
|
||||
|
||||
// Error state
|
||||
interface ErrorStateProps {
|
||||
message: string
|
||||
onRetry?: () => void
|
||||
}
|
||||
|
||||
export const ErrorState = ({ message, onRetry }: ErrorStateProps) => (
|
||||
<div className="error-state">
|
||||
<p>{message}</p>
|
||||
{onRetry && <button className="retry-btn" onClick={onRetry}>Retry</button>}
|
||||
</div>
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
import { useState, useEffect, useCallback } from 'react'
|
||||
|
||||
export const SERVER_URL = import.meta.env.VITE_SERVER_URL || 'http://localhost:8000'
|
||||
export const SERVER_URL = import.meta.env.VITE_SERVER_URL || ''
|
||||
export const SESSION_COOKIE_NAME = import.meta.env.VITE_SESSION_COOKIE_NAME || 'session_id'
|
||||
|
||||
// Cookie utilities
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { useState, useCallback } from 'react'
|
||||
|
||||
const SERVER_URL = import.meta.env.VITE_SERVER_URL || 'http://localhost:8000'
|
||||
const SERVER_URL = import.meta.env.VITE_SERVER_URL || ''
|
||||
const SESSION_COOKIE_NAME = import.meta.env.VITE_SESSION_COOKIE_NAME || 'session_id'
|
||||
const REDIRECT_URI = `${window.location.origin}/ui`
|
||||
|
||||
@ -94,8 +94,8 @@ export const useOAuth = () => {
|
||||
localStorage.setItem('oauth_state', state)
|
||||
localStorage.setItem('code_verifier', codeVerifier)
|
||||
|
||||
// Build authorization URL
|
||||
const authUrl = new URL(`${SERVER_URL}/authorize`)
|
||||
// Build authorization URL (needs full URL for browser redirect)
|
||||
const authUrl = new URL(`${window.location.origin}/authorize`)
|
||||
authUrl.searchParams.set('response_type', 'code')
|
||||
authUrl.searchParams.set('client_id', clientId)
|
||||
authUrl.searchParams.set('redirect_uri', REDIRECT_URI)
|
||||
|
||||
566
frontend/src/hooks/useSources.ts
Normal file
566
frontend/src/hooks/useSources.ts
Normal file
@ -0,0 +1,566 @@
|
||||
import { useCallback } from 'react'
|
||||
import { useAuth, SERVER_URL } from './useAuth'
|
||||
|
||||
// Types for Email Accounts
|
||||
export interface EmailAccount {
|
||||
id: number
|
||||
name: string
|
||||
email_address: string
|
||||
imap_server: string
|
||||
imap_port: number
|
||||
username: string
|
||||
use_ssl: boolean
|
||||
folders: string[]
|
||||
tags: string[]
|
||||
last_sync_at: string | null
|
||||
active: boolean
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
export interface EmailAccountCreate {
|
||||
name: string
|
||||
email_address: string
|
||||
imap_server: string
|
||||
imap_port?: number
|
||||
username: string
|
||||
password: string
|
||||
use_ssl?: boolean
|
||||
folders?: string[]
|
||||
tags?: string[]
|
||||
}
|
||||
|
||||
export interface EmailAccountUpdate {
|
||||
name?: string
|
||||
imap_server?: string
|
||||
imap_port?: number
|
||||
username?: string
|
||||
password?: string
|
||||
use_ssl?: boolean
|
||||
folders?: string[]
|
||||
tags?: string[]
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
// Types for Article Feeds
|
||||
export interface ArticleFeed {
|
||||
id: number
|
||||
url: string
|
||||
title: string | null
|
||||
description: string | null
|
||||
tags: string[]
|
||||
check_interval: number
|
||||
last_checked_at: string | null
|
||||
active: boolean
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
export interface ArticleFeedCreate {
|
||||
url: string
|
||||
title?: string
|
||||
description?: string
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
export interface ArticleFeedUpdate {
|
||||
title?: string
|
||||
description?: string
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
// Types for GitHub
|
||||
export interface GithubRepo {
|
||||
id: number
|
||||
account_id: number
|
||||
owner: string
|
||||
name: string
|
||||
repo_path: string
|
||||
track_issues: boolean
|
||||
track_prs: boolean
|
||||
track_comments: boolean
|
||||
track_project_fields: boolean
|
||||
labels_filter: string[]
|
||||
state_filter: string | null
|
||||
tags: string[]
|
||||
check_interval: number
|
||||
full_sync_interval: number
|
||||
last_sync_at: string | null
|
||||
last_full_sync_at: string | null
|
||||
active: boolean
|
||||
created_at: string
|
||||
}
|
||||
|
||||
export interface GithubAccount {
|
||||
id: number
|
||||
name: string
|
||||
auth_type: 'pat' | 'app'
|
||||
has_access_token: boolean
|
||||
has_private_key: boolean
|
||||
app_id: number | null
|
||||
installation_id: number | null
|
||||
active: boolean
|
||||
last_sync_at: string | null
|
||||
created_at: string
|
||||
updated_at: string
|
||||
repos: GithubRepo[]
|
||||
}
|
||||
|
||||
export interface GithubAccountCreate {
|
||||
name: string
|
||||
auth_type: 'pat' | 'app'
|
||||
access_token?: string
|
||||
app_id?: number
|
||||
installation_id?: number
|
||||
private_key?: string
|
||||
}
|
||||
|
||||
export interface GithubAccountUpdate {
|
||||
name?: string
|
||||
access_token?: string
|
||||
app_id?: number
|
||||
installation_id?: number
|
||||
private_key?: string
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
export interface GithubRepoCreate {
|
||||
owner: string
|
||||
name: string
|
||||
track_issues?: boolean
|
||||
track_prs?: boolean
|
||||
track_comments?: boolean
|
||||
track_project_fields?: boolean
|
||||
labels_filter?: string[]
|
||||
state_filter?: string
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
full_sync_interval?: number
|
||||
}
|
||||
|
||||
export interface GithubRepoUpdate {
|
||||
track_issues?: boolean
|
||||
track_prs?: boolean
|
||||
track_comments?: boolean
|
||||
track_project_fields?: boolean
|
||||
labels_filter?: string[]
|
||||
state_filter?: string
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
full_sync_interval?: number
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
// Types for Google OAuth Config
|
||||
export interface GoogleOAuthConfig {
|
||||
id: number
|
||||
name: string
|
||||
client_id: string
|
||||
project_id: string | null
|
||||
redirect_uris: string[]
|
||||
created_at: string
|
||||
}
|
||||
|
||||
// Types for Google Drive
|
||||
export interface GoogleFolder {
|
||||
id: number
|
||||
folder_id: string
|
||||
folder_name: string
|
||||
folder_path: string | null
|
||||
recursive: boolean
|
||||
include_shared: boolean
|
||||
tags: string[]
|
||||
check_interval: number
|
||||
last_sync_at: string | null
|
||||
active: boolean
|
||||
}
|
||||
|
||||
export interface GoogleAccount {
|
||||
id: number
|
||||
name: string
|
||||
email: string
|
||||
active: boolean
|
||||
last_sync_at: string | null
|
||||
sync_error: string | null
|
||||
folders: GoogleFolder[]
|
||||
}
|
||||
|
||||
export interface GoogleFolderCreate {
|
||||
folder_id: string
|
||||
folder_name: string
|
||||
recursive?: boolean
|
||||
include_shared?: boolean
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
}
|
||||
|
||||
export interface GoogleFolderUpdate {
|
||||
folder_name?: string
|
||||
recursive?: boolean
|
||||
include_shared?: boolean
|
||||
tags?: string[]
|
||||
check_interval?: number
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
// Types for Google Drive browsing
|
||||
export interface DriveItem {
|
||||
id: string
|
||||
name: string
|
||||
mime_type: string
|
||||
is_folder: boolean
|
||||
size: number | null
|
||||
modified_at: string | null
|
||||
}
|
||||
|
||||
export interface BrowseResponse {
|
||||
folder_id: string
|
||||
folder_name: string
|
||||
parent_id: string | null
|
||||
items: DriveItem[]
|
||||
next_page_token: string | null
|
||||
}
|
||||
|
||||
// Task response
|
||||
export interface TaskResponse {
|
||||
task_id: string
|
||||
status: string
|
||||
}
|
||||
|
||||
export const useSources = () => {
|
||||
const { apiCall } = useAuth()
|
||||
|
||||
// === Email Accounts ===
|
||||
|
||||
const listEmailAccounts = useCallback(async (): Promise<EmailAccount[]> => {
|
||||
const response = await apiCall('/email-accounts')
|
||||
if (!response.ok) throw new Error('Failed to fetch email accounts')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const createEmailAccount = useCallback(async (data: EmailAccountCreate): Promise<EmailAccount> => {
|
||||
const response = await apiCall('/email-accounts', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to create email account')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const updateEmailAccount = useCallback(async (id: number, data: EmailAccountUpdate): Promise<EmailAccount> => {
|
||||
const response = await apiCall(`/email-accounts/${id}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to update email account')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteEmailAccount = useCallback(async (id: number): Promise<void> => {
|
||||
const response = await apiCall(`/email-accounts/${id}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete email account')
|
||||
}, [apiCall])
|
||||
|
||||
const syncEmailAccount = useCallback(async (id: number): Promise<TaskResponse> => {
|
||||
const response = await apiCall(`/email-accounts/${id}/sync`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to sync email account')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const testEmailAccount = useCallback(async (id: number): Promise<{ status: string; message: string }> => {
|
||||
const response = await apiCall(`/email-accounts/${id}/test`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to test email account')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
// === Article Feeds ===
|
||||
|
||||
const listArticleFeeds = useCallback(async (): Promise<ArticleFeed[]> => {
|
||||
const response = await apiCall('/article-feeds')
|
||||
if (!response.ok) throw new Error('Failed to fetch article feeds')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const createArticleFeed = useCallback(async (data: ArticleFeedCreate): Promise<ArticleFeed> => {
|
||||
const response = await apiCall('/article-feeds', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to create article feed')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const updateArticleFeed = useCallback(async (id: number, data: ArticleFeedUpdate): Promise<ArticleFeed> => {
|
||||
const response = await apiCall(`/article-feeds/${id}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to update article feed')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteArticleFeed = useCallback(async (id: number): Promise<void> => {
|
||||
const response = await apiCall(`/article-feeds/${id}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete article feed')
|
||||
}, [apiCall])
|
||||
|
||||
const syncArticleFeed = useCallback(async (id: number): Promise<TaskResponse> => {
|
||||
const response = await apiCall(`/article-feeds/${id}/sync`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to sync article feed')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const discoverFeed = useCallback(async (url: string): Promise<{ url: string; title: string | null; description: string | null }> => {
|
||||
const response = await apiCall(`/article-feeds/discover?url=${encodeURIComponent(url)}`, {
|
||||
method: 'POST',
|
||||
})
|
||||
if (!response.ok) throw new Error('Failed to discover feed')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
// === GitHub Accounts ===
|
||||
|
||||
const listGithubAccounts = useCallback(async (): Promise<GithubAccount[]> => {
|
||||
const response = await apiCall('/github/accounts')
|
||||
if (!response.ok) throw new Error('Failed to fetch GitHub accounts')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const createGithubAccount = useCallback(async (data: GithubAccountCreate): Promise<GithubAccount> => {
|
||||
const response = await apiCall('/github/accounts', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to create GitHub account')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const updateGithubAccount = useCallback(async (id: number, data: GithubAccountUpdate): Promise<GithubAccount> => {
|
||||
const response = await apiCall(`/github/accounts/${id}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to update GitHub account')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteGithubAccount = useCallback(async (id: number): Promise<void> => {
|
||||
const response = await apiCall(`/github/accounts/${id}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete GitHub account')
|
||||
}, [apiCall])
|
||||
|
||||
const validateGithubAccount = useCallback(async (id: number): Promise<{ status: string; message: string }> => {
|
||||
const response = await apiCall(`/github/accounts/${id}/validate`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to validate GitHub account')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
// === GitHub Repos ===
|
||||
|
||||
const addGithubRepo = useCallback(async (accountId: number, data: GithubRepoCreate): Promise<GithubRepo> => {
|
||||
const response = await apiCall(`/github/accounts/${accountId}/repos`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to add GitHub repo')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const updateGithubRepo = useCallback(async (accountId: number, repoId: number, data: GithubRepoUpdate): Promise<GithubRepo> => {
|
||||
const response = await apiCall(`/github/accounts/${accountId}/repos/${repoId}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to update GitHub repo')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteGithubRepo = useCallback(async (accountId: number, repoId: number): Promise<void> => {
|
||||
const response = await apiCall(`/github/accounts/${accountId}/repos/${repoId}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete GitHub repo')
|
||||
}, [apiCall])
|
||||
|
||||
const syncGithubRepo = useCallback(async (accountId: number, repoId: number, forceFull = false): Promise<TaskResponse> => {
|
||||
const response = await apiCall(`/github/accounts/${accountId}/repos/${repoId}/sync?force_full=${forceFull}`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to sync GitHub repo')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
// === Google Drive ===
|
||||
|
||||
const listGoogleAccounts = useCallback(async (): Promise<GoogleAccount[]> => {
|
||||
const response = await apiCall('/google-drive/accounts')
|
||||
if (!response.ok) throw new Error('Failed to fetch Google accounts')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const getGoogleAuthUrl = useCallback(async (): Promise<{ authorization_url: string }> => {
|
||||
const response = await apiCall('/google-drive/authorize')
|
||||
if (!response.ok) throw new Error('Failed to get Google auth URL')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteGoogleAccount = useCallback(async (id: number): Promise<void> => {
|
||||
const response = await apiCall(`/google-drive/accounts/${id}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete Google account')
|
||||
}, [apiCall])
|
||||
|
||||
const browseGoogleDrive = useCallback(async (
|
||||
accountId: number,
|
||||
folderId: string = 'root',
|
||||
pageToken?: string
|
||||
): Promise<BrowseResponse> => {
|
||||
const params = new URLSearchParams({ folder_id: folderId })
|
||||
if (pageToken) params.append('page_token', pageToken)
|
||||
const response = await apiCall(`/google-drive/accounts/${accountId}/browse?${params}`)
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to browse Google Drive')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const addGoogleFolder = useCallback(async (accountId: number, data: GoogleFolderCreate): Promise<GoogleFolder> => {
|
||||
const response = await apiCall(`/google-drive/accounts/${accountId}/folders`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to add Google folder')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const updateGoogleFolder = useCallback(async (accountId: number, folderId: number, data: GoogleFolderUpdate): Promise<GoogleFolder> => {
|
||||
const response = await apiCall(`/google-drive/accounts/${accountId}/folders/${folderId}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to update Google folder')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const deleteGoogleFolder = useCallback(async (accountId: number, folderId: number): Promise<void> => {
|
||||
const response = await apiCall(`/google-drive/accounts/${accountId}/folders/${folderId}`, { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete Google folder')
|
||||
}, [apiCall])
|
||||
|
||||
const syncGoogleFolder = useCallback(async (accountId: number, folderId: number, forceFull = false): Promise<TaskResponse> => {
|
||||
const response = await apiCall(`/google-drive/accounts/${accountId}/folders/${folderId}/sync?force_full=${forceFull}`, { method: 'POST' })
|
||||
if (!response.ok) throw new Error('Failed to sync Google folder')
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
// === Google OAuth Config ===
|
||||
|
||||
const getGoogleOAuthConfig = useCallback(async (): Promise<GoogleOAuthConfig | null> => {
|
||||
const response = await apiCall('/google-drive/config')
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error('Failed to fetch Google OAuth config')
|
||||
}
|
||||
return response.json()
|
||||
}, [apiCall])
|
||||
|
||||
const uploadGoogleOAuthConfig = useCallback(async (file: File): Promise<GoogleOAuthConfig> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
|
||||
const accessToken = document.cookie
|
||||
.split('; ')
|
||||
.find(row => row.startsWith('access_token='))
|
||||
?.split('=')[1]
|
||||
|
||||
const response = await fetch('/google-drive/config', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${accessToken}`,
|
||||
},
|
||||
body: formData,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json()
|
||||
throw new Error(error.detail || 'Failed to upload OAuth config')
|
||||
}
|
||||
return response.json()
|
||||
}, [])
|
||||
|
||||
const deleteGoogleOAuthConfig = useCallback(async (): Promise<void> => {
|
||||
const response = await apiCall('/google-drive/config', { method: 'DELETE' })
|
||||
if (!response.ok) throw new Error('Failed to delete Google OAuth config')
|
||||
}, [apiCall])
|
||||
|
||||
return {
|
||||
// Email
|
||||
listEmailAccounts,
|
||||
createEmailAccount,
|
||||
updateEmailAccount,
|
||||
deleteEmailAccount,
|
||||
syncEmailAccount,
|
||||
testEmailAccount,
|
||||
// Article Feeds
|
||||
listArticleFeeds,
|
||||
createArticleFeed,
|
||||
updateArticleFeed,
|
||||
deleteArticleFeed,
|
||||
syncArticleFeed,
|
||||
discoverFeed,
|
||||
// GitHub Accounts
|
||||
listGithubAccounts,
|
||||
createGithubAccount,
|
||||
updateGithubAccount,
|
||||
deleteGithubAccount,
|
||||
validateGithubAccount,
|
||||
// GitHub Repos
|
||||
addGithubRepo,
|
||||
updateGithubRepo,
|
||||
deleteGithubRepo,
|
||||
syncGithubRepo,
|
||||
// Google Drive
|
||||
listGoogleAccounts,
|
||||
getGoogleAuthUrl,
|
||||
deleteGoogleAccount,
|
||||
browseGoogleDrive,
|
||||
addGoogleFolder,
|
||||
updateGoogleFolder,
|
||||
deleteGoogleFolder,
|
||||
syncGoogleFolder,
|
||||
// Google OAuth Config
|
||||
getGoogleOAuthConfig,
|
||||
uploadGoogleOAuthConfig,
|
||||
deleteGoogleOAuthConfig,
|
||||
}
|
||||
}
|
||||
@ -11,4 +11,18 @@ export default defineConfig({
|
||||
'@': path.resolve(__dirname, './src'),
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
// Proxy API requests to backend during development
|
||||
'/register': 'http://localhost:8000',
|
||||
'/authorize': 'http://localhost:8000',
|
||||
'/token': 'http://localhost:8000',
|
||||
'/auth': 'http://localhost:8000',
|
||||
'/health': 'http://localhost:8000',
|
||||
'/email-accounts': 'http://localhost:8000',
|
||||
'/article-feeds': 'http://localhost:8000',
|
||||
'/github': 'http://localhost:8000',
|
||||
'/google-drive': 'http://localhost:8000',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@ -1,3 +1,2 @@
|
||||
discord.py==2.3.2
|
||||
uvicorn==0.29.0
|
||||
fastapi==0.112.2
|
||||
uvicorn==0.29.0
|
||||
@ -5,3 +5,8 @@ markdownify==0.13.1
|
||||
pillow==10.3.0
|
||||
pypandoc==1.15
|
||||
feedparser==6.0.10
|
||||
|
||||
# Google Drive API
|
||||
google-auth>=2.0.0
|
||||
google-auth-oauthlib>=1.0.0
|
||||
google-api-python-client>=2.0.0
|
||||
|
||||
@ -23,6 +23,10 @@ from memory.api.auth import (
|
||||
AuthenticationMiddleware,
|
||||
router as auth_router,
|
||||
)
|
||||
from memory.api.google_drive import router as google_drive_router
|
||||
from memory.api.email_accounts import router as email_accounts_router
|
||||
from memory.api.article_feeds import router as article_feeds_router
|
||||
from memory.api.github_sources import router as github_sources_router
|
||||
from memory.api.MCP.base import mcp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -60,7 +64,7 @@ app.add_middleware(AuthenticationMiddleware)
|
||||
# allow_credentials=True requires specific origins, not wildcards.
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.SERVER_URL],
|
||||
allow_origins=[settings.SERVER_URL, "http://localhost:5173"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
@ -149,6 +153,10 @@ admin = Admin(app, engine)
|
||||
# Setup admin with OAuth protection using existing OAuth provider
|
||||
setup_admin(admin)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(google_drive_router)
|
||||
app.include_router(email_accounts_router)
|
||||
app.include_router(article_feeds_router)
|
||||
app.include_router(github_sources_router)
|
||||
|
||||
|
||||
# Add health check to MCP server instead of main app
|
||||
|
||||
203
src/memory/api/article_feeds.py
Normal file
203
src/memory/api/article_feeds.py
Normal file
@ -0,0 +1,203 @@
|
||||
"""API endpoints for Article Feed management."""
|
||||
|
||||
from typing import cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, HttpUrl
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from memory.common.db.connection import get_session
|
||||
from memory.common.db.models import User
|
||||
from memory.common.db.models.sources import ArticleFeed
|
||||
from memory.api.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/article-feeds", tags=["article-feeds"])
|
||||
|
||||
|
||||
class ArticleFeedCreate(BaseModel):
|
||||
url: HttpUrl
|
||||
title: str | None = None
|
||||
description: str | None = None
|
||||
tags: list[str] = []
|
||||
check_interval: int = 1440 # 24 hours in minutes
|
||||
active: bool = True
|
||||
|
||||
|
||||
class ArticleFeedUpdate(BaseModel):
|
||||
title: str | None = None
|
||||
description: str | None = None
|
||||
tags: list[str] | None = None
|
||||
check_interval: int | None = None
|
||||
active: bool | None = None
|
||||
|
||||
|
||||
class ArticleFeedResponse(BaseModel):
|
||||
id: int
|
||||
url: str
|
||||
title: str | None
|
||||
description: str | None
|
||||
tags: list[str]
|
||||
check_interval: int
|
||||
last_checked_at: str | None
|
||||
active: bool
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class FeedDiscoveryResponse(BaseModel):
|
||||
url: str
|
||||
title: str | None
|
||||
description: str | None
|
||||
|
||||
|
||||
def feed_to_response(feed: ArticleFeed) -> ArticleFeedResponse:
|
||||
"""Convert an ArticleFeed model to a response model."""
|
||||
return ArticleFeedResponse(
|
||||
id=cast(int, feed.id),
|
||||
url=cast(str, feed.url),
|
||||
title=cast(str | None, feed.title),
|
||||
description=cast(str | None, feed.description),
|
||||
tags=list(feed.tags or []),
|
||||
check_interval=cast(int, feed.check_interval),
|
||||
last_checked_at=feed.last_checked_at.isoformat() if feed.last_checked_at else None,
|
||||
active=cast(bool, feed.active),
|
||||
created_at=feed.created_at.isoformat() if feed.created_at else "",
|
||||
updated_at=feed.updated_at.isoformat() if feed.updated_at else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_feeds(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> list[ArticleFeedResponse]:
|
||||
"""List all article feeds."""
|
||||
feeds = db.query(ArticleFeed).all()
|
||||
return [feed_to_response(feed) for feed in feeds]
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_feed(
|
||||
data: ArticleFeedCreate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> ArticleFeedResponse:
|
||||
"""Create a new article feed."""
|
||||
url_str = str(data.url)
|
||||
|
||||
# Check for duplicate URL
|
||||
existing = db.query(ArticleFeed).filter(ArticleFeed.url == url_str).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Feed with this URL already exists")
|
||||
|
||||
feed = ArticleFeed(
|
||||
url=url_str,
|
||||
title=data.title or url_str,
|
||||
description=data.description,
|
||||
tags=data.tags,
|
||||
check_interval=data.check_interval,
|
||||
active=data.active,
|
||||
)
|
||||
db.add(feed)
|
||||
db.commit()
|
||||
db.refresh(feed)
|
||||
|
||||
return feed_to_response(feed)
|
||||
|
||||
|
||||
@router.get("/{feed_id}")
|
||||
def get_feed(
|
||||
feed_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> ArticleFeedResponse:
|
||||
"""Get a single article feed."""
|
||||
feed = db.get(ArticleFeed, feed_id)
|
||||
if not feed:
|
||||
raise HTTPException(status_code=404, detail="Feed not found")
|
||||
return feed_to_response(feed)
|
||||
|
||||
|
||||
@router.patch("/{feed_id}")
|
||||
def update_feed(
|
||||
feed_id: int,
|
||||
updates: ArticleFeedUpdate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> ArticleFeedResponse:
|
||||
"""Update an article feed."""
|
||||
feed = db.get(ArticleFeed, feed_id)
|
||||
if not feed:
|
||||
raise HTTPException(status_code=404, detail="Feed not found")
|
||||
|
||||
if updates.title is not None:
|
||||
feed.title = updates.title
|
||||
if updates.description is not None:
|
||||
feed.description = updates.description
|
||||
if updates.tags is not None:
|
||||
feed.tags = updates.tags
|
||||
if updates.check_interval is not None:
|
||||
feed.check_interval = updates.check_interval
|
||||
if updates.active is not None:
|
||||
feed.active = updates.active
|
||||
|
||||
db.commit()
|
||||
db.refresh(feed)
|
||||
|
||||
return feed_to_response(feed)
|
||||
|
||||
|
||||
@router.delete("/{feed_id}")
|
||||
def delete_feed(
|
||||
feed_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Delete an article feed."""
|
||||
feed = db.get(ArticleFeed, feed_id)
|
||||
if not feed:
|
||||
raise HTTPException(status_code=404, detail="Feed not found")
|
||||
|
||||
db.delete(feed)
|
||||
db.commit()
|
||||
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/{feed_id}/sync")
|
||||
def trigger_sync(
|
||||
feed_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Manually trigger a sync for an article feed."""
|
||||
from memory.workers.tasks.blogs import sync_article_feed
|
||||
|
||||
feed = db.get(ArticleFeed, feed_id)
|
||||
if not feed:
|
||||
raise HTTPException(status_code=404, detail="Feed not found")
|
||||
|
||||
task = sync_article_feed.delay(feed_id)
|
||||
|
||||
return {"task_id": task.id, "status": "scheduled"}
|
||||
|
||||
|
||||
@router.post("/discover")
|
||||
def discover_feed(
|
||||
url: HttpUrl,
|
||||
user: User = Depends(get_current_user),
|
||||
) -> FeedDiscoveryResponse:
|
||||
"""Auto-discover feed metadata from a URL."""
|
||||
from memory.parsers.feeds import get_feed_parser
|
||||
|
||||
url_str = str(url)
|
||||
parser = get_feed_parser(url_str)
|
||||
|
||||
if not parser:
|
||||
raise HTTPException(status_code=400, detail="Could not parse feed from URL")
|
||||
|
||||
return FeedDiscoveryResponse(
|
||||
url=url_str,
|
||||
title=parser.title,
|
||||
description=parser.description,
|
||||
)
|
||||
@ -37,6 +37,7 @@ WHITELIST = {
|
||||
"/.well-known/",
|
||||
"/ui",
|
||||
"/admin/statics/", # SQLAdmin static resources
|
||||
"/google-drive/callback", # Google OAuth callback
|
||||
}
|
||||
|
||||
|
||||
|
||||
233
src/memory/api/email_accounts.py
Normal file
233
src/memory/api/email_accounts.py
Normal file
@ -0,0 +1,233 @@
|
||||
"""API endpoints for Email Account management."""
|
||||
|
||||
from typing import cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, EmailStr
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from memory.common.db.connection import get_session
|
||||
from memory.common.db.models import User
|
||||
from memory.common.db.models.sources import EmailAccount
|
||||
from memory.api.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/email-accounts", tags=["email-accounts"])
|
||||
|
||||
|
||||
class EmailAccountCreate(BaseModel):
|
||||
name: str
|
||||
email_address: EmailStr
|
||||
imap_server: str
|
||||
imap_port: int = 993
|
||||
username: str
|
||||
password: str
|
||||
use_ssl: bool = True
|
||||
folders: list[str] = []
|
||||
tags: list[str] = []
|
||||
|
||||
|
||||
class EmailAccountUpdate(BaseModel):
|
||||
name: str | None = None
|
||||
imap_server: str | None = None
|
||||
imap_port: int | None = None
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
use_ssl: bool | None = None
|
||||
folders: list[str] | None = None
|
||||
tags: list[str] | None = None
|
||||
active: bool | None = None
|
||||
|
||||
|
||||
class EmailAccountResponse(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
email_address: str
|
||||
imap_server: str
|
||||
imap_port: int
|
||||
username: str
|
||||
use_ssl: bool
|
||||
folders: list[str]
|
||||
tags: list[str]
|
||||
last_sync_at: str | None
|
||||
active: bool
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
def account_to_response(account: EmailAccount) -> EmailAccountResponse:
|
||||
"""Convert an EmailAccount model to a response model."""
|
||||
return EmailAccountResponse(
|
||||
id=cast(int, account.id),
|
||||
name=cast(str, account.name),
|
||||
email_address=cast(str, account.email_address),
|
||||
imap_server=cast(str, account.imap_server),
|
||||
imap_port=cast(int, account.imap_port),
|
||||
username=cast(str, account.username),
|
||||
use_ssl=cast(bool, account.use_ssl),
|
||||
folders=list(account.folders or []),
|
||||
tags=list(account.tags or []),
|
||||
last_sync_at=account.last_sync_at.isoformat() if account.last_sync_at else None,
|
||||
active=cast(bool, account.active),
|
||||
created_at=account.created_at.isoformat() if account.created_at else "",
|
||||
updated_at=account.updated_at.isoformat() if account.updated_at else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_accounts(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> list[EmailAccountResponse]:
|
||||
"""List all email accounts."""
|
||||
accounts = db.query(EmailAccount).all()
|
||||
return [account_to_response(account) for account in accounts]
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_account(
|
||||
data: EmailAccountCreate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> EmailAccountResponse:
|
||||
"""Create a new email account."""
|
||||
# Check for duplicate email address
|
||||
existing = (
|
||||
db.query(EmailAccount)
|
||||
.filter(EmailAccount.email_address == data.email_address)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Email account already exists")
|
||||
|
||||
account = EmailAccount(
|
||||
name=data.name,
|
||||
email_address=data.email_address,
|
||||
imap_server=data.imap_server,
|
||||
imap_port=data.imap_port,
|
||||
username=data.username,
|
||||
password=data.password,
|
||||
use_ssl=data.use_ssl,
|
||||
folders=data.folders,
|
||||
tags=data.tags,
|
||||
)
|
||||
db.add(account)
|
||||
db.commit()
|
||||
db.refresh(account)
|
||||
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.get("/{account_id}")
|
||||
def get_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> EmailAccountResponse:
|
||||
"""Get a single email account."""
|
||||
account = db.get(EmailAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.patch("/{account_id}")
|
||||
def update_account(
|
||||
account_id: int,
|
||||
updates: EmailAccountUpdate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> EmailAccountResponse:
|
||||
"""Update an email account."""
|
||||
account = db.get(EmailAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
if updates.name is not None:
|
||||
account.name = updates.name
|
||||
if updates.imap_server is not None:
|
||||
account.imap_server = updates.imap_server
|
||||
if updates.imap_port is not None:
|
||||
account.imap_port = updates.imap_port
|
||||
if updates.username is not None:
|
||||
account.username = updates.username
|
||||
if updates.password is not None:
|
||||
account.password = updates.password
|
||||
if updates.use_ssl is not None:
|
||||
account.use_ssl = updates.use_ssl
|
||||
if updates.folders is not None:
|
||||
account.folders = updates.folders
|
||||
if updates.tags is not None:
|
||||
account.tags = updates.tags
|
||||
if updates.active is not None:
|
||||
account.active = updates.active
|
||||
|
||||
db.commit()
|
||||
db.refresh(account)
|
||||
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.delete("/{account_id}")
|
||||
def delete_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Delete an email account."""
|
||||
account = db.get(EmailAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
db.delete(account)
|
||||
db.commit()
|
||||
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/{account_id}/sync")
|
||||
def trigger_sync(
|
||||
account_id: int,
|
||||
since_date: str | None = None,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Manually trigger a sync for an email account."""
|
||||
from memory.workers.tasks.email import sync_account
|
||||
|
||||
account = db.get(EmailAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
task = sync_account.delay(account_id, since_date=since_date)
|
||||
|
||||
return {"task_id": task.id, "status": "scheduled"}
|
||||
|
||||
|
||||
@router.post("/{account_id}/test")
|
||||
def test_connection(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Test IMAP connection for an email account."""
|
||||
from memory.workers.email import imap_connection
|
||||
|
||||
account = db.get(EmailAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
try:
|
||||
with imap_connection(account) as conn:
|
||||
# List folders to verify connection works
|
||||
status, folders = conn.list()
|
||||
if status != "OK":
|
||||
return {"status": "error", "message": "Failed to list folders"}
|
||||
|
||||
folder_count = len(folders) if folders else 0
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Connected successfully. Found {folder_count} folders.",
|
||||
"folders": folder_count,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": str(e)}
|
||||
447
src/memory/api/github_sources.py
Normal file
447
src/memory/api/github_sources.py
Normal file
@ -0,0 +1,447 @@
|
||||
"""API endpoints for GitHub Account and Repo management."""
|
||||
|
||||
from typing import Literal, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from memory.common.db.connection import get_session
|
||||
from memory.common.db.models import User
|
||||
from memory.common.db.models.sources import GithubAccount, GithubRepo
|
||||
from memory.api.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/github", tags=["github"])
|
||||
|
||||
|
||||
# --- Account Models ---
|
||||
|
||||
|
||||
class GithubAccountCreate(BaseModel):
|
||||
name: str
|
||||
auth_type: Literal["pat", "app"]
|
||||
# For PAT auth
|
||||
access_token: str | None = None
|
||||
# For App auth
|
||||
app_id: int | None = None
|
||||
installation_id: int | None = None
|
||||
private_key: str | None = None
|
||||
|
||||
|
||||
class GithubAccountUpdate(BaseModel):
|
||||
name: str | None = None
|
||||
access_token: str | None = None
|
||||
app_id: int | None = None
|
||||
installation_id: int | None = None
|
||||
private_key: str | None = None
|
||||
active: bool | None = None
|
||||
|
||||
|
||||
class GithubRepoResponse(BaseModel):
|
||||
id: int
|
||||
account_id: int
|
||||
owner: str
|
||||
name: str
|
||||
repo_path: str
|
||||
track_issues: bool
|
||||
track_prs: bool
|
||||
track_comments: bool
|
||||
track_project_fields: bool
|
||||
labels_filter: list[str]
|
||||
state_filter: str | None
|
||||
tags: list[str]
|
||||
check_interval: int
|
||||
full_sync_interval: int
|
||||
last_sync_at: str | None
|
||||
last_full_sync_at: str | None
|
||||
active: bool
|
||||
created_at: str
|
||||
|
||||
|
||||
class GithubAccountResponse(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
auth_type: str
|
||||
has_access_token: bool
|
||||
has_private_key: bool
|
||||
app_id: int | None
|
||||
installation_id: int | None
|
||||
active: bool
|
||||
last_sync_at: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
repos: list[GithubRepoResponse]
|
||||
|
||||
|
||||
# --- Repo Models ---
|
||||
|
||||
|
||||
class GithubRepoCreate(BaseModel):
|
||||
owner: str
|
||||
name: str
|
||||
track_issues: bool = True
|
||||
track_prs: bool = True
|
||||
track_comments: bool = True
|
||||
track_project_fields: bool = False
|
||||
labels_filter: list[str] = []
|
||||
state_filter: str | None = None
|
||||
tags: list[str] = []
|
||||
check_interval: int = 60
|
||||
full_sync_interval: int = 1440
|
||||
|
||||
|
||||
class GithubRepoUpdate(BaseModel):
|
||||
track_issues: bool | None = None
|
||||
track_prs: bool | None = None
|
||||
track_comments: bool | None = None
|
||||
track_project_fields: bool | None = None
|
||||
labels_filter: list[str] | None = None
|
||||
state_filter: str | None = None
|
||||
tags: list[str] | None = None
|
||||
check_interval: int | None = None
|
||||
full_sync_interval: int | None = None
|
||||
active: bool | None = None
|
||||
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
|
||||
def repo_to_response(repo: GithubRepo) -> GithubRepoResponse:
|
||||
"""Convert a GithubRepo model to a response model."""
|
||||
return GithubRepoResponse(
|
||||
id=cast(int, repo.id),
|
||||
account_id=cast(int, repo.account_id),
|
||||
owner=cast(str, repo.owner),
|
||||
name=cast(str, repo.name),
|
||||
repo_path=repo.repo_path,
|
||||
track_issues=cast(bool, repo.track_issues),
|
||||
track_prs=cast(bool, repo.track_prs),
|
||||
track_comments=cast(bool, repo.track_comments),
|
||||
track_project_fields=cast(bool, repo.track_project_fields),
|
||||
labels_filter=list(repo.labels_filter or []),
|
||||
state_filter=cast(str | None, repo.state_filter),
|
||||
tags=list(repo.tags or []),
|
||||
check_interval=cast(int, repo.check_interval),
|
||||
full_sync_interval=cast(int, repo.full_sync_interval),
|
||||
last_sync_at=repo.last_sync_at.isoformat() if repo.last_sync_at else None,
|
||||
last_full_sync_at=repo.last_full_sync_at.isoformat() if repo.last_full_sync_at else None,
|
||||
active=cast(bool, repo.active),
|
||||
created_at=repo.created_at.isoformat() if repo.created_at else "",
|
||||
)
|
||||
|
||||
|
||||
def account_to_response(account: GithubAccount) -> GithubAccountResponse:
|
||||
"""Convert a GithubAccount model to a response model."""
|
||||
return GithubAccountResponse(
|
||||
id=cast(int, account.id),
|
||||
name=cast(str, account.name),
|
||||
auth_type=cast(str, account.auth_type),
|
||||
has_access_token=bool(account.access_token),
|
||||
has_private_key=bool(account.private_key),
|
||||
app_id=cast(int | None, account.app_id),
|
||||
installation_id=cast(int | None, account.installation_id),
|
||||
active=cast(bool, account.active),
|
||||
last_sync_at=account.last_sync_at.isoformat() if account.last_sync_at else None,
|
||||
created_at=account.created_at.isoformat() if account.created_at else "",
|
||||
updated_at=account.updated_at.isoformat() if account.updated_at else "",
|
||||
repos=[repo_to_response(repo) for repo in account.repos],
|
||||
)
|
||||
|
||||
|
||||
# --- Account Endpoints ---
|
||||
|
||||
|
||||
@router.get("/accounts")
|
||||
def list_accounts(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> list[GithubAccountResponse]:
|
||||
"""List all GitHub accounts with their repos."""
|
||||
accounts = db.query(GithubAccount).all()
|
||||
return [account_to_response(account) for account in accounts]
|
||||
|
||||
|
||||
@router.post("/accounts")
|
||||
def create_account(
|
||||
data: GithubAccountCreate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> GithubAccountResponse:
|
||||
"""Create a new GitHub account."""
|
||||
# Validate auth configuration
|
||||
if data.auth_type == "pat":
|
||||
if not data.access_token:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="access_token is required for PAT authentication",
|
||||
)
|
||||
elif data.auth_type == "app":
|
||||
if not all([data.app_id, data.installation_id, data.private_key]):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="app_id, installation_id, and private_key are required for App authentication",
|
||||
)
|
||||
|
||||
account = GithubAccount(
|
||||
name=data.name,
|
||||
auth_type=data.auth_type,
|
||||
access_token=data.access_token,
|
||||
app_id=data.app_id,
|
||||
installation_id=data.installation_id,
|
||||
private_key=data.private_key,
|
||||
)
|
||||
db.add(account)
|
||||
db.commit()
|
||||
db.refresh(account)
|
||||
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.get("/accounts/{account_id}")
|
||||
def get_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> GithubAccountResponse:
|
||||
"""Get a single GitHub account."""
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.patch("/accounts/{account_id}")
|
||||
def update_account(
|
||||
account_id: int,
|
||||
updates: GithubAccountUpdate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> GithubAccountResponse:
|
||||
"""Update a GitHub account."""
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
if updates.name is not None:
|
||||
account.name = updates.name
|
||||
if updates.access_token is not None:
|
||||
account.access_token = updates.access_token
|
||||
if updates.app_id is not None:
|
||||
account.app_id = updates.app_id
|
||||
if updates.installation_id is not None:
|
||||
account.installation_id = updates.installation_id
|
||||
if updates.private_key is not None:
|
||||
account.private_key = updates.private_key
|
||||
if updates.active is not None:
|
||||
account.active = updates.active
|
||||
|
||||
db.commit()
|
||||
db.refresh(account)
|
||||
|
||||
return account_to_response(account)
|
||||
|
||||
|
||||
@router.delete("/accounts/{account_id}")
|
||||
def delete_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Delete a GitHub account and all its repos."""
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
db.delete(account)
|
||||
db.commit()
|
||||
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/accounts/{account_id}/validate")
|
||||
def validate_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Validate GitHub API access for an account."""
|
||||
from memory.parsers.github import GithubClient, GithubCredentials
|
||||
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
try:
|
||||
credentials = GithubCredentials(
|
||||
auth_type=cast(str, account.auth_type),
|
||||
access_token=cast(str | None, account.access_token),
|
||||
app_id=cast(int | None, account.app_id),
|
||||
installation_id=cast(int | None, account.installation_id),
|
||||
private_key=cast(str | None, account.private_key),
|
||||
)
|
||||
client = GithubClient(credentials)
|
||||
|
||||
# Test by getting authenticated user info
|
||||
user_info = client.get_authenticated_user()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Authenticated as {user_info.get('login', 'unknown')}",
|
||||
"user": user_info.get("login"),
|
||||
"scopes": user_info.get("scopes", []),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
# --- Repo Endpoints ---
|
||||
|
||||
|
||||
@router.get("/accounts/{account_id}/repos")
|
||||
def list_repos(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> list[GithubRepoResponse]:
|
||||
"""List all repos for a GitHub account."""
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
return [repo_to_response(repo) for repo in account.repos]
|
||||
|
||||
|
||||
@router.post("/accounts/{account_id}/repos")
|
||||
def add_repo(
|
||||
account_id: int,
|
||||
data: GithubRepoCreate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> GithubRepoResponse:
|
||||
"""Add a repo to track for a GitHub account."""
|
||||
account = db.get(GithubAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
# Check for duplicate
|
||||
existing = (
|
||||
db.query(GithubRepo)
|
||||
.filter(
|
||||
GithubRepo.account_id == account_id,
|
||||
GithubRepo.owner == data.owner,
|
||||
GithubRepo.name == data.name,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Repo already tracked")
|
||||
|
||||
repo = GithubRepo(
|
||||
account_id=account_id,
|
||||
owner=data.owner,
|
||||
name=data.name,
|
||||
track_issues=data.track_issues,
|
||||
track_prs=data.track_prs,
|
||||
track_comments=data.track_comments,
|
||||
track_project_fields=data.track_project_fields,
|
||||
labels_filter=data.labels_filter,
|
||||
state_filter=data.state_filter,
|
||||
tags=data.tags,
|
||||
check_interval=data.check_interval,
|
||||
full_sync_interval=data.full_sync_interval,
|
||||
)
|
||||
db.add(repo)
|
||||
db.commit()
|
||||
db.refresh(repo)
|
||||
|
||||
return repo_to_response(repo)
|
||||
|
||||
|
||||
@router.patch("/accounts/{account_id}/repos/{repo_id}")
|
||||
def update_repo(
|
||||
account_id: int,
|
||||
repo_id: int,
|
||||
updates: GithubRepoUpdate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> GithubRepoResponse:
|
||||
"""Update a repo's tracking configuration."""
|
||||
repo = (
|
||||
db.query(GithubRepo)
|
||||
.filter(GithubRepo.account_id == account_id, GithubRepo.id == repo_id)
|
||||
.first()
|
||||
)
|
||||
if not repo:
|
||||
raise HTTPException(status_code=404, detail="Repo not found")
|
||||
|
||||
if updates.track_issues is not None:
|
||||
repo.track_issues = updates.track_issues
|
||||
if updates.track_prs is not None:
|
||||
repo.track_prs = updates.track_prs
|
||||
if updates.track_comments is not None:
|
||||
repo.track_comments = updates.track_comments
|
||||
if updates.track_project_fields is not None:
|
||||
repo.track_project_fields = updates.track_project_fields
|
||||
if updates.labels_filter is not None:
|
||||
repo.labels_filter = updates.labels_filter
|
||||
if updates.state_filter is not None:
|
||||
repo.state_filter = updates.state_filter
|
||||
if updates.tags is not None:
|
||||
repo.tags = updates.tags
|
||||
if updates.check_interval is not None:
|
||||
repo.check_interval = updates.check_interval
|
||||
if updates.full_sync_interval is not None:
|
||||
repo.full_sync_interval = updates.full_sync_interval
|
||||
if updates.active is not None:
|
||||
repo.active = updates.active
|
||||
|
||||
db.commit()
|
||||
db.refresh(repo)
|
||||
|
||||
return repo_to_response(repo)
|
||||
|
||||
|
||||
@router.delete("/accounts/{account_id}/repos/{repo_id}")
|
||||
def remove_repo(
|
||||
account_id: int,
|
||||
repo_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Remove a repo from tracking."""
|
||||
repo = (
|
||||
db.query(GithubRepo)
|
||||
.filter(GithubRepo.account_id == account_id, GithubRepo.id == repo_id)
|
||||
.first()
|
||||
)
|
||||
if not repo:
|
||||
raise HTTPException(status_code=404, detail="Repo not found")
|
||||
|
||||
db.delete(repo)
|
||||
db.commit()
|
||||
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/accounts/{account_id}/repos/{repo_id}/sync")
|
||||
def trigger_repo_sync(
|
||||
account_id: int,
|
||||
repo_id: int,
|
||||
force_full: bool = False,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Manually trigger a sync for a repo."""
|
||||
from memory.workers.tasks.github import sync_github_repo
|
||||
|
||||
repo = (
|
||||
db.query(GithubRepo)
|
||||
.filter(GithubRepo.account_id == account_id, GithubRepo.id == repo_id)
|
||||
.first()
|
||||
)
|
||||
if not repo:
|
||||
raise HTTPException(status_code=404, detail="Repo not found")
|
||||
|
||||
task = sync_github_repo.delay(repo_id, force_full=force_full)
|
||||
|
||||
return {"task_id": task.id, "status": "scheduled"}
|
||||
658
src/memory/api/google_drive.py
Normal file
658
src/memory/api/google_drive.py
Normal file
@ -0,0 +1,658 @@
|
||||
"""API endpoints for Google Drive configuration."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import cast
|
||||
|
||||
# Allow Google to return additional scopes (like 'openid') without raising an error
|
||||
os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1"
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from memory.common import settings
|
||||
from memory.common.db.connection import get_session, make_session
|
||||
from memory.common.db.models import User
|
||||
from memory.common.db.models.sources import GoogleAccount, GoogleFolder, GoogleOAuthConfig
|
||||
from memory.api.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/google-drive", tags=["google-drive"])
|
||||
|
||||
|
||||
def get_oauth_config(session: Session) -> GoogleOAuthConfig:
|
||||
"""Get the OAuth config from database, falling back to env vars if not found."""
|
||||
config = session.query(GoogleOAuthConfig).filter(GoogleOAuthConfig.name == "default").first()
|
||||
if config:
|
||||
return config
|
||||
|
||||
# Fall back to environment variables for backwards compatibility
|
||||
if settings.GOOGLE_CLIENT_ID and settings.GOOGLE_CLIENT_SECRET:
|
||||
return GoogleOAuthConfig(
|
||||
name="default",
|
||||
client_id=settings.GOOGLE_CLIENT_ID,
|
||||
client_secret=settings.GOOGLE_CLIENT_SECRET,
|
||||
redirect_uris=[settings.GOOGLE_REDIRECT_URI],
|
||||
)
|
||||
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Google OAuth not configured. Upload credentials JSON or set GOOGLE_CLIENT_ID/SECRET.",
|
||||
)
|
||||
|
||||
|
||||
class FolderCreate(BaseModel):
|
||||
folder_id: str # Google Drive folder ID
|
||||
folder_name: str
|
||||
recursive: bool = True
|
||||
include_shared: bool = False
|
||||
tags: list[str] = []
|
||||
check_interval: int = 60 # Minutes
|
||||
|
||||
|
||||
class FolderUpdate(BaseModel):
|
||||
folder_name: str | None = None
|
||||
recursive: bool | None = None
|
||||
include_shared: bool | None = None
|
||||
tags: list[str] | None = None
|
||||
check_interval: int | None = None
|
||||
active: bool | None = None
|
||||
|
||||
|
||||
class FolderResponse(BaseModel):
|
||||
id: int
|
||||
folder_id: str
|
||||
folder_name: str
|
||||
folder_path: str | None
|
||||
recursive: bool
|
||||
include_shared: bool
|
||||
tags: list[str]
|
||||
check_interval: int
|
||||
last_sync_at: str | None
|
||||
active: bool
|
||||
|
||||
|
||||
class AccountResponse(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
email: str
|
||||
active: bool
|
||||
last_sync_at: str | None
|
||||
sync_error: str | None
|
||||
folders: list[FolderResponse]
|
||||
|
||||
|
||||
# OAuth State storage (temporary, for OAuth flow)
|
||||
class GoogleOAuthState:
|
||||
"""In-memory OAuth state storage. In production, use the database."""
|
||||
|
||||
_states: dict[str, int] = {} # state -> user_id
|
||||
|
||||
@classmethod
|
||||
def create(cls, user_id: int) -> str:
|
||||
state = secrets.token_urlsafe(32)
|
||||
cls._states[state] = user_id
|
||||
return state
|
||||
|
||||
@classmethod
|
||||
def validate(cls, state: str) -> int | None:
|
||||
return cls._states.pop(state, None)
|
||||
|
||||
|
||||
class OAuthConfigResponse(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
client_id: str
|
||||
project_id: str | None
|
||||
redirect_uris: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
# Browse endpoint models
|
||||
class DriveItem(BaseModel):
|
||||
"""A file or folder in Google Drive."""
|
||||
id: str
|
||||
name: str
|
||||
mime_type: str
|
||||
is_folder: bool
|
||||
size: int | None = None
|
||||
modified_at: str | None = None
|
||||
|
||||
|
||||
class BrowseResponse(BaseModel):
|
||||
"""Response from browsing a Google Drive folder."""
|
||||
folder_id: str
|
||||
folder_name: str
|
||||
parent_id: str | None = None
|
||||
items: list[DriveItem]
|
||||
next_page_token: str | None = None
|
||||
|
||||
|
||||
@router.post("/config")
|
||||
async def upload_oauth_config(
|
||||
file: UploadFile,
|
||||
name: str = "default",
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> OAuthConfigResponse:
|
||||
"""Upload Google OAuth credentials JSON file."""
|
||||
if not file.filename or not file.filename.endswith(".json"):
|
||||
raise HTTPException(status_code=400, detail="File must be a JSON file")
|
||||
|
||||
try:
|
||||
content = await file.read()
|
||||
json_data = json.loads(content.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid JSON file: {e}")
|
||||
|
||||
# Check if config already exists
|
||||
existing = db.query(GoogleOAuthConfig).filter(GoogleOAuthConfig.name == name).first()
|
||||
if existing:
|
||||
# Update existing config
|
||||
creds = json_data.get("web") or json_data.get("installed") or json_data
|
||||
existing.client_id = creds["client_id"]
|
||||
existing.client_secret = creds["client_secret"]
|
||||
existing.project_id = creds.get("project_id")
|
||||
existing.auth_uri = creds.get("auth_uri", "https://accounts.google.com/o/oauth2/auth")
|
||||
existing.token_uri = creds.get("token_uri", "https://oauth2.googleapis.com/token")
|
||||
existing.redirect_uris = creds.get("redirect_uris", [])
|
||||
existing.javascript_origins = creds.get("javascript_origins", [])
|
||||
db.commit()
|
||||
db.refresh(existing)
|
||||
config = existing
|
||||
else:
|
||||
# Create new config
|
||||
config = GoogleOAuthConfig.from_json(json_data, name=name)
|
||||
db.add(config)
|
||||
db.commit()
|
||||
db.refresh(config)
|
||||
|
||||
return OAuthConfigResponse(
|
||||
id=cast(int, config.id),
|
||||
name=cast(str, config.name),
|
||||
client_id=cast(str, config.client_id),
|
||||
project_id=cast(str | None, config.project_id),
|
||||
redirect_uris=list(config.redirect_uris or []),
|
||||
created_at=config.created_at.isoformat() if config.created_at else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def get_config(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> OAuthConfigResponse | None:
|
||||
"""Get current OAuth configuration (without secrets)."""
|
||||
config = db.query(GoogleOAuthConfig).filter(GoogleOAuthConfig.name == "default").first()
|
||||
if not config:
|
||||
return None
|
||||
|
||||
return OAuthConfigResponse(
|
||||
id=cast(int, config.id),
|
||||
name=cast(str, config.name),
|
||||
client_id=cast(str, config.client_id),
|
||||
project_id=cast(str | None, config.project_id),
|
||||
redirect_uris=list(config.redirect_uris or []),
|
||||
created_at=config.created_at.isoformat() if config.created_at else "",
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/config")
|
||||
def delete_config(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Delete OAuth configuration."""
|
||||
config = db.query(GoogleOAuthConfig).filter(GoogleOAuthConfig.name == "default").first()
|
||||
if not config:
|
||||
raise HTTPException(status_code=404, detail="Config not found")
|
||||
|
||||
db.delete(config)
|
||||
db.commit()
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.get("/authorize")
|
||||
def google_authorize(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> dict:
|
||||
"""Initiate Google OAuth2 flow. Returns the authorization URL."""
|
||||
oauth_config = get_oauth_config(db)
|
||||
|
||||
from google_auth_oauthlib.flow import Flow
|
||||
|
||||
# Determine redirect URI - prefer one from config, fall back to settings
|
||||
redirect_uri = (
|
||||
oauth_config.redirect_uris[0]
|
||||
if oauth_config.redirect_uris
|
||||
else settings.GOOGLE_REDIRECT_URI
|
||||
)
|
||||
|
||||
flow = Flow.from_client_config(
|
||||
oauth_config.to_client_config(),
|
||||
scopes=settings.GOOGLE_SCOPES,
|
||||
redirect_uri=redirect_uri,
|
||||
)
|
||||
|
||||
# Generate state token with user ID
|
||||
state = GoogleOAuthState.create(user.id)
|
||||
|
||||
authorization_url, _ = flow.authorization_url(
|
||||
access_type="offline",
|
||||
include_granted_scopes="true",
|
||||
state=state,
|
||||
prompt="consent", # Force consent to get refresh token
|
||||
)
|
||||
|
||||
return {"authorization_url": authorization_url}
|
||||
|
||||
|
||||
@router.get("/callback", response_class=HTMLResponse)
|
||||
def google_callback(
|
||||
request: Request,
|
||||
code: str | None = None,
|
||||
state: str | None = None,
|
||||
error: str | None = None,
|
||||
):
|
||||
"""Handle Google OAuth2 callback."""
|
||||
if error:
|
||||
return HTMLResponse(
|
||||
content=f"<html><body><h1>Authorization Failed</h1><p>{error}</p></body></html>",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
if not code or not state:
|
||||
return HTMLResponse(
|
||||
content="<html><body><h1>Missing Parameters</h1></body></html>",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
# Validate state
|
||||
user_id = GoogleOAuthState.validate(state)
|
||||
if not user_id:
|
||||
return HTMLResponse(
|
||||
content="<html><body><h1>Invalid or Expired State</h1></body></html>",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
from google_auth_oauthlib.flow import Flow
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
with make_session() as session:
|
||||
oauth_config = get_oauth_config(session)
|
||||
redirect_uri = (
|
||||
oauth_config.redirect_uris[0]
|
||||
if oauth_config.redirect_uris
|
||||
else settings.GOOGLE_REDIRECT_URI
|
||||
)
|
||||
|
||||
flow = Flow.from_client_config(
|
||||
oauth_config.to_client_config(),
|
||||
scopes=settings.GOOGLE_SCOPES,
|
||||
redirect_uri=redirect_uri,
|
||||
)
|
||||
flow.fetch_token(code=code)
|
||||
credentials = flow.credentials
|
||||
|
||||
# Get user info from Google
|
||||
service = build("oauth2", "v2", credentials=credentials)
|
||||
user_info = service.userinfo().get().execute()
|
||||
|
||||
# Create or update GoogleAccount
|
||||
account = (
|
||||
session.query(GoogleAccount)
|
||||
.filter(GoogleAccount.email == user_info["email"])
|
||||
.first()
|
||||
)
|
||||
|
||||
if not account:
|
||||
account = GoogleAccount(
|
||||
name=user_info.get("name", user_info["email"]),
|
||||
email=user_info["email"],
|
||||
)
|
||||
session.add(account)
|
||||
|
||||
account.access_token = credentials.token
|
||||
account.refresh_token = credentials.refresh_token
|
||||
account.token_expires_at = credentials.expiry
|
||||
account.scopes = list(credentials.scopes or [])
|
||||
account.active = True
|
||||
account.sync_error = None
|
||||
|
||||
session.commit()
|
||||
|
||||
return HTMLResponse(
|
||||
content="""
|
||||
<html>
|
||||
<body>
|
||||
<h1>Google Drive Connected Successfully!</h1>
|
||||
<p>You can close this window and configure folders to sync.</p>
|
||||
<script>window.close();</script>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/accounts")
|
||||
def list_accounts(
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> list[AccountResponse]:
|
||||
"""List all connected Google accounts with their folders."""
|
||||
accounts = db.query(GoogleAccount).all()
|
||||
return [
|
||||
AccountResponse(
|
||||
id=cast(int, account.id),
|
||||
name=cast(str, account.name),
|
||||
email=cast(str, account.email),
|
||||
active=cast(bool, account.active),
|
||||
last_sync_at=(
|
||||
account.last_sync_at.isoformat() if account.last_sync_at else None
|
||||
),
|
||||
sync_error=cast(str | None, account.sync_error),
|
||||
folders=[
|
||||
FolderResponse(
|
||||
id=cast(int, folder.id),
|
||||
folder_id=cast(str, folder.folder_id),
|
||||
folder_name=cast(str, folder.folder_name),
|
||||
folder_path=cast(str | None, folder.folder_path),
|
||||
recursive=cast(bool, folder.recursive),
|
||||
include_shared=cast(bool, folder.include_shared),
|
||||
tags=cast(list[str], folder.tags) or [],
|
||||
check_interval=cast(int, folder.check_interval),
|
||||
last_sync_at=(
|
||||
folder.last_sync_at.isoformat() if folder.last_sync_at else None
|
||||
),
|
||||
active=cast(bool, folder.active),
|
||||
)
|
||||
for folder in account.folders
|
||||
],
|
||||
)
|
||||
for account in accounts
|
||||
]
|
||||
|
||||
|
||||
@router.get("/accounts/{account_id}/browse")
|
||||
def browse_folder(
|
||||
account_id: int,
|
||||
folder_id: str = "root",
|
||||
page_size: int = 100,
|
||||
page_token: str | None = None,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> BrowseResponse:
|
||||
"""Browse a Google Drive folder to list its contents.
|
||||
|
||||
Special folder_id values:
|
||||
- "root": User's My Drive root
|
||||
- "shared": Files shared with the user (Shared with me)
|
||||
"""
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
from memory.parsers.google_drive import refresh_credentials
|
||||
|
||||
account = db.get(GoogleAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
if not account.active:
|
||||
raise HTTPException(status_code=400, detail="Account is not active")
|
||||
|
||||
# Refresh credentials if needed
|
||||
credentials = refresh_credentials(account, db)
|
||||
|
||||
# Build the Drive service
|
||||
service = build("drive", "v3", credentials=credentials)
|
||||
|
||||
# Handle special folder IDs
|
||||
if folder_id == "shared":
|
||||
# List files shared with the user
|
||||
folder_name = "Shared with me"
|
||||
parent_id = "root" # Allow navigating back to root
|
||||
query = "sharedWithMe=true and trashed=false"
|
||||
elif folder_id == "root":
|
||||
folder_name = "My Drive"
|
||||
parent_id = None
|
||||
query = "'root' in parents and trashed=false"
|
||||
else:
|
||||
# Get folder info for a specific folder
|
||||
try:
|
||||
folder_info = service.files().get(
|
||||
fileId=folder_id,
|
||||
fields="name, parents",
|
||||
supportsAllDrives=True,
|
||||
).execute()
|
||||
folder_name = folder_info.get("name", folder_id)
|
||||
parents = folder_info.get("parents", [])
|
||||
parent_id = parents[0] if parents else None
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
query = f"'{folder_id}' in parents and trashed=false"
|
||||
|
||||
try:
|
||||
response = service.files().list(
|
||||
q=query,
|
||||
spaces="drive",
|
||||
fields="nextPageToken, files(id, name, mimeType, size, modifiedTime)",
|
||||
pageToken=page_token,
|
||||
pageSize=page_size,
|
||||
orderBy="folder,name", # Folders first, then by name
|
||||
includeItemsFromAllDrives=True,
|
||||
supportsAllDrives=True,
|
||||
).execute()
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to list folder: {e}")
|
||||
|
||||
# Convert to response items
|
||||
items = []
|
||||
|
||||
# Add "Shared with me" as a virtual folder when at root
|
||||
if folder_id == "root":
|
||||
items.append(DriveItem(
|
||||
id="shared",
|
||||
name="Shared with me",
|
||||
mime_type="application/vnd.google-apps.folder",
|
||||
is_folder=True,
|
||||
size=None,
|
||||
modified_at=None,
|
||||
))
|
||||
|
||||
for file in response.get("files", []):
|
||||
is_folder = file["mimeType"] == "application/vnd.google-apps.folder"
|
||||
items.append(DriveItem(
|
||||
id=file["id"],
|
||||
name=file["name"],
|
||||
mime_type=file["mimeType"],
|
||||
is_folder=is_folder,
|
||||
size=file.get("size"),
|
||||
modified_at=file.get("modifiedTime"),
|
||||
))
|
||||
|
||||
return BrowseResponse(
|
||||
folder_id=folder_id,
|
||||
folder_name=folder_name,
|
||||
parent_id=parent_id,
|
||||
items=items,
|
||||
next_page_token=response.get("nextPageToken"),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/accounts/{account_id}/folders")
|
||||
def add_folder(
|
||||
account_id: int,
|
||||
folder: FolderCreate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> FolderResponse:
|
||||
"""Add a folder to sync for a Google account."""
|
||||
account = db.get(GoogleAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
# Check for duplicate
|
||||
existing = (
|
||||
db.query(GoogleFolder)
|
||||
.filter(
|
||||
GoogleFolder.account_id == account_id,
|
||||
GoogleFolder.folder_id == folder.folder_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Folder already added")
|
||||
|
||||
new_folder = GoogleFolder(
|
||||
account_id=account_id,
|
||||
folder_id=folder.folder_id,
|
||||
folder_name=folder.folder_name,
|
||||
recursive=folder.recursive,
|
||||
include_shared=folder.include_shared,
|
||||
tags=folder.tags,
|
||||
check_interval=folder.check_interval,
|
||||
)
|
||||
db.add(new_folder)
|
||||
db.commit()
|
||||
db.refresh(new_folder)
|
||||
|
||||
return FolderResponse(
|
||||
id=cast(int, new_folder.id),
|
||||
folder_id=cast(str, new_folder.folder_id),
|
||||
folder_name=cast(str, new_folder.folder_name),
|
||||
folder_path=None,
|
||||
recursive=cast(bool, new_folder.recursive),
|
||||
include_shared=cast(bool, new_folder.include_shared),
|
||||
tags=cast(list[str], new_folder.tags) or [],
|
||||
check_interval=cast(int, new_folder.check_interval),
|
||||
last_sync_at=None,
|
||||
active=cast(bool, new_folder.active),
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/accounts/{account_id}/folders/{folder_id}")
|
||||
def update_folder(
|
||||
account_id: int,
|
||||
folder_id: int,
|
||||
updates: FolderUpdate,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
) -> FolderResponse:
|
||||
"""Update a folder's configuration."""
|
||||
folder = (
|
||||
db.query(GoogleFolder)
|
||||
.filter(
|
||||
GoogleFolder.account_id == account_id,
|
||||
GoogleFolder.id == folder_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
if updates.folder_name is not None:
|
||||
folder.folder_name = updates.folder_name
|
||||
if updates.recursive is not None:
|
||||
folder.recursive = updates.recursive
|
||||
if updates.include_shared is not None:
|
||||
folder.include_shared = updates.include_shared
|
||||
if updates.tags is not None:
|
||||
folder.tags = updates.tags
|
||||
if updates.check_interval is not None:
|
||||
folder.check_interval = updates.check_interval
|
||||
if updates.active is not None:
|
||||
folder.active = updates.active
|
||||
|
||||
db.commit()
|
||||
db.refresh(folder)
|
||||
|
||||
return FolderResponse(
|
||||
id=cast(int, folder.id),
|
||||
folder_id=cast(str, folder.folder_id),
|
||||
folder_name=cast(str, folder.folder_name),
|
||||
folder_path=cast(str | None, folder.folder_path),
|
||||
recursive=cast(bool, folder.recursive),
|
||||
include_shared=cast(bool, folder.include_shared),
|
||||
tags=cast(list[str], folder.tags) or [],
|
||||
check_interval=cast(int, folder.check_interval),
|
||||
last_sync_at=(
|
||||
folder.last_sync_at.isoformat() if folder.last_sync_at else None
|
||||
),
|
||||
active=cast(bool, folder.active),
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/accounts/{account_id}/folders/{folder_id}")
|
||||
def remove_folder(
|
||||
account_id: int,
|
||||
folder_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Remove a folder from sync."""
|
||||
folder = (
|
||||
db.query(GoogleFolder)
|
||||
.filter(
|
||||
GoogleFolder.account_id == account_id,
|
||||
GoogleFolder.id == folder_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
db.delete(folder)
|
||||
db.commit()
|
||||
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/accounts/{account_id}/folders/{folder_id}/sync")
|
||||
def trigger_sync(
|
||||
account_id: int,
|
||||
folder_id: int,
|
||||
force_full: bool = False,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Manually trigger a sync for a folder."""
|
||||
from memory.workers.tasks.google_drive import sync_google_folder
|
||||
|
||||
folder = (
|
||||
db.query(GoogleFolder)
|
||||
.filter(
|
||||
GoogleFolder.account_id == account_id,
|
||||
GoogleFolder.id == folder_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
task = sync_google_folder.delay(folder.id, force_full=force_full)
|
||||
|
||||
return {"task_id": task.id, "status": "scheduled"}
|
||||
|
||||
|
||||
@router.delete("/accounts/{account_id}")
|
||||
def disconnect_account(
|
||||
account_id: int,
|
||||
user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_session),
|
||||
):
|
||||
"""Disconnect a Google account (removes account and all folders)."""
|
||||
account = db.get(GoogleAccount, account_id)
|
||||
if not account:
|
||||
raise HTTPException(status_code=404, detail="Account not found")
|
||||
|
||||
db.delete(account)
|
||||
db.commit()
|
||||
|
||||
return {"status": "disconnected"}
|
||||
@ -18,6 +18,7 @@ BACKUP_ROOT = "memory.workers.tasks.backup"
|
||||
GITHUB_ROOT = "memory.workers.tasks.github"
|
||||
PEOPLE_ROOT = "memory.workers.tasks.people"
|
||||
PROACTIVE_ROOT = "memory.workers.tasks.proactive"
|
||||
GOOGLE_ROOT = "memory.workers.tasks.google_drive"
|
||||
ADD_DISCORD_MESSAGE = f"{DISCORD_ROOT}.add_discord_message"
|
||||
EDIT_DISCORD_MESSAGE = f"{DISCORD_ROOT}.edit_discord_message"
|
||||
PROCESS_DISCORD_MESSAGE = f"{DISCORD_ROOT}.process_discord_message"
|
||||
@ -77,6 +78,10 @@ SYNC_PROFILE_FROM_FILE = f"{PEOPLE_ROOT}.sync_profile_from_file"
|
||||
# Proactive check-in tasks
|
||||
EVALUATE_PROACTIVE_CHECKINS = f"{PROACTIVE_ROOT}.evaluate_proactive_checkins"
|
||||
EXECUTE_PROACTIVE_CHECKIN = f"{PROACTIVE_ROOT}.execute_proactive_checkin"
|
||||
# Google Drive tasks
|
||||
SYNC_GOOGLE_FOLDER = f"{GOOGLE_ROOT}.sync_google_folder"
|
||||
SYNC_GOOGLE_DOC = f"{GOOGLE_ROOT}.sync_google_doc"
|
||||
SYNC_ALL_GOOGLE_ACCOUNTS = f"{GOOGLE_ROOT}.sync_all_google_accounts"
|
||||
|
||||
|
||||
def get_broker_url() -> str:
|
||||
@ -136,6 +141,7 @@ app.conf.update(
|
||||
f"{GITHUB_ROOT}.*": {"queue": f"{settings.CELERY_QUEUE_PREFIX}-github"},
|
||||
f"{PEOPLE_ROOT}.*": {"queue": f"{settings.CELERY_QUEUE_PREFIX}-people"},
|
||||
f"{PROACTIVE_ROOT}.*": {"queue": f"{settings.CELERY_QUEUE_PREFIX}-discord"},
|
||||
f"{GOOGLE_ROOT}.*": {"queue": f"{settings.CELERY_QUEUE_PREFIX}-google"},
|
||||
},
|
||||
beat_schedule={
|
||||
"sync-github-repos-hourly": {
|
||||
@ -146,6 +152,10 @@ app.conf.update(
|
||||
"task": EVALUATE_PROACTIVE_CHECKINS,
|
||||
"schedule": crontab(), # Every minute
|
||||
},
|
||||
"sync-google-drive-hourly": {
|
||||
"task": SYNC_ALL_GOOGLE_ACCOUNTS,
|
||||
"schedule": crontab(minute=30), # Every hour at :30
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ from memory.common.db.models.source_items import (
|
||||
Photo,
|
||||
MiscDoc,
|
||||
Note,
|
||||
GoogleDoc,
|
||||
MailMessagePayload,
|
||||
EmailAttachmentPayload,
|
||||
AgentObservationPayload,
|
||||
@ -30,6 +31,7 @@ from memory.common.db.models.source_items import (
|
||||
BookSectionPayload,
|
||||
NotePayload,
|
||||
ForumPostPayload,
|
||||
GoogleDocPayload,
|
||||
)
|
||||
from memory.common.db.models.discord import (
|
||||
DiscordServer,
|
||||
@ -57,6 +59,9 @@ from memory.common.db.models.sources import (
|
||||
EmailAccount,
|
||||
GithubAccount,
|
||||
GithubRepo,
|
||||
GoogleOAuthConfig,
|
||||
GoogleAccount,
|
||||
GoogleFolder,
|
||||
)
|
||||
from memory.common.db.models.users import (
|
||||
User,
|
||||
@ -83,6 +88,7 @@ Payload = (
|
||||
| EmailAttachmentPayload
|
||||
| MailMessagePayload
|
||||
| PersonPayload
|
||||
| GoogleDocPayload
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@ -106,6 +112,8 @@ __all__ = [
|
||||
"Photo",
|
||||
"MiscDoc",
|
||||
"Note",
|
||||
"GoogleDoc",
|
||||
"GoogleDocPayload",
|
||||
# Observations
|
||||
"ObservationContradiction",
|
||||
"ReactionPattern",
|
||||
@ -121,6 +129,9 @@ __all__ = [
|
||||
"EmailAccount",
|
||||
"GithubAccount",
|
||||
"GithubRepo",
|
||||
"GoogleOAuthConfig",
|
||||
"GoogleAccount",
|
||||
"GoogleFolder",
|
||||
"DiscordServer",
|
||||
"DiscordChannel",
|
||||
"DiscordUser",
|
||||
|
||||
@ -1203,3 +1203,86 @@ class AgentObservation(SourceItem):
|
||||
@classmethod
|
||||
def get_collections(cls) -> list[str]:
|
||||
return ["semantic", "temporal"]
|
||||
|
||||
|
||||
class GoogleDocPayload(SourceItemPayload):
|
||||
google_file_id: Annotated[str, "Google Drive file ID"]
|
||||
title: Annotated[str, "Document title"]
|
||||
original_mime_type: Annotated[str | None, "Original Google/MIME type"]
|
||||
folder_path: Annotated[str | None, "Path in Google Drive"]
|
||||
owner: Annotated[str | None, "Document owner email"]
|
||||
last_modified_by: Annotated[str | None, "Last modifier email"]
|
||||
google_modified_at: Annotated[str | None, "Last modified time from Google"]
|
||||
word_count: Annotated[int | None, "Approximate word count"]
|
||||
|
||||
|
||||
class GoogleDoc(SourceItem):
|
||||
"""Google Drive document (Docs, PDFs, Word files, text)."""
|
||||
|
||||
__tablename__ = "google_doc"
|
||||
|
||||
id = Column(
|
||||
BigInteger, ForeignKey("source_item.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
# Google-specific identifiers
|
||||
google_file_id = Column(Text, nullable=False, unique=True) # Drive file ID
|
||||
google_modified_at = Column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
) # For change detection
|
||||
|
||||
# Document metadata
|
||||
title = Column(Text, nullable=False)
|
||||
original_mime_type = Column(
|
||||
Text, nullable=True
|
||||
) # e.g., "application/vnd.google-apps.document"
|
||||
folder_id = Column(
|
||||
BigInteger, ForeignKey("google_folders.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
folder_path = Column(Text, nullable=True) # e.g., "My Drive/Work/Projects"
|
||||
|
||||
# Authorship tracking
|
||||
owner = Column(Text, nullable=True) # Email of owner
|
||||
last_modified_by = Column(Text, nullable=True) # Email of last modifier
|
||||
|
||||
# Content stats
|
||||
word_count = Column(Integer, nullable=True)
|
||||
|
||||
# Content hash for change detection
|
||||
content_hash = Column(Text, nullable=True)
|
||||
|
||||
__mapper_args__ = {
|
||||
"polymorphic_identity": "google_doc",
|
||||
}
|
||||
|
||||
__table_args__ = (
|
||||
Index("google_doc_file_id_idx", "google_file_id", unique=True),
|
||||
Index("google_doc_folder_idx", "folder_id"),
|
||||
Index("google_doc_modified_idx", "google_modified_at"),
|
||||
Index("google_doc_title_idx", "title"),
|
||||
)
|
||||
|
||||
def as_payload(self) -> GoogleDocPayload:
|
||||
return GoogleDocPayload(
|
||||
**super().as_payload(),
|
||||
google_file_id=cast(str, self.google_file_id),
|
||||
title=cast(str, self.title),
|
||||
original_mime_type=cast(str | None, self.original_mime_type),
|
||||
folder_path=cast(str | None, self.folder_path),
|
||||
owner=cast(str | None, self.owner),
|
||||
last_modified_by=cast(str | None, self.last_modified_by),
|
||||
google_modified_at=(
|
||||
self.google_modified_at and self.google_modified_at.isoformat()
|
||||
),
|
||||
word_count=cast(int | None, self.word_count),
|
||||
)
|
||||
|
||||
def _chunk_contents(self) -> Sequence[extract.DataChunk]:
|
||||
content = cast(str | None, self.content)
|
||||
if content:
|
||||
return extract.extract_text(content, modality="doc")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def get_collections(cls) -> list[str]:
|
||||
return ["doc"]
|
||||
|
||||
@ -226,3 +226,150 @@ class GithubRepo(Base):
|
||||
@property
|
||||
def repo_path(self) -> str:
|
||||
return f"{self.owner}/{self.name}"
|
||||
|
||||
|
||||
class GoogleOAuthConfig(Base):
|
||||
"""OAuth client configuration for Google APIs (from credentials JSON)."""
|
||||
|
||||
__tablename__ = "google_oauth_config"
|
||||
|
||||
id = Column(BigInteger, primary_key=True)
|
||||
name = Column(Text, nullable=False, unique=True, default="default")
|
||||
client_id = Column(Text, nullable=False)
|
||||
client_secret = Column(Text, nullable=False)
|
||||
project_id = Column(Text, nullable=True)
|
||||
auth_uri = Column(
|
||||
Text, nullable=False, server_default="https://accounts.google.com/o/oauth2/auth"
|
||||
)
|
||||
token_uri = Column(
|
||||
Text, nullable=False, server_default="https://oauth2.googleapis.com/token"
|
||||
)
|
||||
redirect_uris = Column(ARRAY(Text), nullable=False, server_default="{}")
|
||||
javascript_origins = Column(ARRAY(Text), nullable=False, server_default="{}")
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_data: dict, name: str = "default") -> "GoogleOAuthConfig":
|
||||
"""Create from Google credentials JSON file content."""
|
||||
# Handle both "web" and "installed" credential types
|
||||
creds = json_data.get("web") or json_data.get("installed") or json_data
|
||||
return cls(
|
||||
name=name,
|
||||
client_id=creds["client_id"],
|
||||
client_secret=creds["client_secret"],
|
||||
project_id=creds.get("project_id"),
|
||||
auth_uri=creds.get("auth_uri", "https://accounts.google.com/o/oauth2/auth"),
|
||||
token_uri=creds.get("token_uri", "https://oauth2.googleapis.com/token"),
|
||||
redirect_uris=creds.get("redirect_uris", []),
|
||||
javascript_origins=creds.get("javascript_origins", []),
|
||||
)
|
||||
|
||||
def to_client_config(self) -> dict:
|
||||
"""Convert to format expected by google_auth_oauthlib.flow.Flow."""
|
||||
return {
|
||||
"web": {
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret,
|
||||
"project_id": self.project_id,
|
||||
"auth_uri": self.auth_uri,
|
||||
"token_uri": self.token_uri,
|
||||
"redirect_uris": list(self.redirect_uris or []),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class GoogleAccount(Base):
|
||||
"""Google authentication credentials for Drive API access."""
|
||||
|
||||
__tablename__ = "google_accounts"
|
||||
|
||||
id = Column(BigInteger, primary_key=True)
|
||||
name = Column(Text, nullable=False) # Display name
|
||||
email = Column(Text, nullable=False, unique=True) # Google account email
|
||||
|
||||
# OAuth2 tokens
|
||||
access_token = Column(Text, nullable=True)
|
||||
refresh_token = Column(Text, nullable=True)
|
||||
token_expires_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
# Scopes granted
|
||||
scopes = Column(ARRAY(Text), nullable=False, server_default="{}")
|
||||
|
||||
# Status
|
||||
active = Column(Boolean, nullable=False, server_default="true")
|
||||
last_sync_at = Column(DateTime(timezone=True), nullable=True)
|
||||
sync_error = Column(Text, nullable=True) # Last error message if any
|
||||
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
# Relationship to folders
|
||||
folders = relationship(
|
||||
"GoogleFolder", back_populates="account", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("google_accounts_active_idx", "active", "last_sync_at"),
|
||||
Index("google_accounts_email_idx", "email"),
|
||||
)
|
||||
|
||||
|
||||
class GoogleFolder(Base):
|
||||
"""Tracked Google Drive folder configuration."""
|
||||
|
||||
__tablename__ = "google_folders"
|
||||
|
||||
id = Column(BigInteger, primary_key=True)
|
||||
account_id = Column(
|
||||
BigInteger, ForeignKey("google_accounts.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
|
||||
# Folder identification
|
||||
folder_id = Column(Text, nullable=False) # Google Drive folder ID
|
||||
folder_name = Column(Text, nullable=False) # Display name
|
||||
folder_path = Column(Text, nullable=True) # Full path for display
|
||||
|
||||
# Sync options
|
||||
recursive = Column(Boolean, nullable=False, server_default="true") # Include subfolders
|
||||
include_shared = Column(
|
||||
Boolean, nullable=False, server_default="false"
|
||||
) # Include shared files
|
||||
|
||||
# File type filters (empty = all text documents)
|
||||
mime_type_filter = Column(ARRAY(Text), nullable=False, server_default="{}")
|
||||
|
||||
# Tags to apply to all documents from this folder
|
||||
tags = Column(ARRAY(Text), nullable=False, server_default="{}")
|
||||
|
||||
# Sync configuration
|
||||
check_interval = Column(Integer, nullable=False, server_default="60") # Minutes
|
||||
last_sync_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
# Status
|
||||
active = Column(Boolean, nullable=False, server_default="true")
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
# Relationships
|
||||
account = relationship("GoogleAccount", back_populates="folders")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("account_id", "folder_id", name="unique_folder_per_account"),
|
||||
Index("google_folders_active_idx", "active", "last_sync_at"),
|
||||
)
|
||||
|
||||
@ -244,3 +244,20 @@ S3_BACKUP_ENABLED = boolean_env("S3_BACKUP_ENABLED", bool(BACKUP_ENCRYPTION_KEY)
|
||||
S3_BACKUP_INTERVAL = int(
|
||||
os.getenv("S3_BACKUP_INTERVAL", 60 * 60 * 24)
|
||||
) # Daily by default
|
||||
|
||||
# Google OAuth settings
|
||||
GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID", "")
|
||||
GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "")
|
||||
GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI", f"{SERVER_URL}/auth/callback/google")
|
||||
GOOGLE_SCOPES = [
|
||||
"https://www.googleapis.com/auth/drive.readonly",
|
||||
"https://www.googleapis.com/auth/userinfo.email",
|
||||
"https://www.googleapis.com/auth/userinfo.profile",
|
||||
]
|
||||
|
||||
# Google Drive sync settings
|
||||
GOOGLE_DRIVE_STORAGE_DIR = pathlib.Path(
|
||||
os.getenv("GOOGLE_DRIVE_STORAGE_DIR", str(FILE_STORAGE_DIR / "google_drive"))
|
||||
)
|
||||
GOOGLE_DRIVE_STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
GOOGLE_SYNC_INTERVAL = int(os.getenv("GOOGLE_SYNC_INTERVAL", 60 * 60)) # 1 hour default
|
||||
|
||||
339
src/memory/parsers/google_drive.py
Normal file
339
src/memory/parsers/google_drive.py
Normal file
@ -0,0 +1,339 @@
|
||||
"""Google Drive API client for fetching and exporting documents."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Generator, TypedDict
|
||||
|
||||
from memory.common import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# MIME types we support
|
||||
SUPPORTED_GOOGLE_MIMES = {
|
||||
"application/vnd.google-apps.document", # Google Docs
|
||||
}
|
||||
|
||||
SUPPORTED_FILE_MIMES = {
|
||||
"application/pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/msword",
|
||||
"text/plain",
|
||||
"text/markdown",
|
||||
"text/html",
|
||||
"text/csv",
|
||||
}
|
||||
|
||||
# Export mappings for Google native formats
|
||||
EXPORT_MIMES = {
|
||||
"application/vnd.google-apps.document": "text/plain",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class GoogleCredentials:
|
||||
"""Credentials for Google Drive API access."""
|
||||
|
||||
access_token: str
|
||||
refresh_token: str | None
|
||||
token_expires_at: datetime | None
|
||||
scopes: list[str]
|
||||
|
||||
|
||||
class GoogleFileData(TypedDict):
|
||||
"""Parsed file data ready for storage."""
|
||||
|
||||
file_id: str
|
||||
title: str
|
||||
mime_type: str
|
||||
original_mime_type: str
|
||||
folder_path: str | None
|
||||
owner: str | None
|
||||
last_modified_by: str | None
|
||||
modified_at: datetime | None
|
||||
created_at: datetime | None
|
||||
content: str
|
||||
content_hash: str
|
||||
size: int
|
||||
word_count: int
|
||||
|
||||
|
||||
def parse_google_date(date_str: str | None) -> datetime | None:
|
||||
"""Parse RFC 3339 date string from Google API."""
|
||||
if not date_str:
|
||||
return None
|
||||
return datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def compute_content_hash(content: str) -> str:
|
||||
"""Compute SHA256 hash of content for change detection."""
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
|
||||
class GoogleDriveClient:
|
||||
"""Client for Google Drive API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
credentials: GoogleCredentials,
|
||||
client_id: str | None = None,
|
||||
client_secret: str | None = None,
|
||||
token_uri: str = "https://oauth2.googleapis.com/token",
|
||||
):
|
||||
self.credentials = credentials
|
||||
self._service = None
|
||||
# Use provided values or fall back to settings
|
||||
self._client_id = client_id or settings.GOOGLE_CLIENT_ID
|
||||
self._client_secret = client_secret or settings.GOOGLE_CLIENT_SECRET
|
||||
self._token_uri = token_uri
|
||||
|
||||
def _get_service(self):
|
||||
"""Lazily build the Drive service."""
|
||||
if self._service is None:
|
||||
from google.oauth2.credentials import Credentials
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
creds = Credentials(
|
||||
token=self.credentials.access_token,
|
||||
refresh_token=self.credentials.refresh_token,
|
||||
token_uri=self._token_uri,
|
||||
client_id=self._client_id,
|
||||
client_secret=self._client_secret,
|
||||
scopes=self.credentials.scopes,
|
||||
)
|
||||
self._service = build("drive", "v3", credentials=creds)
|
||||
return self._service
|
||||
|
||||
def list_files_in_folder(
|
||||
self,
|
||||
folder_id: str,
|
||||
recursive: bool = True,
|
||||
since: datetime | None = None,
|
||||
page_size: int = 100,
|
||||
) -> Generator[dict, None, None]:
|
||||
"""List all supported files in a folder with pagination."""
|
||||
service = self._get_service()
|
||||
|
||||
# Build query for supported file types
|
||||
all_mimes = SUPPORTED_GOOGLE_MIMES | SUPPORTED_FILE_MIMES
|
||||
mime_conditions = " or ".join(f"mimeType='{mime}'" for mime in all_mimes)
|
||||
|
||||
query_parts = [
|
||||
f"'{folder_id}' in parents",
|
||||
"trashed=false",
|
||||
f"({mime_conditions} or mimeType='application/vnd.google-apps.folder')",
|
||||
]
|
||||
|
||||
if since:
|
||||
query_parts.append(f"modifiedTime > '{since.isoformat()}'")
|
||||
|
||||
query = " and ".join(query_parts)
|
||||
|
||||
page_token = None
|
||||
while True:
|
||||
response = (
|
||||
service.files()
|
||||
.list(
|
||||
q=query,
|
||||
spaces="drive",
|
||||
fields="nextPageToken, files(id, name, mimeType, modifiedTime, createdTime, owners, lastModifyingUser, parents, size)",
|
||||
pageToken=page_token,
|
||||
pageSize=page_size,
|
||||
)
|
||||
.execute()
|
||||
)
|
||||
|
||||
for file in response.get("files", []):
|
||||
if file["mimeType"] == "application/vnd.google-apps.folder":
|
||||
if recursive:
|
||||
# Recursively list files in subfolder
|
||||
yield from self.list_files_in_folder(
|
||||
file["id"], recursive=True, since=since
|
||||
)
|
||||
else:
|
||||
yield file
|
||||
|
||||
page_token = response.get("nextPageToken")
|
||||
if not page_token:
|
||||
break
|
||||
|
||||
def export_file(self, file_id: str, mime_type: str) -> bytes:
|
||||
"""Export a Google native file to the specified format."""
|
||||
from googleapiclient.http import MediaIoBaseDownload
|
||||
|
||||
service = self._get_service()
|
||||
|
||||
if mime_type in SUPPORTED_GOOGLE_MIMES:
|
||||
# Export Google Docs to text
|
||||
export_mime = EXPORT_MIMES.get(mime_type, "text/plain")
|
||||
request = service.files().export_media(fileId=file_id, mimeType=export_mime)
|
||||
else:
|
||||
# Download regular files directly
|
||||
request = service.files().get_media(fileId=file_id)
|
||||
|
||||
buffer = io.BytesIO()
|
||||
downloader = MediaIoBaseDownload(buffer, request)
|
||||
|
||||
done = False
|
||||
while not done:
|
||||
_, done = downloader.next_chunk()
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
def get_folder_path(self, file_id: str) -> str:
|
||||
"""Build the full folder path for a file."""
|
||||
service = self._get_service()
|
||||
path_parts = []
|
||||
|
||||
current_id = file_id
|
||||
while current_id:
|
||||
try:
|
||||
file = (
|
||||
service.files().get(fileId=current_id, fields="name, parents").execute()
|
||||
)
|
||||
path_parts.insert(0, file["name"])
|
||||
parents = file.get("parents", [])
|
||||
current_id = parents[0] if parents else None
|
||||
except Exception:
|
||||
break
|
||||
|
||||
return "/".join(path_parts)
|
||||
|
||||
def fetch_file(
|
||||
self, file_metadata: dict, folder_path: str | None = None
|
||||
) -> GoogleFileData:
|
||||
"""Fetch and parse a single file."""
|
||||
file_id = file_metadata["id"]
|
||||
mime_type = file_metadata["mimeType"]
|
||||
|
||||
logger.info(f"Fetching file: {file_metadata['name']} ({mime_type})")
|
||||
|
||||
# Download/export content
|
||||
content_bytes = self.export_file(file_id, mime_type)
|
||||
|
||||
# Handle encoding
|
||||
try:
|
||||
content = content_bytes.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
content = content_bytes.decode("latin-1")
|
||||
|
||||
# For PDFs and Word docs, we need to extract text
|
||||
if mime_type == "application/pdf":
|
||||
content = self._extract_pdf_text(content_bytes)
|
||||
elif mime_type in (
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/msword",
|
||||
):
|
||||
content = self._extract_docx_text(content_bytes)
|
||||
|
||||
# Extract owner info
|
||||
owners = file_metadata.get("owners", [])
|
||||
owner = owners[0].get("emailAddress") if owners else None
|
||||
|
||||
last_modifier = file_metadata.get("lastModifyingUser", {})
|
||||
last_modified_by = last_modifier.get("emailAddress")
|
||||
|
||||
return GoogleFileData(
|
||||
file_id=file_id,
|
||||
title=file_metadata["name"],
|
||||
mime_type=EXPORT_MIMES.get(mime_type, mime_type),
|
||||
original_mime_type=mime_type,
|
||||
folder_path=folder_path,
|
||||
owner=owner,
|
||||
last_modified_by=last_modified_by,
|
||||
modified_at=parse_google_date(file_metadata.get("modifiedTime")),
|
||||
created_at=parse_google_date(file_metadata.get("createdTime")),
|
||||
content=content,
|
||||
content_hash=compute_content_hash(content),
|
||||
size=len(content.encode("utf-8")),
|
||||
word_count=len(content.split()),
|
||||
)
|
||||
|
||||
def _extract_pdf_text(self, pdf_bytes: bytes) -> str:
|
||||
"""Extract text from PDF using PyMuPDF."""
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
||||
text_parts = []
|
||||
for page in doc:
|
||||
text_parts.append(page.get_text())
|
||||
doc.close()
|
||||
return "\n\n".join(text_parts)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract PDF text: {e}")
|
||||
return ""
|
||||
|
||||
def _extract_docx_text(self, docx_bytes: bytes) -> str:
|
||||
"""Extract text from Word document."""
|
||||
try:
|
||||
import zipfile
|
||||
from xml.etree import ElementTree
|
||||
|
||||
# docx is a zip file containing XML
|
||||
with zipfile.ZipFile(io.BytesIO(docx_bytes)) as zf:
|
||||
with zf.open("word/document.xml") as doc_xml:
|
||||
tree = ElementTree.parse(doc_xml)
|
||||
root = tree.getroot()
|
||||
|
||||
# Word uses namespaces
|
||||
ns = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
|
||||
|
||||
text_parts = []
|
||||
for para in root.findall(".//w:p", ns):
|
||||
para_text = "".join(
|
||||
node.text or ""
|
||||
for node in para.findall(".//w:t", ns)
|
||||
)
|
||||
if para_text:
|
||||
text_parts.append(para_text)
|
||||
|
||||
return "\n\n".join(text_parts)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract docx text: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def _get_oauth_config(session: Any) -> tuple[str, str, str]:
|
||||
"""Get OAuth client credentials from database or settings."""
|
||||
from memory.common.db.models.sources import GoogleOAuthConfig
|
||||
|
||||
config = session.query(GoogleOAuthConfig).filter(GoogleOAuthConfig.name == "default").first()
|
||||
if config:
|
||||
return config.client_id, config.client_secret, config.token_uri
|
||||
|
||||
# Fall back to environment variables
|
||||
return (
|
||||
settings.GOOGLE_CLIENT_ID,
|
||||
settings.GOOGLE_CLIENT_SECRET,
|
||||
"https://oauth2.googleapis.com/token",
|
||||
)
|
||||
|
||||
|
||||
def refresh_credentials(account: Any, session: Any) -> Any:
|
||||
"""Refresh OAuth2 credentials if expired."""
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2.credentials import Credentials
|
||||
|
||||
client_id, client_secret, token_uri = _get_oauth_config(session)
|
||||
|
||||
credentials = Credentials(
|
||||
token=account.access_token,
|
||||
refresh_token=account.refresh_token,
|
||||
token_uri=token_uri,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
scopes=account.scopes or [],
|
||||
)
|
||||
|
||||
if credentials.expired and credentials.refresh_token:
|
||||
credentials.refresh(Request())
|
||||
|
||||
# Update stored tokens
|
||||
account.access_token = credentials.token
|
||||
account.token_expires_at = credentials.expiry
|
||||
session.commit()
|
||||
|
||||
return credentials
|
||||
305
src/memory/workers/tasks/google_drive.py
Normal file
305
src/memory/workers/tasks/google_drive.py
Normal file
@ -0,0 +1,305 @@
|
||||
"""Celery tasks for Google Drive document syncing."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, cast
|
||||
|
||||
from memory.common import qdrant
|
||||
from memory.common.celery_app import app
|
||||
from memory.common.db.connection import make_session
|
||||
from memory.common.db.models import GoogleDoc
|
||||
from memory.common.db.models.sources import GoogleAccount, GoogleFolder
|
||||
from memory.parsers.google_drive import (
|
||||
GoogleDriveClient,
|
||||
GoogleCredentials,
|
||||
GoogleFileData,
|
||||
refresh_credentials,
|
||||
_get_oauth_config,
|
||||
)
|
||||
from memory.workers.tasks.content_processing import (
|
||||
create_content_hash,
|
||||
create_task_result,
|
||||
process_content_item,
|
||||
safe_task_execution,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Task name constants
|
||||
GOOGLE_ROOT = "memory.workers.tasks.google_drive"
|
||||
SYNC_GOOGLE_FOLDER = f"{GOOGLE_ROOT}.sync_google_folder"
|
||||
SYNC_GOOGLE_DOC = f"{GOOGLE_ROOT}.sync_google_doc"
|
||||
SYNC_ALL_GOOGLE_ACCOUNTS = f"{GOOGLE_ROOT}.sync_all_google_accounts"
|
||||
|
||||
|
||||
def _build_credentials(account: GoogleAccount, session) -> GoogleCredentials:
|
||||
"""Build credentials from account, refreshing if needed."""
|
||||
credentials = refresh_credentials(account, session)
|
||||
return GoogleCredentials(
|
||||
access_token=credentials.token,
|
||||
refresh_token=credentials.refresh_token,
|
||||
token_expires_at=credentials.expiry,
|
||||
scopes=list(credentials.scopes or []),
|
||||
)
|
||||
|
||||
|
||||
def _serialize_file_data(data: GoogleFileData) -> dict[str, Any]:
|
||||
"""Serialize GoogleFileData for Celery task passing."""
|
||||
return {
|
||||
**data,
|
||||
"modified_at": data["modified_at"].isoformat() if data["modified_at"] else None,
|
||||
"created_at": data["created_at"].isoformat() if data["created_at"] else None,
|
||||
}
|
||||
|
||||
|
||||
def _deserialize_file_data(data: dict[str, Any]) -> GoogleFileData:
|
||||
"""Deserialize file data from Celery task."""
|
||||
from memory.parsers.google_drive import parse_google_date
|
||||
|
||||
return GoogleFileData(
|
||||
file_id=data["file_id"],
|
||||
title=data["title"],
|
||||
mime_type=data["mime_type"],
|
||||
original_mime_type=data["original_mime_type"],
|
||||
folder_path=data["folder_path"],
|
||||
owner=data["owner"],
|
||||
last_modified_by=data["last_modified_by"],
|
||||
modified_at=parse_google_date(data.get("modified_at")),
|
||||
created_at=parse_google_date(data.get("created_at")),
|
||||
content=data["content"],
|
||||
content_hash=data["content_hash"],
|
||||
size=data["size"],
|
||||
word_count=data["word_count"],
|
||||
)
|
||||
|
||||
|
||||
def _needs_reindex(existing: GoogleDoc, new_data: GoogleFileData) -> bool:
|
||||
"""Check if an existing document needs reindexing."""
|
||||
# Compare content hash
|
||||
if existing.content_hash != new_data["content_hash"]:
|
||||
return True
|
||||
|
||||
# Check if modified time is newer
|
||||
existing_modified = cast(datetime | None, existing.google_modified_at)
|
||||
if existing_modified and new_data["modified_at"]:
|
||||
if new_data["modified_at"] > existing_modified:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _create_google_doc(
|
||||
folder: GoogleFolder,
|
||||
file_data: GoogleFileData,
|
||||
) -> GoogleDoc:
|
||||
"""Create a GoogleDoc from parsed file data."""
|
||||
folder_tags = cast(list[str], folder.tags) or []
|
||||
|
||||
return GoogleDoc(
|
||||
modality="doc",
|
||||
sha256=create_content_hash(file_data["content"]),
|
||||
content=file_data["content"],
|
||||
google_file_id=file_data["file_id"],
|
||||
title=file_data["title"],
|
||||
original_mime_type=file_data["original_mime_type"],
|
||||
folder_id=folder.id,
|
||||
folder_path=file_data["folder_path"],
|
||||
owner=file_data["owner"],
|
||||
last_modified_by=file_data["last_modified_by"],
|
||||
google_modified_at=file_data["modified_at"],
|
||||
word_count=file_data["word_count"],
|
||||
content_hash=file_data["content_hash"],
|
||||
tags=folder_tags,
|
||||
size=file_data["size"],
|
||||
mime_type=file_data["mime_type"],
|
||||
)
|
||||
|
||||
|
||||
def _update_existing_doc(
|
||||
session: Any,
|
||||
existing: GoogleDoc,
|
||||
folder: GoogleFolder,
|
||||
file_data: GoogleFileData,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing GoogleDoc and reindex if content changed."""
|
||||
if not _needs_reindex(existing, file_data):
|
||||
return create_task_result(existing, "unchanged")
|
||||
|
||||
logger.info(f"Content changed for {file_data['title']}, reindexing")
|
||||
|
||||
# Delete old chunks from Qdrant
|
||||
chunk_ids = [str(c.id) for c in existing.chunks if c.id]
|
||||
if chunk_ids:
|
||||
try:
|
||||
client = qdrant.get_qdrant_client()
|
||||
qdrant.delete_points(client, cast(str, existing.modality), chunk_ids)
|
||||
except IOError as e:
|
||||
logger.error(f"Error deleting chunks: {e}")
|
||||
|
||||
# Delete chunks from database
|
||||
for chunk in existing.chunks:
|
||||
session.delete(chunk)
|
||||
|
||||
# Update the existing item
|
||||
existing.content = file_data["content"]
|
||||
existing.sha256 = create_content_hash(file_data["content"])
|
||||
existing.title = file_data["title"]
|
||||
existing.google_modified_at = file_data["modified_at"]
|
||||
existing.last_modified_by = file_data["last_modified_by"]
|
||||
existing.word_count = file_data["word_count"]
|
||||
existing.content_hash = file_data["content_hash"]
|
||||
existing.size = file_data["size"]
|
||||
existing.folder_path = file_data["folder_path"]
|
||||
|
||||
# Update tags
|
||||
folder_tags = cast(list[str], folder.tags) or []
|
||||
existing.tags = folder_tags
|
||||
|
||||
session.flush()
|
||||
|
||||
return process_content_item(existing, session)
|
||||
|
||||
|
||||
@app.task(name=SYNC_GOOGLE_DOC)
|
||||
@safe_task_execution
|
||||
def sync_google_doc(
|
||||
folder_id: int,
|
||||
file_data_serialized: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Sync a single Google Drive document."""
|
||||
file_data = _deserialize_file_data(file_data_serialized)
|
||||
logger.info(f"Syncing Google Doc: {file_data['title']}")
|
||||
|
||||
with make_session() as session:
|
||||
folder = session.get(GoogleFolder, folder_id)
|
||||
if not folder:
|
||||
return {"status": "error", "error": "Folder not found"}
|
||||
|
||||
# Check for existing document by Google file ID
|
||||
existing = (
|
||||
session.query(GoogleDoc)
|
||||
.filter(GoogleDoc.google_file_id == file_data["file_id"])
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
return _update_existing_doc(session, existing, folder, file_data)
|
||||
|
||||
# Create new document
|
||||
google_doc = _create_google_doc(folder, file_data)
|
||||
return process_content_item(google_doc, session)
|
||||
|
||||
|
||||
@app.task(name=SYNC_GOOGLE_FOLDER)
|
||||
@safe_task_execution
|
||||
def sync_google_folder(folder_id: int, force_full: bool = False) -> dict[str, Any]:
|
||||
"""Sync all documents in a Google Drive folder."""
|
||||
logger.info(f"Syncing Google folder {folder_id}")
|
||||
|
||||
with make_session() as session:
|
||||
folder = session.get(GoogleFolder, folder_id)
|
||||
if not folder or not cast(bool, folder.active):
|
||||
return {"status": "error", "error": "Folder not found or inactive"}
|
||||
|
||||
account = folder.account
|
||||
if not account or not cast(bool, account.active):
|
||||
return {"status": "error", "error": "Account not found or inactive"}
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
last_sync = cast(datetime | None, folder.last_sync_at)
|
||||
|
||||
# Check if sync is needed based on interval
|
||||
if last_sync and not force_full:
|
||||
check_interval = cast(int, folder.check_interval)
|
||||
if now - last_sync < timedelta(minutes=check_interval):
|
||||
return {"status": "skipped_recent_check", "folder_id": folder_id}
|
||||
|
||||
# Build credentials (refresh if needed)
|
||||
try:
|
||||
credentials = _build_credentials(account, session)
|
||||
except Exception as e:
|
||||
account.sync_error = str(e)
|
||||
account.active = False # Disable until re-auth
|
||||
session.commit()
|
||||
return {"status": "error", "error": f"Token refresh failed: {e}"}
|
||||
|
||||
# Get OAuth config for client
|
||||
client_id, client_secret, token_uri = _get_oauth_config(session)
|
||||
client = GoogleDriveClient(
|
||||
credentials,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
token_uri=token_uri,
|
||||
)
|
||||
|
||||
# Determine sync window
|
||||
since = None if force_full else last_sync
|
||||
|
||||
docs_synced = 0
|
||||
task_ids = []
|
||||
|
||||
try:
|
||||
# Get folder path for context
|
||||
folder_path = client.get_folder_path(cast(str, folder.folder_id))
|
||||
|
||||
for file_meta in client.list_files_in_folder(
|
||||
cast(str, folder.folder_id),
|
||||
recursive=cast(bool, folder.recursive),
|
||||
since=since,
|
||||
):
|
||||
try:
|
||||
file_data = client.fetch_file(file_meta, folder_path)
|
||||
serialized = _serialize_file_data(file_data)
|
||||
task = sync_google_doc.delay(folder.id, serialized)
|
||||
task_ids.append(task.id)
|
||||
docs_synced += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching file {file_meta.get('name')}: {e}")
|
||||
continue
|
||||
|
||||
# Update sync timestamps
|
||||
folder.last_sync_at = now
|
||||
account.last_sync_at = now
|
||||
account.sync_error = None
|
||||
session.commit()
|
||||
|
||||
except Exception as e:
|
||||
account.sync_error = str(e)
|
||||
session.commit()
|
||||
raise
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"sync_type": "full" if force_full else "incremental",
|
||||
"folder_id": folder_id,
|
||||
"folder_name": folder.folder_name,
|
||||
"docs_synced": docs_synced,
|
||||
"task_ids": task_ids,
|
||||
}
|
||||
|
||||
|
||||
@app.task(name=SYNC_ALL_GOOGLE_ACCOUNTS)
|
||||
def sync_all_google_accounts(force_full: bool = False) -> list[dict[str, Any]]:
|
||||
"""Trigger sync for all active Google Drive folders."""
|
||||
with make_session() as session:
|
||||
active_folders = (
|
||||
session.query(GoogleFolder)
|
||||
.join(GoogleAccount)
|
||||
.filter(GoogleFolder.active, GoogleAccount.active)
|
||||
.all()
|
||||
)
|
||||
|
||||
results = [
|
||||
{
|
||||
"folder_id": folder.id,
|
||||
"folder_name": folder.folder_name,
|
||||
"task_id": sync_google_folder.delay(folder.id, force_full=force_full).id,
|
||||
}
|
||||
for folder in active_folders
|
||||
]
|
||||
|
||||
logger.info(
|
||||
f"Scheduled {'full' if force_full else 'incremental'} sync "
|
||||
f"for {len(results)} active Google folders"
|
||||
)
|
||||
return results
|
||||
Loading…
x
Reference in New Issue
Block a user