Text content
@@ -409,40 +410,37 @@ def test_process_images_basic(mock_file_storage_dir, mock_process_image): content = cast(Tag, soup.find("div")) base_url = "https://example.com" - with tempfile.TemporaryDirectory() as temp_dir: - image_dir = pathlib.Path(temp_dir) - mock_file_storage_dir.resolve.return_value = pathlib.Path(temp_dir) + # Mock successful image processing with proper filenames + mock_images = [] + for i in range(3): + mock_img = MagicMock(spec=PILImage.Image) + mock_img.filename = str(settings.WEBPAGE_STORAGE_DIR / f"image{i + 1}.jpg") + mock_images.append(mock_img) - # Mock successful image processing with proper filenames - mock_images = [] - for i in range(3): - mock_img = MagicMock(spec=PILImage.Image) - mock_img.filename = str(pathlib.Path(temp_dir) / f"image{i + 1}.jpg") - mock_images.append(mock_img) + mock_process_image.side_effect = mock_images - mock_process_image.side_effect = mock_images + updated_content, images = process_images( + content, base_url, settings.WEBPAGE_STORAGE_DIR + ) - updated_content, images = process_images(content, base_url, image_dir) - - # Should have processed 3 images (skipping the one without src) - assert len(images) == 3 - assert mock_process_image.call_count == 3 - - # Check that img src attributes were updated to relative paths - img_tags = [ - tag - for tag in (updated_content.find_all("img") if updated_content else []) - if isinstance(tag, Tag) - ] - src_values = [] - for img in img_tags: - src = img.get("src") - if src and isinstance(src, str): - src_values.append(src) - - # Should have relative paths to the processed images - for src in src_values[:3]: # First 3 have src - assert not src.startswith("http") # Should be relative paths + expected = BeautifulSoup( + """
+ 
+ 
+ 
+ ![No src]()
+
+ """,
+ "html.parser",
+ )
+ assert updated_content.prettify() == expected.prettify() # type: ignore
+ assert images == {
+ "images/image1.jpg": mock_images[0],
+ "images/image2.jpg": mock_images[1],
+ "images/image3.jpg": mock_images[2],
+ }
def test_process_images_empty():
@@ -454,8 +452,7 @@ def test_process_images_empty():
@patch("memory.common.parsers.html.process_image")
-@patch("memory.common.parsers.html.FILE_STORAGE_DIR")
-def test_process_images_with_failures(mock_file_storage_dir, mock_process_image):
+def test_process_images_with_failures(mock_process_image):
html = """
Text content
+
+
+
+ More text
+
@@ -465,22 +462,20 @@ def test_process_images_with_failures(mock_file_storage_dir, mock_process_image)
soup = BeautifulSoup(html, "html.parser")
content = cast(Tag, soup.find("div"))
- with tempfile.TemporaryDirectory() as temp_dir:
- image_dir = pathlib.Path(temp_dir)
- mock_file_storage_dir.resolve.return_value = pathlib.Path(temp_dir)
+ # First image succeeds, second fails
+ mock_good_image = MagicMock(spec=PILImage.Image)
+ mock_good_image.filename = settings.WEBPAGE_STORAGE_DIR / "good.jpg"
+ mock_process_image.side_effect = [mock_good_image, None]
- # First image succeeds, second fails
- mock_good_image = MagicMock(spec=PILImage.Image)
- mock_good_image.filename = str(pathlib.Path(temp_dir) / "good.jpg")
- mock_process_image.side_effect = [mock_good_image, None]
+ updated_content, images = process_images(
+ content, "https://example.com", settings.WEBPAGE_STORAGE_DIR
+ )
- updated_content, images = process_images(
- content, "https://example.com", image_dir
- )
-
- # Should only return successful image
- assert len(images) == 1
- assert images[0] == mock_good_image
+ expected = BeautifulSoup(
+ html.replace("good.jpg", "images/good.jpg"), "html.parser"
+ ).prettify()
+ assert updated_content.prettify() == expected # type: ignore
+ assert images == {"images/good.jpg": mock_good_image}
@patch("memory.common.parsers.html.process_image")
@@ -494,15 +489,12 @@ def test_process_images_no_filename(mock_process_image):
mock_image.filename = None
mock_process_image.return_value = mock_image
- with tempfile.TemporaryDirectory() as temp_dir:
- image_dir = pathlib.Path(temp_dir)
+ updated_content, images = process_images(
+ content, "https://example.com", settings.WEBPAGE_STORAGE_DIR
+ )
- updated_content, images = process_images(
- content, "https://example.com", image_dir
- )
-
- # Should skip image without filename
- assert len(images) == 0
+ # Should skip image without filename
+ assert not images
class TestBaseHTMLParser:
@@ -541,7 +533,7 @@ class TestBaseHTMLParser:
assert article.title == "Article Title"
assert article.author == "John Smith" # Should prefer content over meta
- assert article.published_date == "2023-01-15T00:00:00"
+ assert article.published_date == datetime(2023, 1, 15)
assert article.url == "https://example.com/article"
assert "This is the main content" in article.content
assert article.metadata["author"] == "Jane Doe"
diff --git a/tests/memory/common/test_embedding.py b/tests/memory/common/test_embedding.py
index adf3602..f921802 100644
--- a/tests/memory/common/test_embedding.py
+++ b/tests/memory/common/test_embedding.py
@@ -5,14 +5,10 @@ from typing import cast
import pytest
from PIL import Image
-from memory.common import settings
+from memory.common import settings, collections
from memory.common.embedding import (
- embed,
- embed_file,
embed_mixed,
- embed_page,
embed_text,
- get_modality,
make_chunk,
write_to_file,
)
@@ -50,7 +46,7 @@ def mock_embed(mock_voyage_client):
],
)
def test_get_modality(mime_type, expected_modality):
- assert get_modality(mime_type) == expected_modality
+ assert collections.get_modality(mime_type) == expected_modality
def test_embed_text(mock_embed):
@@ -58,59 +54,11 @@ def test_embed_text(mock_embed):
assert embed_text(texts) == [[0], [1]]
-def test_embed_file(mock_embed, tmp_path):
- mock_file = tmp_path / "test.txt"
- mock_file.write_text("file content")
-
- assert embed_file(mock_file) == [[0]]
-
-
def test_embed_mixed(mock_embed):
items = ["text", {"type": "image", "data": "base64"}]
assert embed_mixed(items) == [[0]]
-def test_embed_page_text_only(mock_embed):
- page = {"contents": ["text1", "text2"]}
- assert embed_page(page) == [[0], [1]] # type: ignore
-
-
-def test_embed_page_mixed_content(mock_embed):
- page = {"contents": ["text", {"type": "image", "data": "base64"}]}
- assert embed_page(page) == [[0]] # type: ignore
-
-
-def test_embed(mock_embed):
- mime_type = "text/plain"
- content = "sample content"
- metadata = {"source": "test"}
-
- with patch.object(uuid, "uuid4", return_value="id1"):
- modality, chunks = embed(mime_type, content, metadata)
-
- assert modality == "text"
- assert [
- {
- "id": c.id, # type: ignore
- "file_path": c.file_path, # type: ignore
- "content": c.content, # type: ignore
- "embedding_model": c.embedding_model, # type: ignore
- "vector": c.vector, # type: ignore
- "item_metadata": c.item_metadata, # type: ignore
- }
- for c in chunks
- ] == [
- {
- "content": "sample content",
- "embedding_model": "voyage-3-large",
- "file_path": None,
- "id": "id1",
- "item_metadata": {"source": "test"},
- "vector": [0],
- },
- ]
-
-
def test_write_to_file_text(mock_file_storage):
"""Test writing a string to a file."""
chunk_id = "test-chunk-id"
@@ -160,17 +108,14 @@ def test_write_to_file_unsupported_type(mock_file_storage):
def test_make_chunk_text_only(mock_file_storage, db_session):
"""Test creating a chunk from string content."""
- page = {
- "contents": ["text content 1", "text content 2"],
- "metadata": {"source": "test"},
- }
+ contents = ["text content 1", "text content 2"]
vector = [0.1, 0.2, 0.3]
metadata = {"doc_type": "test", "source": "unit-test"}
with patch.object(
uuid, "uuid4", return_value=uuid.UUID("00000000-0000-0000-0000-000000000001")
):
- chunk = make_chunk(page, vector, metadata) # type: ignore
+ chunk = make_chunk(contents, vector, metadata) # type: ignore
assert cast(str, chunk.id) == "00000000-0000-0000-0000-000000000001"
assert cast(str, chunk.content) == "text content 1\n\ntext content 2"
@@ -183,14 +128,14 @@ def test_make_chunk_text_only(mock_file_storage, db_session):
def test_make_chunk_single_image(mock_file_storage, db_session):
"""Test creating a chunk from a single image."""
img = Image.new("RGB", (100, 100), color=(73, 109, 137))
- page = {"contents": [img], "metadata": {"source": "test"}}
+ contents = [img]
vector = [0.1, 0.2, 0.3]
metadata = {"doc_type": "test", "source": "unit-test"}
with patch.object(
uuid, "uuid4", return_value=uuid.UUID("00000000-0000-0000-0000-000000000002")
):
- chunk = make_chunk(page, vector, metadata) # type: ignore
+ chunk = make_chunk(contents, vector, metadata) # type: ignore
assert cast(str, chunk.id) == "00000000-0000-0000-0000-000000000002"
assert chunk.content is None
@@ -208,14 +153,14 @@ def test_make_chunk_single_image(mock_file_storage, db_session):
def test_make_chunk_mixed_content(mock_file_storage, db_session):
"""Test creating a chunk from mixed content (string and image)."""
img = Image.new("RGB", (100, 100), color=(73, 109, 137))
- page = {"contents": ["text content", img], "metadata": {"source": "test"}}
+ contents = ["text content", img]
vector = [0.1, 0.2, 0.3]
metadata = {"doc_type": "test", "source": "unit-test"}
with patch.object(
uuid, "uuid4", return_value=uuid.UUID("00000000-0000-0000-0000-000000000003")
):
- chunk = make_chunk(page, vector, metadata) # type: ignore
+ chunk = make_chunk(contents, vector, metadata) # type: ignore
assert cast(str, chunk.id) == "00000000-0000-0000-0000-000000000003"
assert chunk.content is None
@@ -233,3 +178,181 @@ def test_make_chunk_mixed_content(mock_file_storage, db_session):
assert (
settings.CHUNK_STORAGE_DIR / "00000000-0000-0000-0000-000000000003_1.png"
).exists()
+
+
+@pytest.mark.parametrize(
+ "data,embedding_model,collection,expected_model,expected_count,expected_has_content",
+ [
+ # Text-only with default model
+ (
+ ["text content 1", "text content 2"],
+ None,
+ None,
+ settings.TEXT_EMBEDDING_MODEL,
+ 2,
+ True,
+ ),
+ # Text with explicit mixed model - but make_chunk still uses TEXT_EMBEDDING_MODEL for text-only content
+ (
+ ["text content"],
+ settings.MIXED_EMBEDDING_MODEL,
+ None,
+ settings.TEXT_EMBEDDING_MODEL,
+ 1,
+ True,
+ ),
+ # Text collection model selection - make_chunk uses TEXT_EMBEDDING_MODEL for text-only content
+ (["text content"], None, "mail", settings.TEXT_EMBEDDING_MODEL, 1, True),
+ (["text content"], None, "photo", settings.TEXT_EMBEDDING_MODEL, 1, True),
+ (["text content"], None, "doc", settings.TEXT_EMBEDDING_MODEL, 1, True),
+ # Unknown collection falls back to default
+ (["text content"], None, "unknown", settings.TEXT_EMBEDDING_MODEL, 1, True),
+ # Explicit model takes precedence over collection
+ (
+ ["text content"],
+ settings.TEXT_EMBEDDING_MODEL,
+ "photo",
+ settings.TEXT_EMBEDDING_MODEL,
+ 1,
+ True,
+ ),
+ ],
+)
+def test_embed_data_chunk_scenarios(
+ data,
+ embedding_model,
+ collection,
+ expected_model,
+ expected_count,
+ expected_has_content,
+ mock_embed,
+ mock_file_storage,
+):
+ """Test various embedding scenarios for data chunks."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ chunk = DataChunk(
+ data=data,
+ embedding_model=embedding_model,
+ collection=collection,
+ metadata={"source": "test"},
+ )
+
+ result = embed_data_chunk(chunk, {"doc_type": "test"})
+
+ assert len(result) == expected_count
+ assert all(cast(str, c.embedding_model) == expected_model for c in result)
+ if expected_has_content:
+ assert all(c.content is not None for c in result)
+ assert all(c.file_path is None for c in result)
+ else:
+ assert all(c.content is None for c in result)
+ assert all(c.file_path is not None for c in result)
+ assert all(
+ c.item_metadata == {"source": "test", "doc_type": "test"} for c in result
+ )
+
+
+def test_embed_data_chunk_mixed_content(mock_embed, mock_file_storage):
+ """Test embedding mixed content (text and images)."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ img = Image.new("RGB", (100, 100), color=(73, 109, 137))
+ chunk = DataChunk(
+ data=["text content", img],
+ embedding_model=settings.MIXED_EMBEDDING_MODEL,
+ metadata={"source": "test"},
+ )
+
+ result = embed_data_chunk(chunk)
+
+ assert len(result) == 1 # Mixed content returns single vector
+ assert result[0].content is None # Mixed content stored in files
+ assert result[0].file_path is not None
+ assert cast(str, result[0].embedding_model) == settings.MIXED_EMBEDDING_MODEL
+
+
+@pytest.mark.parametrize(
+ "chunk_max_size,chunk_size_param,expected_chunk_size",
+ [
+ (512, 1024, 512), # chunk.max_size takes precedence
+ (None, 2048, 2048), # chunk_size parameter used when max_size is None
+ (256, None, 256), # chunk.max_size used when parameter is None
+ ],
+)
+def test_embed_data_chunk_chunk_size_handling(
+ chunk_max_size, chunk_size_param, expected_chunk_size, mock_embed, mock_file_storage
+):
+ """Test chunk size parameter handling."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ chunk = DataChunk(
+ data=["text content"], max_size=chunk_max_size, metadata={"source": "test"}
+ )
+
+ with patch("memory.common.embedding.embed_text") as mock_embed_text:
+ mock_embed_text.return_value = [[0.1, 0.2, 0.3]]
+
+ result = embed_data_chunk(chunk, chunk_size=chunk_size_param)
+
+ mock_embed_text.assert_called_once()
+ args, kwargs = mock_embed_text.call_args
+ assert kwargs["chunk_size"] == expected_chunk_size
+
+
+def test_embed_data_chunk_metadata_merging(mock_embed, mock_file_storage):
+ """Test that chunk metadata and parameter metadata are properly merged."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ chunk = DataChunk(
+ data=["text content"], metadata={"source": "test", "type": "chunk"}
+ )
+ metadata = {
+ "doc_type": "test",
+ "source": "override",
+ } # chunk.metadata takes precedence over parameter metadata
+
+ result = embed_data_chunk(chunk, metadata)
+
+ assert len(result) == 1
+ expected_metadata = {
+ "source": "test",
+ "type": "chunk",
+ "doc_type": "test",
+ } # chunk source wins
+ assert result[0].item_metadata == expected_metadata
+
+
+def test_embed_data_chunk_unsupported_model(mock_embed, mock_file_storage):
+ """Test error handling for unsupported embedding model."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ chunk = DataChunk(
+ data=["text content"],
+ embedding_model="unsupported-model",
+ metadata={"source": "test"},
+ )
+
+ with pytest.raises(ValueError, match="Unsupported model: unsupported-model"):
+ embed_data_chunk(chunk)
+
+
+def test_embed_data_chunk_empty_data(mock_embed, mock_file_storage):
+ """Test handling of empty data."""
+ from memory.common.extract import DataChunk
+ from memory.common.embedding import embed_data_chunk
+
+ chunk = DataChunk(data=[], metadata={"source": "test"})
+
+ # Should handle empty data gracefully
+ with patch("memory.common.embedding.embed_text") as mock_embed_text:
+ mock_embed_text.return_value = []
+
+ result = embed_data_chunk(chunk)
+
+ assert result == []
diff --git a/tests/memory/common/test_extract.py b/tests/memory/common/test_extract.py
index 8be3aef..6a40bdc 100644
--- a/tests/memory/common/test_extract.py
+++ b/tests/memory/common/test_extract.py
@@ -7,7 +7,6 @@ import shutil
from memory.common.extract import (
as_file,
extract_text,
- extract_content,
doc_to_images,
extract_image,
docx_to_pdf,
@@ -107,52 +106,6 @@ def test_extract_image_with_str():
extract_image("test")
-@pytest.mark.parametrize(
- "mime_type,content",
- [
- ("text/plain", "Text content"),
- ("text/html", "content"),
- ("text/markdown", "# Heading"),
- ("text/csv", "a,b,c"),
- ],
-)
-def test_extract_content_different_text_types(mime_type, content):
- assert extract_content(mime_type, content) == [
- {"contents": [content], "metadata": {}}
- ]
-
-
-def test_extract_content_pdf():
- result = extract_content("application/pdf", REGULAMIN)
-
- assert len(result) == 2
- assert all(
- isinstance(page["contents"], list)
- and all(isinstance(c, Image.Image) for c in page["contents"])
- for page in result
- )
- assert all("page" in page["metadata"] for page in result)
- assert all("width" in page["metadata"] for page in result)
- assert all("height" in page["metadata"] for page in result)
-
-
-def test_extract_content_image(tmp_path):
- # Create a test image
- img = Image.new("RGB", (100, 100), color="red")
- img_path = tmp_path / "test_img.png"
- img.save(img_path)
-
- (result,) = extract_content("image/png", img_path)
-
- assert isinstance(result["contents"][0], Image.Image)
- assert result["contents"][0].size == (100, 100)
- assert result["metadata"] == {}
-
-
-def test_extract_content_unsupported_type():
- assert extract_content("unsupported/type", "content") == []
-
-
@pytest.mark.skipif(not is_pdflatex_available(), reason="pdflatex not installed")
def test_docx_to_pdf(tmp_path):
output_path = tmp_path / "output.pdf"
diff --git a/tests/memory/workers/tasks/test_comic_tasks.py b/tests/memory/workers/tasks/test_comic_tasks.py
new file mode 100644
index 0000000..6b9f5db
--- /dev/null
+++ b/tests/memory/workers/tasks/test_comic_tasks.py
@@ -0,0 +1,431 @@
+import pytest
+from datetime import datetime
+from unittest.mock import Mock, patch
+
+from memory.common.db.models import Comic
+from memory.workers.tasks import comic
+import requests
+
+
+@pytest.fixture
+def mock_comic_info():
+ """Mock comic info data for testing."""
+ return {
+ "title": "Test Comic",
+ "image_url": "https://example.com/comic.png",
+ "url": "https://example.com/comic/1",
+ "published_date": "2024-01-01T12:00:00Z",
+ }
+
+
+@pytest.fixture
+def mock_feed_data():
+ """Mock RSS feed data."""
+ return {
+ "entries": [
+ {"link": "https://example.com/comic/1", "id": None},
+ {"link": "https://example.com/comic/2", "id": None},
+ {"link": None, "id": "https://example.com/comic/3"},
+ ]
+ }
+
+
+@pytest.fixture
+def mock_image_response():
+ """Mock HTTP response for comic image."""
+ # 1x1 PNG image (smallest valid PNG)
+ png_data = (
+ b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01"
+ b"\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\tpHYs\x00\x00\x0b\x13"
+ b"\x00\x00\x0b\x13\x01\x00\x9a\x9c\x18\x00\x00\x00\nIDATx\x9cc```"
+ b"\x00\x00\x00\x02\x00\x01\xe2!\xbc3\x00\x00\x00\x00IEND\xaeB`\x82"
+ )
+ response = Mock()
+ response.status_code = 200
+ response.content = png_data
+ with patch.object(requests, "get", return_value=response):
+ yield response
+
+
+@patch("memory.workers.tasks.comic.feedparser.parse")
+def test_find_new_urls_success(mock_parse, mock_feed_data, db_session):
+ """Test successful URL discovery from RSS feed."""
+ mock_parse.return_value = Mock(entries=mock_feed_data["entries"])
+
+ result = comic.find_new_urls("https://example.com", "https://example.com/rss")
+
+ assert result == {
+ "https://example.com/comic/1",
+ "https://example.com/comic/2",
+ "https://example.com/comic/3",
+ }
+ mock_parse.assert_called_once_with("https://example.com/rss")
+
+
+@patch("memory.workers.tasks.comic.feedparser.parse")
+def test_find_new_urls_with_existing_comics(mock_parse, mock_feed_data, db_session):
+ """Test URL discovery when some comics already exist."""
+ mock_parse.return_value = Mock(entries=mock_feed_data["entries"])
+
+ # Add existing comic to database
+ existing_comic = Comic(
+ title="Existing Comic",
+ url="https://example.com/comic/1",
+ author="https://example.com",
+ filename="/test/path",
+ sha256=b"test_hash",
+ modality="comic",
+ tags=["comic"],
+ )
+ db_session.add(existing_comic)
+ db_session.commit()
+
+ result = comic.find_new_urls("https://example.com", "https://example.com/rss")
+
+ # Should only return URLs not in database
+ assert result == {
+ "https://example.com/comic/2",
+ "https://example.com/comic/3",
+ }
+
+
+@patch("memory.workers.tasks.comic.feedparser.parse")
+def test_find_new_urls_parse_error(mock_parse):
+ """Test handling of RSS feed parsing errors."""
+ mock_parse.side_effect = Exception("Parse error")
+
+ assert (
+ comic.find_new_urls("https://example.com", "https://example.com/rss") == set()
+ )
+
+
+@patch("memory.workers.tasks.comic.feedparser.parse")
+def test_find_new_urls_empty_feed(mock_parse):
+ """Test handling of empty RSS feed."""
+ mock_parse.return_value = Mock(entries=[])
+
+ result = comic.find_new_urls("https://example.com", "https://example.com/rss")
+
+ assert result == set()
+
+
+@patch("memory.workers.tasks.comic.feedparser.parse")
+def test_find_new_urls_malformed_entries(mock_parse):
+ """Test handling of malformed RSS entries."""
+ mock_parse.return_value = Mock(
+ entries=[
+ {"link": None, "id": None}, # Both None
+ {}, # Missing keys
+ ]
+ )
+
+ result = comic.find_new_urls("https://example.com", "https://example.com/rss")
+
+ assert result == set()
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.find_new_urls")
+def test_fetch_new_comics_success(mock_find_urls, mock_sync_delay, mock_comic_info):
+ """Test successful comic fetching."""
+ mock_find_urls.return_value = {"https://example.com/comic/1"}
+ mock_parser = Mock(return_value=mock_comic_info)
+
+ result = comic.fetch_new_comics(
+ "https://example.com", "https://example.com/rss", mock_parser
+ )
+
+ assert result == {"https://example.com/comic/1"}
+ mock_parser.assert_called_once_with("https://example.com/comic/1")
+ expected_call_args = {
+ **mock_comic_info,
+ "author": "https://example.com",
+ "url": "https://example.com/comic/1",
+ }
+ mock_sync_delay.assert_called_once_with(**expected_call_args)
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.find_new_urls")
+def test_fetch_new_comics_no_new_urls(mock_find_urls, mock_sync_delay):
+ """Test when no new URLs are found."""
+ mock_find_urls.return_value = set()
+ mock_parser = Mock()
+
+ result = comic.fetch_new_comics(
+ "https://example.com", "https://example.com/rss", mock_parser
+ )
+
+ assert result == set()
+ mock_parser.assert_not_called()
+ mock_sync_delay.assert_not_called()
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.find_new_urls")
+def test_fetch_new_comics_multiple_urls(
+ mock_find_urls, mock_sync_delay, mock_comic_info
+):
+ """Test fetching multiple new comics."""
+ urls = {"https://example.com/comic/1", "https://example.com/comic/2"}
+ mock_find_urls.return_value = urls
+ mock_parser = Mock(return_value=mock_comic_info)
+
+ result = comic.fetch_new_comics(
+ "https://example.com", "https://example.com/rss", mock_parser
+ )
+
+ assert result == urls
+ assert mock_parser.call_count == 2
+ assert mock_sync_delay.call_count == 2
+
+
+@patch("memory.workers.tasks.comic.requests.get")
+def test_sync_comic_success(mock_get, mock_image_response, db_session, qdrant):
+ """Test successful comic synchronization."""
+ mock_get.return_value = mock_image_response
+
+ comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test Comic",
+ author="https://example.com",
+ published_date=datetime(2024, 1, 1, 12, 0, 0),
+ )
+
+ # Verify comic was created in database
+ saved_comic = (
+ db_session.query(Comic)
+ .filter(Comic.url == "https://example.com/comic/1")
+ .first()
+ )
+ assert saved_comic is not None
+ assert saved_comic.title == "Test Comic"
+ assert saved_comic.author == "https://example.com"
+ assert saved_comic.mime_type == "image/png"
+ assert saved_comic.size == len(mock_image_response.content)
+ assert "comic" in saved_comic.tags
+ assert "https://example.com" in saved_comic.tags
+
+ # Verify vectors were added to Qdrant
+ vectors, _ = qdrant.scroll(collection_name="comic")
+ expected_vectors = [
+ (
+ {
+ "author": "https://example.com",
+ "published": "2024-01-01T12:00:00",
+ "tags": ["comic", "https://example.com"],
+ "title": "Test Comic",
+ "url": "https://example.com/comic/1",
+ "source_id": 1,
+ },
+ None,
+ )
+ ]
+ assert [
+ ({**v.payload, "tags": sorted(v.payload["tags"])}, v.vector) for v in vectors
+ ] == expected_vectors
+
+
+def test_sync_comic_already_exists(db_session):
+ """Test that duplicate comics are not processed."""
+ # Add existing comic
+ existing_comic = Comic(
+ title="Existing Comic",
+ url="https://example.com/comic/1",
+ author="https://example.com",
+ filename="/test/path",
+ sha256=b"test_hash",
+ modality="comic",
+ tags=["comic"],
+ )
+ db_session.add(existing_comic)
+ db_session.commit()
+
+ with patch("memory.workers.tasks.comic.requests.get") as mock_get:
+ result = comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test Comic",
+ author="https://example.com",
+ )
+
+ # Should return early without making HTTP request
+ mock_get.assert_not_called()
+ assert result == {"comic_id": 1, "status": "already_exists"}
+
+
+@patch("memory.workers.tasks.comic.requests.get")
+def test_sync_comic_http_error(mock_get, db_session, qdrant):
+ """Test handling of HTTP errors when downloading image."""
+ mock_response = Mock()
+ mock_response.status_code = 404
+ mock_response.content = b""
+ mock_get.return_value = mock_response
+
+ comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test Comic",
+ author="https://example.com",
+ )
+
+ assert not (
+ db_session.query(Comic)
+ .filter(Comic.url == "https://example.com/comic/1")
+ .first()
+ )
+
+
+@patch("memory.workers.tasks.comic.requests.get")
+def test_sync_comic_no_published_date(
+ mock_get, mock_image_response, db_session, qdrant
+):
+ """Test comic sync without published date."""
+ mock_get.return_value = mock_image_response
+
+ comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test Comic",
+ author="https://example.com",
+ published_date=None,
+ )
+
+ saved_comic = (
+ db_session.query(Comic)
+ .filter(Comic.url == "https://example.com/comic/1")
+ .first()
+ )
+ assert saved_comic is not None
+ assert saved_comic.published is None
+
+
+@patch("memory.workers.tasks.comic.requests.get")
+def test_sync_comic_special_characters_in_title(
+ mock_get, mock_image_response, db_session, qdrant
+):
+ """Test comic sync with special characters in title."""
+ mock_get.return_value = mock_image_response
+
+ comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test/Comic: With*Special?Characters",
+ author="https://example.com",
+ )
+
+ # Verify comic was created with cleaned title
+ saved_comic = (
+ db_session.query(Comic)
+ .filter(Comic.url == "https://example.com/comic/1")
+ .first()
+ )
+ assert saved_comic is not None
+ assert saved_comic.title == "Test/Comic: With*Special?Characters"
+
+
+@patch(
+ "memory.common.embedding.embed_source_item",
+ side_effect=Exception("Embedding failed"),
+)
+def test_sync_comic_embedding_failure(
+ mock_embed_source_item, mock_image_response, db_session, qdrant
+):
+ """Test handling of embedding failures."""
+ result = comic.sync_comic(
+ url="https://example.com/comic/1",
+ image_url="https://example.com/image.png",
+ title="Test Comic",
+ author="https://example.com",
+ )
+ assert result == {
+ "comic_id": 1,
+ "title": "Test Comic",
+ "status": "processed",
+ "chunks_count": 0,
+ "embed_status": "FAILED",
+ "content_length": 90,
+ }
+
+
+@patch("memory.workers.tasks.comic.sync_xkcd.delay")
+@patch("memory.workers.tasks.comic.sync_smbc.delay")
+def test_sync_all_comics(mock_smbc_delay, mock_xkcd_delay):
+ """Test synchronization of all comics."""
+ comic.sync_all_comics()
+
+ mock_smbc_delay.assert_called_once()
+ mock_xkcd_delay.assert_called_once()
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.comics.extract_xkcd")
+@patch("memory.workers.tasks.comic.comics.extract_smbc")
+@patch("requests.get")
+def test_trigger_comic_sync_smbc_navigation(
+ mock_get, mock_extract_smbc, mock_extract_xkcd, mock_sync_delay, mock_comic_info
+):
+ """Test full SMBC comic sync with navigation."""
+ # Mock HTML responses for navigation
+ mock_responses = [
+ Mock(text=''),
+ Mock(text=''),
+ Mock(text="No prev link
"), # End of navigation
+ ]
+ mock_get.side_effect = mock_responses
+ mock_extract_smbc.return_value = mock_comic_info
+ mock_extract_xkcd.return_value = mock_comic_info
+
+ comic.trigger_comic_sync()
+
+ # Should have called extract_smbc for each discovered URL
+ assert mock_extract_smbc.call_count == 2
+ mock_extract_smbc.assert_any_call("https://smbc.com/comic/2")
+ mock_extract_smbc.assert_any_call("https://smbc.com/comic/1")
+
+ # Should have called extract_xkcd for range 1-307
+ assert mock_extract_xkcd.call_count == 307
+
+ # Should have queued sync tasks
+ assert mock_sync_delay.call_count == 2 + 307 # SMBC + XKCD
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.comics.extract_smbc")
+@patch("requests.get")
+def test_trigger_comic_sync_smbc_extraction_error(
+ mock_get, mock_extract_smbc, mock_sync_delay
+):
+ """Test handling of extraction errors during full sync."""
+ # Mock responses: first one has a prev link, second one doesn't
+ mock_responses = [
+ Mock(text=''),
+ Mock(text="No prev link
"),
+ ]
+ mock_get.side_effect = mock_responses
+ mock_extract_smbc.side_effect = Exception("Extraction failed")
+
+ # Should not raise exception, just log error
+ comic.trigger_comic_sync()
+
+ mock_extract_smbc.assert_called_once_with("https://smbc.com/comic/1")
+ mock_sync_delay.assert_not_called()
+
+
+@patch("memory.workers.tasks.comic.sync_comic.delay")
+@patch("memory.workers.tasks.comic.comics.extract_xkcd")
+@patch("requests.get")
+def test_trigger_comic_sync_xkcd_extraction_error(
+ mock_get, mock_extract_xkcd, mock_sync_delay
+):
+ """Test handling of XKCD extraction errors during full sync."""
+ mock_get.return_value = Mock(text="No prev link
")
+ mock_extract_xkcd.side_effect = Exception("XKCD extraction failed")
+
+ # Should not raise exception, just log errors
+ comic.trigger_comic_sync()
+
+ # Should attempt all 307 XKCD comics
+ assert mock_extract_xkcd.call_count == 307
+ mock_sync_delay.assert_not_called()
diff --git a/tests/memory/workers/tasks/test_ebook_tasks.py b/tests/memory/workers/tasks/test_ebook_tasks.py
index 6ace439..9dd4de4 100644
--- a/tests/memory/workers/tasks/test_ebook_tasks.py
+++ b/tests/memory/workers/tasks/test_ebook_tasks.py
@@ -51,24 +51,6 @@ def mock_ebook():
)
-@pytest.fixture
-def mock_embedding():
- """Mock the embedding function to return dummy vectors."""
- with patch("memory.workers.tasks.ebook.embedding.embed") as mock:
- mock.return_value = (
- "book",
- [
- Chunk(
- vector=[0.1] * 1024,
- item_metadata={"test": "data"},
- content="Test content",
- embedding_model="model",
- )
- ],
- )
- yield mock
-
-
@pytest.fixture
def mock_qdrant():
"""Mock Qdrant operations."""
@@ -151,7 +133,7 @@ def test_create_book_and_sections(mock_ebook, db_session):
assert getattr(chapter1, "parent_section_id") is None
-def test_embed_sections(db_session, mock_embedding):
+def test_embed_sections(db_session):
"""Test basic embedding sections workflow."""
# Create a test book first
book = Book(
@@ -187,31 +169,6 @@ def test_embed_sections(db_session, mock_embedding):
assert hasattr(sections[0], "embed_status")
-def test_push_to_qdrant(qdrant):
- """Test pushing embeddings to Qdrant."""
- # Create test sections with chunks
- mock_chunk = Mock(
- id="00000000-0000-0000-0000-000000000000",
- vector=[0.1] * 1024,
- item_metadata={"test": "data"},
- )
-
- mock_section = Mock(spec=BookSection)
- mock_section.embed_status = "QUEUED"
- mock_section.chunks = [mock_chunk]
-
- sections = [mock_section]
-
- ebook.push_to_qdrant(sections) # type: ignore
-
- assert {r.id: r.payload for r in qdrant.scroll(collection_name="book")[0]} == {
- "00000000-0000-0000-0000-000000000000": {
- "test": "data",
- }
- }
- assert mock_section.embed_status == "STORED"
-
-
@patch("memory.workers.tasks.ebook.parse_ebook")
def test_sync_book_success(mock_parse, mock_ebook, db_session, tmp_path):
"""Test successful book synchronization."""
@@ -229,7 +186,7 @@ def test_sync_book_success(mock_parse, mock_ebook, db_session, tmp_path):
"author": "Test Author",
"status": "processed",
"total_sections": 4,
- "sections_embedded": 4,
+ "sections_embedded": 8,
}
book = db_session.query(Book).filter(Book.title == "Test Book").first()
@@ -270,8 +227,9 @@ def test_sync_book_already_exists(mock_parse, mock_ebook, db_session, tmp_path):
@patch("memory.workers.tasks.ebook.parse_ebook")
+@patch("memory.common.embedding.embed_source_item")
def test_sync_book_embedding_failure(
- mock_parse, mock_ebook, db_session, tmp_path, mock_embedding
+ mock_embedding, mock_parse, mock_ebook, db_session, tmp_path
):
"""Test handling of embedding failures."""
book_file = tmp_path / "test.epub"
@@ -307,14 +265,18 @@ def test_sync_book_qdrant_failure(mock_parse, mock_ebook, db_session, tmp_path):
# Since embedding is already failing, this test will complete without hitting Qdrant
# So let's just verify that the function completes without raising an exception
with patch.object(ebook, "push_to_qdrant", side_effect=Exception("Qdrant failed")):
- with pytest.raises(Exception, match="Qdrant failed"):
- ebook.sync_book(str(book_file))
+ assert ebook.sync_book(str(book_file)) == {
+ "status": "error",
+ "error": "Qdrant failed",
+ }
def test_sync_book_file_not_found():
"""Test handling of missing files."""
- with pytest.raises(FileNotFoundError):
- ebook.sync_book("/nonexistent/file.epub")
+ assert ebook.sync_book("/nonexistent/file.epub") == {
+ "status": "error",
+ "error": "Book file not found: /nonexistent/file.epub",
+ }
def test_embed_sections_uses_correct_chunk_size(db_session, mock_voyage_client):
@@ -363,7 +325,7 @@ def test_embed_sections_uses_correct_chunk_size(db_session, mock_voyage_client):
calls = mock_voyage_client.embed.call_args_list
texts = [c[0][0] for c in calls]
assert texts == [
- [large_section_content.strip()],
[large_page_1.strip()],
[large_page_2.strip()],
+ [large_section_content.strip()],
]
diff --git a/tests/memory/workers/tasks/test_email_tasks.py b/tests/memory/workers/tasks/test_email_tasks.py
index 8ae3db3..ca8f921 100644
--- a/tests/memory/workers/tasks/test_email_tasks.py
+++ b/tests/memory/workers/tasks/test_email_tasks.py
@@ -69,13 +69,21 @@ def test_email_account(db_session):
def test_process_simple_email(db_session, test_email_account, qdrant):
"""Test processing a simple email message."""
- mail_message_id = process_message(
+ res = process_message(
account_id=test_email_account.id,
message_id="101",
folder="INBOX",
raw_email=SIMPLE_EMAIL_RAW,
)
+ mail_message_id = res["mail_message_id"]
+ assert res == {
+ "status": "processed",
+ "mail_message_id": mail_message_id,
+ "message_id": "101",
+ "chunks_count": 1,
+ "attachments_count": 0,
+ }
mail_message = (
db_session.query(MailMessage).filter(MailMessage.id == mail_message_id).one()
)
@@ -98,7 +106,7 @@ def test_process_email_with_attachment(db_session, test_email_account, qdrant):
message_id="302",
folder="Archive",
raw_email=EMAIL_WITH_ATTACHMENT_RAW,
- )
+ )["mail_message_id"]
# Check mail message specifics
mail_message = (
db_session.query(MailMessage).filter(MailMessage.id == mail_message_id).one()
@@ -125,25 +133,25 @@ def test_process_email_with_attachment(db_session, test_email_account, qdrant):
def test_process_empty_message(db_session, test_email_account, qdrant):
"""Test processing an empty/invalid message."""
- source_id = process_message(
+ res = process_message(
account_id=test_email_account.id,
message_id="999",
folder="Archive",
raw_email="",
)
-
- assert source_id is None
+ assert res == {"reason": "empty_content", "status": "skipped"}
def test_process_duplicate_message(db_session, test_email_account, qdrant):
"""Test that duplicate messages are detected and not stored again."""
# First call should succeed and create records
- source_id_1 = process_message(
+ res = process_message(
account_id=test_email_account.id,
message_id="101",
folder="INBOX",
raw_email=SIMPLE_EMAIL_RAW,
)
+ source_id_1 = res.get("mail_message_id")
assert source_id_1 is not None, "First call should return a source_id"
@@ -157,7 +165,7 @@ def test_process_duplicate_message(db_session, test_email_account, qdrant):
message_id="101",
folder="INBOX",
raw_email=SIMPLE_EMAIL_RAW,
- )
+ ).get("mail_message_id")
assert source_id_2 is None, "Second call should return None for duplicate message"
diff --git a/tests/memory/workers/test_email.py b/tests/memory/workers/test_email.py
index 154aca0..e17898b 100644
--- a/tests/memory/workers/test_email.py
+++ b/tests/memory/workers/test_email.py
@@ -12,7 +12,7 @@ from memory.common.db.models import (
EmailAttachment,
MailMessage,
)
-from memory.common.parsers.email import Attachment
+from memory.common.parsers.email import Attachment, parse_email_message
from memory.workers.email import (
create_mail_message,
extract_email_uid,
@@ -226,14 +226,14 @@ def test_create_mail_message(db_session):
"--boundary--"
)
folder = "INBOX"
+ parsed_email = parse_email_message(raw_email, "321")
# Call function
mail_message = create_mail_message(
db_session=db_session,
- raw_email=raw_email,
folder=folder,
tags=["test"],
- message_id="123",
+ parsed_email=parsed_email,
)
db_session.commit()
@@ -344,6 +344,7 @@ def test_vectorize_email_basic(db_session, qdrant, mock_uuid4):
recipients=["recipient@example.com"],
content="This is a test email for vectorization",
folder="INBOX",
+ modality="mail",
)
db_session.add(mail_message)
db_session.flush()
@@ -373,6 +374,7 @@ def test_vectorize_email_with_attachments(db_session, qdrant, mock_uuid4):
recipients=["recipient@example.com"],
content="This is a test email with attachments",
folder="INBOX",
+ modality="mail",
)
db_session.add(mail_message)
db_session.flush()