Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions src/crawlee/storage_clients/_memory/_request_queue_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,6 @@ async def add_batch_of_requests(
# Add updated request back to queue.
self._pending_requests.appendleft(request)

processed_requests.append(
ProcessedRequest(
unique_key=request.unique_key,
was_already_present=True,
was_already_handled=False,
)
)

# Add the new request to the queue.
else:
if forefront:
Expand Down
17 changes: 17 additions & 0 deletions tests/unit/storage_clients/_memory/test_memory_rq_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,23 @@ async def test_memory_specific_purge_behavior() -> None:
assert await rq_client2.is_empty() is True


async def test_add_existing_pending_request_returns_single_processed_request(
rq_client: MemoryRequestQueueClient,
) -> None:
"""Test that re-adding a pending (not handled) request yields exactly one `ProcessedRequest` entry."""
request = Request.from_url('https://example.com')
await rq_client.add_batch_of_requests([request])

# Re-add the same request while it is still pending (not handled, not in progress).
response = await rq_client.add_batch_of_requests([request])

assert len(response.processed_requests) == 1
processed_request = response.processed_requests[0]
assert processed_request.unique_key == request.unique_key
assert processed_request.was_already_present is True
assert processed_request.was_already_handled is False


async def test_memory_metadata_updates(rq_client: MemoryRequestQueueClient) -> None:
"""Test that metadata timestamps are updated correctly in memory storage."""
# Record initial timestamps
Expand Down
Loading