From a84a29f229d8db7f6c7351880da5698dd4e33f4a Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Thu, 28 May 2026 11:45:31 -0400 Subject: [PATCH 01/11] feat(incidents): Create postmortem doc at incident declaration Move Notion postmortem page creation from the dumpslack flow to on_incident_created so the doc exists from the start. The page is added as a Slack bookmark without posting a message. When dumpslack runs later (on status change to mitigated/done/postmortem), it finds the existing page and populates it with channel content. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/4jYVXERt_otp5j1rF_YAEgnh2RY8c8mk3U9J19IuwUI --- src/firetower/incidents/hooks.py | 94 ++++++++ src/firetower/incidents/tests/test_hooks.py | 244 ++++++++++++++++++++ 2 files changed, 338 insertions(+) diff --git a/src/firetower/incidents/hooks.py b/src/firetower/incidents/hooks.py index de425bcf..59746cb3 100644 --- a/src/firetower/incidents/hooks.py +++ b/src/firetower/incidents/hooks.py @@ -749,6 +749,99 @@ def _create_troubleshooting_doc(incident: Incident, channel_id: str) -> None: ).delete() +def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: + try: + if incident.is_private: + logger.info("Skipping postmortem doc for private incident %s", incident.id) + return + + if not NotionService.is_configured(): + logger.info( + "Notion not configured, skipping postmortem doc for incident %s", + incident.id, + ) + return + + with transaction.atomic(): + link, created = ExternalLink.objects.select_for_update().get_or_create( + incident=incident, + type=ExternalLinkType.NOTION, + defaults={"url": ""}, + ) + if not created: + if link.url: + logger.info( + "Incident %s already has a postmortem doc, skipping", + incident.id, + ) + else: + logger.info( + "Incident %s postmortem doc creation already in progress, skipping", + incident.id, + ) + return + + notion = NotionService.from_settings() + if not notion: + ExternalLink.objects.filter( + incident=incident, + type=ExternalLinkType.NOTION, + url="", + ).delete() + return + + incident_url = _build_incident_url(incident) + captain_email = incident.captain.email if incident.captain else None + page = notion.create_postmortem_page( + incident_number=incident.incident_number, + incident_title=incident.title, + incident_url=incident_url, + incident_date=incident.created_at, + severity=incident.severity, + captain_email=captain_email, + ) + if not page or not page.get("url"): + logger.error( + "Postmortem doc creation returned no URL for %s", + incident.incident_number, + ) + ExternalLink.objects.filter( + incident=incident, + type=ExternalLinkType.NOTION, + url="", + ).delete() + return + + with transaction.atomic(): + link = ExternalLink.objects.select_for_update().get( + incident=incident, + type=ExternalLinkType.NOTION, + ) + if link.url: + logger.warning( + "Race condition: concurrent call already created postmortem doc for %s", + incident.incident_number, + ) + return + link.url = page["url"] + link.save(update_fields=["url"]) + + try: + _slack_service.add_bookmark(channel_id, "Postmortem Doc", page["url"]) + except Exception: + logger.exception( + "Failed to add postmortem doc bookmark for %s", + incident.incident_number, + ) + except Exception: + logger.exception("Failed to create postmortem doc for incident %s", incident.id) + ExternalLink.objects.filter( + incident=incident, + type=ExternalLinkType.NOTION, + url="", + ).delete() + + def decorate_incident_channel( ctx: ChannelSetupContext, slack_service: SlackService, @@ -1219,6 +1312,7 @@ def on_incident_created(incident: Incident) -> None: # appears first in the channel. _create_datadog_notebook(incident, channel_id) _create_troubleshooting_doc(incident, channel_id) + _create_postmortem_doc(incident, channel_id) try: create_linear_parent_issue(incident, channel_id=channel_id) diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index ec8a67cc..f20fc4df 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -10,6 +10,7 @@ from firetower.auth.models import ExternalProfile, ExternalProfileType from firetower.incidents.hooks import ( DEFAULT_STATUSPAGE_WARNING_BUFFER_MINUTES, + _create_postmortem_doc, _create_status_channel, _create_troubleshooting_doc, _invite_oncall_users, @@ -2601,6 +2602,249 @@ def test_creates_troubleshooting_doc( mock_create_ts.assert_called_once_with(incident, "C99999") +@pytest.mark.django_db +class TestCreatePostmortemDoc: + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_skips_for_private_incident(self, mock_slack, mock_notion_cls): + incident = Incident.objects.create( + title="Sensitive", + severity=IncidentSeverity.P1, + is_private=True, + ) + + _create_postmortem_doc(incident, "C99999") + + mock_notion_cls.is_configured.assert_not_called() + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + mock_slack.post_message.assert_not_called() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_creates_doc_and_adds_bookmark_without_message( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.create_postmortem_page.return_value = { + "id": "page-456", + "url": "https://notion.so/page-456", + } + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Production is down", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + mock_service.create_postmortem_page.assert_called_once_with( + incident_number=incident.incident_number, + incident_title=incident.title, + incident_url=f"https://firetower.example.com/{incident.incident_number}", + incident_date=incident.created_at, + severity=incident.severity, + captain_email=None, + ) + link = ExternalLink.objects.get(incident=incident, type=ExternalLinkType.NOTION) + assert link.url == "https://notion.so/page-456" + + bookmark_calls = [ + c + for c in mock_slack.add_bookmark.call_args_list + if c[0][1] == "Postmortem Doc" + ] + assert len(bookmark_calls) == 1 + assert bookmark_calls[0][0][2] == "https://notion.so/page-456" + + mock_slack.post_message.assert_not_called() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_skips_when_not_configured(self, mock_slack, mock_notion_cls): + mock_notion_cls.is_configured.return_value = False + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + mock_notion_cls.from_settings.assert_not_called() + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_idempotent_when_link_exists(self, mock_slack, mock_notion_cls, settings): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + ExternalLink.objects.create( + incident=incident, + type=ExternalLinkType.NOTION, + url="https://notion.so/existing", + ) + + _create_postmortem_doc(incident, "C99999") + + mock_service.create_postmortem_page.assert_not_called() + link = ExternalLink.objects.get(incident=incident, type=ExternalLinkType.NOTION) + assert link.url == "https://notion.so/existing" + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_skips_when_placeholder_exists(self, mock_slack, mock_notion_cls, settings): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + ExternalLink.objects.create( + incident=incident, + type=ExternalLinkType.NOTION, + url="", + ) + + _create_postmortem_doc(incident, "C99999") + + mock_service.create_postmortem_page.assert_not_called() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_deletes_placeholder_when_api_returns_no_url( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.create_postmortem_page.return_value = {"id": "page-456"} + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_api_error_cleans_up_placeholder( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.create_postmortem_page.side_effect = RuntimeError("boom") + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_bookmark_failure_does_not_propagate( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.create_postmortem_page.return_value = { + "id": "page-456", + "url": "https://notion.so/page-456", + } + mock_notion_cls.from_settings.return_value = mock_service + mock_slack.add_bookmark.side_effect = RuntimeError("bookmark boom") + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + link = ExternalLink.objects.get(incident=incident, type=ExternalLinkType.NOTION) + assert link.url == "https://notion.so/page-456" + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_deletes_placeholder_when_notion_service_returns_none( + self, mock_slack, mock_notion_cls + ): + mock_notion_cls.is_configured.return_value = True + mock_notion_cls.from_settings.return_value = None + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + + +@pytest.mark.django_db +class TestOnIncidentCreatedPostmortemDoc: + @patch("firetower.incidents.hooks._create_postmortem_doc") + @patch("firetower.incidents.hooks._create_troubleshooting_doc") + @patch("firetower.incidents.hooks._create_status_channel_for_context") + @patch("firetower.incidents.hooks._invite_oncall_to_channel") + @patch("firetower.incidents.hooks._page_if_needed") + @patch("firetower.incidents.hooks._slack_service") + def test_creates_postmortem_doc( + self, + mock_slack, + mock_page, + mock_invite_oncall, + mock_status_channel, + mock_create_ts, + mock_create_pm, + ): + mock_slack.create_channel.return_value = "C99999" + mock_slack.build_channel_url.return_value = "https://slack.com/archives/C99999" + + incident = Incident.objects.create( + title="Production is down", + severity=IncidentSeverity.P1, + ) + + on_incident_created(incident) + + mock_create_pm.assert_called_once_with(incident, "C99999") + + @pytest.mark.django_db class TestLinearIssueTitle: def test_public_incident_without_sync_identifiers(self): From 4c14fc3cb74417f614c79f78f8c21da14c1c02fb Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 1 Jun 2026 10:30:27 -0400 Subject: [PATCH 02/11] fix(incidents): Prevent duplicate Notion page when dumpslack races with declaration When _create_postmortem_doc commits a placeholder row (url="") and is still calling the Notion API, _trigger_slack_dump could see the empty URL and fall through to create a second page. Poll for the URL to appear before proceeding, avoiding orphaned Notion pages. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/zceqUSIMWMxN-_wwsojxMiKzfo3l8cdsSTarMkl9IOM --- src/firetower/slack_app/handlers/dumpslack.py | 29 ++++++++ .../tests/handlers/test_dumpslack.py | 71 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 3f41811b..5b72bd0d 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -1,6 +1,7 @@ import logging import re import threading +import time from datetime import UTC, datetime from typing import Any @@ -61,6 +62,34 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: ) existing_url = notion_link.url + # Another process (e.g. _create_postmortem_doc) may have claimed the + # placeholder row but not yet saved the Notion URL. Wait for it. + if not db_record_created and not existing_url: + for _ in range(15): + time.sleep(1) + notion_link.refresh_from_db() + if notion_link.url: + existing_url = notion_link.url + break + + if not existing_url: + logger.warning( + "Timed out waiting for in-progress postmortem doc for %s", + incident.incident_number, + ) + try: + client.chat_postMessage( + channel=channel_id, + text="Postmortem doc creation is still in progress. " + "Please try `/ft dumpslack` again shortly.", + ) + except Exception: + logger.exception( + "Failed to post in-progress message to channel %s", + channel_id, + ) + return + # Notion API calls happen outside the transaction to avoid holding the # SELECT FOR UPDATE lock while making slow external requests. if not db_record_created and existing_url: diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 758a970b..9648e7bd 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -544,6 +544,77 @@ def test_chat_post_message_failure_does_not_raise(self): client.chat_postMessage.assert_called() + def test_waits_for_in_progress_postmortem_doc(self): + notion_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url="") + mock_notion = MagicMock() + call_count = 0 + + def simulate_url_appearing(): + nonlocal call_count + call_count += 1 + if call_count >= 2: + mock_notion_link.url = notion_url + + mock_notion_link.refresh_from_db = simulate_url_appearing + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch("firetower.slack_app.handlers.dumpslack.time") as mock_time, + ): + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + False, + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_notion.create_postmortem_page.assert_not_called() + mock_time.sleep.assert_called() + assert client.chat_postMessage.call_count == 1 + posted = client.chat_postMessage.call_args[1]["text"] + assert notion_url in posted + + def test_times_out_waiting_for_in_progress_postmortem_doc(self): + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url="") + mock_notion_link.refresh_from_db = MagicMock() + mock_notion = MagicMock() + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch("firetower.slack_app.handlers.dumpslack.time") as mock_time, + ): + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + False, + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_notion.create_postmortem_page.assert_not_called() + assert mock_time.sleep.call_count == 15 + assert client.chat_postMessage.call_count == 1 + posted = client.chat_postMessage.call_args[1]["text"] + assert "in progress" in posted.lower() + class TestHandleDumpslackCommand: def _make_args(self, notion_config=None, channel_id="C123"): From 9f456d22fbaab8471ed46773593dcf67dae2af8c Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 1 Jun 2026 10:54:22 -0400 Subject: [PATCH 03/11] fix(incidents): Handle deleted/orphaned placeholder in dumpslack polling loop When _create_postmortem_doc fails and cleans up the placeholder row, refresh_from_db() raises DoesNotExist. When the process crashes without cleanup, the empty placeholder blocks all retries indefinitely. Now catch DoesNotExist during polling and re-acquire the row on either deletion or timeout, taking ownership of orphaned placeholders instead of telling the user to retry manually. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/gaiK6_e-BFE9nt5GSu3LgLCnqJ_a_AIUTpycqehKAUs --- src/firetower/slack_app/handlers/dumpslack.py | 34 ++++++----- .../tests/handlers/test_dumpslack.py | 58 +++++++++++++++++-- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 5b72bd0d..ca3a52ff 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -67,28 +67,30 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: if not db_record_created and not existing_url: for _ in range(15): time.sleep(1) - notion_link.refresh_from_db() + try: + notion_link.refresh_from_db() + except ExternalLink.DoesNotExist: + break if notion_link.url: existing_url = notion_link.url break if not existing_url: - logger.warning( - "Timed out waiting for in-progress postmortem doc for %s", - incident.incident_number, - ) - try: - client.chat_postMessage( - channel=channel_id, - text="Postmortem doc creation is still in progress. " - "Please try `/ft dumpslack` again shortly.", + # Placeholder was deleted (creation failed) or timed out + # (process crashed without cleanup). Re-acquire under lock + # so we can take over an orphaned placeholder or create a + # fresh row. + with transaction.atomic(): + notion_link, db_record_created = ( + ExternalLink.objects.select_for_update().get_or_create( + incident=incident, + type=ExternalLinkType.NOTION, + defaults={"url": ""}, + ) ) - except Exception: - logger.exception( - "Failed to post in-progress message to channel %s", - channel_id, - ) - return + existing_url = notion_link.url + if not db_record_created and not existing_url: + db_record_created = True # Notion API calls happen outside the transaction to avoid holding the # SELECT FOR UPDATE lock while making slow external requests. diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 9648e7bd..15f97f86 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -586,34 +586,80 @@ def simulate_url_appearing(): posted = client.chat_postMessage.call_args[1]["text"] assert notion_url in posted - def test_times_out_waiting_for_in_progress_postmortem_doc(self): + def test_times_out_waiting_and_takes_over_orphaned_placeholder(self): client = MagicMock() mock_incident = MagicMock(is_private=False) mock_incident.captain = None mock_notion_link = MagicMock(url="") - mock_notion_link.refresh_from_db = MagicMock() mock_notion = MagicMock() + mock_page = {"id": "page-id", "url": "https://notion.so/page-id"} + mock_notion.create_postmortem_page.return_value = mock_page with ( patch( "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", return_value=mock_notion, ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, patch("firetower.slack_app.handlers.dumpslack.transaction"), patch("firetower.slack_app.handlers.dumpslack.time") as mock_time, + patch("firetower.slack_app.handlers.dumpslack.settings") as mock_settings, ): + mock_settings.FIRETOWER_BASE_URL = "https://firetower.example.com" mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( mock_notion_link, False, ) + mock_el.objects.select_for_update.return_value.get.return_value = ( + mock_notion_link + ) _trigger_slack_dump(client, "C123", mock_incident) - mock_notion.create_postmortem_page.assert_not_called() assert mock_time.sleep.call_count == 15 - assert client.chat_postMessage.call_count == 1 - posted = client.chat_postMessage.call_args[1]["text"] - assert "in progress" in posted.lower() + mock_notion.create_postmortem_page.assert_called_once() + + def test_recovers_when_placeholder_deleted_during_poll(self): + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url="") + fresh_notion_link = MagicMock(url="") + mock_notion = MagicMock() + mock_page = {"id": "page-id", "url": "https://notion.so/page-id"} + mock_notion.create_postmortem_page.return_value = mock_page + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch("firetower.slack_app.handlers.dumpslack.time") as mock_time, + patch("firetower.slack_app.handlers.dumpslack.settings") as mock_settings, + ): + mock_settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_el.DoesNotExist = type("DoesNotExist", (Exception,), {}) + mock_notion_link.refresh_from_db.side_effect = mock_el.DoesNotExist + mock_el.objects.select_for_update.return_value.get_or_create.side_effect = [ + (mock_notion_link, False), + (fresh_notion_link, True), + ] + mock_el.objects.select_for_update.return_value.get.return_value = ( + fresh_notion_link + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_time.sleep.assert_called_once() + mock_notion.create_postmortem_page.assert_called_once() class TestHandleDumpslackCommand: From 6c93a662b549af959eb76f0d1aa1a2bdda87e65c Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 1 Jun 2026 11:12:28 -0400 Subject: [PATCH 04/11] fix(incidents): Adopt winner's page on dumpslack race instead of returning empty When dumpslack loses the re-lock race (another process saved a Notion URL first), use the winner's page URL and continue with apply_template to populate it with Slack history. Previously the loser returned immediately, leaving the winning page empty. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/cl-NMNVbu5jWyVJ1ATQwcvv_niIuI5oowy27M4fC3KY --- src/firetower/slack_app/handlers/dumpslack.py | 17 +++++--- .../tests/handlers/test_dumpslack.py | 41 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index ca3a52ff..c1009c0e 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -125,8 +125,8 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: page_id = page["id"] page_url = page["url"] # Re-acquire lock before saving to detect concurrent callers that - # also saw url="" and raced to create a page. The loser exits here; - # the winner's URL and apply_template call stand. + # also saw url="" and raced to create a page. The loser adopts the + # winner's page so apply_template still populates it. with transaction.atomic(): notion_link = ExternalLink.objects.select_for_update().get( incident=incident, @@ -137,10 +137,15 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: "Race condition: concurrent call already created Notion page for %s", incident.incident_number, ) - return - notion_link.url = page_url - notion_link.save(update_fields=["url"]) - notion_page_created = True + page_id = _extract_notion_page_id(notion_link.url) + page_url = notion_link.url + if not page_id: + return + update_slack = True + else: + notion_link.url = page_url + notion_link.save(update_fields=["url"]) + notion_page_created = True except Exception: logger.exception( "Failed to create Notion postmortem page for %s", diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 15f97f86..fea50d8a 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -661,6 +661,47 @@ def test_recovers_when_placeholder_deleted_during_poll(self): mock_time.sleep.assert_called_once() mock_notion.create_postmortem_page.assert_called_once() + def test_race_loser_adopts_winner_page_and_populates(self): + winner_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_page = {"id": "our-page", "url": "https://notion.so/our-page-id"} + mock_notion_link = MagicMock(url="") + winner_link = MagicMock(url=winner_url) + mock_notion = MagicMock() + mock_notion.create_postmortem_page.return_value = mock_page + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch("firetower.slack_app.handlers.dumpslack.settings") as mock_settings, + ): + mock_settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + True, + ) + mock_el.objects.select_for_update.return_value.get.return_value = ( + winner_link + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_notion.apply_template.assert_called_once() + call_args = mock_notion.apply_template.call_args + assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" + assert call_args[1]["update_slack"] is True + posted = client.chat_postMessage.call_args[1]["text"] + assert winner_url in posted + class TestHandleDumpslackCommand: def _make_args(self, notion_config=None, channel_id="C123"): From 48ea6dc43d20edada284a4dc3370d9d21f17d3d6 Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Fri, 5 Jun 2026 10:24:25 -0400 Subject: [PATCH 05/11] fix(dumpslack): Clean up orphaned placeholder on failure and add missing tests _trigger_slack_dump left an empty-URL ExternalLink placeholder when the Notion API call failed, causing a 15-second polling delay on retry and blocking _create_postmortem_doc from creating the page. Add the same cleanup that _create_postmortem_doc already performs. Also add tests for the pre-existing URL branch (dumpslack finds an already-populated Notion URL and calls apply_template with update_slack=True) and for the placeholder cleanup on failure. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/3OqlwFB6o1EuMYtSikhAnJGwfLF5tR9hl2uTJbVDUTQ --- src/firetower/slack_app/handlers/dumpslack.py | 5 ++ .../tests/handlers/test_dumpslack.py | 71 ++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 471b5881..11c5706d 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -152,6 +152,11 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: "Failed to create Notion postmortem page for %s", incident.incident_number, ) + ExternalLink.objects.filter( + incident=incident, + type=ExternalLinkType.NOTION, + url="", + ).delete() try: client.chat_postMessage( channel=channel_id, diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 34e99cd2..4223fa40 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import User from slack_sdk.errors import SlackApiError -from firetower.incidents.models import Incident, IncidentSeverity +from firetower.incidents.models import ExternalLinkType, Incident, IncidentSeverity from firetower.integrations.services.slack import SlackService, is_slack_url from firetower.slack_app.handlers.dumpslack import ( _backfill_milestones, @@ -665,6 +665,41 @@ def test_recovers_when_placeholder_deleted_during_poll(self): mock_time.sleep.assert_called_once() mock_notion.create_postmortem_page.assert_called_once() + def test_uses_existing_url_and_updates_slack(self): + existing_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url=existing_url) + mock_notion = MagicMock() + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + ): + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + False, + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_notion.create_postmortem_page.assert_not_called() + mock_notion.apply_template.assert_called_once() + call_args = mock_notion.apply_template.call_args + assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" + assert call_args[1]["update_slack"] is True + posted = client.chat_postMessage.call_args[1]["text"] + assert "Updated" in posted + assert existing_url in posted + def test_race_loser_adopts_winner_page_and_populates(self): winner_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" client = MagicMock() @@ -706,6 +741,40 @@ def test_race_loser_adopts_winner_page_and_populates(self): posted = client.chat_postMessage.call_args[1]["text"] assert winner_url in posted + def test_cleans_up_placeholder_on_failure(self): + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url="") + mock_notion = MagicMock() + mock_notion.create_postmortem_page.side_effect = RuntimeError("API error") + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch("firetower.slack_app.handlers.dumpslack.settings") as mock_settings, + ): + mock_settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + True, + ) + mock_el.objects.filter.return_value.delete.return_value = (1, {}) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_el.objects.filter.assert_called_once_with( + incident=mock_incident, + type=ExternalLinkType.NOTION, + url="", + ) + mock_el.objects.filter.return_value.delete.assert_called_once() + posted = client.chat_postMessage.call_args[1]["text"] + assert "Failed" in posted + class TestHandleDumpslackCommand: def _make_args(self, notion_config=None, channel_id="C123"): From 06610132f2cd0c85f18a8f7e9e5d09618ccecfbe Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Fri, 5 Jun 2026 15:10:56 -0400 Subject: [PATCH 06/11] latest round of fixes --- src/firetower/incidents/hooks.py | 54 ++++++++------- src/firetower/incidents/tests/test_hooks.py | 69 +++++++++++++++++-- src/firetower/integrations/services/notion.py | 6 ++ .../integrations/tests/test_notion.py | 16 +++++ src/firetower/slack_app/handlers/dumpslack.py | 2 + .../tests/handlers/test_dumpslack.py | 3 +- 6 files changed, 120 insertions(+), 30 deletions(-) diff --git a/src/firetower/incidents/hooks.py b/src/firetower/incidents/hooks.py index ee60eae8..b6bae3c2 100644 --- a/src/firetower/incidents/hooks.py +++ b/src/firetower/incidents/hooks.py @@ -763,10 +763,6 @@ def _create_troubleshooting_doc(incident: Incident, channel_id: str) -> None: def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: try: - if incident.is_private: - logger.info("Skipping postmortem doc for private incident %s", incident.id) - return - if not NotionService.is_configured(): logger.info( "Notion not configured, skipping postmortem doc for incident %s", @@ -774,6 +770,14 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: ) return + notion = NotionService.from_settings() + if not notion: + return + + if incident.is_private: + logger.info("Skipping postmortem doc for private incident %s", incident.id) + return + with transaction.atomic(): link, created = ExternalLink.objects.select_for_update().get_or_create( incident=incident, @@ -793,15 +797,6 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: ) return - notion = NotionService.from_settings() - if not notion: - ExternalLink.objects.filter( - incident=incident, - type=ExternalLinkType.NOTION, - url="", - ).delete() - return - incident_url = _build_incident_url(incident) captain_email = incident.captain.email if incident.captain else None page = notion.create_postmortem_page( @@ -824,19 +819,28 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: ).delete() return - with transaction.atomic(): - link = ExternalLink.objects.select_for_update().get( - incident=incident, - type=ExternalLinkType.NOTION, - ) - if link.url: - logger.warning( - "Race condition: concurrent call already created postmortem doc for %s", - incident.incident_number, + try: + with transaction.atomic(): + link = ExternalLink.objects.select_for_update().get( + incident=incident, + type=ExternalLinkType.NOTION, ) - return - link.url = page["url"] - link.save(update_fields=["url"]) + if link.url: + logger.warning( + "Race condition: concurrent call already created postmortem doc for %s", + incident.incident_number, + ) + notion.archive_page(page["id"]) + return + link.url = page["url"] + link.save(update_fields=["url"]) + except ExternalLink.DoesNotExist: + logger.warning( + "Placeholder deleted before postmortem URL could be saved for %s", + incident.incident_number, + ) + notion.archive_page(page["id"]) + return try: _slack_service.add_bookmark(channel_id, "Postmortem Doc", page["url"]) diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index f14f2afb..1ea1376c 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -2685,6 +2685,9 @@ class TestCreatePostmortemDoc: @patch("firetower.incidents.hooks.NotionService") @patch("firetower.incidents.hooks._slack_service") def test_skips_for_private_incident(self, mock_slack, mock_notion_cls): + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_notion_cls.from_settings.return_value = mock_service incident = Incident.objects.create( title="Sensitive", severity=IncidentSeverity.P1, @@ -2693,7 +2696,7 @@ def test_skips_for_private_incident(self, mock_slack, mock_notion_cls): _create_postmortem_doc(incident, "C99999") - mock_notion_cls.is_configured.assert_not_called() + mock_service.create_postmortem_page.assert_not_called() assert not ExternalLink.objects.filter( incident=incident, type=ExternalLinkType.NOTION ).exists() @@ -2875,9 +2878,7 @@ def test_bookmark_failure_does_not_propagate( @patch("firetower.incidents.hooks.NotionService") @patch("firetower.incidents.hooks._slack_service") - def test_deletes_placeholder_when_notion_service_returns_none( - self, mock_slack, mock_notion_cls - ): + def test_skips_when_notion_service_returns_none(self, mock_slack, mock_notion_cls): mock_notion_cls.is_configured.return_value = True mock_notion_cls.from_settings.return_value = None @@ -2892,6 +2893,66 @@ def test_deletes_placeholder_when_notion_service_returns_none( incident=incident, type=ExternalLinkType.NOTION ).exists() + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_race_loser_archives_orphan_page( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + def create_page_and_simulate_winner(**kwargs): + ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION, url="" + ).update(url="https://notion.so/winner-page") + return {"id": "orphan-page", "url": "https://notion.so/orphan-page"} + + mock_service.create_postmortem_page.side_effect = ( + create_page_and_simulate_winner + ) + + _create_postmortem_doc(incident, "C99999") + + mock_service.archive_page.assert_called_once_with("orphan-page") + link = ExternalLink.objects.get(incident=incident, type=ExternalLinkType.NOTION) + assert link.url == "https://notion.so/winner-page" + + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_archives_page_when_placeholder_deleted( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + + def create_page_and_delete_placeholder(**kwargs): + ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION, url="" + ).delete() + return {"id": "orphan-page", "url": "https://notion.so/orphan-page"} + + mock_service.create_postmortem_page.side_effect = ( + create_page_and_delete_placeholder + ) + + _create_postmortem_doc(incident, "C99999") + + mock_service.archive_page.assert_called_once_with("orphan-page") + @pytest.mark.django_db class TestOnIncidentCreatedPostmortemDoc: diff --git a/src/firetower/integrations/services/notion.py b/src/firetower/integrations/services/notion.py index b2a35dfa..c5883971 100644 --- a/src/firetower/integrations/services/notion.py +++ b/src/firetower/integrations/services/notion.py @@ -153,6 +153,12 @@ def get_users(self) -> dict[str, dict[str, str]]: self._users = users return users + def archive_page(self, page_id: str) -> None: + try: + self.client.pages.update(page_id=page_id, archived=True) + except Exception: + logger.exception("Failed to archive orphaned Notion page %s", page_id) + def create_postmortem_page( self, incident_number: str, diff --git a/src/firetower/integrations/tests/test_notion.py b/src/firetower/integrations/tests/test_notion.py index 90508ddc..ca08d6f6 100644 --- a/src/firetower/integrations/tests/test_notion.py +++ b/src/firetower/integrations/tests/test_notion.py @@ -152,6 +152,22 @@ def test_tolerates_user_with_missing_name(self, notion): assert users == {"a@sentry.io": {"name": "", "id": "U1"}} +class TestArchivePage: + def test_archives_page_successfully(self, notion): + notion.archive_page("page-123") + + notion.client.pages.update.assert_called_once_with( + page_id="page-123", archived=True + ) + + def test_logs_and_swallows_exception(self, notion): + notion.client.pages.update.side_effect = Exception("API error") + + notion.archive_page("page-123") + + notion.client.pages.update.assert_called_once() + + class TestSendMarkdown: def test_sends_insert_content_patch(self, notion): with patch( diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 11c5706d..71a02212 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -128,6 +128,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: # Re-acquire lock before saving to detect concurrent callers that # also saw url="" and raced to create a page. The loser adopts the # winner's page so apply_template still populates it. + orphan_page_id = page["id"] with transaction.atomic(): notion_link = ExternalLink.objects.select_for_update().get( incident=incident, @@ -138,6 +139,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: "Race condition: concurrent call already created Notion page for %s", incident.incident_number, ) + notion.archive_page(orphan_page_id) page_id = _extract_notion_page_id(notion_link.url) page_url = notion_link.url if not page_id: diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 4223fa40..7566332d 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -700,7 +700,7 @@ def test_uses_existing_url_and_updates_slack(self): assert "Updated" in posted assert existing_url in posted - def test_race_loser_adopts_winner_page_and_populates(self): + def test_race_loser_adopts_winner_page_and_archives_orphan(self): winner_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" client = MagicMock() mock_incident = MagicMock(is_private=False) @@ -734,6 +734,7 @@ def test_race_loser_adopts_winner_page_and_populates(self): ) _trigger_slack_dump(client, "C123", mock_incident) + mock_notion.archive_page.assert_called_once_with("our-page") mock_notion.apply_template.assert_called_once() call_args = mock_notion.apply_template.call_args assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" From 7de74bd42c6dba1ba11f2eea8ce12407d0fd7fb4 Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 8 Jun 2026 11:18:02 -0400 Subject: [PATCH 07/11] fix(postmortem): Apply template at creation and always retry bookmark _create_postmortem_doc now renders and sends the postmortem template markdown after page creation, matching the troubleshooting doc pattern. Previously the template was never applied because dumpslack set update_slack=True when finding an existing URL, which skipped the template in apply_template. Also moves archive_page calls outside transaction blocks to avoid holding SELECT FOR UPDATE locks during external API calls, and removes the notion_page_created guard on add_bookmark in dumpslack so the bookmark is retried if it failed during initial creation. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/NM6HrlV9VN1emYV0tYeQpLcIBCEpOKuxvRwAjLyU-CQ --- src/firetower/incidents/hooks.py | 21 +++++++++--- src/firetower/incidents/tests/test_hooks.py | 32 +++++++++++++++++++ src/firetower/slack_app/handlers/dumpslack.py | 20 ++++++------ .../tests/handlers/test_dumpslack.py | 8 +++++ 4 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/firetower/incidents/hooks.py b/src/firetower/incidents/hooks.py index b6bae3c2..957c255d 100644 --- a/src/firetower/incidents/hooks.py +++ b/src/firetower/incidents/hooks.py @@ -819,6 +819,7 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: ).delete() return + orphan_page_id = None try: with transaction.atomic(): link = ExternalLink.objects.select_for_update().get( @@ -830,10 +831,10 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: "Race condition: concurrent call already created postmortem doc for %s", incident.incident_number, ) - notion.archive_page(page["id"]) - return - link.url = page["url"] - link.save(update_fields=["url"]) + orphan_page_id = page["id"] + else: + link.url = page["url"] + link.save(update_fields=["url"]) except ExternalLink.DoesNotExist: logger.warning( "Placeholder deleted before postmortem URL could be saved for %s", @@ -842,6 +843,18 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: notion.archive_page(page["id"]) return + if orphan_page_id: + notion.archive_page(orphan_page_id) + return + + if notion.template_markdown: + content = NotionService._render_template(notion.template_markdown, incident) + if not notion._send_markdown(page["id"], content): + logger.error( + "Failed to apply postmortem template to page %s", + page["id"], + ) + try: _slack_service.add_bookmark(channel_id, "Postmortem Doc", page["url"]) except Exception: diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index 1ea1376c..1ae5abb8 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -2744,6 +2744,38 @@ def test_creates_doc_and_adds_bookmark_without_message( mock_slack.post_message.assert_not_called() + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_applies_postmortem_template_after_creation( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.template_markdown = "# Postmortem\n{linear_url}" + mock_service.create_postmortem_page.return_value = { + "id": "page-456", + "url": "https://notion.so/page-456", + } + mock_notion_cls.from_settings.return_value = mock_service + mock_notion_cls._render_template.return_value = ( + "# Postmortem\nhttps://linear.app/issue" + ) + + incident = Incident.objects.create( + title="Production is down", + severity=IncidentSeverity.P1, + ) + + _create_postmortem_doc(incident, "C99999") + + mock_notion_cls._render_template.assert_called_once_with( + "# Postmortem\n{linear_url}", incident + ) + mock_service._send_markdown.assert_called_once_with( + "page-456", "# Postmortem\nhttps://linear.app/issue" + ) + @patch("firetower.incidents.hooks.NotionService") @patch("firetower.incidents.hooks._slack_service") def test_skips_when_not_configured(self, mock_slack, mock_notion_cls): diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 71a02212..9c40b57f 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -129,6 +129,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: # also saw url="" and raced to create a page. The loser adopts the # winner's page so apply_template still populates it. orphan_page_id = page["id"] + race_lost = False with transaction.atomic(): notion_link = ExternalLink.objects.select_for_update().get( incident=incident, @@ -139,16 +140,18 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: "Race condition: concurrent call already created Notion page for %s", incident.incident_number, ) - notion.archive_page(orphan_page_id) + race_lost = True page_id = _extract_notion_page_id(notion_link.url) page_url = notion_link.url - if not page_id: - return - update_slack = True else: notion_link.url = page_url notion_link.save(update_fields=["url"]) notion_page_created = True + if race_lost: + notion.archive_page(orphan_page_id) + if not page_id: + return + update_slack = True except Exception: logger.exception( "Failed to create Notion postmortem page for %s", @@ -204,11 +207,10 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: except Exception: logger.exception("Failed to add AI timeline to Notion page %s", page_id) - if notion_page_created: - try: - slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) - except Exception: - logger.exception("Failed to add Notion bookmark to channel %s", channel_id) + try: + slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) + except Exception: + logger.exception("Failed to add Notion bookmark to channel %s", channel_id) try: client.chat_postMessage( diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 7566332d..5b6da55f 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -672,6 +672,7 @@ def test_uses_existing_url_and_updates_slack(self): mock_incident.captain = None mock_notion_link = MagicMock(url=existing_url) mock_notion = MagicMock() + mock_slack_service = MagicMock() with ( patch( @@ -684,6 +685,10 @@ def test_uses_existing_url_and_updates_slack(self): ), patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch( + "firetower.slack_app.handlers.dumpslack.SlackService", + return_value=mock_slack_service, + ), ): mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( mock_notion_link, @@ -696,6 +701,9 @@ def test_uses_existing_url_and_updates_slack(self): call_args = mock_notion.apply_template.call_args assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" assert call_args[1]["update_slack"] is True + mock_slack_service.add_bookmark.assert_called_once_with( + "C123", "Postmortem Doc", existing_url + ) posted = client.chat_postMessage.call_args[1]["text"] assert "Updated" in posted assert existing_url in posted From bc90a159284573b319e48a926fd8134fa72bf02c Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 8 Jun 2026 11:25:59 -0400 Subject: [PATCH 08/11] fix(dumpslack): Add assertion for page_id to satisfy mypy page_id is guaranteed non-None at this point since all None paths return early, but mypy cannot narrow across the branching logic. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/2SzUTaoLPySZkmO8sOLmQWGNoEW2pY0lIjleNhsJ7NQ --- src/firetower/slack_app/handlers/dumpslack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 9c40b57f..571353d9 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -175,6 +175,8 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: action = "Created" if notion_page_created else "Updated" + assert page_id is not None + slack_service = SlackService() messages = _get_channel_messages(slack_service, channel_id) From 30720e4e8401d00aa134ce313bc0928fbfede11a Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 8 Jun 2026 12:58:01 -0400 Subject: [PATCH 09/11] fix(postmortem): Apply template at creation and always retry bookmark Move template application from _create_postmortem_doc to apply_template using a blocks-based idempotency check: the template is only sent when the page has no existing blocks. This ensures the template is applied exactly once by dumpslack (after the Linear issue exists) rather than too early during incident creation. Also archive orphaned Notion pages in dumpslack exception handler, and deduplicate bookmarks by checking bookmarks_list before adding. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/wh_HvrZWRy7fnDnoSmn1C_mLmzeK2uRVjwn4hnUFysY --- src/firetower/incidents/hooks.py | 8 --- src/firetower/incidents/tests/test_hooks.py | 32 ----------- src/firetower/integrations/services/notion.py | 19 ++++--- .../integrations/tests/test_notion.py | 56 ++++++++++++++----- src/firetower/slack_app/handlers/dumpslack.py | 23 +++++--- .../tests/handlers/test_dumpslack.py | 41 +++++++++++++- 6 files changed, 108 insertions(+), 71 deletions(-) diff --git a/src/firetower/incidents/hooks.py b/src/firetower/incidents/hooks.py index 957c255d..cea5dfb8 100644 --- a/src/firetower/incidents/hooks.py +++ b/src/firetower/incidents/hooks.py @@ -847,14 +847,6 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: notion.archive_page(orphan_page_id) return - if notion.template_markdown: - content = NotionService._render_template(notion.template_markdown, incident) - if not notion._send_markdown(page["id"], content): - logger.error( - "Failed to apply postmortem template to page %s", - page["id"], - ) - try: _slack_service.add_bookmark(channel_id, "Postmortem Doc", page["url"]) except Exception: diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index 1ae5abb8..1ea1376c 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -2744,38 +2744,6 @@ def test_creates_doc_and_adds_bookmark_without_message( mock_slack.post_message.assert_not_called() - @patch("firetower.incidents.hooks.NotionService") - @patch("firetower.incidents.hooks._slack_service") - def test_applies_postmortem_template_after_creation( - self, mock_slack, mock_notion_cls, settings - ): - settings.FIRETOWER_BASE_URL = "https://firetower.example.com" - mock_notion_cls.is_configured.return_value = True - mock_service = MagicMock() - mock_service.template_markdown = "# Postmortem\n{linear_url}" - mock_service.create_postmortem_page.return_value = { - "id": "page-456", - "url": "https://notion.so/page-456", - } - mock_notion_cls.from_settings.return_value = mock_service - mock_notion_cls._render_template.return_value = ( - "# Postmortem\nhttps://linear.app/issue" - ) - - incident = Incident.objects.create( - title="Production is down", - severity=IncidentSeverity.P1, - ) - - _create_postmortem_doc(incident, "C99999") - - mock_notion_cls._render_template.assert_called_once_with( - "# Postmortem\n{linear_url}", incident - ) - mock_service._send_markdown.assert_called_once_with( - "page-456", "# Postmortem\nhttps://linear.app/issue" - ) - @patch("firetower.incidents.hooks.NotionService") @patch("firetower.incidents.hooks._slack_service") def test_skips_when_not_configured(self, mock_slack, mock_notion_cls): diff --git a/src/firetower/integrations/services/notion.py b/src/firetower/integrations/services/notion.py index c5883971..a011fbd1 100644 --- a/src/firetower/integrations/services/notion.py +++ b/src/firetower/integrations/services/notion.py @@ -250,15 +250,20 @@ def apply_template( self, page_id: str, messages: list[dict[str, Any]], - update_slack: bool = False, incident: Any | None = None, ) -> None: - if not update_slack and self.template_markdown: - content = self._render_template(self.template_markdown, incident) - if not self._send_markdown(page_id, content): - raise RuntimeError( - f"Failed to apply markdown template to Notion page {page_id}" - ) + if self.template_markdown: + try: + existing = self.client.blocks.children.list(block_id=page_id) + has_content = bool(existing.get("results")) + except Exception: + has_content = False + if not has_content: + content = self._render_template(self.template_markdown, incident) + if not self._send_markdown(page_id, content): + raise RuntimeError( + f"Failed to apply markdown template to Notion page {page_id}" + ) timestamp = datetime.now(tz=UTC).strftime("%Y-%m-%d %H:%M:%S UTC") toggle: dict[str, Any] = { diff --git a/src/firetower/integrations/tests/test_notion.py b/src/firetower/integrations/tests/test_notion.py index ca08d6f6..1e047d2d 100644 --- a/src/firetower/integrations/tests/test_notion.py +++ b/src/firetower/integrations/tests/test_notion.py @@ -409,12 +409,13 @@ def _make_append_response(self, block_id: str) -> dict: return {"results": [{"id": block_id}]} def test_new_page_sends_markdown_then_slack_blocks(self, notion): + notion.client.blocks.children.list.return_value = {"results": []} notion.client.blocks.children.append.return_value = self._make_append_response( "toggle-id" ) with patch.object(notion, "_send_markdown", return_value=True) as mock_md: - notion.apply_template("page-id", messages=[], update_slack=False) + notion.apply_template("page-id", messages=[]) mock_md.assert_called_once_with("page-id", "# Template\n\nSome content.") notion.client.blocks.children.append.assert_called_once() @@ -424,6 +425,7 @@ def test_new_page_sends_markdown_then_slack_blocks(self, notion): assert toggle_block["type"] == "toggle" def test_raises_when_send_markdown_fails(self, notion): + notion.client.blocks.children.list.return_value = {"results": []} notion.client.blocks.children.append.return_value = self._make_append_response( "toggle-id" ) @@ -432,15 +434,16 @@ def test_raises_when_send_markdown_fails(self, notion): patch.object(notion, "_send_markdown", return_value=False), pytest.raises(RuntimeError, match="template"), ): - notion.apply_template("page-id", messages=[], update_slack=False) + notion.apply_template("page-id", messages=[]) def test_raises_when_toggle_creation_fails(self, notion): + notion.client.blocks.children.list.return_value = {"results": []} with ( patch.object(notion, "_send_markdown", return_value=True), patch.object(notion, "_append_children", return_value=None), pytest.raises(RuntimeError, match="toggle"), ): - notion.apply_template("page-id", messages=[], update_slack=False) + notion.apply_template("page-id", messages=[]) def test_interpolates_incident_into_template(self): svc = NotionService( @@ -449,6 +452,7 @@ def test_interpolates_incident_into_template(self): template_markdown="# PM\n[Action Items]({linear_url})", ) svc.client = MagicMock() + svc.client.blocks.children.list.return_value = {"results": []} svc.client.blocks.children.append.return_value = self._make_append_response( "toggle-id" ) @@ -458,21 +462,22 @@ def test_interpolates_incident_into_template(self): } with patch.object(svc, "_send_markdown", return_value=True) as mock_md: - svc.apply_template( - "page-id", messages=[], update_slack=False, incident=incident - ) + svc.apply_template("page-id", messages=[], incident=incident) mock_md.assert_called_once_with( "page-id", "# PM\n[Action Items](https://linear.app/team/issue/INC-42)" ) - def test_update_slack_skips_markdown_template(self, notion): + def test_skips_template_when_page_has_content(self, notion): + notion.client.blocks.children.list.return_value = { + "results": [{"id": "existing-block"}] + } notion.client.blocks.children.append.return_value = self._make_append_response( "toggle-id" ) with patch.object(notion, "_send_markdown", return_value=True) as mock_md: - notion.apply_template("page-id", messages=[], update_slack=True) + notion.apply_template("page-id", messages=[]) mock_md.assert_not_called() @@ -499,7 +504,10 @@ def test_appends_replies_as_children_of_parent_bullet(self, notion): ] with patch.object(notion, "_send_markdown", return_value=True): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) reply_call = notion.client.blocks.children.append.call_args_list[2] assert reply_call.kwargs["block_id"] == "bullet-id" @@ -534,7 +542,10 @@ def test_appends_images_before_replies_as_children_of_bullet(self, notion): patch.object(notion, "_send_markdown", return_value=True), patch.object(notion, "_create_image_block", return_value=image_block), ): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) children_call = notion.client.blocks.children.append.call_args_list[2] assert children_call.kwargs["block_id"] == "bullet-id" @@ -561,7 +572,10 @@ def test_skips_failed_image_uploads(self, notion): patch.object(notion, "_send_markdown", return_value=True), patch.object(notion, "_create_image_block", return_value=None), ): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) # Only 2 appends: toggle creation + bullet batch. No children call since image failed. assert notion.client.blocks.children.append.call_count == 2 @@ -589,7 +603,10 @@ def test_batches_children_at_block_limit(self, notion): ] with patch.object(notion, "_send_markdown", return_value=True): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) # 4 total: toggle, bullet batch, first 85 replies, last 5 replies assert notion.client.blocks.children.append.call_count == 4 @@ -650,7 +667,10 @@ def test_logs_warning_when_notion_returns_fewer_ids_than_batch(self, notion): patch.object(notion, "_send_markdown", return_value=True), patch("firetower.integrations.services.notion.logger") as mock_logger, ): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) # Confirm the mismatch warning was emitted. warning_args = [call[0][0] for call in mock_logger.warning.call_args_list] @@ -681,7 +701,10 @@ def test_aborts_and_logs_error_when_create_slack_content_makes_no_progress( ), patch("firetower.integrations.services.notion.logger") as mock_logger, ): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) error_args = [call[0][0] for call in mock_logger.error.call_args_list] assert any("no progress" in msg for msg in error_args) @@ -708,7 +731,10 @@ def test_logs_warning_when_batch_append_returns_none(self, notion): patch.object(notion, "_send_markdown", return_value=True), patch("firetower.integrations.services.notion.logger") as mock_logger, ): - notion.apply_template("page-id", messages=messages, update_slack=True) + notion.apply_template( + "page-id", + messages=messages, + ) warning_args = [call[0][0] for call in mock_logger.warning.call_args_list] assert any("absent from the Notion dump" in msg for msg in warning_args) diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 571353d9..f1206e47 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -49,8 +49,8 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: page_id: str | None = None page_url: str = "" - update_slack: bool = False notion_page_created: bool = False + created_page_id: str | None = None try: with transaction.atomic(): @@ -110,7 +110,6 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: ) return page_url = existing_url - update_slack = True else: base_url = settings.FIRETOWER_BASE_URL incident_url = f"{base_url}/{incident.incident_number}" @@ -125,6 +124,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: ) page_id = page["id"] page_url = page["url"] + created_page_id = page["id"] # Re-acquire lock before saving to detect concurrent callers that # also saw url="" and raced to create a page. The loser adopts the # winner's page so apply_template still populates it. @@ -151,12 +151,18 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: notion.archive_page(orphan_page_id) if not page_id: return - update_slack = True except Exception: logger.exception( "Failed to create Notion postmortem page for %s", incident.incident_number, ) + if created_page_id and not notion_page_created: + try: + notion.archive_page(created_page_id) + except Exception: + logger.exception( + "Failed to archive orphaned Notion page %s", created_page_id + ) ExternalLink.objects.filter( incident=incident, type=ExternalLinkType.NOTION, @@ -181,9 +187,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: messages = _get_channel_messages(slack_service, channel_id) try: - notion.apply_template( - page_id, messages, update_slack=update_slack, incident=incident - ) + notion.apply_template(page_id, messages, incident=incident) except Exception: logger.exception("Failed to populate Notion page %s", page_id) try: @@ -210,7 +214,12 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: logger.exception("Failed to add AI timeline to Notion page %s", page_id) try: - slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) + existing = slack_service.client.bookmarks_list(channel_id=channel_id) + has_bookmark = any( + b.get("title") == "Postmortem Doc" for b in existing.get("bookmarks", []) + ) + if not has_bookmark: + slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) except Exception: logger.exception("Failed to add Notion bookmark to channel %s", channel_id) diff --git a/src/firetower/slack_app/tests/handlers/test_dumpslack.py b/src/firetower/slack_app/tests/handlers/test_dumpslack.py index 5b6da55f..6c734b8b 100644 --- a/src/firetower/slack_app/tests/handlers/test_dumpslack.py +++ b/src/firetower/slack_app/tests/handlers/test_dumpslack.py @@ -673,6 +673,7 @@ def test_uses_existing_url_and_updates_slack(self): mock_notion_link = MagicMock(url=existing_url) mock_notion = MagicMock() mock_slack_service = MagicMock() + mock_slack_service.client.bookmarks_list.return_value = {"bookmarks": []} with ( patch( @@ -700,7 +701,7 @@ def test_uses_existing_url_and_updates_slack(self): mock_notion.apply_template.assert_called_once() call_args = mock_notion.apply_template.call_args assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" - assert call_args[1]["update_slack"] is True + assert call_args[1]["incident"] == mock_incident mock_slack_service.add_bookmark.assert_called_once_with( "C123", "Postmortem Doc", existing_url ) @@ -708,6 +709,42 @@ def test_uses_existing_url_and_updates_slack(self): assert "Updated" in posted assert existing_url in posted + def test_skips_bookmark_when_already_exists(self): + existing_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" + client = MagicMock() + mock_incident = MagicMock(is_private=False) + mock_incident.captain = None + mock_notion_link = MagicMock(url=existing_url) + mock_notion = MagicMock() + mock_slack_service = MagicMock() + mock_slack_service.client.bookmarks_list.return_value = { + "bookmarks": [{"title": "Postmortem Doc", "link": existing_url}] + } + + with ( + patch( + "firetower.slack_app.handlers.dumpslack.NotionService.from_settings", + return_value=mock_notion, + ), + patch( + "firetower.slack_app.handlers.dumpslack._get_channel_messages", + return_value=[], + ), + patch("firetower.slack_app.handlers.dumpslack.ExternalLink") as mock_el, + patch("firetower.slack_app.handlers.dumpslack.transaction"), + patch( + "firetower.slack_app.handlers.dumpslack.SlackService", + return_value=mock_slack_service, + ), + ): + mock_el.objects.select_for_update.return_value.get_or_create.return_value = ( + mock_notion_link, + False, + ) + _trigger_slack_dump(client, "C123", mock_incident) + + mock_slack_service.add_bookmark.assert_not_called() + def test_race_loser_adopts_winner_page_and_archives_orphan(self): winner_url = "https://notion.so/12345678-1234-1234-1234-123456789abc" client = MagicMock() @@ -746,7 +783,7 @@ def test_race_loser_adopts_winner_page_and_archives_orphan(self): mock_notion.apply_template.assert_called_once() call_args = mock_notion.apply_template.call_args assert call_args[0][0] == "12345678-1234-1234-1234-123456789abc" - assert call_args[1]["update_slack"] is True + assert call_args[1]["incident"] == mock_incident posted = client.chat_postMessage.call_args[1]["text"] assert winner_url in posted From c55fd01758fba1e7d5fb8935a5e4c6420c72d1fa Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Mon, 8 Jun 2026 13:46:28 -0400 Subject: [PATCH 10/11] fix: Resolve mypy errors and address review bot findings Fix union-attr mypy errors in notion.py and dumpslack.py. Archive orphaned Notion pages on unexpected DB errors in _create_postmortem_doc. Treat transient API errors in apply_template as "has content" to prevent duplicate template application. Co-Authored-By: Claude Opus 4.6 Agent transcript: https://claudescope.sentry.dev/share/6wZMYD_w5Q44B88Gyky9rTHQSH61cCc7pC8_AKSNs7k --- src/firetower/incidents/hooks.py | 11 +++++++ src/firetower/incidents/tests/test_hooks.py | 29 +++++++++++++++++++ src/firetower/integrations/services/notion.py | 11 +++++-- .../integrations/tests/test_notion.py | 11 +++++++ src/firetower/slack_app/handlers/dumpslack.py | 14 +++++---- 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/firetower/incidents/hooks.py b/src/firetower/incidents/hooks.py index cea5dfb8..d9f0ed95 100644 --- a/src/firetower/incidents/hooks.py +++ b/src/firetower/incidents/hooks.py @@ -762,6 +762,8 @@ def _create_troubleshooting_doc(incident: Incident, channel_id: str) -> None: def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: + notion = None + page = None try: if not NotionService.is_configured(): logger.info( @@ -856,6 +858,15 @@ def _create_postmortem_doc(incident: Incident, channel_id: str) -> None: ) except Exception: logger.exception("Failed to create postmortem doc for incident %s", incident.id) + if notion and page and page.get("id"): + try: + notion.archive_page(page["id"]) + except Exception: + logger.exception( + "Failed to archive orphaned Notion page %s for incident %s", + page["id"], + incident.id, + ) ExternalLink.objects.filter( incident=incident, type=ExternalLinkType.NOTION, diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index 1ea1376c..72769f7d 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -2851,6 +2851,35 @@ def test_api_error_cleans_up_placeholder( incident=incident, type=ExternalLinkType.NOTION ).exists() + @patch("firetower.incidents.hooks.NotionService") + @patch("firetower.incidents.hooks._slack_service") + def test_db_error_archives_orphan_notion_page( + self, mock_slack, mock_notion_cls, settings + ): + settings.FIRETOWER_BASE_URL = "https://firetower.example.com" + mock_notion_cls.is_configured.return_value = True + mock_service = MagicMock() + mock_service.create_postmortem_page.return_value = { + "id": "orphan-page-id", + "url": "https://notion.so/orphan-page", + } + mock_notion_cls.from_settings.return_value = mock_service + + incident = Incident.objects.create( + title="Test", + severity=IncidentSeverity.P1, + ) + ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).delete() + + _create_postmortem_doc(incident, "C99999") + + assert not ExternalLink.objects.filter( + incident=incident, type=ExternalLinkType.NOTION + ).exists() + mock_service.archive_page.assert_called_once_with("orphan-page-id") + @patch("firetower.incidents.hooks.NotionService") @patch("firetower.incidents.hooks._slack_service") def test_bookmark_failure_does_not_propagate( diff --git a/src/firetower/integrations/services/notion.py b/src/firetower/integrations/services/notion.py index a011fbd1..9b4385d7 100644 --- a/src/firetower/integrations/services/notion.py +++ b/src/firetower/integrations/services/notion.py @@ -254,10 +254,17 @@ def apply_template( ) -> None: if self.template_markdown: try: - existing = self.client.blocks.children.list(block_id=page_id) + existing = cast( + dict[str, Any], + self.client.blocks.children.list(block_id=page_id), + ) has_content = bool(existing.get("results")) except Exception: - has_content = False + logger.warning( + "Failed to check existing content for page %s, skipping template", + page_id, + ) + has_content = True if not has_content: content = self._render_template(self.template_markdown, incident) if not self._send_markdown(page_id, content): diff --git a/src/firetower/integrations/tests/test_notion.py b/src/firetower/integrations/tests/test_notion.py index 1e047d2d..90b3020a 100644 --- a/src/firetower/integrations/tests/test_notion.py +++ b/src/firetower/integrations/tests/test_notion.py @@ -481,6 +481,17 @@ def test_skips_template_when_page_has_content(self, notion): mock_md.assert_not_called() + def test_skips_template_on_transient_api_error(self, notion): + notion.client.blocks.children.list.side_effect = Exception("connection reset") + notion.client.blocks.children.append.return_value = self._make_append_response( + "toggle-id" + ) + + with patch.object(notion, "_send_markdown", return_value=True) as mock_md: + notion.apply_template("page-id", messages=[]) + + mock_md.assert_not_called() + def test_appends_replies_as_children_of_parent_bullet(self, notion): notion.client.blocks.children.append.side_effect = [ {"results": [{"id": "toggle-id"}]}, diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index f1206e47..2d2f7173 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -214,12 +214,14 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: logger.exception("Failed to add AI timeline to Notion page %s", page_id) try: - existing = slack_service.client.bookmarks_list(channel_id=channel_id) - has_bookmark = any( - b.get("title") == "Postmortem Doc" for b in existing.get("bookmarks", []) - ) - if not has_bookmark: - slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) + if slack_service.client: + existing = slack_service.client.bookmarks_list(channel_id=channel_id) + bookmarks: list[dict[str, Any]] = existing.get("bookmarks", []) + has_bookmark = any( + b.get("title") == "Postmortem Doc" for b in bookmarks + ) + if not has_bookmark: + slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) except Exception: logger.exception("Failed to add Notion bookmark to channel %s", channel_id) From 1c274dc4044a7af369281ed201bfcb807340c1eb Mon Sep 17 00:00:00 2001 From: Richard Gibert Date: Tue, 9 Jun 2026 15:43:46 -0400 Subject: [PATCH 11/11] fix(postmortem): Re-raise transient API errors and fix broken test apply_template silently set has_content=True on Notion API failure, permanently skipping template application. Re-raise instead so callers can retry. Fix test_db_error_archives_orphan_notion_page which deleted non-existent ExternalLink records (no-op) instead of simulating a DB error. Use patch.object on ExternalLink.save to raise during URL update, properly exercising the outer exception handler's cleanup path. Co-Authored-By: Claude Agent transcript: https://claudescope.sentry.dev/share/iyF3yLBeORDyqOkG5o2PqyXy3DuvoRTZuVdblGqXkCU --- src/firetower/incidents/tests/test_hooks.py | 13 +++++++++---- src/firetower/integrations/services/notion.py | 4 ++-- src/firetower/integrations/tests/test_notion.py | 9 ++------- src/firetower/slack_app/handlers/dumpslack.py | 4 +--- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/firetower/incidents/tests/test_hooks.py b/src/firetower/incidents/tests/test_hooks.py index 72769f7d..d53de6b9 100644 --- a/src/firetower/incidents/tests/test_hooks.py +++ b/src/firetower/incidents/tests/test_hooks.py @@ -2869,11 +2869,16 @@ def test_db_error_archives_orphan_notion_page( title="Test", severity=IncidentSeverity.P1, ) - ExternalLink.objects.filter( - incident=incident, type=ExternalLinkType.NOTION - ).delete() - _create_postmortem_doc(incident, "C99999") + original_save = ExternalLink.save + + def save_raises_on_url_update(self, *args, **kwargs): + if kwargs.get("update_fields") == ["url"]: + raise RuntimeError("simulated db error") + return original_save(self, *args, **kwargs) + + with patch.object(ExternalLink, "save", save_raises_on_url_update): + _create_postmortem_doc(incident, "C99999") assert not ExternalLink.objects.filter( incident=incident, type=ExternalLinkType.NOTION diff --git a/src/firetower/integrations/services/notion.py b/src/firetower/integrations/services/notion.py index 9b4385d7..5b57d793 100644 --- a/src/firetower/integrations/services/notion.py +++ b/src/firetower/integrations/services/notion.py @@ -261,10 +261,10 @@ def apply_template( has_content = bool(existing.get("results")) except Exception: logger.warning( - "Failed to check existing content for page %s, skipping template", + "Failed to check existing content for page %s", page_id, ) - has_content = True + raise if not has_content: content = self._render_template(self.template_markdown, incident) if not self._send_markdown(page_id, content): diff --git a/src/firetower/integrations/tests/test_notion.py b/src/firetower/integrations/tests/test_notion.py index 90b3020a..ed1ae1e0 100644 --- a/src/firetower/integrations/tests/test_notion.py +++ b/src/firetower/integrations/tests/test_notion.py @@ -481,17 +481,12 @@ def test_skips_template_when_page_has_content(self, notion): mock_md.assert_not_called() - def test_skips_template_on_transient_api_error(self, notion): + def test_raises_on_transient_api_error_checking_content(self, notion): notion.client.blocks.children.list.side_effect = Exception("connection reset") - notion.client.blocks.children.append.return_value = self._make_append_response( - "toggle-id" - ) - with patch.object(notion, "_send_markdown", return_value=True) as mock_md: + with pytest.raises(Exception, match="connection reset"): notion.apply_template("page-id", messages=[]) - mock_md.assert_not_called() - def test_appends_replies_as_children_of_parent_bullet(self, notion): notion.client.blocks.children.append.side_effect = [ {"results": [{"id": "toggle-id"}]}, diff --git a/src/firetower/slack_app/handlers/dumpslack.py b/src/firetower/slack_app/handlers/dumpslack.py index 2d2f7173..cf9c5bac 100644 --- a/src/firetower/slack_app/handlers/dumpslack.py +++ b/src/firetower/slack_app/handlers/dumpslack.py @@ -217,9 +217,7 @@ def _trigger_slack_dump(client: Any, channel_id: str, incident: Any) -> None: if slack_service.client: existing = slack_service.client.bookmarks_list(channel_id=channel_id) bookmarks: list[dict[str, Any]] = existing.get("bookmarks", []) - has_bookmark = any( - b.get("title") == "Postmortem Doc" for b in bookmarks - ) + has_bookmark = any(b.get("title") == "Postmortem Doc" for b in bookmarks) if not has_bookmark: slack_service.add_bookmark(channel_id, "Postmortem Doc", page_url) except Exception: