From 0bab58336bbda6e09176fc5a1a1d94042be483c2 Mon Sep 17 00:00:00 2001 From: Ruiying-Ma Date: Tue, 23 Jun 2026 17:22:24 -0700 Subject: [PATCH] Recompute civic_unstructured ground truth with entity resolution - Update query1-10 ground_truth.csv to entity-resolved values (query3 and query7 resolve to empty sets after merging name variants). - Update validate.py with the new values; embed an explicit ER map in the project-name validators (query1, query2, query8) so answers are entity-resolved before checking; rewrite query3/query7 to verify an empty-result answer instead of trivially passing. - Add hints to db_description_withhint.txt: entity resolution, funding-row noise, and the possibly-incorrect agenda meeting date. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JT6eUF1WfnHt7ERCGL4yew --- .../db_description_withhint.txt | 7 ++- .../query1/ground_truth.csv | 2 - query_civic_unstructured/query1/validate.py | 51 +++++++++++++++++-- .../query10/ground_truth.csv | 2 +- query_civic_unstructured/query10/validate.py | 2 +- .../query2/ground_truth.csv | 2 +- query_civic_unstructured/query2/validate.py | 25 +++++++-- .../query3/ground_truth.csv | 3 -- query_civic_unstructured/query3/validate.py | 39 ++++++++++---- .../query4/ground_truth.csv | 2 +- query_civic_unstructured/query4/validate.py | 2 +- .../query5/ground_truth.csv | 2 +- query_civic_unstructured/query5/validate.py | 2 +- .../query6/ground_truth.csv | 2 +- query_civic_unstructured/query6/validate.py | 2 +- .../query7/ground_truth.csv | 7 --- query_civic_unstructured/query7/validate.py | 40 +++++++-------- query_civic_unstructured/query8/validate.py | 21 +++++++- .../query9/ground_truth.csv | 3 -- query_civic_unstructured/query9/validate.py | 3 -- 20 files changed, 150 insertions(+), 69 deletions(-) diff --git a/query_civic_unstructured/db_description_withhint.txt b/query_civic_unstructured/db_description_withhint.txt index 95ce65df5f..efbdb47890 100644 --- a/query_civic_unstructured/db_description_withhint.txt +++ b/query_civic_unstructured/db_description_withhint.txt @@ -1,5 +1,8 @@ HINTS: - Projects have two types: "capital" and "disaster". - Projects have three statuses: "design", "completed", and "not started". -- Project name formats vary across sources and may need to be canonicalized before comparison. -- Disaster project names often include suffixes like "(FEMA Project)", "(CalJPIA Project)", or "(CalOES Project)". \ No newline at end of file +- The same project may appear under different names, so entity resolution is required. + - Two names likely refer to the same project if their names or agenda descriptions are semantically similar. + - Two names listed separately in the same agenda are likely distinct projects, as a project usually appears once per agenda. +- Some Funding records have a project_name that is not a real civic project; treat these as noise. +- An agenda's stated meeting date may be wrong, so cross-check it against the dates its own text mentions. \ No newline at end of file diff --git a/query_civic_unstructured/query1/ground_truth.csv b/query_civic_unstructured/query1/ground_truth.csv index 25f32c8e99..0942645b4b 100644 --- a/query_civic_unstructured/query1/ground_truth.csv +++ b/query_civic_unstructured/query1/ground_truth.csv @@ -1,4 +1,3 @@ -2022 Annual Street Maintenance Annual Street Maintenance Civic Center Water Treatment Facility Phase 2 Marie Canyon Green Streets @@ -7,5 +6,4 @@ PCH Median Improvements Project PCH Signal Synchronization System Improvements Project PCH at Trancas Canyon Road Right Turn Lane Permanent Skate Park -Westward Beach Road Improvements Project Westward Beach Road Repair Project diff --git a/query_civic_unstructured/query1/validate.py b/query_civic_unstructured/query1/validate.py index 200e9ed1ba..6a04eca361 100644 --- a/query_civic_unstructured/query1/validate.py +++ b/query_civic_unstructured/query1/validate.py @@ -1,7 +1,7 @@ import re +# Ground truth = canonical (entity-resolved) project names. GROUND_TRUTH = [ - "2022 Annual Street Maintenance", "Annual Street Maintenance", "Civic Center Water Treatment Facility Phase 2", "Marie Canyon Green Streets", @@ -10,10 +10,47 @@ "PCH Signal Synchronization System Improvements Project", "PCH at Trancas Canyon Road Right Turn Lane", "Permanent Skate Park", - "Westward Beach Road Improvements Project", "Westward Beach Road Repair Project", ] +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant that refers to +# the same project. The answer is entity-resolved against this map, so naming a +# project by ANY of its variants counts as naming the canonical project. +ER = { + "Annual Street Maintenance": [ + "Annual Street Maintenance", + "2021 Annual Street Maintenance", + "2022 Annual Street Maintenance", + ], + "Civic Center Water Treatment Facility Phase 2": [ + "Civic Center Water Treatment Facility Phase 2", + ], + "Marie Canyon Green Streets": [ + "Marie Canyon Green Streets", + ], + "Michael Landon Center Roof Replacement Project": [ + "Michael Landon Center Roof Replacement Project", + "Malibu Bluffs Park Roof Replacement Project", + ], + "PCH Median Improvements Project": [ + "PCH Median Improvements Project", + ], + "PCH Signal Synchronization System Improvements Project": [ + "PCH Signal Synchronization System Improvements Project", + ], + "PCH at Trancas Canyon Road Right Turn Lane": [ + "PCH at Trancas Canyon Road Right Turn Lane", + ], + "Permanent Skate Park": [ + "Permanent Skate Park", + ], + "Westward Beach Road Repair Project": [ + "Westward Beach Road Repair Project", + "Westward Beach Road Improvements Project", + ], +} + def _norm(s): """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace.""" @@ -22,10 +59,16 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm] + missing = [p for p in GROUND_TRUTH if not _mentions(text_norm, p)] if not missing: - return True, "All ground truth project names found in LLM output." + return True, "All ground truth project names found in LLM output (entity-resolved)." reason = f"Missing project(s) in LLM output: {missing}" return False, reason diff --git a/query_civic_unstructured/query10/ground_truth.csv b/query_civic_unstructured/query10/ground_truth.csv index 9922215085..b04c647450 100644 --- a/query_civic_unstructured/query10/ground_truth.csv +++ b/query_civic_unstructured/query10/ground_truth.csv @@ -1 +1 @@ -0.26 +0.69 diff --git a/query_civic_unstructured/query10/validate.py b/query_civic_unstructured/query10/validate.py index 6bad3bfa71..f9ad4473f5 100644 --- a/query_civic_unstructured/query10/validate.py +++ b/query_civic_unstructured/query10/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 0.26 +GROUND_TRUTH = 0.69 TOL = 1e-2 diff --git a/query_civic_unstructured/query2/ground_truth.csv b/query_civic_unstructured/query2/ground_truth.csv index 079e6cfac7..e952ddc261 100644 --- a/query_civic_unstructured/query2/ground_truth.csv +++ b/query_civic_unstructured/query2/ground_truth.csv @@ -1 +1 @@ -Guardrail Replacement Citywide (FEMA Project),12519000 +Corral Canyon Culvert Repairs,13460000 diff --git a/query_civic_unstructured/query2/validate.py b/query_civic_unstructured/query2/validate.py index 8b50a90ab4..d52995c40f 100644 --- a/query_civic_unstructured/query2/validate.py +++ b/query_civic_unstructured/query2/validate.py @@ -1,7 +1,18 @@ import re -GT_PROJECT = "Guardrail Replacement Citywide (FEMA Project)" -GT_AMOUNT = 12519000 +GT_PROJECT = "Corral Canyon Culvert Repairs" +GT_AMOUNT = 13460000 + +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant. The answer is +# entity-resolved, so naming the project by ANY variant counts. +ER = { + "Corral Canyon Culvert Repairs": [ + "Corral Canyon Culvert Repairs", + "Corral Canyon Culvert Repairs (FEMA Project)", + "Corral Canyon Culvert Repairs (FEMA/CalOES Project)", + ], +} def extract_numeric_values(text): @@ -21,14 +32,20 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - name_found = _norm(GT_PROJECT) in text_norm + name_found = _mentions(text_norm, GT_PROJECT) amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output)) if name_found and amount_found: - return True, "Ground truth project name and amount found in LLM output." + return True, "Ground truth project name (entity-resolved) and amount found in LLM output." missing = [] if not name_found: missing.append(f"project name '{GT_PROJECT}'") diff --git a/query_civic_unstructured/query3/ground_truth.csv b/query_civic_unstructured/query3/ground_truth.csv index 464a647523..e69de29bb2 100644 --- a/query_civic_unstructured/query3/ground_truth.csv +++ b/query_civic_unstructured/query3/ground_truth.csv @@ -1,3 +0,0 @@ -Federal Grant -Municipal Fund -Public-Private Partnership diff --git a/query_civic_unstructured/query3/validate.py b/query_civic_unstructured/query3/validate.py index d5dc9a0e1f..300fac064a 100644 --- a/query_civic_unstructured/query3/validate.py +++ b/query_civic_unstructured/query3/validate.py @@ -1,9 +1,26 @@ import re -GROUND_TRUTH = [ - "Federal Grant", - "Municipal Fund", - "Public-Private Partnership", +# After entity resolution, no funding source has >= 25% of its distinct projects +# at status 'completed' as of 2022-08-01, so the ground-truth answer is the empty +# set. A correct response must therefore state that no funding source qualifies. +GROUND_TRUTH = [] + +# Real funding-source names; if the output lists any, it is not the empty answer. +FUNDING_SOURCES = [ + "Public-Private Partnership", "Municipal Fund", "Municipal Bond", + "Emergency Relief Fund", "Private Donation", "County Grant", + "State Grant", "Federal Grant", +] + +# Phrasings that indicate the (correct) empty result. +EMPTY_PATTERNS = [ + r'\bno\b[^.\n]*\bfunding sources?\b', + r'\bno\b[^.\n]*\bsources?\b', + r'\bnone\b', + r'\bno qualifying\b', + r'\bthere (?:are|were) no\b', + r'\bempty\b', + r'\bzero\b', ] @@ -15,9 +32,13 @@ def _norm(s): def validate(llm_output: str): + text = llm_output.lower() text_norm = _norm(llm_output) - missing = [s for s in GROUND_TRUTH if _norm(s) not in text_norm] - if not missing: - return True, "All ground truth funding sources found in LLM output." - reason = f"Missing funding source(s) in LLM output: {missing}" - return False, reason + + listed = [s for s in FUNDING_SOURCES if _norm(s) in text_norm] + if listed: + return False, f"Ground truth is empty, but output lists funding source(s): {listed}" + if any(re.search(p, text) for p in EMPTY_PATTERNS): + return True, "Correctly reports that no funding source qualifies." + return False, ("Ground truth is empty (no funding source qualifies); the LLM " + "output did not clearly indicate an empty result.") diff --git a/query_civic_unstructured/query4/ground_truth.csv b/query_civic_unstructured/query4/ground_truth.csv index 48b9990e0b..61d2f35767 100644 --- a/query_civic_unstructured/query4/ground_truth.csv +++ b/query_civic_unstructured/query4/ground_truth.csv @@ -1 +1 @@ -0.31 +0.34 diff --git a/query_civic_unstructured/query4/validate.py b/query_civic_unstructured/query4/validate.py index 48e2d807cd..9b0b419c1f 100644 --- a/query_civic_unstructured/query4/validate.py +++ b/query_civic_unstructured/query4/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 0.31 +GROUND_TRUTH = 0.34 TOL = 1e-2 diff --git a/query_civic_unstructured/query5/ground_truth.csv b/query_civic_unstructured/query5/ground_truth.csv index a35402699d..940b7ef7ff 100644 --- a/query_civic_unstructured/query5/ground_truth.csv +++ b/query_civic_unstructured/query5/ground_truth.csv @@ -1 +1 @@ -3722235.29 +4401352.94 diff --git a/query_civic_unstructured/query5/validate.py b/query_civic_unstructured/query5/validate.py index 788e3de642..037ecd8ddc 100644 --- a/query_civic_unstructured/query5/validate.py +++ b/query_civic_unstructured/query5/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 3722235.29 +GROUND_TRUTH = 4401352.94 TOL = 0.1 diff --git a/query_civic_unstructured/query6/ground_truth.csv b/query_civic_unstructured/query6/ground_truth.csv index 418108c61d..83ec11e607 100644 --- a/query_civic_unstructured/query6/ground_truth.csv +++ b/query_civic_unstructured/query6/ground_truth.csv @@ -1 +1 @@ -2023,64171000 +2023,88615000 diff --git a/query_civic_unstructured/query6/validate.py b/query_civic_unstructured/query6/validate.py index a52f8e398a..d0e5e0edfd 100644 --- a/query_civic_unstructured/query6/validate.py +++ b/query_civic_unstructured/query6/validate.py @@ -1,7 +1,7 @@ import re GT_YEAR = 2023 -GT_AMOUNT = 64171000 +GT_AMOUNT = 88615000 def extract_numeric_values(text): diff --git a/query_civic_unstructured/query7/ground_truth.csv b/query_civic_unstructured/query7/ground_truth.csv index 2d09583df9..e69de29bb2 100644 --- a/query_civic_unstructured/query7/ground_truth.csv +++ b/query_civic_unstructured/query7/ground_truth.csv @@ -1,7 +0,0 @@ -Clover Heights Storm Drain -Encinal Canyon Road Drainage Improvements -Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project) -Latigo Canyon Road Culvert Repairs -Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project) -Outdoor Warning Sirens (FEMA) -Storm Drain Master Plan diff --git a/query_civic_unstructured/query7/validate.py b/query_civic_unstructured/query7/validate.py index 2de1aa5fba..77d8b5fe72 100644 --- a/query_civic_unstructured/query7/validate.py +++ b/query_civic_unstructured/query7/validate.py @@ -1,27 +1,25 @@ import re -GROUND_TRUTH = [ - "Clover Heights Storm Drain", - "Encinal Canyon Road Drainage Improvements", - "Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project)", - "Latigo Canyon Road Culvert Repairs", - "Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project)", - "Outdoor Warning Sirens (FEMA)", - "Storm Drain Master Plan", -] - +# After entity resolution, no project is both type 'disaster' and status +# 'not started' with accumulated funding > $1,000,000 as of 2022-11-01, so the +# ground-truth answer is the empty set. A correct response must therefore state +# that no project qualifies. +GROUND_TRUTH = [] -def _norm(s): - """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace.""" - s = s.lower().replace('_', ' ') - s = re.sub(r'[^a-z0-9\s]', ' ', s) - return re.sub(r'\s+', ' ', s).strip() +# Phrasings that indicate the (correct) empty result. +EMPTY_PATTERNS = [ + r'\bno\b[^.\n]*\bprojects?\b', + r'\bnone\b', + r'\bno qualifying\b', + r'\bthere (?:are|were) no\b', + r'\bempty\b', + r'\bzero\b', +] def validate(llm_output: str): - text_norm = _norm(llm_output) - missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm] - if not missing: - return True, "All ground truth project names found in LLM output." - reason = f"Missing project(s) in LLM output: {missing}" - return False, reason + text = llm_output.lower() + if any(re.search(p, text) for p in EMPTY_PATTERNS): + return True, "Correctly reports that no project qualifies." + return False, ("Ground truth is empty (no project qualifies); the LLM output " + "did not clearly indicate an empty result.") diff --git a/query_civic_unstructured/query8/validate.py b/query_civic_unstructured/query8/validate.py index d7c99435a6..88c78a9462 100644 --- a/query_civic_unstructured/query8/validate.py +++ b/query_civic_unstructured/query8/validate.py @@ -3,6 +3,17 @@ GT_PROJECT = "City Hall Roof Replacement" GT_AMOUNT = 1340000 +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant. The answer is +# entity-resolved, so naming the project by ANY variant counts. (This project has +# no variants, but the ER map is kept for consistency across the project-name +# validators.) +ER = { + "City Hall Roof Replacement": [ + "City Hall Roof Replacement", + ], +} + def extract_numeric_values(text): values = [] @@ -21,14 +32,20 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - name_found = _norm(GT_PROJECT) in text_norm + name_found = _mentions(text_norm, GT_PROJECT) amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output)) if name_found and amount_found: - return True, "Ground truth project name and amount found in LLM output." + return True, "Ground truth project name (entity-resolved) and amount found in LLM output." missing = [] if not name_found: missing.append(f"project name '{GT_PROJECT}'") diff --git a/query_civic_unstructured/query9/ground_truth.csv b/query_civic_unstructured/query9/ground_truth.csv index d83f3d2b73..91da05e467 100644 --- a/query_civic_unstructured/query9/ground_truth.csv +++ b/query_civic_unstructured/query9/ground_truth.csv @@ -1,4 +1 @@ -Emergency Relief Fund -Municipal Fund -Private Donation Public-Private Partnership diff --git a/query_civic_unstructured/query9/validate.py b/query_civic_unstructured/query9/validate.py index 3babc4db6b..1b470ca798 100644 --- a/query_civic_unstructured/query9/validate.py +++ b/query_civic_unstructured/query9/validate.py @@ -1,9 +1,6 @@ import re GROUND_TRUTH = [ - "Emergency Relief Fund", - "Municipal Fund", - "Private Donation", "Public-Private Partnership", ]