Recompute civic_unstructured ground truth with entity resolution

query_civic_unstructured/db_description_withhint.txt

@@ -1,5 +1,8 @@
 HINTS:
 - Projects have two types: "capital" and "disaster".
 - Projects have three statuses: "design", "completed", and "not started".
-- Project name formats vary across sources and may need to be canonicalized before comparison.
-- Disaster project names often include suffixes like "(FEMA Project)", "(CalJPIA Project)", or "(CalOES Project)".
+- The same project may appear under different names, so entity resolution is required.
+  - Two names likely refer to the same project if their names or agenda descriptions are semantically similar.
+  - Two names listed separately in the same agenda are likely distinct projects, as a project usually appears once per agenda.
+- Some Funding records have a project_name that is not a real civic project; treat these as noise.
+- An agenda's stated meeting date may be wrong, so cross-check it against the dates its own text mentions.

query_civic_unstructured/query1/ground_truth.csv

@@ -1,4 +1,3 @@
-2022 Annual Street Maintenance
 Annual Street Maintenance
 Civic Center Water Treatment Facility Phase 2
 Marie Canyon Green Streets
@@ -7,5 +6,4 @@ PCH Median Improvements Project
 PCH Signal Synchronization System Improvements Project
 PCH at Trancas Canyon Road Right Turn Lane
 Permanent Skate Park
-Westward Beach Road Improvements Project
 Westward Beach Road Repair Project

query_civic_unstructured/query1/validate.py

@@ -1,7 +1,7 @@
 import re
 
+# Ground truth = canonical (entity-resolved) project names.
 GROUND_TRUTH = [
-    "2022 Annual Street Maintenance",
     "Annual Street Maintenance",
     "Civic Center Water Treatment Facility Phase 2",
     "Marie Canyon Green Streets",
@@ -10,10 +10,47 @@
     "PCH Signal Synchronization System Improvements Project",
     "PCH at Trancas Canyon Road Right Turn Lane",
     "Permanent Skate Park",
-    "Westward Beach Road Improvements Project",
     "Westward Beach Road Repair Project",
 ]
 
+# Entity resolution, listed explicitly here (NOT loaded from the ground-truth
+# dataset): canonical project name -> every surface-name variant that refers to
+# the same project. The answer is entity-resolved against this map, so naming a
+# project by ANY of its variants counts as naming the canonical project.
+ER = {
+    "Annual Street Maintenance": [
+        "Annual Street Maintenance",
+        "2021 Annual Street Maintenance",
+        "2022 Annual Street Maintenance",
+    ],
+    "Civic Center Water Treatment Facility Phase 2": [
+        "Civic Center Water Treatment Facility Phase 2",
+    ],
+    "Marie Canyon Green Streets": [
+        "Marie Canyon Green Streets",
+    ],
+    "Michael Landon Center Roof Replacement Project": [
+        "Michael Landon Center Roof Replacement Project",
+        "Malibu Bluffs Park Roof Replacement Project",
+    ],
+    "PCH Median Improvements Project": [
+        "PCH Median Improvements Project",
+    ],
+    "PCH Signal Synchronization System Improvements Project": [
+        "PCH Signal Synchronization System Improvements Project",
+    ],
+    "PCH at Trancas Canyon Road Right Turn Lane": [
+        "PCH at Trancas Canyon Road Right Turn Lane",
+    ],
+    "Permanent Skate Park": [
+        "Permanent Skate Park",
+    ],
+    "Westward Beach Road Repair Project": [
+        "Westward Beach Road Repair Project",
+        "Westward Beach Road Improvements Project",
+    ],
+}
+
 
 def _norm(s):
     """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace."""
@@ -22,10 +59,16 @@ def _norm(s):
     return re.sub(r'\s+', ' ', s).strip()
 
 
+def _mentions(text_norm, canonical):
+    """Entity-resolve the answer: the canonical project is named if ANY of its
+    variants appears in the output."""
+    return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical]))
+
+
 def validate(llm_output: str):
     text_norm = _norm(llm_output)
-    missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm]
+    missing = [p for p in GROUND_TRUTH if not _mentions(text_norm, p)]
     if not missing:
-        return True, "All ground truth project names found in LLM output."
+        return True, "All ground truth project names found in LLM output (entity-resolved)."
     reason = f"Missing project(s) in LLM output: {missing}"
     return False, reason

query_civic_unstructured/query10/ground_truth.csv

@@ -1 +1 @@
-0.26
+0.69

query_civic_unstructured/query10/validate.py

@@ -1,6 +1,6 @@
 import re
 
-GROUND_TRUTH = 0.26
+GROUND_TRUTH = 0.69
 TOL = 1e-2
 
 

query_civic_unstructured/query2/ground_truth.csv

@@ -1 +1 @@
-Guardrail Replacement Citywide (FEMA Project),12519000
+Corral Canyon Culvert Repairs,13460000

query_civic_unstructured/query2/validate.py

@@ -1,7 +1,18 @@
 import re
 
-GT_PROJECT = "Guardrail Replacement Citywide (FEMA Project)"
-GT_AMOUNT = 12519000
+GT_PROJECT = "Corral Canyon Culvert Repairs"
+GT_AMOUNT = 13460000
+
+# Entity resolution, listed explicitly here (NOT loaded from the ground-truth
+# dataset): canonical project name -> every surface-name variant. The answer is
+# entity-resolved, so naming the project by ANY variant counts.
+ER = {
+    "Corral Canyon Culvert Repairs": [
+        "Corral Canyon Culvert Repairs",
+        "Corral Canyon Culvert Repairs (FEMA Project)",
+        "Corral Canyon Culvert Repairs (FEMA/CalOES Project)",
+    ],
+}
 
 
 def extract_numeric_values(text):
@@ -21,14 +32,20 @@ def _norm(s):
     return re.sub(r'\s+', ' ', s).strip()
 
 
+def _mentions(text_norm, canonical):
+    """Entity-resolve the answer: the canonical project is named if ANY of its
+    variants appears in the output."""
+    return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical]))
+
+
 def validate(llm_output: str):
     text_norm = _norm(llm_output)
 
-    name_found = _norm(GT_PROJECT) in text_norm
+    name_found = _mentions(text_norm, GT_PROJECT)
     amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output))
 
     if name_found and amount_found:
-        return True, "Ground truth project name and amount found in LLM output."
+        return True, "Ground truth project name (entity-resolved) and amount found in LLM output."
     missing = []
     if not name_found:
         missing.append(f"project name '{GT_PROJECT}'")

query_civic_unstructured/query3/ground_truth.csv

@@ -1,3 +0,0 @@
-Federal Grant
-Municipal Fund
-Public-Private Partnership

query_civic_unstructured/query3/validate.py

@@ -1,9 +1,26 @@
 import re
 
-GROUND_TRUTH = [
-    "Federal Grant",
-    "Municipal Fund",
-    "Public-Private Partnership",
+# After entity resolution, no funding source has >= 25% of its distinct projects
+# at status 'completed' as of 2022-08-01, so the ground-truth answer is the empty
+# set. A correct response must therefore state that no funding source qualifies.
+GROUND_TRUTH = []
+
+# Real funding-source names; if the output lists any, it is not the empty answer.
+FUNDING_SOURCES = [
+    "Public-Private Partnership", "Municipal Fund", "Municipal Bond",
+    "Emergency Relief Fund", "Private Donation", "County Grant",
+    "State Grant", "Federal Grant",
+]
+
+# Phrasings that indicate the (correct) empty result.
+EMPTY_PATTERNS = [
+    r'\bno\b[^.\n]*\bfunding sources?\b',
+    r'\bno\b[^.\n]*\bsources?\b',
+    r'\bnone\b',
+    r'\bno qualifying\b',
+    r'\bthere (?:are|were) no\b',
+    r'\bempty\b',
+    r'\bzero\b',
 ]
 
 
@@ -15,9 +32,13 @@ def _norm(s):
 
 
 def validate(llm_output: str):
+    text = llm_output.lower()
     text_norm = _norm(llm_output)
-    missing = [s for s in GROUND_TRUTH if _norm(s) not in text_norm]
-    if not missing:
-        return True, "All ground truth funding sources found in LLM output."
-    reason = f"Missing funding source(s) in LLM output: {missing}"
-    return False, reason
+
+    listed = [s for s in FUNDING_SOURCES if _norm(s) in text_norm]
+    if listed:
+        return False, f"Ground truth is empty, but output lists funding source(s): {listed}"
+    if any(re.search(p, text) for p in EMPTY_PATTERNS):
+        return True, "Correctly reports that no funding source qualifies."
+    return False, ("Ground truth is empty (no funding source qualifies); the LLM "
+                   "output did not clearly indicate an empty result.")

query_civic_unstructured/query4/ground_truth.csv

@@ -1 +1 @@
-0.31
+0.34

query_civic_unstructured/query4/validate.py

@@ -1,6 +1,6 @@
 import re
 
-GROUND_TRUTH = 0.31
+GROUND_TRUTH = 0.34
 TOL = 1e-2
 
 

query_civic_unstructured/query5/ground_truth.csv

@@ -1 +1 @@
-3722235.29
+4401352.94

query_civic_unstructured/query5/validate.py

@@ -1,6 +1,6 @@
 import re
 
-GROUND_TRUTH = 3722235.29
+GROUND_TRUTH = 4401352.94
 TOL = 0.1
 
 

query_civic_unstructured/query6/ground_truth.csv

@@ -1 +1 @@
-2023,64171000
+2023,88615000

query_civic_unstructured/query6/validate.py

@@ -1,7 +1,7 @@
 import re
 
 GT_YEAR = 2023
-GT_AMOUNT = 64171000
+GT_AMOUNT = 88615000
 
 
 def extract_numeric_values(text):

query_civic_unstructured/query7/ground_truth.csv

@@ -1,7 +0,0 @@
-Clover Heights Storm Drain
-Encinal Canyon Road Drainage Improvements
-Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project)
-Latigo Canyon Road Culvert Repairs
-Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project)
-Outdoor Warning Sirens (FEMA)
-Storm Drain Master Plan

query_civic_unstructured/query7/validate.py

@@ -1,27 +1,25 @@
 import re
 
-GROUND_TRUTH = [
-    "Clover Heights Storm Drain",
-    "Encinal Canyon Road Drainage Improvements",
-    "Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project)",
-    "Latigo Canyon Road Culvert Repairs",
-    "Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project)",
-    "Outdoor Warning Sirens (FEMA)",
-    "Storm Drain Master Plan",
-]
-
+# After entity resolution, no project is both type 'disaster' and status
+# 'not started' with accumulated funding > $1,000,000 as of 2022-11-01, so the
+# ground-truth answer is the empty set. A correct response must therefore state
+# that no project qualifies.
+GROUND_TRUTH = []
 
-def _norm(s):
-    """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace."""
-    s = s.lower().replace('_', ' ')
-    s = re.sub(r'[^a-z0-9\s]', ' ', s)
-    return re.sub(r'\s+', ' ', s).strip()
+# Phrasings that indicate the (correct) empty result.
+EMPTY_PATTERNS = [
+    r'\bno\b[^.\n]*\bprojects?\b',
+    r'\bnone\b',
+    r'\bno qualifying\b',
+    r'\bthere (?:are|were) no\b',
+    r'\bempty\b',
+    r'\bzero\b',
+]
 
 
 def validate(llm_output: str):
-    text_norm = _norm(llm_output)
-    missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm]
-    if not missing:
-        return True, "All ground truth project names found in LLM output."
-    reason = f"Missing project(s) in LLM output: {missing}"
-    return False, reason
+    text = llm_output.lower()
+    if any(re.search(p, text) for p in EMPTY_PATTERNS):
+        return True, "Correctly reports that no project qualifies."
+    return False, ("Ground truth is empty (no project qualifies); the LLM output "
+                   "did not clearly indicate an empty result.")

query_civic_unstructured/query8/validate.py

@@ -3,6 +3,17 @@
 GT_PROJECT = "City Hall Roof Replacement"
 GT_AMOUNT = 1340000
 
+# Entity resolution, listed explicitly here (NOT loaded from the ground-truth
+# dataset): canonical project name -> every surface-name variant. The answer is
+# entity-resolved, so naming the project by ANY variant counts. (This project has
+# no variants, but the ER map is kept for consistency across the project-name
+# validators.)
+ER = {
+    "City Hall Roof Replacement": [
+        "City Hall Roof Replacement",
+    ],
+}
+
 
 def extract_numeric_values(text):
     values = []
@@ -21,14 +32,20 @@ def _norm(s):
     return re.sub(r'\s+', ' ', s).strip()
 
 
+def _mentions(text_norm, canonical):
+    """Entity-resolve the answer: the canonical project is named if ANY of its
+    variants appears in the output."""
+    return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical]))
+
+
 def validate(llm_output: str):
     text_norm = _norm(llm_output)
 
-    name_found = _norm(GT_PROJECT) in text_norm
+    name_found = _mentions(text_norm, GT_PROJECT)
     amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output))
 
     if name_found and amount_found:
-        return True, "Ground truth project name and amount found in LLM output."
+        return True, "Ground truth project name (entity-resolved) and amount found in LLM output."
     missing = []
     if not name_found:
         missing.append(f"project name '{GT_PROJECT}'")

query_civic_unstructured/query9/ground_truth.csv

@@ -1,4 +1 @@
-Emergency Relief Fund
-Municipal Fund
-Private Donation
 Public-Private Partnership

query_civic_unstructured/query9/validate.py

@@ -1,9 +1,6 @@
 import re
 
 GROUND_TRUTH = [
-    "Emergency Relief Fund",
-    "Municipal Fund",
-    "Private Donation",
     "Public-Private Partnership",
 ]