diff --git a/query_civic_unstructured/db_description_withhint.txt b/query_civic_unstructured/db_description_withhint.txt index 95ce65df5..efbdb4789 100644 --- a/query_civic_unstructured/db_description_withhint.txt +++ b/query_civic_unstructured/db_description_withhint.txt @@ -1,5 +1,8 @@ HINTS: - Projects have two types: "capital" and "disaster". - Projects have three statuses: "design", "completed", and "not started". -- Project name formats vary across sources and may need to be canonicalized before comparison. -- Disaster project names often include suffixes like "(FEMA Project)", "(CalJPIA Project)", or "(CalOES Project)". \ No newline at end of file +- The same project may appear under different names, so entity resolution is required. + - Two names likely refer to the same project if their names or agenda descriptions are semantically similar. + - Two names listed separately in the same agenda are likely distinct projects, as a project usually appears once per agenda. +- Some Funding records have a project_name that is not a real civic project; treat these as noise. +- An agenda's stated meeting date may be wrong, so cross-check it against the dates its own text mentions. \ No newline at end of file diff --git a/query_civic_unstructured/query1/ground_truth.csv b/query_civic_unstructured/query1/ground_truth.csv index 25f32c8e9..0942645b4 100644 --- a/query_civic_unstructured/query1/ground_truth.csv +++ b/query_civic_unstructured/query1/ground_truth.csv @@ -1,4 +1,3 @@ -2022 Annual Street Maintenance Annual Street Maintenance Civic Center Water Treatment Facility Phase 2 Marie Canyon Green Streets @@ -7,5 +6,4 @@ PCH Median Improvements Project PCH Signal Synchronization System Improvements Project PCH at Trancas Canyon Road Right Turn Lane Permanent Skate Park -Westward Beach Road Improvements Project Westward Beach Road Repair Project diff --git a/query_civic_unstructured/query1/validate.py b/query_civic_unstructured/query1/validate.py index 200e9ed1b..6a04eca36 100644 --- a/query_civic_unstructured/query1/validate.py +++ b/query_civic_unstructured/query1/validate.py @@ -1,7 +1,7 @@ import re +# Ground truth = canonical (entity-resolved) project names. GROUND_TRUTH = [ - "2022 Annual Street Maintenance", "Annual Street Maintenance", "Civic Center Water Treatment Facility Phase 2", "Marie Canyon Green Streets", @@ -10,10 +10,47 @@ "PCH Signal Synchronization System Improvements Project", "PCH at Trancas Canyon Road Right Turn Lane", "Permanent Skate Park", - "Westward Beach Road Improvements Project", "Westward Beach Road Repair Project", ] +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant that refers to +# the same project. The answer is entity-resolved against this map, so naming a +# project by ANY of its variants counts as naming the canonical project. +ER = { + "Annual Street Maintenance": [ + "Annual Street Maintenance", + "2021 Annual Street Maintenance", + "2022 Annual Street Maintenance", + ], + "Civic Center Water Treatment Facility Phase 2": [ + "Civic Center Water Treatment Facility Phase 2", + ], + "Marie Canyon Green Streets": [ + "Marie Canyon Green Streets", + ], + "Michael Landon Center Roof Replacement Project": [ + "Michael Landon Center Roof Replacement Project", + "Malibu Bluffs Park Roof Replacement Project", + ], + "PCH Median Improvements Project": [ + "PCH Median Improvements Project", + ], + "PCH Signal Synchronization System Improvements Project": [ + "PCH Signal Synchronization System Improvements Project", + ], + "PCH at Trancas Canyon Road Right Turn Lane": [ + "PCH at Trancas Canyon Road Right Turn Lane", + ], + "Permanent Skate Park": [ + "Permanent Skate Park", + ], + "Westward Beach Road Repair Project": [ + "Westward Beach Road Repair Project", + "Westward Beach Road Improvements Project", + ], +} + def _norm(s): """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace.""" @@ -22,10 +59,16 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm] + missing = [p for p in GROUND_TRUTH if not _mentions(text_norm, p)] if not missing: - return True, "All ground truth project names found in LLM output." + return True, "All ground truth project names found in LLM output (entity-resolved)." reason = f"Missing project(s) in LLM output: {missing}" return False, reason diff --git a/query_civic_unstructured/query10/ground_truth.csv b/query_civic_unstructured/query10/ground_truth.csv index 992221508..b04c64745 100644 --- a/query_civic_unstructured/query10/ground_truth.csv +++ b/query_civic_unstructured/query10/ground_truth.csv @@ -1 +1 @@ -0.26 +0.69 diff --git a/query_civic_unstructured/query10/validate.py b/query_civic_unstructured/query10/validate.py index 6bad3bfa7..f9ad4473f 100644 --- a/query_civic_unstructured/query10/validate.py +++ b/query_civic_unstructured/query10/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 0.26 +GROUND_TRUTH = 0.69 TOL = 1e-2 diff --git a/query_civic_unstructured/query2/ground_truth.csv b/query_civic_unstructured/query2/ground_truth.csv index 079e6cfac..e952ddc26 100644 --- a/query_civic_unstructured/query2/ground_truth.csv +++ b/query_civic_unstructured/query2/ground_truth.csv @@ -1 +1 @@ -Guardrail Replacement Citywide (FEMA Project),12519000 +Corral Canyon Culvert Repairs,13460000 diff --git a/query_civic_unstructured/query2/validate.py b/query_civic_unstructured/query2/validate.py index 8b50a90ab..d52995c40 100644 --- a/query_civic_unstructured/query2/validate.py +++ b/query_civic_unstructured/query2/validate.py @@ -1,7 +1,18 @@ import re -GT_PROJECT = "Guardrail Replacement Citywide (FEMA Project)" -GT_AMOUNT = 12519000 +GT_PROJECT = "Corral Canyon Culvert Repairs" +GT_AMOUNT = 13460000 + +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant. The answer is +# entity-resolved, so naming the project by ANY variant counts. +ER = { + "Corral Canyon Culvert Repairs": [ + "Corral Canyon Culvert Repairs", + "Corral Canyon Culvert Repairs (FEMA Project)", + "Corral Canyon Culvert Repairs (FEMA/CalOES Project)", + ], +} def extract_numeric_values(text): @@ -21,14 +32,20 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - name_found = _norm(GT_PROJECT) in text_norm + name_found = _mentions(text_norm, GT_PROJECT) amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output)) if name_found and amount_found: - return True, "Ground truth project name and amount found in LLM output." + return True, "Ground truth project name (entity-resolved) and amount found in LLM output." missing = [] if not name_found: missing.append(f"project name '{GT_PROJECT}'") diff --git a/query_civic_unstructured/query3/ground_truth.csv b/query_civic_unstructured/query3/ground_truth.csv index 464a64752..e69de29bb 100644 --- a/query_civic_unstructured/query3/ground_truth.csv +++ b/query_civic_unstructured/query3/ground_truth.csv @@ -1,3 +0,0 @@ -Federal Grant -Municipal Fund -Public-Private Partnership diff --git a/query_civic_unstructured/query3/validate.py b/query_civic_unstructured/query3/validate.py index d5dc9a0e1..300fac064 100644 --- a/query_civic_unstructured/query3/validate.py +++ b/query_civic_unstructured/query3/validate.py @@ -1,9 +1,26 @@ import re -GROUND_TRUTH = [ - "Federal Grant", - "Municipal Fund", - "Public-Private Partnership", +# After entity resolution, no funding source has >= 25% of its distinct projects +# at status 'completed' as of 2022-08-01, so the ground-truth answer is the empty +# set. A correct response must therefore state that no funding source qualifies. +GROUND_TRUTH = [] + +# Real funding-source names; if the output lists any, it is not the empty answer. +FUNDING_SOURCES = [ + "Public-Private Partnership", "Municipal Fund", "Municipal Bond", + "Emergency Relief Fund", "Private Donation", "County Grant", + "State Grant", "Federal Grant", +] + +# Phrasings that indicate the (correct) empty result. +EMPTY_PATTERNS = [ + r'\bno\b[^.\n]*\bfunding sources?\b', + r'\bno\b[^.\n]*\bsources?\b', + r'\bnone\b', + r'\bno qualifying\b', + r'\bthere (?:are|were) no\b', + r'\bempty\b', + r'\bzero\b', ] @@ -15,9 +32,13 @@ def _norm(s): def validate(llm_output: str): + text = llm_output.lower() text_norm = _norm(llm_output) - missing = [s for s in GROUND_TRUTH if _norm(s) not in text_norm] - if not missing: - return True, "All ground truth funding sources found in LLM output." - reason = f"Missing funding source(s) in LLM output: {missing}" - return False, reason + + listed = [s for s in FUNDING_SOURCES if _norm(s) in text_norm] + if listed: + return False, f"Ground truth is empty, but output lists funding source(s): {listed}" + if any(re.search(p, text) for p in EMPTY_PATTERNS): + return True, "Correctly reports that no funding source qualifies." + return False, ("Ground truth is empty (no funding source qualifies); the LLM " + "output did not clearly indicate an empty result.") diff --git a/query_civic_unstructured/query4/ground_truth.csv b/query_civic_unstructured/query4/ground_truth.csv index 48b9990e0..61d2f3576 100644 --- a/query_civic_unstructured/query4/ground_truth.csv +++ b/query_civic_unstructured/query4/ground_truth.csv @@ -1 +1 @@ -0.31 +0.34 diff --git a/query_civic_unstructured/query4/validate.py b/query_civic_unstructured/query4/validate.py index 48e2d807c..9b0b419c1 100644 --- a/query_civic_unstructured/query4/validate.py +++ b/query_civic_unstructured/query4/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 0.31 +GROUND_TRUTH = 0.34 TOL = 1e-2 diff --git a/query_civic_unstructured/query5/ground_truth.csv b/query_civic_unstructured/query5/ground_truth.csv index a35402699..940b7ef7f 100644 --- a/query_civic_unstructured/query5/ground_truth.csv +++ b/query_civic_unstructured/query5/ground_truth.csv @@ -1 +1 @@ -3722235.29 +4401352.94 diff --git a/query_civic_unstructured/query5/validate.py b/query_civic_unstructured/query5/validate.py index 788e3de64..037ecd8dd 100644 --- a/query_civic_unstructured/query5/validate.py +++ b/query_civic_unstructured/query5/validate.py @@ -1,6 +1,6 @@ import re -GROUND_TRUTH = 3722235.29 +GROUND_TRUTH = 4401352.94 TOL = 0.1 diff --git a/query_civic_unstructured/query6/ground_truth.csv b/query_civic_unstructured/query6/ground_truth.csv index 418108c61..83ec11e60 100644 --- a/query_civic_unstructured/query6/ground_truth.csv +++ b/query_civic_unstructured/query6/ground_truth.csv @@ -1 +1 @@ -2023,64171000 +2023,88615000 diff --git a/query_civic_unstructured/query6/validate.py b/query_civic_unstructured/query6/validate.py index a52f8e398..d0e5e0edf 100644 --- a/query_civic_unstructured/query6/validate.py +++ b/query_civic_unstructured/query6/validate.py @@ -1,7 +1,7 @@ import re GT_YEAR = 2023 -GT_AMOUNT = 64171000 +GT_AMOUNT = 88615000 def extract_numeric_values(text): diff --git a/query_civic_unstructured/query7/ground_truth.csv b/query_civic_unstructured/query7/ground_truth.csv index 2d09583df..e69de29bb 100644 --- a/query_civic_unstructured/query7/ground_truth.csv +++ b/query_civic_unstructured/query7/ground_truth.csv @@ -1,7 +0,0 @@ -Clover Heights Storm Drain -Encinal Canyon Road Drainage Improvements -Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project) -Latigo Canyon Road Culvert Repairs -Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project) -Outdoor Warning Sirens (FEMA) -Storm Drain Master Plan diff --git a/query_civic_unstructured/query7/validate.py b/query_civic_unstructured/query7/validate.py index 2de1aa5fb..77d8b5fe7 100644 --- a/query_civic_unstructured/query7/validate.py +++ b/query_civic_unstructured/query7/validate.py @@ -1,27 +1,25 @@ import re -GROUND_TRUTH = [ - "Clover Heights Storm Drain", - "Encinal Canyon Road Drainage Improvements", - "Encinal Canyon Road Drainage Improvements (FEMA/CalOES Project)", - "Latigo Canyon Road Culvert Repairs", - "Latigo Canyon Road Culvert Repairs (FEMA/CalOES Project)", - "Outdoor Warning Sirens (FEMA)", - "Storm Drain Master Plan", -] - +# After entity resolution, no project is both type 'disaster' and status +# 'not started' with accumulated funding > $1,000,000 as of 2022-11-01, so the +# ground-truth answer is the empty set. A correct response must therefore state +# that no project qualifies. +GROUND_TRUTH = [] -def _norm(s): - """Lowercase, underscores→spaces, strip non-alphanumeric, collapse whitespace.""" - s = s.lower().replace('_', ' ') - s = re.sub(r'[^a-z0-9\s]', ' ', s) - return re.sub(r'\s+', ' ', s).strip() +# Phrasings that indicate the (correct) empty result. +EMPTY_PATTERNS = [ + r'\bno\b[^.\n]*\bprojects?\b', + r'\bnone\b', + r'\bno qualifying\b', + r'\bthere (?:are|were) no\b', + r'\bempty\b', + r'\bzero\b', +] def validate(llm_output: str): - text_norm = _norm(llm_output) - missing = [p for p in GROUND_TRUTH if _norm(p) not in text_norm] - if not missing: - return True, "All ground truth project names found in LLM output." - reason = f"Missing project(s) in LLM output: {missing}" - return False, reason + text = llm_output.lower() + if any(re.search(p, text) for p in EMPTY_PATTERNS): + return True, "Correctly reports that no project qualifies." + return False, ("Ground truth is empty (no project qualifies); the LLM output " + "did not clearly indicate an empty result.") diff --git a/query_civic_unstructured/query8/validate.py b/query_civic_unstructured/query8/validate.py index d7c99435a..88c78a946 100644 --- a/query_civic_unstructured/query8/validate.py +++ b/query_civic_unstructured/query8/validate.py @@ -3,6 +3,17 @@ GT_PROJECT = "City Hall Roof Replacement" GT_AMOUNT = 1340000 +# Entity resolution, listed explicitly here (NOT loaded from the ground-truth +# dataset): canonical project name -> every surface-name variant. The answer is +# entity-resolved, so naming the project by ANY variant counts. (This project has +# no variants, but the ER map is kept for consistency across the project-name +# validators.) +ER = { + "City Hall Roof Replacement": [ + "City Hall Roof Replacement", + ], +} + def extract_numeric_values(text): values = [] @@ -21,14 +32,20 @@ def _norm(s): return re.sub(r'\s+', ' ', s).strip() +def _mentions(text_norm, canonical): + """Entity-resolve the answer: the canonical project is named if ANY of its + variants appears in the output.""" + return any(_norm(v) in text_norm for v in ER.get(canonical, [canonical])) + + def validate(llm_output: str): text_norm = _norm(llm_output) - name_found = _norm(GT_PROJECT) in text_norm + name_found = _mentions(text_norm, GT_PROJECT) amount_found = any(abs(v - GT_AMOUNT) == 0 for v in extract_numeric_values(llm_output)) if name_found and amount_found: - return True, "Ground truth project name and amount found in LLM output." + return True, "Ground truth project name (entity-resolved) and amount found in LLM output." missing = [] if not name_found: missing.append(f"project name '{GT_PROJECT}'") diff --git a/query_civic_unstructured/query9/ground_truth.csv b/query_civic_unstructured/query9/ground_truth.csv index d83f3d2b7..91da05e46 100644 --- a/query_civic_unstructured/query9/ground_truth.csv +++ b/query_civic_unstructured/query9/ground_truth.csv @@ -1,4 +1 @@ -Emergency Relief Fund -Municipal Fund -Private Donation Public-Private Partnership diff --git a/query_civic_unstructured/query9/validate.py b/query_civic_unstructured/query9/validate.py index 3babc4db6..1b470ca79 100644 --- a/query_civic_unstructured/query9/validate.py +++ b/query_civic_unstructured/query9/validate.py @@ -1,9 +1,6 @@ import re GROUND_TRUTH = [ - "Emergency Relief Fund", - "Municipal Fund", - "Private Donation", "Public-Private Partnership", ]