Add internal debugging to API

README.md

@@ -3,6 +3,16 @@
 
 Read the API docs at `<api_endpoint>/docs`
 
+The publicly accessible endpoints are actually at the path `/api`, i.e. the standard 
+`GET` request goes to `/api/get`, but this is hidden from the user by the reverse proxy, Mulesoft.
+
+### Debugging
+There is an internal endpoint available for testing. This will only be accessible on the City network, i.e. not by the reverse proxy. 
+
+It's available at `/schemas`, and you can pass `?table=<table>` to see a specific table's schema
+
+### Development
+
 For local development and testing, copy `env.example` to `.env` and populate it. Then run `export $(grep -v '^#' .env | xargs)` to export them as environment variables so the python program can access it.
 
 Running the API locally:
@@ -13,6 +23,6 @@ To run in a docker container, make sure your .env file is setup then run:
 
 `docker-compose up --build -d`
 
-Testing:
+### Testing
 
 `uv run --env-file=.env pytest --maxfail=4 --tb=short -v`

app/abstract.py → app/apis/abstract.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
-from .models import ReturnJson, GeoJsonFeature, TableSchema
+
+import datetime as dt
+import inspect
 from abc import ABC, abstractmethod
+from collections.abc import Callable
+
 from fastapi import Request
 from fastapi.exceptions import HTTPException
-import inspect
-import datetime as dt
-from collections.abc import Callable
+
+from ..utils.models import GeoJsonFeature, ReturnJson, TableSchema
 
 
 class AbstractWorker(ABC):
@@ -41,18 +44,13 @@ async def normalize_rv(self) -> ReturnJson:
         raise NotImplementedError
 
     @abstractmethod
-    def harmonize_timestamp_fields(
-        self, records: list[dict], table_schema: TableSchema
-    ) -> list[dict]:
-        """Return a consistent representation of timestamp fields. Implementation 
+    def harmonize_timestamp_fields(self, records: list[dict], table_schema: TableSchema):
+        """Coerce to a consistent representation of timestamp fields. Implementation 
         is API-specific
 
         Args:
             records (list[dict]): Data records
             table_schema (TableSchema): TableSchema
-
-        Returns:
-            list[dict]: Updated records
         """
         raise NotImplementedError
 

app/ago.py → app/apis/ago.py

@@ -1,9 +1,17 @@
-# ago.py
+import datetime as dt
+
 import aiohttp
 from fastapi import Request
-import datetime as dt
+
+from ..utils.models import (
+    Error,
+    GeoJsonFeatureCollection,
+    Links,
+    Meta,
+    ReturnJson,
+    TableSchema,
+)
 from .abstract import AbstractWorker, check_fields_valid
-from .models import ReturnJson, Meta, Links, Error, GeoJsonFeatureCollection, TableSchema
 
 
 class Ago(AbstractWorker):
@@ -122,16 +130,15 @@ async def normalize_rv(
             data = await response.json()
             if "error" not in data:
                 records = data["features"]
-                records = self.harmonize_timestamp_fields(records, table_schema)
-                gjfc = GeoJsonFeatureCollection(features=records)
+                self.harmonize_timestamp_fields(records, table_schema)
+                gjfc = GeoJsonFeatureCollection(**data)
                 meta.record_count = len(gjfc.features)
                 try:
                     data["properties"]["exceededTransferLimit"]
                     next_url = self.create_next_url(gjfc.features, request)
                     links.next = next_url
                 except KeyError:
                     pass
-                gjfc = GeoJsonFeatureCollection(**data)
                 rv = ReturnJson(data=gjfc, links=links, meta=meta)
                 return rv
             else:
@@ -155,16 +162,13 @@ async def normalize_rv(
             rv = ReturnJson(errors=[error], links=links, meta=meta)
             return rv
 
-    def harmonize_timestamp_fields(self, records: list[dict], table_schema: TableSchema) -> list[dict]: 
-        """Return a consistent representation of timestamp fields. AGO returns 
+    def harmonize_timestamp_fields(self, records: list[dict], table_schema: TableSchema): 
+        """Coerce to a consistent representation of timestamp fields. AGO returns 
         timestamp fields as milliseconds since the epoch
 
         Args:
             records (list[dict]): Data records
             table_schema (TableSchema): TableSchema
-
-        Returns:
-            list[dict]: Updated records
         """        
         for record in records:
             for field in record["properties"]:
@@ -173,7 +177,6 @@ def harmonize_timestamp_fields(self, records: list[dict], table_schema: TableSch
                         record["properties"][field] = dt.datetime.fromtimestamp(
                             record["properties"][field] / 1000
                         )
-        return records
 
     def mask_service_url(
         self, request: Request, response: aiohttp.ClientResponse

app/carto.py → app/apis/carto.py

@@ -1,19 +1,21 @@
-import aiohttp
-import os
 import datetime as dt
+import os
+
+import aiohttp
 from fastapi import Request
 from psycopg import sql as psql  # Redefine to allow "sql" as a query parameter
-from .abstract import AbstractWorker, check_fields_valid
-from .models import (
-    ReturnJson,
-    Meta,
-    Links,
+
+from ..utils.models import (
     Error,
-    GeoJsonFeatureCollection,
     GeoJsonFeature,
-    TableSchema
+    GeoJsonFeatureCollection,
+    Links,
+    Meta,
+    ReturnJson,
+    TableSchema,
 )
-from .utils_carto import FULL_QUERY
+from ..utils.utils_carto import FULL_QUERY
+from .abstract import AbstractWorker, check_fields_valid
 
 
 class Carto(AbstractWorker):
@@ -37,7 +39,7 @@ async def get_count(
         table: str | None,
         where: str | None,
         timeout: float,
-        no_cache: bool,
+        max_age: int,
         session: aiohttp.ClientSession,
         request: Request,
         **kwargs,
@@ -52,8 +54,8 @@ async def get_count(
             query = query + q_where
         params = {"q": query.as_string()}
         headers = self.headers
-        if no_cache: 
-            headers['cache-control'] = "max-age=0"
+        if max_age:
+            headers["cache-control"] = f"max-age={max_age}"
         async with session.get(
             self.base_url, params=params, headers=headers, timeout=timeout
         ) as response:
@@ -89,7 +91,7 @@ async def get(
         out_sr: int | None,
         sql: str | None,
         timeout: float,
-        no_cache: bool,
+        max_age: int,
         session: aiohttp.ClientSession,
         request: Request,
         **kwargs,
@@ -143,8 +145,8 @@ async def get(
             table_schema = None
         params = {"q": query.as_string()}
         headers = self.headers
-        if no_cache: 
-            headers['cache-control'] = "max-age=0"
+        if max_age:
+            headers["cache-control"] = f"max-age={max_age}"
         async with session.get(
             self.base_url, params=params, headers=headers, timeout=timeout
         ) as response:
@@ -172,7 +174,7 @@ async def normalize_rv(
         if response.ok:
             if not sql:
                 records = data["rows"][0]["jsonb_build_object"]['features']
-                records = self.harmonize_timestamp_fields(records, table_schema)
+                self.harmonize_timestamp_fields(records, table_schema)
                 gjfc = GeoJsonFeatureCollection(
                     type="FeatureCollection", features=records
                 )
@@ -198,18 +200,13 @@ async def normalize_rv(
             rv = ReturnJson(errors=[error], links=links, meta=meta)
             return rv
 
-    def harmonize_timestamp_fields(
-        self, records: list[dict], table_schema: TableSchema
-    ) -> list[dict]:
-        """Return a consistent representation of timestamp fields. Carto returns 
+    def harmonize_timestamp_fields(self, records: list[dict], table_schema: TableSchema):
+        """Coerce to a consistent representation of timestamp fields. Carto returns 
         timestamps in ISO format
 
         Args:
             records (list[dict]): Data records
             table_schema (TableSchema): TableSchema
-
-        Returns:
-            list[dict]: Updated records
         """
         for record in records:
             for field in record["properties"]:
@@ -218,4 +215,3 @@ def harmonize_timestamp_fields(
                         record["properties"][field] = dt.datetime.fromisoformat(
                             record["properties"][field]
                         )
-        return records