WIP : Handle dual and reduced cost

.github/workflows/ci.yml

@@ -15,40 +15,29 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.11
-          cache: pip
-          cache-dependency-path: |
-            requirements.txt
-            requirements-dev.txt
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-dev.txt
+        run: uv sync --group dev
 
       - name: Check imports sort order
-        uses: isort/isort-action@v1
-        with:
-          sort-paths: src, tests
-          configuration: --profile black --check-only --diff
-          requirements-files: "requirements.txt requirements-dev.txt"
+        run: uv run isort --profile black --check-only --diff src tests
 
       - name: Check formatting
-        uses: psf/black@stable
-        with:
-          # Version of Black should match the versions set in `requirements-dev.txt`
-          version: "~=23.7.0"
-          options: --check --diff
+        run: uv run black --check --diff src tests
 
       - name: Check typing
-        run: |
-          python -m mypy
+        run: uv run mypy
 
       - name: Test
-        run: |
-          pytest --cov antarest --cov-report xml
+        run: uv run pytest --cov gems --cov-report xml
 
       - name: Archive code coverage results
         uses: actions/upload-artifact@v4
         with:
           name: python-code-coverage-report
           path: coverage.xml
-

.gitignore

@@ -14,3 +14,4 @@ outputs
 build
 src/gems.egg-info/
 src/gemspy.egg-info/
+build_time_scalability.csv

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     hooks:
     -   id: mypy
         name: Run mypy
-        entry: mypy
+        entry: uv run mypy
         language: system
         files: ^src/
         types: [python]
@@ -12,7 +12,7 @@ repos:
     hooks:
     -   id: black
         name: Run black
-        entry: black
+        entry: uv run black
         args: ["--config", "pyproject.toml"]
         language: system
         types: [python]
@@ -21,7 +21,7 @@ repos:
     hooks:
     -   id: isort
         name: Run isort
-        entry: isort
-        args: ["--profile", "black", "--filter-files"]
+        entry: uv run isort
+        args: ["--profile", "black", "--filter-files", "src", "tests"]
         language: system
         types: [python]

.readthedocs.yml

@@ -13,7 +13,10 @@ build:
 mkdocs:
   configuration: mkdocs.yml
 
-# Optionally declare the Python requirements required to build your docs
+# Install the package with doc optional dependencies
 python:
   install:
-    - requirements: requirements-doc.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc

AGENTS.md

@@ -17,7 +17,7 @@ pip install -r requirements.txt -r requirements-dev.txt
 ```bash
 pytest                                          # run all tests
 pytest tests/path/to/test_file.py::test_name   # run a single test
-pytest --cov gems --cov-report xml         # with coverage
+pytest --cov gems --cov-report xml             # with coverage
 ```
 
 **Lint & Format:**
@@ -37,51 +37,65 @@ gemspy \
   --duration     8760 \
   --scenarios    1
 
-# Python API (minimal example)
-from gems.main.main import build_problem
+# Python API — directory-based study
+from gems.study.folder import load_study, run_study
+from gems.simulation import TimeBlock
+
+study = load_study(Path("path/to/study_dir"))   # reads input/, model-libraries/, data-series/
+problem = run_study(Path("path/to/study_dir"), scenarios=1, time_block=TimeBlock(1, list(range(8760))))
+
+# Python API — programmatic study
+from gems.study import Study
+from gems.simulation import build_problem, TimeBlock
+
+study = Study(system=system, database=database)
+problem = build_problem(study, TimeBlock(1, list(range(8760))), scenarios=1)
+problem.solve(solver_name="highs")
 ```
 
 ## Architecture
 
-The pipeline flows: **YAML input → parsing → model resolution → network building → optimization problem → OR-Tools solver → results**
+The pipeline flows: **YAML input → parsing → model resolution → system instantiation → optimization problem → HiGHS solver (via linopy) → results**
+
+An optional `optim-config.yml` activates decomposition: variables and constraints are split across a master problem and subproblems, with either sequential resolution or full Benders decomposition.
 
 ### Core Modules (`src/gems/`)
 
 **`model/`** — Immutable model templates.
 - `Model`: defines component behavior (parameters, variables, constraints, ports)
 - `Library`: a collection of models, loaded from YAML
-- Models are never instantiated directly — they are referenced by components
 
 **`expression/`** — Mathematical expression language and AST.
 - `ExpressionNode`: base frozen dataclass for all expression tree nodes
-- Node types cover: arithmetic (`+`, `-`, `*`, `/`), comparisons (`<=`, `>=`, `==`), time/scenario operators (`time_sum()`), and functions (`max()`, `min()`, `ceil()`, `floor()`)
-- Grammar is defined in `/grammar/` and parsed via ANTLR4 (generated files live in `expression/parsing/antlr/`)
+- Grammar is defined in `grammar/Expr.g4` and parsed via ANTLR4 (generated files live in `expression/parsing/antlr/` — do not edit directly)
 - `ExpressionVisitor` is the dominant pattern for traversing and transforming expression trees (evaluation, linearization, printing, degree analysis)
-- Expressions support operator overloading: `var('x') + 5 * param('p')`
 
-**`study/`** — Study definition and network instantiation.
-- `System`: top-level structure parsed from YAML (before instantiation)
-- `Network`: instantiated graph of `Node`s, `Component`s, and connections
-- `Component`: an instance of a `Model` with concrete parameter values
-- `DataBase`: manages time series and scenario data
+**`study/`** — Study definition and instantiation.
+- `System` (`system.py`): resolved topology — graph of `Component`s, `PortRef`s, and `PortsConnection`s after library references are substituted
+- `Study` (`study.py`): dataclass pairing a `System` with a `DataBase`; validates that the database supplies every parameter required by the system
+- `DataBase` (`data.py`): manages time-series and scenario data;
+- `load_study` / `run_study` (`folder.py`): convenience functions for directory-based studies (`input/system.yml`, `input/model-libraries/`, `input/data-series/`)
 
 **`simulation/`** — Optimization problem construction and solving.
-- `OptimizationProblem`: main interface; translates network + database into OR-Tools constraints
-- `LinearExpression`: the linearized form of model constraints used by the solver
-- `BendersDecomposedProblem`: temporal decomposition strategy for large problems
-- `TimeBlock`: structure for defining temporal decomposition
-- `OutputValues`: result extraction and formatting
+- `OptimizationProblem` (`optimization.py`): main interface; translates a `Study` into a linopy model solved by HiGHS
+- `DecomposedProblems` (`optimization.py`): holds the master problem and subproblem produced by temporal decomposition
+- `VectorizedLinearExprBuilder` (`linearize.py`): `ExpressionVisitor` subclass that converts an expression AST into a `VectorizedExpr`
+- `VectorizedBuilderBase` (`vectorized_builder.py`): shared base for all vectorized visitors (used by both `linearize.py` and `extra_output.py`)
+- `TimeBlock` (`time_block.py`): defines the temporal window for one solve
+- `SimulationTableBuilder` / `SimulationTableWriter` (`simulation_table.py`): result extraction as a flat pandas `DataFrame`
 
-### Key Design Patterns
+**`optim_config/`** — Optional decomposition configuration.
+- `OptimConfig` (`parsing.py`): top-level config loaded from `optim-config.yml`
+- `ResolutionMode` (`parsing.py`): `SEQUENTIAL_SUBPROBLEMS` (default) or `BENDERS_DECOMPOSITION`
+- `ModelDecompositionConfig` (`parsing.py`): per-model assignment of variables/constraints/objective contributions to master or subproblems
 
-- **Frozen dataclasses** throughout for immutability (models, expressions, constraints)
-- **Visitor pattern** for all expression tree operations (`ExpressionVisitor` subclasses)
-- **Indexing dimensions**: parameters and variables carry time and scenario indices explicitly; expressions track these automatically
-- **`ValueType`** enum (`INTEGER`, `CONTINUOUS`, `BOOLEAN`) for variable typing
+**`libs/`** — Resolves the path to bundled YAML model libraries shipped with the package.
 
-### Type Checking
+### Key Design Patterns
 
-Strict mypy is enforced (`disallow_untyped_defs`, `disallow_untyped_calls`). All new code must be fully typed. Configuration is in `mypy.ini`.
+- **Visitor pattern** for all expression tree operations (`ExpressionVisitor` subclasses). Use `ExpressionVisitorOperations` as a base when the return type supports `+, -, *, /` — it provides those four method implementations for free.
+- **Template-method via single abstract method**: `VectorizedBuilderBase` implements all 18+ visitor methods once with `xr.DataArray`-compatible semantics; concrete subclasses only override `variable()` (and optionally a few linopy-specific methods).
+- **Indexing dimensions**: parameters and variables carry time and scenario indices explicitly via `IndexingStructure`; expressions track these automatically.
 
 ## Further Reading
 

docs/agents/python-convention.md

@@ -2,18 +2,19 @@
 
 ## Code Style & Conventions
 
-- **Formatter**: Black, line-length 88. Never adjust line breaks manually—let Black decide.
-- **Import order**: isort with `profile = "black"`. One `import` block, no manual blank lines between
-  groups.
-- **Type annotations**: All functions and methods **must** have full type annotations. mypy is run in
-  strict mode (`disallow_untyped_defs`, `disallow_untyped_calls`).
+- **Formatter**: Black `23.7.0`, line-length 88. Never adjust line breaks manually—let Black decide.
+- **Import order**: isort `5.12.0` with `profile = "black"`. One `import` block, no manual blank
+  lines between groups.
+- **Type annotations**: All functions and methods **must** have full type annotations. mypy `1.7.1`
+  is run with `disallow_untyped_defs = true` and `disallow_untyped_calls = true` (see `mypy.ini`).
 - **Dataclasses**: Prefer `@dataclass(frozen=True)` for value objects (expression nodes, model
   definitions). Mutability must be justified explicitly.
 - **Pydantic**: Use `ConfigDict(alias_generator=to_camel)` or kebab-case alias generation for YAML
   round-tripping; use Pydantic v2 APIs only.
 - **Naming**:
   - Classes: `PascalCase`
-  - Functions / variables: `snake_case`
+  - Functions / variables: `snake_case` — use descriptive names; avoid single-letter or two-letter
+    names except for well-established loop indices (`i`, `j`) or type parameters (`T`).
   - Constants / type-level aliases: `UPPER_SNAKE_CASE`
   - YAML keys: `kebab-case`
 - **Commit messages**: Follow Conventional Commits — `<type>(<scope>): <summary>`, e.g.

docs/agents/testing.md

@@ -3,13 +3,19 @@
 ## Philosophy
 
 Tests live alongside the module they exercise. Fixtures (YAML snippets, small networks) are kept
-in `tests/` sub-directories. No mocking of the solver—tests use real OR-Tools calls.
+in `tests/` sub-directories. No mocking of the solver—tests use real HiGHS calls via linopy
+(`problem.solve(solver_name="highs")`).
 
 ## Layers
 
 | Layer | Location | Description |
 |---|---|---|
-| Unit | `tests/unittests/` | One file per module area; covers AST visitors, parsing, model loading, linearisation |
-| Integration | `tests/unittests/simulation/` | Full problem build + solve on tiny networks |
-| End-to-end | `tests/e2e/` | CLI-level tests reading real YAML fixtures |
-| Converter | `tests/input_converter/`, `tests/pypsa_converter/`, `tests/antares_historic/` | Format-specific conversion tests |
+| Unit — expressions | `tests/unittests/expressions/` | AST visitors and expression parsing (parsing/, visitor/) |
+| Unit — libraries | `tests/unittests/lib_parsing/` | Model library YAML parsing |
+| Unit — system | `tests/unittests/system/` | Model, network, and port object behaviour |
+| Unit — system parsing | `tests/unittests/system_parsing/` | System YAML parsing |
+| Unit — scenario builder | `tests/unittests/scenario_builder/` | Scenario and time-series builder |
+| Integration | `tests/unittests/simulation/` | Full problem build + solve on small networks |
+| End-to-end — functional | `tests/e2e/functional/` | Cross-cutting tests: library/system combinations, stochastic, investment, scenario builder |
+| End-to-end — models | `tests/e2e/models/` | Model-level tests (andromede-v1 models, operator tests, proof-of-concept models) |
+| End-to-end — studies | `tests/e2e/studies/` | Study-level tests reading full YAML study fixtures |

docs/getting-started.md

@@ -174,19 +174,18 @@ with open("system.yml") as lib_file:
     input_libraries = [parse_yaml_library(lib_file)]
 
 result_lib = resolve_library(input_libraries)
-components_input = resolve_system(input_system, result_lib)
+system_input = resolve_system(input_system, result_lib)
 database = build_data_base(input_system, Path(series_dir))
 ~~~
 
 ### Building the optimisation problem
 
 ~~~ python
 
-network = build_network(components_input)
+network = build_network(system_input)
 
 problem = build_problem(
-    network,
-    database,
+    Study(network, database),
     TimeBlock(1, [i for i in range(0, timespan)]),
     scenarios,
 )

docs/user-guide/inputs.md

@@ -13,7 +13,7 @@ with open("simple_library.yml") as lib_file:
     input_libraries = [parse_yaml_library(lib_file)]
 
 result_lib = resolve_library(input_libraries)
-components_input = resolve_system(input_system, result_lib)
+system_input = resolve_system(input_system, result_lib)
 database = build_data_base(input_system, Path(series_dir))
 ~~~
 

docs/user-guide/optimisation.md

@@ -7,11 +7,10 @@
 
 ~~~ python
 
-network = build_network(components_input)
+network = build_network(system_input)
 
 problem = build_problem(
-    network,
-    database,
+    Study(network, database),
     TimeBlock(1, [i for i in range(0, timespan)]),
     scenarios,
 )

docs/user-guide/outputs.md

@@ -4,13 +4,25 @@
 Once the optimisation problem was built and solved, one can retrieve the results as follows:
 
 ~~~ python
-problem.solver.Solve()
-results = OutputValues(problem)
+from gems.simulation.simulation_table import SimulationTableBuilder
+
+df = SimulationTableBuilder().build(problem)
 ~~~
 
+The result is a `pandas.DataFrame` with columns: `block`, `component`, `output`,
+`absolute-time-index`, `block-time-index`, `scenario-index`, `value`, `basis-status`.
+
+Reading the value of the optimisation variable `var_id` of component `component_id`
+for a single time step and scenario reads:
+
+~~~ python
+value = df[(df["component"] == component_id) & (df["output"] == var_id)]["value"].iloc[0]
+~~~
 
-Reading the timeseries of the optimisation variable ''var_id'' of component "component_id" reads:
+For multi-time or multi-scenario results, filter additionally by `block-time-index`
+and `scenario-index`:
 
 ~~~ python
-var_timeseries = results.component(component_id).var(var_id).value
-~~~
+sub = df[(df["component"] == component_id) & (df["output"] == var_id)]
+value_s0_t1 = sub[(sub["scenario-index"] == 0) & (sub["block-time-index"] == 1)]["value"].iloc[0]
+~~~