Remove graph dependency from src/constraints/consumer.jl (#944)

Remove graph dependency from src/constraints/consumer.jl.
Create profiles structure with lookup to the profile vector. 

Part of #942 and #642 
Closes #943

src/constraints/consumer.jl

@@ -11,29 +11,77 @@ add_consumer_constraints!(model,
 
 Adds the consumer asset constraints to the model.
 """
-
-function add_consumer_constraints!(model, constraints, graph)
+function add_consumer_constraints!(connection, model, constraints, profiles)
     cons = constraints[:balance_consumer]
-    incoming_flow_highest_in_out_resolution = cons.expressions[:incoming]
-    outgoing_flow_highest_in_out_resolution = cons.expressions[:outgoing]
+
+    table = _create_consumer_table(connection)
 
     # - Balance constraint (using the lowest temporal resolution)
-    model[:consumer_balance] = [
-        @constraint(
-            model,
-            incoming_flow_highest_in_out_resolution[row.index] -
-            outgoing_flow_highest_in_out_resolution[row.index] -
-            profile_aggregation(
-                Statistics.mean,
-                graph[row.asset].rep_periods_profiles,
-                row.year,
-                row.year,
-                ("demand", row.rep_period),
-                row.time_block_start:row.time_block_end,
-                1.0,
-            ) * graph[row.asset].peak_demand[row.year] in
-            graph[row.asset].consumer_balance_sense,
-            base_name = "consumer_balance[$(row.asset),$(row.year),$(row.rep_period),$(row.time_block_start):$(row.time_block_end)]"
-        ) for row in eachrow(cons.indices)
-    ]
+    attach_constraint!(
+        model,
+        cons,
+        :balance_consumer,
+        [
+            begin
+                consumer_balance_sense = if ismissing(row.consumer_balance_sense)
+                    MathOptInterface.EqualTo(0.0)
+                else
+                    MathOptInterface.GreaterThan(0.0)
+                end
+                # On demand computation of the mean
+                demand_agg = _profile_aggregate(
+                    profiles.rep_period,
+                    (row.profile_name, row.year, row.rep_period),
+                    row.time_block_start:row.time_block_end,
+                    Statistics.mean,
+                    1.0,
+                )
+                @constraint(
+                    model,
+                    incoming_flow - outgoing_flow - demand_agg * row.peak_demand in
+                    consumer_balance_sense,
+                    base_name = "consumer_balance[$(row.asset),$(row.year),$(row.rep_period),$(row.time_block_start):$(row.time_block_end)]"
+                )
+            end for (row, incoming_flow, outgoing_flow) in
+            zip(table, cons.expressions[:incoming], cons.expressions[:outgoing])
+        ],
+    )
+
+    return
+end
+
+function _create_consumer_table(connection)
+    #=
+        In the query below, the "filtering" by profile_type = 'demand' must
+        happen at the join clause, i.e., in the ON ... AND ... list. This is
+        necessary because we are using an OUTER join with the result, because
+        we want to propagate the information that some combinations of (asset,
+        year, rep_period) don't have a profile for the given profile_type.
+
+        If we use a WHERE condition, all combination with all the profile_type
+        would be created, and only after that it would be filtered (which would
+        probably leave the table with a different number of rows, and thus
+        impossible to match the constraints table.
+    =#
+    return DuckDB.query(
+        connection,
+        "SELECT
+            cons.*,
+            asset.type,
+            asset.consumer_balance_sense,
+            asset_milestone.peak_demand,
+            assets_profiles.profile_name,
+        FROM cons_balance_consumer AS cons
+        LEFT JOIN asset
+            ON cons.asset = asset.asset
+        LEFT JOIN asset_milestone
+            ON cons.asset = asset_milestone.asset
+            AND cons.year = asset_milestone.milestone_year
+        LEFT OUTER JOIN assets_profiles
+            ON cons.asset = assets_profiles.asset
+            AND cons.year = assets_profiles.commission_year
+            AND assets_profiles.profile_type = 'demand' -- This must be a ON condition not a where (note 1)
+        ORDER BY cons.index -- order is important
+        ",
+    )
 end

src/create-model.jl

@@ -6,26 +6,19 @@ export create_model!, create_model
 Create the internal model of an [`TulipaEnergyModel.EnergyProblem`](@ref).
 """
 function create_model!(energy_problem; kwargs...)
-    graph = energy_problem.graph
-    representative_periods = energy_problem.representative_periods
-    variables = energy_problem.variables
-    constraints = energy_problem.constraints
-    timeframe = energy_problem.timeframe
-    groups = energy_problem.groups
-    model_parameters = energy_problem.model_parameters
-    years = energy_problem.years
-    sets = @timeit to "create_sets" create_sets(graph, years)
+    sets = @timeit to "create_sets" create_sets(energy_problem.graph, energy_problem.years)
     energy_problem.model = @timeit to "create_model" create_model(
         energy_problem.db_connection,
-        graph,
+        energy_problem.graph,
         sets,
-        variables,
-        constraints,
-        representative_periods,
-        years,
-        timeframe,
-        groups,
-        model_parameters;
+        energy_problem.variables,
+        energy_problem.constraints,
+        energy_problem.profiles,
+        energy_problem.representative_periods,
+        energy_problem.years,
+        energy_problem.timeframe,
+        energy_problem.groups,
+        energy_problem.model_parameters;
         kwargs...,
     )
     energy_problem.termination_status = JuMP.OPTIMIZE_NOT_CALLED
@@ -36,7 +29,7 @@ function create_model!(energy_problem; kwargs...)
 end
 
 """
-    model = create_model(graph, representative_periods, dataframes, timeframe, groups; write_lp_file = false, enable_names = true)
+    model = create_model(connection, graph, representative_periods, dataframes, timeframe, groups; write_lp_file = false, enable_names = true)
 
 Create the energy model given the `graph`, `representative_periods`, dictionary of `dataframes` (created by [`construct_dataframes`](@ref)), timeframe, and groups.
 """
@@ -46,6 +39,7 @@ function create_model(
     sets,
     variables,
     constraints,
+    profiles,
     representative_periods,
     years,
     timeframe,
@@ -122,7 +116,12 @@ function create_model(
 
     @timeit to "add_energy_constraints!" add_energy_constraints!(model, constraints, graph)
 
-    @timeit to "add_consumer_constraints!" add_consumer_constraints!(model, constraints, graph)
+    @timeit to "add_consumer_constraints!" add_consumer_constraints!(
+        connection,
+        model,
+        constraints,
+        profiles,
+    )
 
     @timeit to "add_storage_constraints!" add_storage_constraints!(
         model,

src/model-preparation.jl

@@ -1,5 +1,5 @@
 # Tools to prepare data and structures to the model creation
-export create_sets
+export create_sets, prepare_profiles_structure
 
 """
     add_expression_terms_intra_rp_constraints!(df_cons,
@@ -600,3 +600,51 @@ function create_sets(graph, years)
         starting_year_using_simple_method,
     )
 end
+
+function prepare_profiles_structure(connection)
+    rep_period = Dict(
+        (row.profile_name, row.year, row.rep_period) => [
+            row.value for row in DuckDB.query(
+                connection,
+                "SELECT profile.value
+                FROM profiles_rep_periods AS profile
+                WHERE
+                    profile.profile_name = '$(row.profile_name)'
+                    AND profile.year = $(row.year)
+                    AND profile.rep_period = $(row.rep_period)
+                ",
+            )
+        ] for row in DuckDB.query(
+            connection,
+            "SELECT DISTINCT
+                profiles.profile_name,
+                profiles.year,
+                profiles.rep_period
+            FROM profiles_rep_periods AS profiles
+            ",
+        )
+    )
+
+    over_clustered_year = Dict(
+        (row.profile_name, row.year) => [
+            row.value for row in DuckDB.query(
+                connection,
+                "SELECT profile.value
+                FROM profiles_timeframe AS profile
+                WHERE
+                    profile.profile_name = '$(row.profile_name)'
+                    AND profile.year = $(row.year)
+                ",
+            )
+        ] for row in DuckDB.query(
+            connection,
+            "SELECT DISTINCT
+                profiles.profile_name,
+                profiles.year
+            FROM profiles_timeframe AS profiles
+            ",
+        )
+    )
+
+    return ProfileLookup(rep_period, over_clustered_year)
+end

src/structures.jl

@@ -331,6 +331,16 @@ mutable struct Solution
     duals::Union{Nothing,Dict{Symbol,Vector{Float64}}}
 end
 
+mutable struct ProfileLookup
+    # The integers here are Int32 because they are obtained directly from DuckDB
+    #
+    # rep_period[(asset, year, rep_period)]
+    rep_period::Dict{Tuple{String,Int32,Int32},Vector{Float64}}
+
+    # over_clustered_year[(asset, year)]
+    over_clustered_year::Dict{Tuple{String,Int32},Vector{Float64}}
+end
+
 """
 Structure to hold all parts of an energy problem. It is a wrapper around various other relevant structures.
 It hides the complexity behind the energy problem, making the usage more friendly, although more verbose.
@@ -367,6 +377,7 @@ mutable struct EnergyProblem
     }
     variables::Dict{Symbol,TulipaVariable}
     constraints::Dict{Symbol,TulipaConstraint}
+    profiles::ProfileLookup
     representative_periods::Dict{Int,Vector{RepresentativePeriod}}
     timeframe::Timeframe
     groups::Vector{Group}
@@ -395,11 +406,14 @@ mutable struct EnergyProblem
         constraints =
             @timeit to "compute_constraints_indices" compute_constraints_indices(connection)
 
+        profiles = @timeit to "prepare_profiles_structure" prepare_profiles_structure(connection)
+
         energy_problem = new(
             connection,
             graph,
             variables,
             constraints,
+            profiles,
             representative_periods,
             timeframe,
             groups,

src/utils.jl

@@ -116,6 +116,14 @@ function profile_aggregation(agg, profiles, year, commission_year, key, block, d
     end
 end
 
+function _profile_aggregate(profiles, tuple_key, time_block, agg_function, default_value)
+    if !haskey(profiles, tuple_key)
+        return agg_function(Iterators.repeated(default_value, length(time_block)))
+    end
+    profile_value = profiles[tuple_key]
+    return agg_function(skipmissing(profile_value[time_block]))
+end
+
 """
     create_intervals(years)
 

test/test-case-studies.jl

@@ -85,8 +85,16 @@ end
     dir = joinpath(INPUT_FOLDER, "Tiny")
     connection = DBInterface.connect(DuckDB.DB)
     _read_csv_folder(connection, dir)
+    DuckDB.execute( # Make it infeasible
+        connection,
+        "UPDATE asset_milestone
+            SET peak_demand = -1
+            WHERE
+                asset = 'demand'
+                AND milestone_year = 2030
+        ",
+    )
     energy_problem = EnergyProblem(connection)
-    energy_problem.graph["demand"].peak_demand[2030] = -1 # make it infeasible
     create_model!(energy_problem)
     @test_logs (:warn, "Model status different from optimal") solve_model!(energy_problem)
     @test energy_problem.termination_status == JuMP.INFEASIBLE

test/test-pipeline.jl

@@ -26,6 +26,7 @@ end
     sets = create_sets(graph, years)
     variables = compute_variables_indices(connection)
     constraints = compute_constraints_indices(connection)
+    profiles = prepare_profiles_structure(connection)
 
     # Create model
     model = create_model(
@@ -34,6 +35,7 @@ end
         sets,
         variables,
         constraints,
+        profiles,
         representative_periods,
         years,
         timeframe,