From c5c68cab4d43e9f51b9caf04907e958c2da4971a Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 16 Oct 2025 11:11:02 +0200 Subject: [PATCH 01/21] Simple single-variable hacky ad hoc connector test ; as a model it's awkward --- src/Abstract_model_structs.jl | 45 +++++++++- src/mtg/GraphSimulation.jl | 8 +- src/run.jl | 20 ++++- test/test-simulation.jl | 151 +++++++++++++++++++++++++++++++++- 4 files changed, 217 insertions(+), 7 deletions(-) diff --git a/src/Abstract_model_structs.jl b/src/Abstract_model_structs.jl index 9cef7b08..59405e93 100644 --- a/src/Abstract_model_structs.jl +++ b/src/Abstract_model_structs.jl @@ -25,4 +25,47 @@ model_(m::AbstractModel) = m get_models(m::AbstractModel) = [model_(m)] # Get the models of an AbstractModel # Note: it is returning a vector of models, because in this case the user provided a single model instead of a vector of. get_status(m::AbstractModel) = nothing -get_mapped_variables(m::AbstractModel) = Pair{Symbol,String}[] \ No newline at end of file +get_mapped_variables(m::AbstractModel) = Pair{Symbol,String}[] + + +#using Dates +struct TimestepRange + lower_bound::Period + upper_bound::Period +end + +# Default, no specified range, meaning the model either doesn't depend on time or uses the simulation's default (eg smallest) timestep +TimestepRange() = TimestepRange(Second(0), Second(0)) +# Only a single timestep type possible +TimestepRange(p::Period) = TimestepRange(p, p) + +""" + timestep_range_(tsr::TimestepRange) + +Return the model's valid range for timesteps (which corresponds to the simulation base timestep in the default case). +""" +function timestep_range_(model::AbstractModel) + return TimestepRange() +end + +""" + timestep_valid(tsr::TimestepRange) + +Checks whether a TimestepRange +""" +timestep_valid(tsr::TimestepRange) = tsr.lower_bound <= tsr.upper_bound + +function model_timestep_range_compatible_with_timestep(tsr::TimestepRange, p::Period) + if !timestep_valid(tsr) + return false + end + + # 0 means any timestep is valid, no timestep constraints + if tsr.upper_bound == Seconds(0) + return true + end + + return p >= tsr.lower_bound && p <= tsr.lower_bound +end + +# TODO should i set all timestep ranges to default and hope the modeler gets it right or should i force them to write something ? \ No newline at end of file diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 93376240..3aa491c9 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -16,7 +16,7 @@ A type that holds all information for a simulation over a graph. - `models`: a dictionary of models - `outputs`: a dictionary of outputs """ -struct GraphSimulation{T,S,U,O,V} +struct GraphSimulation{T,S,U,O,V,W} graph::T statuses::S status_templates::Dict{String,Dict{Symbol,Any}} @@ -26,10 +26,12 @@ struct GraphSimulation{T,S,U,O,V} models::Dict{String,U} outputs::Dict{String,O} outputs_index::Dict{String, Int} + default_timestep::Int # TODO make it a period ? + model_timesteps::Dict{W, Int} #where {W <: AbstractModel} end -function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false) - GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)...) +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, default_timestep=1, model_timesteps=Dict{String, Int}()) + GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)..., default_timestep, model_timesteps) end dep(g::GraphSimulation) = g.dependency_graph diff --git a/src/run.jl b/src/run.jl index d3312473..03974654 100644 --- a/src/run.jl +++ b/src/run.jl @@ -365,17 +365,21 @@ function run!( meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; + orchestrator::Orchestrator=nothing, nsteps=nothing, tracked_outputs=nothing, check=true, - executor=ThreadedEx() + executor=ThreadedEx(), + default_timestep::Int, + model_timesteps::Dict{T, Int} where {T} + ) isnothing(nsteps) && (nsteps = get_nsteps(meteo)) meteo_adjusted = adjust_weather_timesteps_to_given_length(nsteps, meteo) # NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used # otherwise there might be vector length conflicts with timesteps - sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=tracked_outputs) + sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=tracked_outputs, default_timestep=default_timestep, model_timesteps=model_timesteps) run!( sim, meteo_adjusted, @@ -455,6 +459,18 @@ function run_node_multiscale!( return nothing end + model_timestep = object.model_timesteps[typeof(node.value)] + + if model_timestep != object.default_timestep + # do accumulation + + + # run if necessary + if i % model_timestep != 0 + return nothing + end + end + node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale models_at_scale = models[node.scale] diff --git a/test/test-simulation.jl b/test/test-simulation.jl index 4ff83a64..c4439308 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -292,4 +292,153 @@ end; end end end -end \ No newline at end of file +end + + + + +using PlantSimEngine +# Include the example dummy processes: +using PlantSimEngine.Examples +using Test, Aqua +using Tables, DataFrames, CSV +using MultiScaleTreeGraph +using PlantMeteo, Statistics +using Documenter # for doctests + +using PlantMeteo.Dates +include("helper-functions.jl") + + + +# These models might be worth exposing in the future ? +PlantSimEngine.@process "basic_current_timestep" verbose = false + +struct HelperCurrentTimestepModel <: AbstractBasic_Current_TimestepModel +end + +PlantSimEngine.inputs_(::HelperCurrentTimestepModel) = (next_timestep=1,) +PlantSimEngine.outputs_(m::HelperCurrentTimestepModel) = (current_timestep=1,) + +function PlantSimEngine.run!(m::HelperCurrentTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.current_timestep = status.next_timestep + end + + PlantSimEngine.ObjectDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsObjectDependent() + PlantSimEngine.TimeStepDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsTimeStepDependent() + +PlantSimEngine.timestep_range_(m::HelperCurrentTimestepModel) = Day(1) + + + PlantSimEngine.@process "basic_next_timestep" verbose = false + struct HelperNextTimestepModel <: AbstractBasic_Next_TimestepModel + end + + PlantSimEngine.inputs_(::HelperNextTimestepModel) = (current_timestep=1,) + PlantSimEngine.outputs_(m::HelperNextTimestepModel) = (next_timestep=1,) + + function PlantSimEngine.run!(m::HelperNextTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.next_timestep = status.current_timestep + 1 + end + +PlantSimEngine.timestep_range_(m::HelperNextTimestepModel) = Day(1) + + + + + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=-Inf,) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = Week(1) + + + +PlantSimEngine.@process "DWConnector" verbose = false + +struct MyDwconnectorModel <: AbstractDwconnectorModel + T_daily::Array{Float64} +end + +MyDwconnectorModel() = MyDwconnectorModel(Array{Float64}(undef, 7)) + +function PlantSimEngine.inputs_(::MyDwconnectorModel) + (daily_temperature=-Inf, current_timestep=1,) +end +PlantSimEngine.outputs_(m::MyDwconnectorModel) = (weekly_max_temperature = 0.0,) + +function PlantSimEngine.run!(m::MyDwconnectorModel, models, status, meteo, constants=nothing, extra=nothing) + m.T_daily[1 + (status.current_timestep % 7)] = status.daily_temperature + + if(status.current_timestep % 7 == 1) + status.weekly_max_temperature = sum(m.T_daily)/7.0 + else + status.weekly_max_temperature = 0 + end +end + + PlantSimEngine.timestep_range_(m::MyDwconnectorModel) = Day(1) + + + + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m = Dict("Default" => ( + MyToyDayModel(), + MyToyWeekModel(), + MyDwconnectorModel(), + HelperNextTimestepModel(), + MultiScaleModel( + model=HelperCurrentTimestepModel(), + mapped_variables=[PreviousTimeStep(:next_timestep),], + ), + Status(a=1,))) + +to_initialize(m) + +models_timestep = Dict(MyToyDayModel=>1, MyDwconnectorModel => 1, MyToyWeekModel =>7, HelperNextTimestepModel => 1, HelperCurrentTimestepModel => 1) + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) + +out = run!(mtg, m, meteo_day, default_timestep=1, model_timesteps=models_timestep) + +@testset "Test varying timestep" begin + + + @test + @test + +end + + + # NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used + # otherwise there might be vector length conflicts with timesteps + sim = @enter PlantSimEngine.GraphSimulation(mtg, m, nsteps=nothing, check=true, outputs=nothing, default_timestep=1, model_timesteps=models_timestep) + +using PlantSimEngine From a3c89735b6d6c193de6d76b19c2573f511b3729a Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 16 Oct 2025 11:15:38 +0200 Subject: [PATCH 02/21] Start to implement an orchestrator struct that centralizes timestep issues and connector data --- src/PlantSimEngine.jl | 5 ++++ src/mtg/GraphSimulation.jl | 11 ++++----- src/mtg/initialisation.jl | 2 +- src/timestep/timestep_mapping.jl | 42 ++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 src/timestep/timestep_mapping.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index ebf59dcb..b31d9847 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -26,6 +26,7 @@ import Statistics import SHA: sha1 using PlantMeteo +using PlantMeteo.Dates # UninitializedVar + PreviousTimeStep: include("variables_wrappers.jl") @@ -65,6 +66,9 @@ include("dependencies/printing.jl") include("dependencies/dependencies.jl") include("dependencies/get_model_in_dependency_graph.jl") +# Timesteps. : +include("timestep/timestep_mapping.jl") + # MTG compatibility: include("mtg/GraphSimulation.jl") include("mtg/mapping/getters.jl") @@ -103,6 +107,7 @@ include("examples_import.jl") export PreviousTimeStep export AbstractModel export ModelList, MultiScaleModel +export Orchestrator export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 3aa491c9..001a7f3f 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -14,9 +14,10 @@ A type that holds all information for a simulation over a graph. - `var_need_init`: a dictionary indicating if a variable needs to be initialized - `dependency_graph`: the dependency graph of the models applied to the graph - `models`: a dictionary of models +- `Orchestrator : the structure that handles timestep peculiarities - `outputs`: a dictionary of outputs """ -struct GraphSimulation{T,S,U,O,V,W} +struct GraphSimulation{T,S,U,O,V} graph::T statuses::S status_templates::Dict{String,Dict{Symbol,Any}} @@ -24,14 +25,12 @@ struct GraphSimulation{T,S,U,O,V,W} var_need_init::Dict{String,V} dependency_graph::DependencyGraph models::Dict{String,U} + orchestrator::Orchestrator outputs::Dict{String,O} - outputs_index::Dict{String, Int} - default_timestep::Int # TODO make it a period ? - model_timesteps::Dict{W, Int} #where {W <: AbstractModel} end -function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, default_timestep=1, model_timesteps=Dict{String, Int}()) - GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)..., default_timestep, model_timesteps) +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) + GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)..., orchestrator) end dep(g::GraphSimulation) = g.dependency_graph diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 57aef81c..95fb6e5c 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -305,7 +305,7 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false) +function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) # Ensure the user called the model generation function to handle vectors passed into a status # before we keep going diff --git a/src/timestep/timestep_mapping.jl b/src/timestep/timestep_mapping.jl new file mode 100644 index 00000000..e30fc97c --- /dev/null +++ b/src/timestep/timestep_mapping.jl @@ -0,0 +1,42 @@ + +# Those names all suck, need to change them +# Some of them are probably not ideal for new users, too + +# Some types can also be constrained a lot more, probably + +struct TimestepMapper#{V} + variable_from#::V + timestep_from::Int + mapping_function +end + +struct SimulationTimestepHandler#{W,V} + model_timesteps::Dict{Any, Int} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range + timestep_variable_mapping::Dict{Any, TimestepMapper} #where {V} +end + +SimulationTimestepHandler() = SimulationTimestepHandler(Dict(), Dict()) #Dict{W, Int}(), Dict{V, TimestepMapper}()) where {W, V} + +mutable struct Orchestrator + # This is actually a general simulation parameter, not-scale specific + # todo change to Period + default_timestep::Int64 + + # This needs to be per-scale : if a model is used at two different scales, + # and the same variable of that model maps to some other timestep to two *different* variables + # then I believe we can only rely on the different scale to disambiguate + non_default_timestep_data_per_scale::Dict{String, SimulationTimestepHandler} + + function Orchestrator(default::Int64, per_scale::Dict{String, SimulationTimestepHandler}) + @assert default >= 0 "The default_timestep should be greater than or equal to 0." + return new(default, per_scale) + end +end + +# TODO have a default constructor take in a meteo or something, and set up the default timestep automagically to be the finest weather timestep +# Other options are possible +Orchestrator() = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) + + +#o = Orchestrator() +#oo = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) \ No newline at end of file From cb77476e7288f3d0cffcae946c04e7c18c41d3f5 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Tue, 28 Oct 2025 15:03:51 +0100 Subject: [PATCH 03/21] Many changes : new orchestrator attempt, many more fixes. Multiple timesteps handled, but bad meshing with refvalue/refvectors cause overwrites if we use multiscale mappings, and unexplored issues if we avoid them. Also, many-node to many-node issues if multiscale mapping. Some TODO comments, structs, commented out code and notes are outdated. More exploration and cleanup required, but current state is much more interesting than the previous commit. --- src/Abstract_model_structs.jl | 6 +- src/PlantSimEngine.jl | 8 +- src/dependencies/dependencies.jl | 6 +- src/dependencies/dependency_graph.jl | 12 + src/dependencies/hard_dependencies.jl | 42 ++- src/dependencies/soft_dependencies.jl | 101 +++++- src/mtg/GraphSimulation.jl | 8 +- src/mtg/add_organ.jl | 5 + src/mtg/initialisation.jl | 120 ++++++- src/mtg/mapping/compute_mapping.jl | 4 +- src/mtg/mapping/reverse_mapping.jl | 2 +- src/processes/model_initialisation.jl | 2 +- src/run.jl | 85 +++-- src/timestep/timestep_mapping.jl | 93 +++++- test/test_multitimestep.jl | 457 ++++++++++++++++++++++++++ 15 files changed, 887 insertions(+), 64 deletions(-) create mode 100644 test/test_multitimestep.jl diff --git a/src/Abstract_model_structs.jl b/src/Abstract_model_structs.jl index 59405e93..f8a079e0 100644 --- a/src/Abstract_model_structs.jl +++ b/src/Abstract_model_structs.jl @@ -50,18 +50,16 @@ end """ timestep_valid(tsr::TimestepRange) - -Checks whether a TimestepRange """ timestep_valid(tsr::TimestepRange) = tsr.lower_bound <= tsr.upper_bound -function model_timestep_range_compatible_with_timestep(tsr::TimestepRange, p::Period) +function is_timestep_in_range(tsr::TimestepRange, p::Period) if !timestep_valid(tsr) return false end # 0 means any timestep is valid, no timestep constraints - if tsr.upper_bound == Seconds(0) + if tsr.upper_bound == Second(0) return true end diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index b31d9847..5ffaea10 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -57,6 +57,9 @@ include("component_models/get_status.jl") # Transform into a dataframe: include("dataframe.jl") +# Timesteps. : +include("timestep/timestep_mapping.jl") + # Computing model dependencies: include("dependencies/soft_dependencies.jl") include("dependencies/hard_dependencies.jl") @@ -66,9 +69,6 @@ include("dependencies/printing.jl") include("dependencies/dependencies.jl") include("dependencies/get_model_in_dependency_graph.jl") -# Timesteps. : -include("timestep/timestep_mapping.jl") - # MTG compatibility: include("mtg/GraphSimulation.jl") include("mtg/mapping/getters.jl") @@ -107,7 +107,7 @@ include("examples_import.jl") export PreviousTimeStep export AbstractModel export ModelList, MultiScaleModel -export Orchestrator +export Orchestrator, Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 9252e7b0..27b9fcde 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -100,16 +100,16 @@ function dep(m::NamedTuple, nsteps=1; verbose::Bool=true) dep(nsteps; verbose=verbose, m...) end -function dep(mapping::Dict{String,T}; verbose::Bool=true) where {T} +function dep(mapping::Dict{String,T}; verbose::Bool=true, orchestrator=Orchestrator2()) where {T} # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they # are independant. - soft_dep_graphs_roots, hard_dep_dict = hard_dependencies(mapping; verbose=verbose) + soft_dep_graphs_roots, hard_dep_dict = hard_dependencies(mapping; verbose=verbose, orchestrator=Orchestrator2()) # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children # of the nodes that they depend on. - dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, mapping, hard_dep_dict) + dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, mapping, hard_dep_dict, orchestrator=orchestrator) # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale. # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale. diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 2be06b64..33b5fc62 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -12,6 +12,16 @@ mutable struct HardDependencyNode{T} <: AbstractDependencyNode children::Vector{HardDependencyNode} end +mutable struct TimestepMapping + variable_from::Symbol + variable_to::Symbol + node_to # SoftDependencyNode causes a circular reference # TODO could it be a harddependencynode... ? + mapping_function::Function + mapping_data_template + mapping_data::Dict{Int, Any} # TODO Any's type is the variable's type, also, is Int good here ? Prob not +end + +# can hard dependency nodes also handle timestep mapped variables... ? mutable struct SoftDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol @@ -23,6 +33,8 @@ mutable struct SoftDependencyNode{T} <: AbstractDependencyNode parent_vars::Union{Nothing,NamedTuple} children::Vector{SoftDependencyNode} simulation_id::Vector{Int} # id of the simulation + timestep::Period + timestep_mapping_data::Union{Nothing, Vector{TimestepMapping}} # TODO : this approach might not play too well with parallelisation over MTG nodes end # Add methods to check if a node is parallelizable: diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 0d1fa5b3..eda0573f 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -112,7 +112,7 @@ end # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): -function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T} +function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T} full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping) soft_dep_graphs = Dict{String,Any}() not_found = Dict{Symbol,DataType}() @@ -226,6 +226,42 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T end end +#= + # TODO check whether this is a bit late in the game + # maybe the timestep mapping should be done before we enter this function + if length(orchestrator.non_default_timestep_data_per_scale) > 0 + if haskey(orchestrator.non_default_timestep_data_per_scale, symbol(node)) + tvm = orchestrator.non_default_timestep_data_per_scale[symbol(node)].timestep_variable_mapping + if haskey(twm, var) + + end + end + end + error("Variable `$(var)` is not computed by any model, not mapped from a different scale or timestep not initialised by the user in the status, and not found in the MTG at scale $(symbol(node)) (checked for MTG node $(node_id(node))).") +=# + + + # Once multiscale mapping has been dealt with, check if any variable has a timestep mapping + # Which will add potential new dependencies + #=if !isempty(orchestrator.non_default_timestep_data_per_scale) + # TODO the user can get away with not declaring the model, only the scale if necessary + # a prepass that recomputes everything might simplify code here and make the simulation require less variable digging + for (scale, tsh) in non_default_timestep_data_per_scale + # TODO find which model the variable is pulled from + # TODO check the variable exists + for (model, timestep) in tsh.model_timesteps + # TODO check the timestep is within the model's accepted timestep range + # TODO recover the right variables + end + + for (variable, tvm) in tsh.timestep_variable_mapping + # TODO check the variable isn't already mapped + # If it is, ensure there are no name conflicts + # and the model of the variable it is taken from has the expected timestep + # If it isn't, create a new link + end + end + end=# for (organ, model) in mapping soft_dep_graph = Dict( process_ => SoftDependencyNode( @@ -238,7 +274,9 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T nothing, nothing, SoftDependencyNode[], - [0] # Vector of zeros of length = number of time-steps + [0], # Vector of zeros of length = number of time-steps + orchestrator.default_timestep, + nothing ) for (process_, soft_dep_vars) in hard_deps[organ].roots # proc_ = :carbon_assimilation ; soft_dep_vars = hard_deps.roots[proc_] ) diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index b772a207..241a63bf 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -70,7 +70,9 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, nothing, nothing, SoftDependencyNode[], - fill(0, nsteps) + fill(0, nsteps), + Day(1), # TODO + nothing ) for (process_, soft_dep_vars) in d.roots ) @@ -138,8 +140,18 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, return DependencyGraph(independant_process_root, d.not_found) end +function timestep_mapped_variables(orchestrator) + +#=struct SimulationTimestepHandler#{W,V} + model_timesteps::Dict{Any, Period} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range + timestep_variable_mapping::Dict{Any, TimestepMapper} #where {V} +end + non_default_timestep_data_per_scale::Dict{String, SimulationTimestepHandler} +=# +end + # For multiscale mapping: -function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{String,Any}}, mapping::Dict{String,A}, hard_dep_dict::Dict{Pair{Symbol,String},HardDependencyNode}) where {A<:Any} +function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{String,Any}}, mapping::Dict{String,A}, hard_dep_dict::Dict{Pair{Symbol,String},HardDependencyNode}; orchestrator::Orchestrator2=Orchestrator2()) where {A<:Any} mapped_vars = mapped_variables(mapping, soft_dep_graphs_roots, verbose=false) rev_mapping = reverse_mapping(mapped_vars, all=false) @@ -324,10 +336,93 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic end end - return DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found) + dep_graph = DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found) + traverse_dependency_graph!(x -> set_non_default_timestep_in_node(x, orchestrator), dep_graph, visit_hard_dep=false) + traverse_dependency_graph!(x -> add_timestep_data_to_node(x, orchestrator), dep_graph, visit_hard_dep=false) + + return dep_graph +end + +# # set the timestep for everyone first, else we might not use the correct timestep when looking at the parents later +function set_non_default_timestep_in_node(soft_dependency_node, orchestrator::Orchestrator2) + for mtsm in orchestrator.non_default_timestep_mapping + if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value)) + soft_dependency_node.timestep = mtsm.timestep + end + end +end + +function add_timestep_data_to_node(soft_dependency_node, orchestrator::Orchestrator2) + + # now we can create the mapping + for mtsm in orchestrator.non_default_timestep_mapping + if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value)) + for (var_to, var_from) in mtsm.var_to_var + if !isnothing(soft_dependency_node.parent) + parent = nothing + variable_mapping = nothing + for parent_node in soft_dependency_node.parent + if typeof(parent_node.value) == var_from.model && parent_node.scale == var_from.scale + parent = parent_node + variable_mapping = create_timestep_mapping(soft_dependency_node, parent, var_to, var_from) + break + end + end + if isnothing(parent) + #error + end + if isnothing(parent.timestep_mapping_data) + parent.timestep_mapping_data = Vector{TimestepMapping}() + end + push!(parent.timestep_mapping_data, variable_mapping) + else + # Error + end + end + end + end end +# TODO this is incorrect, there may be multiple variables mapped between the two nodes +function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependencyNode, var_to::Var_to, var_from::Var_from) + + @assert parent.timestep != 0 "Error : node timestep internally set to 0" + + timestep_ratio = node.timestep / parent.timestep + + # Keeping things simple for now, only integers allowed + @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio" + + # TODO ensure type compatibility between var_to and var_from + # Simplification probably possible by doing the check earlier + + # TODO test previoustimestep + var_type = DataType + + for (symbol, var_dump) in node.inputs + for var in var_dump + if isa(var, MappedVar) + # check the source variable, because the sink one might be a vector...? + # TODO multinode mapping + if var.source_variable == var_from.name + # This should be a fixed size array, ideally + var_type = eltype(mapped_default(var)) + break + end + else + if var.symbol == var_from.name + @assert "untested" + var_type = eltype(mapped_default(var)) + break + end + end + end + end + mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio)) + # TODO : type shouldn't be Any but Vector{var_type} + return TimestepMapping(var_from.name, var_to.name, node, var_from.mapping_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG, Any}()) +end """ drop_process(proc_vars, process) diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 001a7f3f..efe17ec5 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -25,12 +25,14 @@ struct GraphSimulation{T,S,U,O,V} var_need_init::Dict{String,V} dependency_graph::DependencyGraph models::Dict{String,U} - orchestrator::Orchestrator outputs::Dict{String,O} + outputs_index::Dict{String, Int} + orchestrator::Orchestrator2 + end -function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) - GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)..., orchestrator) +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) + GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose, orchestrator=orchestrator)...) end dep(g::GraphSimulation) = g.dependency_graph diff --git a/src/mtg/add_organ.jl b/src/mtg/add_organ.jl index 1c2925ef..58c71a9e 100644 --- a/src/mtg/add_organ.jl +++ b/src/mtg/add_organ.jl @@ -33,5 +33,10 @@ function add_organ!(node::MultiScaleTreeGraph.Node, sim_object, link, symbol, sc new_node = MultiScaleTreeGraph.Node(id, node, MultiScaleTreeGraph.NodeMTG(link, symbol, index, scale), attributes) st = init_node_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.reverse_multiscale_mapping, sim_object.var_need_init, check=check) + # TODO add the node to the timestep mappings + # TODO initialise the MTG nodes in the timestep mappings + # NOTE : this isn't ideal, as it constrains the add_organ! function usage + init_timestep_mapping_data(new_node, sim_object.dependency_graph) + return st end \ No newline at end of file diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 95fb6e5c..eac56d0c 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -21,7 +21,7 @@ a dictionary of variables that need to be initialised or computed by other model `(;statuses, status_templates, reverse_multiscale_mapping, vars_need_init, nodes_with_models)` """ -function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false)); type_promotion=nothing, verbose=false, check=true) +function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())); type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator2()) # We compute the variables mapping for each scale: mapped_vars = mapped_variables(mapping, dependency_graph, verbose=verbose) @@ -38,9 +38,17 @@ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(ma convert_reference_values!(mapped_vars) # Get the variables that are not initialised or computed by other models in the output: - vars_need_init = Dict(org => filter(x -> isa(last(x), UninitializedVar), vars) |> keys for (org, vars) in mapped_vars) |> + vars_need_init = Dict(org => filter(x -> isa(last(x), UninitializedVar), vars) |> keys |> collect for (org, vars) in mapped_vars) |> filter(x -> length(last(x)) > 0) + # Filter out variables that are timestep-mapped by the user, + # as those variables are initialized by another model, but are currently flagged as needing initialization + # At this stage, data present in the orchestrator is expected to be valid, so we can take it into account + # A model with a different timestep can still have unitialized vars found in a node, the meteo, or to be initialized by the user + # in which case it'll be absent from the timestep mapping, but this needs testing + #filter_timestep_mapped_variables!(vars_need_init, orchestrator) + + # Note: these variables may be present in the MTG attributes, we check that below when traversing the MTG. # We traverse the MTG to initialise the statuses linked to the nodes: @@ -91,7 +99,7 @@ The `check` argument is a boolean indicating if variables initialisation should in the node attributes (using the variable name). If `true`, the function returns an error if the attribute is missing, otherwise it uses the default value from the model. """ -function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init=Dict{String,Any}(), type_promotion=nothing; check=true, attribute_name=:plantsimengine_status) +function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init=Dict{String,Any}(), type_promotion=nothing; check=true, attribute_name=:plantsimengine_status, orchestrator=Orchestrator2()) # Check if the node has a model defined for its symbol, if not, no need to compute symbol(node) ∉ collect(keys(mapped_vars)) && return @@ -113,7 +121,6 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi end continue end - error("Variable `$(var)` is not computed by any model, not initialised by the user in the status, and not found in the MTG at scale $(symbol(node)) (checked for MTG node $(node_id(node))).") end # Applying the type promotion to the node attribute if needed: if isnothing(type_promotion) @@ -305,7 +312,7 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) +function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) # Ensure the user called the model generation function to handle vectors passed into a status # before we keep going @@ -314,9 +321,11 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion @assert false "Error : Mapping status at $organ_with_vector level contains a vector. If this was intentional, call the function generate_models_from_status_vectors on your mapping before calling run!. And bear in mind this is not meant for production. If this wasn't intentional, then it's likely an issue on the mapping definition, or an unusual model." end + # preliminary_check_timestep_data(mapping, orchestrator) + # Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses, status_templates, reverse_multiscale_mapping, vars_need_init = - init_statuses(mtg, mapping, first(hard_dependencies(mapping; verbose=false)); type_promotion=type_promotion, verbose=verbose, check=check) + init_statuses(mtg, mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=orchestrator)); type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) @@ -329,5 +338,102 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion outputs = pre_allocate_outputs(statuses, status_templates, reverse_multiscale_mapping, vars_need_init, outputs, nsteps, type_promotion=type_promotion, check=check) outputs_index = Dict{String, Int}(s => 1 for s in keys(outputs)) - return (; mtg, statuses, status_templates, reverse_multiscale_mapping, vars_need_init, dependency_graph=dep(mapping, verbose=verbose), models, outputs, outputs_index) + + dependency_graph = dep(mapping, verbose=verbose, orchestrator=orchestrator) + + # Samuel : Once the dependency graph is created, and the timestep mappings are added into it + # We need to register the existing MTG nodes to initialize their individual data + # The current implementation is heavy, it may be quite slow for MTGs that already contain many nodes + # A faster way would be to count nodes by type once, store their ids, and only traverse the dependency graph once to add them + MultiScaleTreeGraph.traverse!(mtg, init_timestep_mapping_data, dependency_graph) + + return (; mtg, statuses, status_templates, reverse_multiscale_mapping, vars_need_init, dependency_graph=dependency_graph, models, outputs, outputs_index, orchestrator) +end + +function preliminary_check_timestep_data(mapping, orchestrator) + + if isempty(orchestrator.non_default_timestep_data_per_scale) + return + end + + # First, check timesteps are within models' accepted ranges + for (organ, models_status) in mapping + models = get_models(models_status) + + for model in models + checked = false + + if haskey(orchestrator.non_default_timestep_data_per_scale, organ) + tsh = orchestrator.non_default_timestep_data_per_scale[organ] + + if typeof(model) in collect(keys(tsh.model_timesteps)) + checked = true + # Check the timestep is within the model's accepted timestep range + if !is_timestep_in_range(timestep_range_(model), tsh.model_timesteps[typeof(model)]) + # TODO return error + end + end + end + # if it wasn't found, it means it's a model set to the default timestep + if !checked + if !is_timestep_in_range(timestep_range_(model), orchestrator.default_timestep) + # TODO return error + end + end + end + end + + # Next, check timestep mapped variables : + # They should all exist as input/output somewhere (not dealing with mtg stuff for now) + # If they are already mapped in a MultiScaleModel, there should be no differences + # They should not cause name conflicts + # The scales should all be in the mapping + # They should map to different timesteps, and if the timestep isn't the default, then the downstream model should be in the list as well + for (organ, tsh) in orchestrator.non_default_timestep_data_per_scale + #if !organ in collect(keys(mapping)) + # TODO error + #end + + for (model, tvm) in tsh.model_timesteps + end + end> + + return +end + + + + +#= +function filter_timestep_mapped_variables!(vars_need_init, orchestrator) + for (org, vars) in vars_need_init + if haskey(orchestrator.non_default_timestep_data_per_scale, org) + + filter!(o -> !haskey(orchestrator.non_default_timestep_data_per_scale[org].timestep_variable_mapping, o), vars) + #for var in vars + # if haskey(orchestrator.non_default_timestep_data_per_scale[org].timestep_variable_mapping, var) + # delete!(vars, var) + # end + #end + if isempty(vars) + delete!(vars_need_init, org) + end + end + end +end=# + +function filter_timestep_mapped_variables!(vars_need_init, orchestrator) + for (org, vars) in vars_need_init + for tmst in orchestrator.non_default_timestep_mapping + if tmst.scale == org + + for (var_to, var_from) in tmst.var_to_var + filter!(o -> o == var_to.name || o == var_from.name, vars) + end + if isempty(vars) + delete!(vars_need_init, org) + end + end + end + end end \ No newline at end of file diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index 21d64008..029b9c3d 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -11,7 +11,7 @@ However, models that are identified as hard-dependencies are not given individua nodes under other models. - `verbose::Bool`: whether to print the stacktrace of the search for the default value in the mapping. """ -function mapped_variables(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false)); verbose=false) +function mapped_variables(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())); verbose=false) # Initialise a dict that defines the multiscale variables for each organ type: mapped_vars = mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph) @@ -54,7 +54,7 @@ This function returns a dictionary with the (multiscale-) inputs and outputs var Note that this function does not include the variables that are outputs from another scale and not computed by this scale, see `mapped_variables_with_outputs_as_inputs` for that. """ -function mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false))) +function mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2()))) nodes_insouts = Dict(organ => (inputs=ins, outputs=outs) for (organ, (soft_dep_graph, ins, outs)) in dependency_graph.roots) ins = Dict{String,NamedTuple}(organ => flatten_vars(vcat(values(ins)...)) for (organ, (ins, outs)) in nodes_insouts) outs = Dict{String,NamedTuple}(organ => flatten_vars(vcat(values(outs)...)) for (organ, (ins, outs)) in nodes_insouts) diff --git a/src/mtg/mapping/reverse_mapping.jl b/src/mtg/mapping/reverse_mapping.jl index 059b4294..71ba46df 100644 --- a/src/mtg/mapping/reverse_mapping.jl +++ b/src/mtg/mapping/reverse_mapping.jl @@ -69,7 +69,7 @@ Dict{String, Dict{String, Dict{Symbol, Any}}} with 3 entries: """ function reverse_mapping(mapping::Dict{String,T}; all=true) where {T<:Any} # Method for the reverse mapping applied directly on the mapping (not used in the code base) - mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false)), verbose=false) + mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())), verbose=false) reverse_mapping(mapped_vars, all=all) end diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 95a8cc76..04702072 100755 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -148,7 +148,7 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} end to_init = Dict(org => Symbol[] for org in keys(mapping)) - mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false)), verbose=false) + mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())), verbose=false) for (org, vars) in mapped_vars for (var, val) in vars if isa(val, UninitializedVar) && var ∉ vars_in_mtg diff --git a/src/run.jl b/src/run.jl index 03974654..8a62a200 100644 --- a/src/run.jl +++ b/src/run.jl @@ -365,21 +365,18 @@ function run!( meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; - orchestrator::Orchestrator=nothing, nsteps=nothing, tracked_outputs=nothing, check=true, - executor=ThreadedEx(), - default_timestep::Int, - model_timesteps::Dict{T, Int} where {T} - + executor=ThreadedEx(), + orchestrator::Orchestrator2=Orchestrator2(), ) isnothing(nsteps) && (nsteps = get_nsteps(meteo)) meteo_adjusted = adjust_weather_timesteps_to_given_length(nsteps, meteo) # NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used # otherwise there might be vector length conflicts with timesteps - sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=tracked_outputs, default_timestep=default_timestep, model_timesteps=model_timesteps) + sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=tracked_outputs, orchestrator=orchestrator) run!( sim, meteo_adjusted, @@ -452,6 +449,15 @@ function run_node_multiscale!( executor ) where {T<:GraphSimulation} # T is the status of each node by organ type + # Hack until default timestep is inferred from the meteo + node_ratio = node.timestep / Day(1) + + # Check if the model needs to run this timestep + if (1 + (i-1) % node_ratio) != node_ratio + node.simulation_id[1] += 1 + return nothing + end + # run!(status(object), dependency_node, meteo, constants, extra) # Check if all the parents have been called before the child: if !AbstractTrees.isroot(node) && any([p.simulation_id[1] <= node.simulation_id[1] for p in node.parent]) @@ -459,33 +465,60 @@ function run_node_multiscale!( return nothing end - model_timestep = object.model_timesteps[typeof(node.value)] - - if model_timestep != object.default_timestep - # do accumulation - - - # run if necessary - if i % model_timestep != 0 - return nothing - end - end - node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale models_at_scale = models[node.scale] - for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) - # Actual call to the model: - run!(node.value, models_at_scale, st, meteo, constants, extra) - end + if isnothing(node.timestep_mapping_data) || isempty(node.timestep_mapping_data) + # Samuel : this is the happy path, no further timestep mapping checks needed - node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already + for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) + # Actual call to the model: + run!(node.value, models_at_scale, st, meteo, constants, extra) + end + node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already - # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): - for child in node.children + # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): + for child in node.children #! check if we can run this safely in a @floop loop. I would say no, #! because we are running a parallel computation above already, modifying the node.simulation_id, #! which is not thread-safe yet. - run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + end + else + # we have a child with a different timestep than the parent, + # which requires accumulation/reduce and running only some of the time + + for st in node_statuses + run!(node.value, models_at_scale, st, meteo, constants, extra) + # TODO do the accumulation + # then write into the child's status if need be ? + for tmst in node.timestep_mapping_data + ratio = Int(tmst.node_to.timestep / node.timestep) + # TODO assert etc. + # do the accumulation for each variable + accumulation_index = 1 + ((i-1)%ratio) + tmst.mapping_data[node_id(st.node)][accumulation_index] = st[tmst.variable_from] + + # if we have completed a *full* cycle, then presumably we need to write out the value to + # the mapped model + # A full cycle isn't just the ratio to the parent, it's the ratio to the finest-grained timestep + if accumulation_index == ratio + node_statuses_to = status(object)[tmst.node_to.scale] + + # TODO : INCORRECT in a scale with multiple mtg nodes + for st_to in node_statuses_to + # TODO might be able to catch mapping_function type incompatibility errors and make them clearer + st_to[tmst.variable_to] = tmst.mapping_function(tmst.mapping_data[node_id(st.node)]) + end + end + end + end + + node.simulation_id[1] += 1 + + for child in node.children + + run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + end end end \ No newline at end of file diff --git a/src/timestep/timestep_mapping.jl b/src/timestep/timestep_mapping.jl index e30fc97c..20ad038e 100644 --- a/src/timestep/timestep_mapping.jl +++ b/src/timestep/timestep_mapping.jl @@ -4,39 +4,116 @@ # Some types can also be constrained a lot more, probably +# Many shortcuts will be taken, I'll try and comment what's missing/implicit/etc. + +# TODO specify scale ? struct TimestepMapper#{V} variable_from#::V - timestep_from::Int + timestep_from::Period # ? Not sure whether this is the best bit of info... Also, to or from ? And it should be a Period, no ? mapping_function + mapping_data # TODO this should be internal end struct SimulationTimestepHandler#{W,V} - model_timesteps::Dict{Any, Int} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range + model_timesteps::Dict{Any, Period} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range timestep_variable_mapping::Dict{Any, TimestepMapper} #where {V} end -SimulationTimestepHandler() = SimulationTimestepHandler(Dict(), Dict()) #Dict{W, Int}(), Dict{V, TimestepMapper}()) where {W, V} +SimulationTimestepHandler() = SimulationTimestepHandler(Dict{Any, Int}(), Dict{Any, TimestepMapper}()) #Dict{W, Int}(), Dict{V, TimestepMapper}()) where {W, V} mutable struct Orchestrator # This is actually a general simulation parameter, not-scale specific # todo change to Period - default_timestep::Int64 + default_timestep::Period # This needs to be per-scale : if a model is used at two different scales, # and the same variable of that model maps to some other timestep to two *different* variables # then I believe we can only rely on the different scale to disambiguate non_default_timestep_data_per_scale::Dict{String, SimulationTimestepHandler} - function Orchestrator(default::Int64, per_scale::Dict{String, SimulationTimestepHandler}) - @assert default >= 0 "The default_timestep should be greater than or equal to 0." + function Orchestrator(default::Period, per_scale::Dict{String, SimulationTimestepHandler}) + @assert default >= Second(0) "The default_timestep should be greater than or equal to 0." return new(default, per_scale) end end # TODO have a default constructor take in a meteo or something, and set up the default timestep automagically to be the finest weather timestep # Other options are possible -Orchestrator() = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) +Orchestrator() = Orchestrator(Day(1), Dict{String, SimulationTimestepHandler}()) #o = Orchestrator() -#oo = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) \ No newline at end of file +#oo = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) + + +# TODO issue : what if the user wants to force initialize a variable that isn't at the default timestep ? +# This requires the ability to fit data with a vector that isn't at the default timestep +# Automated model generation does not have that feature +# As is, the current workaround is for the user to write their own model, I think, which is not ideal + +# TODO check for cycles (and other errors) before timestep mapping, then do it again afterwards, as the new mapping dependencies might cause specific cycles. + + +# TODO status initialisation ? +# TODO type promotion for mapped timestep variables ? +# TODO check type if a variable is timestep mapped and scale mapped +# TODO simulation_id change consequences ? + + + +# TODO prev timestep ? Vector mapping ? +struct Var_from + model + scale::String + name::Symbol + mapping_function::Function + # mapping_data::Dict{NodeMTG, Vector{stuff}} +end + +struct Var_to + name::Symbol +end + +struct ModelTimestepMapping + model + scale::String + timestep::Period + var_to_var::Dict{Var_to, Var_from} +end + +mutable struct Orchestrator2 + default_timestep::Period + non_default_timestep_mapping::Vector{ModelTimestepMapping} + + function Orchestrator2(default::Period, non_default_timestep_mapping::Vector{ModelTimestepMapping}) + @assert default >= Second(0) "The default_timestep should be greater than or equal to 0." + return new(default, non_default_timestep_mapping) + end +end + +Orchestrator2() = Orchestrator2(Day(1), Vector{ModelTimestepMapping}()) + + +# TODO parallelization, + +function init_timestep_mapping_data(node_mtg::MultiScaleTreeGraph.Node, dependency_graph) + traverse_dependency_graph!(x -> register_mtg_node_in_timestep_mapping(x, node_mtg), dependency_graph, visit_hard_dep=false) +end + +function register_mtg_node_in_timestep_mapping(node_dep::SoftDependencyNode, node_mtg::MultiScaleTreeGraph.Node) + if isnothing(node_dep.timestep_mapping_data) + return + end + + # no need to check the current softdependencynode's scale, I think + # only the mapped downstream softdependencynodes + + # TODO this structure doesn't play well with parallelisation... ? + # TODO having an extra level of indirection, mapping the MTG node to an index into a vector + # Allows one to resize! the vector when it lacks space, saving in terms of # of memory allocations/copies + for mtsm in node_dep.timestep_mapping_data + if node_dep.scale == symbol(node_mtg) + push!(mtsm.mapping_data, node_id(node_mtg) => deepcopy(mtsm.mapping_data_template)) + end + end +end \ No newline at end of file diff --git a/test/test_multitimestep.jl b/test/test_multitimestep.jl new file mode 100644 index 00000000..9e25f1ce --- /dev/null +++ b/test/test_multitimestep.jl @@ -0,0 +1,457 @@ + +########################### +# Simple test using an ad hoc connector model +# Broken by subsequent changes, left just in case for now (TODO remove once prototyping is over) +########################### + +#= +using PlantSimEngine +# Include the example dummy processes: +using PlantSimEngine.Examples +using Test, Aqua +using Tables, DataFrames, CSV +using MultiScaleTreeGraph +using PlantMeteo, Statistics +using Documenter # for doctests + +using PlantMeteo.Dates +include("helper-functions.jl") + + + +# These models might be worth exposing in the future ? +PlantSimEngine.@process "basic_current_timestep" verbose = false + +struct HelperCurrentTimestepModel <: AbstractBasic_Current_TimestepModel +end + +PlantSimEngine.inputs_(::HelperCurrentTimestepModel) = (next_timestep=1,) +PlantSimEngine.outputs_(m::HelperCurrentTimestepModel) = (current_timestep=1,) + +function PlantSimEngine.run!(m::HelperCurrentTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.current_timestep = status.next_timestep + end + + PlantSimEngine.ObjectDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsObjectDependent() + PlantSimEngine.TimeStepDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsTimeStepDependent() + +PlantSimEngine.timestep_range_(m::HelperCurrentTimestepModel) = Day(1) + + + PlantSimEngine.@process "basic_next_timestep" verbose = false + struct HelperNextTimestepModel <: AbstractBasic_Next_TimestepModel + end + + PlantSimEngine.inputs_(::HelperNextTimestepModel) = (current_timestep=1,) + PlantSimEngine.outputs_(m::HelperNextTimestepModel) = (next_timestep=1,) + + function PlantSimEngine.run!(m::HelperNextTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.next_timestep = status.current_timestep + 1 + end + +PlantSimEngine.timestep_range_(m::HelperNextTimestepModel) = Day(1) + + + + + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=-Inf,) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = Week(1) + + + +PlantSimEngine.@process "DWConnector" verbose = false + +struct MyDwconnectorModel <: AbstractDwconnectorModel + T_daily::Array{Float64} +end + +MyDwconnectorModel() = MyDwconnectorModel(Array{Float64}(undef, 7)) + +function PlantSimEngine.inputs_(::MyDwconnectorModel) + (daily_temperature=-Inf, current_timestep=1,) +end +PlantSimEngine.outputs_(m::MyDwconnectorModel) = (weekly_max_temperature = 0.0,) + +function PlantSimEngine.run!(m::MyDwconnectorModel, models, status, meteo, constants=nothing, extra=nothing) + m.T_daily[1 + (status.current_timestep % 7)] = status.daily_temperature + + if(status.current_timestep % 7 == 1) + status.weekly_max_temperature = sum(m.T_daily)/7.0 + else + status.weekly_max_temperature = 0 + end +end + + PlantSimEngine.timestep_range_(m::MyDwconnectorModel) = Day(1) + + + + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m = Dict("Default" => ( + MyToyDayModel(), + MyToyWeekModel(), + MyDwconnectorModel(), + HelperNextTimestepModel(), + MultiScaleModel( + model=HelperCurrentTimestepModel(), + mapped_variables=[PreviousTimeStep(:next_timestep),], + ), + Status(a=1,))) + +to_initialize(m) + +models_timestep = Dict(MyToyDayModel=>1, MyDwconnectorModel => 1, MyToyWeekModel =>7, HelperNextTimestepModel => 1, HelperCurrentTimestepModel => 1) + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) + +out = run!(mtg, m, meteo_day, default_timestep=1, model_timesteps=models_timestep) + + +# NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used +# otherwise there might be vector length conflicts with timesteps +sim = PlantSimEngine.GraphSimulation(mtg, m, nsteps=nothing, check=true, outputs=nothing, default_timestep=1, model_timesteps=models_timestep) + +=# + + +########################### +# First attempt at an orchetrator, broken by subsequent changes +########################### +#= +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=-Inf,) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +m = Dict("Default" => ( + MyToyDayModel(), + MyToyWeekModel(), + Status(a=1,))) + + meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) + + + model_timesteps_defaultscale = Dict(MyToyWeekModel =>Week(1)) + tsm = PlantSimEngine.TimestepMapper(:daily_temperature, Day(1), max, nothing) + sth = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:weekly_max_temperature => tsm )) + + orchestrator = Orchestrator(Day(1), Dict("Default" => sth)) + + #out = @enter run!(mtg, m, meteo_day, orchestrator=orchestrator) + +# TODO could some mapping happen automatically for variables directly taken from weather data ? +# Does this happen often in a typical model ? + +#=m_multiscale = Dict("Default" => ( + MyToyDayModel(), + Status(a=1,) + ), + "Default2" => ( + MyToyWeekModel(), + ),) +=# +m_multiscale = Dict("Default" => ( + MyToyDayModel(), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + + orchestrator_multiscale = Orchestrator(Day(1), Dict("Default2" => sth)) + +#out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orchestrator_multiscale) +=# + + +########################### +# Simple test with a second attempt at an orchestrator +# Functional, except for the revalue/refvector overwriting issues +########################### +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=-Inf,) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + + meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + + model_timesteps_defaultscale = Dict(MyToyWeekModel =>Week(1)) + tsm = PlantSimEngine.TimestepMapper(:daily_temperature, Day(1), max, nothing) + sth = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:weekly_max_temperature => tsm )) + +m_multiscale = Dict("Default" => ( + MyToyDayModel(), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + + +to = PlantSimEngine.Var_to(:weekly_max_temperature) +from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) + +dict_to_from = Dict(to => from) +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) + +orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) + +out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + + +########################### +# Test with three timesteps, multiscale +# Issues with data overwriting (refvector/refvalue) +########################### + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay2" verbose = false + +struct MyToyDay2Model <: AbstractToyday2Model end + +PlantSimEngine.inputs_(m::MyToyDay2Model) = NamedTuple() +PlantSimEngine.outputs_(m::MyToyDay2Model) = (out_day=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day = meteo.data +end + +PlantSimEngine.@process "ToyWeek2" verbose = false + +struct MyToyWeek2Model <: AbstractToyweek2Model end + +PlantSimEngine.inputs_(::MyToyWeek2Model) = (in_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyWeek2Model) = (out_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_week = status.in_week +end + +PlantSimEngine.timestep_range_(m::MyToyWeek2Model) = TimestepRange(Week(1)) + + +PlantSimEngine.@process "ToyFourWeek2" verbose = false + +struct MyToyFourWeek2Model <: AbstractToyfourweek2Model end + +PlantSimEngine.inputs_(::MyToyFourWeek2Model) = (in_four_week_from_week=-Inf, in_four_week_from_day=-Inf,) +PlantSimEngine.outputs_(m::MyToyFourWeek2Model) = (inputs_agreement=false,) + +function PlantSimEngine.run!(m::MyToyFourWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.inputs_agreement = status.in_four_week_from_week == status.in_four_week_from_day +end + +PlantSimEngine.timestep_range_(m::MyToyFourWeek2Model) = TimestepRange(Week(4)) + + + +df = DataFrame(:data => [1 for i in 1:365], ) + + # TODO can make this optional if the timestep range is actually a single value + model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) + sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) + + tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) + sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) + + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(to_w => from_d) + + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default2", :out_week, sum) + + dict_to_from_w4 = Dict(to_w4_d => from_d, to_w4_w => from_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) + + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + + +m_multiscale = Dict("Default" => ( + MyToyDay2Model(), + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[:in_week => "Default" => :out_day], + ), + ), + "Default3" => ( + MultiScaleModel(model=MyToyFourWeek2Model(), + mapped_variables=[ + :in_four_week_from_day => "Default" => :out_day, + :in_four_week_from_week => "Default2" => :out_week, + ], + ), + ),) + + +# TODO test with multiple nodes +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) +#out = @enter run!(mtg, m_multiscale, df, orchestrator=orch2) + + + +########################### +# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting +# and explore alternatives +# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) +########################### + + m_singlescale = Dict("Default" => ( + MyToyDay2Model(), + MyToyWeek2Model(), + MyToyFourWeek2Model(), + ),) + + + model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) + sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) + + tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) + sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) + + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(to_w => from_d) + + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default", :out_week, sum) + + dict_to_from_w4 = Dict(to_w4_d => from_d, to_w4_w => from_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) + + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + +mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) + + + +using Test + @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [0.0, 28.0] + @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [0.0, 7.0] + @test unique([out["Default3"][i].inputs_agreement for i in 1:length(out["Default3"])]) == [1] + + unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) + unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) + unique([out["Default3"][i].inputs_agreement for i in 1:length(out["Default3"])]) \ No newline at end of file From 2d343f666a99fc04bebe225fac861fb03a448bbe Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Tue, 28 Oct 2025 15:13:02 +0100 Subject: [PATCH 04/21] Addendum : prototype tests moved to a new file, remove them from the old file. --- test/test-simulation.jl | 151 +--------------------------------------- 1 file changed, 1 insertion(+), 150 deletions(-) diff --git a/test/test-simulation.jl b/test/test-simulation.jl index c4439308..4ff83a64 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -292,153 +292,4 @@ end; end end end -end - - - - -using PlantSimEngine -# Include the example dummy processes: -using PlantSimEngine.Examples -using Test, Aqua -using Tables, DataFrames, CSV -using MultiScaleTreeGraph -using PlantMeteo, Statistics -using Documenter # for doctests - -using PlantMeteo.Dates -include("helper-functions.jl") - - - -# These models might be worth exposing in the future ? -PlantSimEngine.@process "basic_current_timestep" verbose = false - -struct HelperCurrentTimestepModel <: AbstractBasic_Current_TimestepModel -end - -PlantSimEngine.inputs_(::HelperCurrentTimestepModel) = (next_timestep=1,) -PlantSimEngine.outputs_(m::HelperCurrentTimestepModel) = (current_timestep=1,) - -function PlantSimEngine.run!(m::HelperCurrentTimestepModel, models, status, meteo, constants=nothing, extra=nothing) - status.current_timestep = status.next_timestep - end - - PlantSimEngine.ObjectDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsObjectDependent() - PlantSimEngine.TimeStepDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsTimeStepDependent() - -PlantSimEngine.timestep_range_(m::HelperCurrentTimestepModel) = Day(1) - - - PlantSimEngine.@process "basic_next_timestep" verbose = false - struct HelperNextTimestepModel <: AbstractBasic_Next_TimestepModel - end - - PlantSimEngine.inputs_(::HelperNextTimestepModel) = (current_timestep=1,) - PlantSimEngine.outputs_(m::HelperNextTimestepModel) = (next_timestep=1,) - - function PlantSimEngine.run!(m::HelperNextTimestepModel, models, status, meteo, constants=nothing, extra=nothing) - status.next_timestep = status.current_timestep + 1 - end - -PlantSimEngine.timestep_range_(m::HelperNextTimestepModel) = Day(1) - - - - - -PlantSimEngine.@process "ToyDay" verbose = false - -struct MyToyDayModel <: AbstractToydayModel end - -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) - -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end - -PlantSimEngine.@process "ToyWeek" verbose = false - -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end - -MyToyWeekModel() = MyToyWeekModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=-Inf,) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) - -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end - -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = Week(1) - - - -PlantSimEngine.@process "DWConnector" verbose = false - -struct MyDwconnectorModel <: AbstractDwconnectorModel - T_daily::Array{Float64} -end - -MyDwconnectorModel() = MyDwconnectorModel(Array{Float64}(undef, 7)) - -function PlantSimEngine.inputs_(::MyDwconnectorModel) - (daily_temperature=-Inf, current_timestep=1,) -end -PlantSimEngine.outputs_(m::MyDwconnectorModel) = (weekly_max_temperature = 0.0,) - -function PlantSimEngine.run!(m::MyDwconnectorModel, models, status, meteo, constants=nothing, extra=nothing) - m.T_daily[1 + (status.current_timestep % 7)] = status.daily_temperature - - if(status.current_timestep % 7 == 1) - status.weekly_max_temperature = sum(m.T_daily)/7.0 - else - status.weekly_max_temperature = 0 - end -end - - PlantSimEngine.timestep_range_(m::MyDwconnectorModel) = Day(1) - - - - - -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - -m = Dict("Default" => ( - MyToyDayModel(), - MyToyWeekModel(), - MyDwconnectorModel(), - HelperNextTimestepModel(), - MultiScaleModel( - model=HelperCurrentTimestepModel(), - mapped_variables=[PreviousTimeStep(:next_timestep),], - ), - Status(a=1,))) - -to_initialize(m) - -models_timestep = Dict(MyToyDayModel=>1, MyDwconnectorModel => 1, MyToyWeekModel =>7, HelperNextTimestepModel => 1, HelperCurrentTimestepModel => 1) - -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) - -out = run!(mtg, m, meteo_day, default_timestep=1, model_timesteps=models_timestep) - -@testset "Test varying timestep" begin - - - @test - @test - -end - - - # NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used - # otherwise there might be vector length conflicts with timesteps - sim = @enter PlantSimEngine.GraphSimulation(mtg, m, nsteps=nothing, check=true, outputs=nothing, default_timestep=1, model_timesteps=models_timestep) - -using PlantSimEngine +end \ No newline at end of file From da65ff7399e331e6834d80b4a5765d394129f765 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Fri, 31 Oct 2025 14:44:48 +0100 Subject: [PATCH 05/21] Fix little issues + attempt at addressing main blocker : disconnect timestep-mapped variables from their source, to avoid overwriting source (inputs to non-default-timestep models are changed by the accumulation function so can't be a simple Ref to source) --- src/dependencies/dependency_graph.jl | 78 -------------- src/dependencies/hard_dependencies.jl | 88 +++++++++++++++- src/dependencies/soft_dependencies.jl | 2 +- src/mtg/initialisation.jl | 50 ++++++--- src/mtg/mapping/compute_mapping.jl | 20 +++- src/mtg/save_results.jl | 10 +- src/run.jl | 7 +- src/timestep/timestep_mapping.jl | 2 +- test/test_multitimestep.jl | 142 ++++++++++++++++++-------- 9 files changed, 249 insertions(+), 150 deletions(-) diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 33b5fc62..59ec29f5 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -89,82 +89,4 @@ function Base.show(io::IO, ::MIME"text/plain", t::DependencyGraph) else draw_dependency_graph(io, t) end -end - -""" - variables_multiscale(node, organ, mapping, st=NamedTuple()) - -Get the variables of a HardDependencyNode, taking into account the multiscale mapping, *i.e.* -defining variables as `MappedVar` if they are mapped to another scale. The default values are -taken from the model if not given by the user (`st`), and are marked as `UninitializedVar` if -they are inputs of the node. - -Return a NamedTuple with the variables and their default values. - -# Arguments - -- `node::HardDependencyNode`: the node to get the variables from. -- `organ::String`: the organ type, *e.g.* "Leaf". -- `vars_mapping::Dict{String,T}`: the mapping of the models (see details below). -- `st::NamedTuple`: an optional named tuple with default values for the variables. - -# Details - -The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is -computed from the user mapping like so: -""" -function variables_multiscale(node, organ, vars_mapping, st=NamedTuple()) - node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,)) - ins = node_vars.inputs - ins_variables = keys(ins) - outs_variables = keys(node_vars.outputs) - defaults = merge(node_vars...) - map((inputs=ins_variables, outputs=outs_variables)) do vars # Map over vars from :inputs and vars from :outputs - vars_ = Vector{Pair{Symbol,Any}}() - for var in vars # e.g. var = :carbon_biomass - if var in keys(st) - #If the user has given a status, we use it as default value. - default = st[var] - elseif var in ins_variables - # Otherwise, we use the default value given by the model: - # If the variable is an input, we mark it as uninitialized: - default = UninitializedVar(var, defaults[var]) - else - # If the variable is an output, we use the default value given by the model: - default = defaults[var] - end - - if haskey(vars_mapping[organ], var) - organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var]) - push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default)) - #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current - #* values, and another one the previous values. - if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) - organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) - push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) - end - elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) - # If not found in the current time step, we check if the variable is mapped to the previous time step: - organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) - push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) - else - # Else we take the default value: - push!(vars_, var => default) - end - end - return (; vars_...,) - end -end - -function _node_mapping(var_mapping::Pair{String,Symbol}) - # One organ is mapped to the variable: - return SingleNodeMapping(first(var_mapping)), last(var_mapping) -end - -function _node_mapping(var_mapping) - # Several organs are mapped to the variable: - organ_mapped = MultiNodeMapping([first(i) for i in var_mapping]) - organ_mapped_var = [last(i) for i in var_mapping] - - return organ_mapped, organ_mapped_var end \ No newline at end of file diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index eda0573f..04fb16f4 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -110,6 +110,92 @@ function initialise_all_as_hard_dependency_node(models, scale) return dep_graph end +# Samuel : this requires the orchestrator, which requires the dependency graph +# Leaving it in dependency_graph.jl causes forward declaration issues, moving it here as a quick protoyping hack, it might not be the ideal spot +""" + variables_multiscale(node, organ, mapping, st=NamedTuple()) + +Get the variables of a HardDependencyNode, taking into account the multiscale mapping, *i.e.* +defining variables as `MappedVar` if they are mapped to another scale. The default values are +taken from the model if not given by the user (`st`), and are marked as `UninitializedVar` if +they are inputs of the node. + +Return a NamedTuple with the variables and their default values. + +# Arguments + +- `node::HardDependencyNode`: the node to get the variables from. +- `organ::String`: the organ type, *e.g.* "Leaf". +- `vars_mapping::Dict{String,T}`: the mapping of the models (see details below). +- `st::NamedTuple`: an optional named tuple with default values for the variables. + +# Details + +The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is +computed from the user mapping like so: +""" +function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator2=Orchestrator2()) + node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,)) + ins = node_vars.inputs + ins_variables = keys(ins) + outs_variables = keys(node_vars.outputs) + defaults = merge(node_vars...) + map((inputs=ins_variables, outputs=outs_variables)) do vars # Map over vars from :inputs and vars from :outputs + vars_ = Vector{Pair{Symbol,Any}}() + for var in vars # e.g. var = :carbon_biomass + if var in keys(st) + #If the user has given a status, we use it as default value. + default = st[var] + elseif var in ins_variables + # Otherwise, we use the default value given by the model: + # If the variable is an input, we mark it as uninitialized: + default = UninitializedVar(var, defaults[var]) + else + # If the variable is an output, we use the default value given by the model: + default = defaults[var] + end + + # TODO no idea how this meshes with refvector situations or previoustimestep + if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true) + + push!(vars_, var => default) + else + + if haskey(vars_mapping[organ], var) + organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var]) + push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default)) + #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current + #* values, and another one the previous values. + if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) + organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) + push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) + end + elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) + # If not found in the current time step, we check if the variable is mapped to the previous time step: + organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) + push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) + else + # Else we take the default value: + push!(vars_, var => default) + end + end + end + return (; vars_...,) + end +end + +function _node_mapping(var_mapping::Pair{String,Symbol}) + # One organ is mapped to the variable: + return SingleNodeMapping(first(var_mapping)), last(var_mapping) +end + +function _node_mapping(var_mapping) + # Several organs are mapped to the variable: + organ_mapped = MultiNodeMapping([first(i) for i in var_mapping]) + organ_mapped_var = [last(i) for i in var_mapping] + + return organ_mapped, organ_mapped_var +end # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T} @@ -148,7 +234,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr status_scale = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() for (procname, node) in hard_deps[organ].roots # procname = :leaf_surface ; node = hard_deps.roots[procname] var = Pair{Symbol,NamedTuple}[] - traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_vars_mapping, st_scale_user), var) + traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_vars_mapping, st_scale_user, orchestrator), var) push!(status_scale, procname => var) end diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 241a63bf..135e32b6 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -357,7 +357,7 @@ function add_timestep_data_to_node(soft_dependency_node, orchestrator::Orchestra # now we can create the mapping for mtsm in orchestrator.non_default_timestep_mapping if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value)) - for (var_to, var_from) in mtsm.var_to_var + for (var_from, var_to) in mtsm.var_to_var if !isnothing(soft_dependency_node.parent) parent = nothing variable_mapping = nothing diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index eac56d0c..a4004ebe 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -35,18 +35,23 @@ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(ma # Note 3: we do it before `convert_reference_values!` because we need the variables to be MappedVar{MultiNodeMapping} to get the reverse mapping. # Convert the MappedVar{SelfNodeMapping} or MappedVar{SingleNodeMapping} to RefValues, and MappedVar{MultiNodeMapping} to RefVectors: - convert_reference_values!(mapped_vars) + convert_reference_values!(mapped_vars, orchestrator) # Get the variables that are not initialised or computed by other models in the output: vars_need_init = Dict(org => filter(x -> isa(last(x), UninitializedVar), vars) |> keys |> collect for (org, vars) in mapped_vars) |> filter(x -> length(last(x)) > 0) # Filter out variables that are timestep-mapped by the user, - # as those variables are initialized by another model, but are currently flagged as needing initialization + # Since we disconnect outputs from the source variable (as values are changed by the accumulation function, + # meaning they differ and we can't just Ref point to the source variable) + # they will be currently flagged as needing initialization # At this stage, data present in the orchestrator is expected to be valid, so we can take it into account # A model with a different timestep can still have unitialized vars found in a node, the meteo, or to be initialized by the user # in which case it'll be absent from the timestep mapping, but this needs testing - #filter_timestep_mapped_variables!(vars_need_init, orchestrator) + # TODO, *however*, this isn't the cleanest in its current state, + # there may be some user initialisation issues that are hidden by this approach + # Needs to be checked + filter_timestep_mapped_variables!(vars_need_init, orchestrator) # Note: these variables may be present in the MTG attributes, we check that below when traversing the MTG. @@ -54,7 +59,7 @@ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(ma # We traverse the MTG to initialise the statuses linked to the nodes: statuses = Dict(i => Status[] for i in collect(keys(mapped_vars))) MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = MultiScaleTreeGraph.get_node(mtg, 5) - init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init, type_promotion, check=check) + init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init, type_promotion, check=check, orchestrator=orchestrator) end return (; statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init) @@ -153,7 +158,7 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi end # Make the node status from the template: - st = status_from_template(st_template) + st = status_from_template(st_template, symbol(node), orchestrator) push!(statuses[symbol(node)], st) @@ -209,17 +214,23 @@ julia> b 2.0 ``` """ -function status_from_template(d::Dict{Symbol,T} where {T}) +function status_from_template(d::Dict{Symbol,T} where {T}, scale::String, orchestrator::Orchestrator2) # Sort vars to put the values that are of type PerStatusRef at the end (we need the pass on the other ones first): sorted_vars = Dict{Symbol,Any}(sort([pairs(d)...], by=v -> last(v) isa RefVariable ? 1 : 0)) # Note: PerStatusRef are used to reference variables in the same status for renaming. # We create the status with the right references for variables, and for PerStatusRef (we reference the reference variable): for (k, v) in sorted_vars - if isa(v, RefVariable) - sorted_vars[k] = sorted_vars[v.reference_variable] - else + if is_timestep_mapped((scale => k), orchestrator, search_inputs_only=true) + # avoid referring to the original variable sorted_vars[k] = ref_var(v) + else + + if isa(v, RefVariable) + sorted_vars[k] = sorted_vars[v.reference_variable] + else + sorted_vars[k] = ref_var(v) + end end end @@ -423,17 +434,22 @@ function filter_timestep_mapped_variables!(vars_need_init, orchestrator) end=# function filter_timestep_mapped_variables!(vars_need_init, orchestrator) - for (org, vars) in vars_need_init - for tmst in orchestrator.non_default_timestep_mapping + for tmst in orchestrator.non_default_timestep_mapping + for (org, vars) in vars_need_init if tmst.scale == org - - for (var_to, var_from) in tmst.var_to_var - filter!(o -> o == var_to.name || o == var_from.name, vars) - end - if isempty(vars) - delete!(vars_need_init, org) + for (var_from, var_to) in tmst.var_to_var + filter!(o -> o == var_to.name, vars) + end + end + for (var_from, var_to) in tmst.var_to_var + if var_from.scale == org + filter!(o -> o == var_from.name, vars) end end + + if isempty(vars) + delete!(vars_need_init, org) + end end end end \ No newline at end of file diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index 029b9c3d..872fa67e 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -279,6 +279,18 @@ function default_variables_from_mapping(mapped_vars, verbose=true) end +function is_timestep_mapped(key, orchestrator::Orchestrator2; search_inputs_only::Bool=false) + for mtsm in orchestrator.non_default_timestep_mapping + for (var_from, var_to) in mtsm.var_to_var + if ((first(key) == mtsm.scale) && last(key) == var_to.name) || + (!search_inputs_only && (first(key) == var_from.scale && last(key) == var_from.name)) + return true + end + end + end + return false +end + """ convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}) @@ -286,7 +298,7 @@ Convert the variables that are `MappedVar{SelfNodeMapping}` or `MappedVar{Single common value for the variable; and convert `MappedVar{MultiNodeMapping}` to RefVectors that reference the values for the variable in the source organs. """ -function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}) +function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}, orchestrator::Orchestrator2) # For the variables that will be RefValues, i.e. referencing a value that exists for different scales, we need to first # create a common reference to the value that we use wherever we need this value. These values are created in the dict_mapped_vars # Dict, and then referenced from there every time we point to it. @@ -303,7 +315,11 @@ function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}) # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: if !haskey(dict_mapped_vars, key) - push!(dict_mapped_vars, key => Ref(mapped_default(vars[k]))) +# if is_timestep_mapped(key, orchestrator) +# push!(dict_mapped_vars, key => (mapped_default(vars[k]))) +# else + push!(dict_mapped_vars, key => Ref(mapped_default(vars[k]))) +# end end # Then we use the value for the particular variable to replace the MappedVar to a RefValue in the mapping: diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 51baffcb..52a94ba8 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -109,8 +109,8 @@ julia> collect(keys(preallocated_vars["Leaf"])) :carbon_demand ``` """ - -function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_mapping, vars_need_init, outs, nsteps; type_promotion=nothing, check=true) +# TODO orchestrator prob shouldn't be a kwarg with a default +function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_mapping, vars_need_init, outs, nsteps; type_promotion=nothing, check=true, orchestrator=Orchestrator2()) outs_ = Dict{String,Vector{Symbol}}() # default behaviour : track everything @@ -211,18 +211,18 @@ function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_ma # I don't know if this function barrier is necessary preallocated_outputs = Dict{String,Vector}() - complete_preallocation_from_types!(preallocated_outputs, nsteps, outs_, node_type, statuses_template) + complete_preallocation_from_types!(preallocated_outputs, nsteps, outs_, node_type, statuses_template, orchestrator) return preallocated_outputs end -function complete_preallocation_from_types!(preallocated_outputs, nsteps, outs_, node_type, statuses_template) +function complete_preallocation_from_types!(preallocated_outputs, nsteps, outs_, node_type, statuses_template, orchestrator) types = Vector{DataType}() for organ in keys(outs_) outs_no_node = filter(x -> x != :node, outs_[organ]) #types = [typeof(status_from_template(statuses_template[organ])[var]) for var in outs[organ]] - values = [status_from_template(statuses_template[organ])[var] for var in outs_no_node] + values = [status_from_template(statuses_template[organ], organ, orchestrator)[var] for var in outs_no_node] #push!(types, node_type) diff --git a/src/run.jl b/src/run.jl index 8a62a200..3bac7595 100644 --- a/src/run.jl +++ b/src/run.jl @@ -454,7 +454,11 @@ function run_node_multiscale!( # Check if the model needs to run this timestep if (1 + (i-1) % node_ratio) != node_ratio - node.simulation_id[1] += 1 + + # TODO : This does prevent the node form being updated by two different parents but probably should be changed + if any([p.simulation_id[1] >= node.simulation_id[1] for p in node.parent]) + node.simulation_id[1] += 1 + end return nothing end @@ -468,6 +472,7 @@ function run_node_multiscale!( node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale models_at_scale = models[node.scale] + # TODO : is empty check should be pre simulation if isnothing(node.timestep_mapping_data) || isempty(node.timestep_mapping_data) # Samuel : this is the happy path, no further timestep mapping checks needed diff --git a/src/timestep/timestep_mapping.jl b/src/timestep/timestep_mapping.jl index 20ad038e..06d40d01 100644 --- a/src/timestep/timestep_mapping.jl +++ b/src/timestep/timestep_mapping.jl @@ -78,7 +78,7 @@ struct ModelTimestepMapping model scale::String timestep::Period - var_to_var::Dict{Var_to, Var_from} + var_to_var::Dict{Var_from, Var_to} end mutable struct Orchestrator2 diff --git a/test/test_multitimestep.jl b/test/test_multitimestep.jl index 9e25f1ce..b9be19da 100644 --- a/test/test_multitimestep.jl +++ b/test/test_multitimestep.jl @@ -297,14 +297,65 @@ mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) to = PlantSimEngine.Var_to(:weekly_max_temperature) from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) -dict_to_from = Dict(to => from) +dict_to_from = Dict(from => to) mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) -out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +########################### +# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting +# and explore alternatives +# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) +########################### + + m_singlescale = Dict("Default" => ( + MyToyDay2Model(), + MyToyWeek2Model(), + MyToyFourWeek2Model(), + ),) + + + model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) + sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) + + tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) + sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) + + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(to_w => from_d) + + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default", :out_week, sum) + + dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) + + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + +mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) + + + + + + + + + + + + + + ########################### # Test with three timesteps, multiscale # Issues with data overwriting (refvector/refvalue) @@ -326,6 +377,28 @@ function PlantSimEngine.run!(m::MyToyDay2Model, models, status, meteo, constants status.out_day = meteo.data end +#=PlantSimEngine.@process "ToyDay3" verbose = false + +struct MyToyDay3Model <: AbstractToyday3Model end + +PlantSimEngine.inputs_(m::MyToyDay3Model) = (in_day=-Inf, in_day_summed_prev_timestep=-Inf) +PlantSimEngine.outputs_(m::MyToyDay3Model) = (out_day_summed=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay3Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day_summed = status.in_day + status.in_day_summed_prev_timestep +end + +PlantSimEngine.@process "ToyDay4" verbose = false + +struct MyToyDay4Model <: AbstractToyday4Model end + +PlantSimEngine.inputs_(m::MyToyDay4Model) = (in_day_summed=-Inf,) +PlantSimEngine.outputs_(m::MyToyDay4Model) = (out_day_summed_2= -Inf,) + +function PlantSimEngine.run!(m::MyToyDay4Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day_summed_2 = status.in_day_summed +end=# + PlantSimEngine.@process "ToyWeek2" verbose = false struct MyToyWeek2Model <: AbstractToyweek2Model end @@ -367,13 +440,13 @@ df = DataFrame(:data => [1 for i in 1:365], ) to_w = PlantSimEngine.Var_to(:in_week) from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) - dict_to_from_w = Dict(to_w => from_d) + dict_to_from_w = Dict(from_d => to_w) to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default2", :out_week, sum) - dict_to_from_w4 = Dict(to_w4_d => from_d, to_w4_w => from_w) + dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) @@ -395,55 +468,36 @@ m_multiscale = Dict("Default" => ( :in_four_week_from_day => "Default" => :out_day, :in_four_week_from_week => "Default2" => :out_week, ], + ),), + #="Default4"=> ( + MultiScaleModel( + model=MyToyDay3Model(), + mapped_variables=[ + PlantSimEngine.PreviousTimeStep(:in_day_summed_prev_timestep) => "Default5" => :out_day_summed_2, + :in_day => "Default" => :out_day, + ]), + Status(in_day_summed_prev_timestep=0,) ), - ),) + "Default5" => ( + MultiScaleModel(model=MyToyDay4Model(), + mapped_variables= [:in_day_summed => "Default4" => :out_day_summed], + ), + Status(in_day_summed=0,out_day_summed_2=0) + ),=# + ) # TODO test with multiple nodes mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) -#out = @enter run!(mtg, m_multiscale, df, orchestrator=orch2) - - - -########################### -# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting -# and explore alternatives -# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) -########################### - - m_singlescale = Dict("Default" => ( - MyToyDay2Model(), - MyToyWeek2Model(), - MyToyFourWeek2Model(), - ),) - - - model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) - tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) - sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) +#mtg4 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default4", 1, 4)) +#mtg5 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default5", 1, 5)) - tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) - sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) - - to_w = PlantSimEngine.Var_to(:in_week) - from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) - dict_to_from_w = Dict(to_w => from_d) + #orch2 = PlantSimEngine.Orchestrator2() - to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) - to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) - from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default", :out_week, sum) - - dict_to_from_w4 = Dict(to_w4_d => from_d, to_w4_w => from_w) - - mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) - mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) - - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) - -mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) +out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) +#out = run!(mtg, m_multiscale, df, orchestrator=orch2) From d964abdb820d537dbdbd13ec6a2c6ac9c047e75e Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Fri, 31 Oct 2025 15:43:36 +0100 Subject: [PATCH 06/21] Fix issues with simulation id and models not running properly. Many loose ends to investigate --- src/run.jl | 119 ++++++++++++++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/src/run.jl b/src/run.jl index 3bac7595..42ee7355 100644 --- a/src/run.jl +++ b/src/run.jl @@ -453,77 +453,76 @@ function run_node_multiscale!( node_ratio = node.timestep / Day(1) # Check if the model needs to run this timestep - if (1 + (i-1) % node_ratio) != node_ratio + if (1 + (i - 1) % node_ratio) != node_ratio # TODO : This does prevent the node form being updated by two different parents but probably should be changed - if any([p.simulation_id[1] >= node.simulation_id[1] for p in node.parent]) + if any([p.simulation_id[1] > node.simulation_id[1] for p in node.parent]) node.simulation_id[1] += 1 end - return nothing - end - - # run!(status(object), dependency_node, meteo, constants, extra) - # Check if all the parents have been called before the child: - if !AbstractTrees.isroot(node) && any([p.simulation_id[1] <= node.simulation_id[1] for p in node.parent]) - # If not, this node should be called via another parent - return nothing - end + else - node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale - models_at_scale = models[node.scale] + # run!(status(object), dependency_node, meteo, constants, extra) + # Check if all the parents have been called before the child: + if !AbstractTrees.isroot(node) && any([p.simulation_id[1] <= node.simulation_id[1] for p in node.parent]) + # If not, this node should be called via another parent + return nothing + end - # TODO : is empty check should be pre simulation - if isnothing(node.timestep_mapping_data) || isempty(node.timestep_mapping_data) - # Samuel : this is the happy path, no further timestep mapping checks needed + node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale + models_at_scale = models[node.scale] - for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) - # Actual call to the model: - run!(node.value, models_at_scale, st, meteo, constants, extra) - end - node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already + # TODO : is empty check should be pre simulation + if isnothing(node.timestep_mapping_data) || isempty(node.timestep_mapping_data) + # Samuel : this is the happy path, no further timestep mapping checks needed - # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): - for child in node.children - #! check if we can run this safely in a @floop loop. I would say no, - #! because we are running a parallel computation above already, modifying the node.simulation_id, - #! which is not thread-safe yet. - run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) - end - else - # we have a child with a different timestep than the parent, - # which requires accumulation/reduce and running only some of the time - - for st in node_statuses - run!(node.value, models_at_scale, st, meteo, constants, extra) - # TODO do the accumulation - # then write into the child's status if need be ? - for tmst in node.timestep_mapping_data - ratio = Int(tmst.node_to.timestep / node.timestep) - # TODO assert etc. - # do the accumulation for each variable - accumulation_index = 1 + ((i-1)%ratio) - tmst.mapping_data[node_id(st.node)][accumulation_index] = st[tmst.variable_from] - - # if we have completed a *full* cycle, then presumably we need to write out the value to - # the mapped model - # A full cycle isn't just the ratio to the parent, it's the ratio to the finest-grained timestep - if accumulation_index == ratio - node_statuses_to = status(object)[tmst.node_to.scale] - - # TODO : INCORRECT in a scale with multiple mtg nodes - for st_to in node_statuses_to - # TODO might be able to catch mapping_function type incompatibility errors and make them clearer - st_to[tmst.variable_to] = tmst.mapping_function(tmst.mapping_data[node_id(st.node)]) + for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) + # Actual call to the model: + run!(node.value, models_at_scale, st, meteo, constants, extra) + end + node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already + + # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): + for child in node.children + #! check if we can run this safely in a @floop loop. I would say no, + #! because we are running a parallel computation above already, modifying the node.simulation_id, + #! which is not thread-safe yet. + run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + end + else + # we have a child with a different timestep than the parent, + # which requires accumulation/reduce and running only some of the time + + for st in node_statuses + run!(node.value, models_at_scale, st, meteo, constants, extra) + # TODO do the accumulation + # then write into the child's status if need be ? + for tmst in node.timestep_mapping_data + ratio = Int(tmst.node_to.timestep / node.timestep) + # TODO assert etc. This is all assuming the ratio is an integer, whereas it can be, like 1/7 + # do the accumulation for each variable + index = Int(i*Day(1) / node.timestep) + accumulation_index = 1 + ((index - 1) % ratio) + tmst.mapping_data[node_id(st.node)][accumulation_index] = st[tmst.variable_from] + + # if we have completed a *full* cycle, then presumably we need to write out the value to + # the mapped model + # A full cycle isn't just the ratio to the parent, it's the ratio to the finest-grained timestep + if accumulation_index == ratio + node_statuses_to = status(object)[tmst.node_to.scale] + + # TODO : INCORRECT in a scale with multiple mtg nodes + for st_to in node_statuses_to + # TODO might be able to catch mapping_function type incompatibility errors and make them clearer + st_to[tmst.variable_to] = tmst.mapping_function(tmst.mapping_data[node_id(st.node)]) + end end end - end - end - - node.simulation_id[1] += 1 + end - for child in node.children - - run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + node.simulation_id[1] += 1 end end + for child in node.children + run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor) + end end \ No newline at end of file From 80e12de17a5a9a3eafae9c04f149df5db68369ca Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Fri, 31 Oct 2025 16:29:39 +0100 Subject: [PATCH 07/21] Reintroduce error message which I may have unintentionally removed earlier --- src/mtg/initialisation.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index a4004ebe..1f604840 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -126,6 +126,7 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi end continue end + error("Variable `$(var)` is not computed by any model, not initialised by the user in the status, and not found in the MTG at scale $(symbol(node)) (checked for MTG node $(node_id(node))).") end # Applying the type promotion to the node attribute if needed: if isnothing(type_promotion) From 2ee5f85ee7dec5d948f1c8e1b100461e1b07fb22 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 3 Nov 2025 15:46:29 +0100 Subject: [PATCH 08/21] Fix bug in timestep-mapped variables filtering from var_need_init --- src/mtg/initialisation.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 1f604840..f738ee42 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -439,12 +439,12 @@ function filter_timestep_mapped_variables!(vars_need_init, orchestrator) for (org, vars) in vars_need_init if tmst.scale == org for (var_from, var_to) in tmst.var_to_var - filter!(o -> o == var_to.name, vars) + filter!(o -> o != var_to.name, vars) end end for (var_from, var_to) in tmst.var_to_var if var_from.scale == org - filter!(o -> o == var_from.name, vars) + filter!(o -> o != var_from.name, vars) end end From 70451054f11d92183a78d6d0c2dbae87b385055f Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 3 Nov 2025 16:25:25 +0100 Subject: [PATCH 09/21] Rename test file to conform properly to test file naming convention --- ...multitimestep.jl => test-multitimestep.jl} | 112 +----------------- 1 file changed, 6 insertions(+), 106 deletions(-) rename test/{test_multitimestep.jl => test-multitimestep.jl} (77%) diff --git a/test/test_multitimestep.jl b/test/test-multitimestep.jl similarity index 77% rename from test/test_multitimestep.jl rename to test/test-multitimestep.jl index b9be19da..5adf4659 100644 --- a/test/test_multitimestep.jl +++ b/test/test-multitimestep.jl @@ -143,98 +143,13 @@ sim = PlantSimEngine.GraphSimulation(mtg, m, nsteps=nothing, check=true, outputs =# - -########################### -# First attempt at an orchetrator, broken by subsequent changes -########################### -#= -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates - -PlantSimEngine.@process "ToyDay" verbose = false - -struct MyToyDayModel <: AbstractToydayModel end - -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) - -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end - -PlantSimEngine.@process "ToyWeek" verbose = false - -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end - -MyToyWeekModel() = MyToyWeekModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=-Inf,) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) - -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end - -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) - - -m = Dict("Default" => ( - MyToyDayModel(), - MyToyWeekModel(), - Status(a=1,))) - - meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) - - - model_timesteps_defaultscale = Dict(MyToyWeekModel =>Week(1)) - tsm = PlantSimEngine.TimestepMapper(:daily_temperature, Day(1), max, nothing) - sth = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:weekly_max_temperature => tsm )) - - orchestrator = Orchestrator(Day(1), Dict("Default" => sth)) - - #out = @enter run!(mtg, m, meteo_day, orchestrator=orchestrator) - # TODO could some mapping happen automatically for variables directly taken from weather data ? # Does this happen often in a typical model ? -#=m_multiscale = Dict("Default" => ( - MyToyDayModel(), - Status(a=1,) - ), - "Default2" => ( - MyToyWeekModel(), - ),) -=# -m_multiscale = Dict("Default" => ( - MyToyDayModel(), - Status(a=1,) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekModel(), - mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], - ), - ),) - - - -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) - - orchestrator_multiscale = Orchestrator(Day(1), Dict("Default2" => sth)) - -#out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orchestrator_multiscale) -=# ########################### -# Simple test with a second attempt at an orchestrator -# Functional, except for the revalue/refvector overwriting issues +# Simple test with an orchestrator ########################### using MultiScaleTreeGraph using PlantSimEngine @@ -271,11 +186,7 @@ end PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) - meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - - model_timesteps_defaultscale = Dict(MyToyWeekModel =>Week(1)) - tsm = PlantSimEngine.TimestepMapper(:daily_temperature, Day(1), max, nothing) - sth = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:weekly_max_temperature => tsm )) +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) m_multiscale = Dict("Default" => ( MyToyDayModel(), @@ -289,11 +200,9 @@ m_multiscale = Dict("Default" => ( ),) - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) - to = PlantSimEngine.Var_to(:weekly_max_temperature) from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) @@ -303,7 +212,6 @@ mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) - ########################### # Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting @@ -319,12 +227,6 @@ out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) - tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) - sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) - - tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) - sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) - to_w = PlantSimEngine.Var_to(:in_week) from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) dict_to_from_w = Dict(to_w => from_d) @@ -358,7 +260,6 @@ out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) ########################### # Test with three timesteps, multiscale -# Issues with data overwriting (refvector/refvalue) ########################### using MultiScaleTreeGraph @@ -433,10 +334,8 @@ df = DataFrame(:data => [1 for i in 1:365], ) # TODO can make this optional if the timestep range is actually a single value model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) - sth_d = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_week => tsm_d )) tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) - sth_w = PlantSimEngine.SimulationTimestepHandler(model_timesteps_defaultscale, Dict(:in_four_week_from_day => tsm_d, :in_four_week_from_week => tsm_w )) to_w = PlantSimEngine.Var_to(:in_week) from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) @@ -497,13 +396,14 @@ mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) #orch2 = PlantSimEngine.Orchestrator2() out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) -#out = run!(mtg, m_multiscale, df, orchestrator=orch2) +out = run!(mtg, m_multiscale, df, orchestrator=orch2) using Test - @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [0.0, 28.0] - @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [0.0, 7.0] +# TODO : fix these tests + @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [-Inf, 28.0] + @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [-Inf, 28.0] @test unique([out["Default3"][i].inputs_agreement for i in 1:length(out["Default3"])]) == [1] unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) From 6f058f9a21bdd8da62591bf9a241ecb9e87402cf Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 24 Nov 2025 14:17:35 +0100 Subject: [PATCH 10/21] Remove discarded data structures --- src/PlantSimEngine.jl | 2 +- src/timestep/timestep_mapping.jl | 40 +------------------------------- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 5ffaea10..f370a946 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -107,7 +107,7 @@ include("examples_import.jl") export PreviousTimeStep export AbstractModel export ModelList, MultiScaleModel -export Orchestrator, Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping +export Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! diff --git a/src/timestep/timestep_mapping.jl b/src/timestep/timestep_mapping.jl index 06d40d01..85c7f4ea 100644 --- a/src/timestep/timestep_mapping.jl +++ b/src/timestep/timestep_mapping.jl @@ -6,45 +6,8 @@ # Many shortcuts will be taken, I'll try and comment what's missing/implicit/etc. -# TODO specify scale ? -struct TimestepMapper#{V} - variable_from#::V - timestep_from::Period # ? Not sure whether this is the best bit of info... Also, to or from ? And it should be a Period, no ? - mapping_function - mapping_data # TODO this should be internal -end - -struct SimulationTimestepHandler#{W,V} - model_timesteps::Dict{Any, Period} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range - timestep_variable_mapping::Dict{Any, TimestepMapper} #where {V} -end - -SimulationTimestepHandler() = SimulationTimestepHandler(Dict{Any, Int}(), Dict{Any, TimestepMapper}()) #Dict{W, Int}(), Dict{V, TimestepMapper}()) where {W, V} - -mutable struct Orchestrator - # This is actually a general simulation parameter, not-scale specific - # todo change to Period - default_timestep::Period - - # This needs to be per-scale : if a model is used at two different scales, - # and the same variable of that model maps to some other timestep to two *different* variables - # then I believe we can only rely on the different scale to disambiguate - non_default_timestep_data_per_scale::Dict{String, SimulationTimestepHandler} - - function Orchestrator(default::Period, per_scale::Dict{String, SimulationTimestepHandler}) - @assert default >= Second(0) "The default_timestep should be greater than or equal to 0." - return new(default, per_scale) - end -end - # TODO have a default constructor take in a meteo or something, and set up the default timestep automagically to be the finest weather timestep # Other options are possible -Orchestrator() = Orchestrator(Day(1), Dict{String, SimulationTimestepHandler}()) - - -#o = Orchestrator() -#oo = Orchestrator(1, Dict{String, SimulationTimestepHandler}()) - # TODO issue : what if the user wants to force initialize a variable that isn't at the default timestep ? # This requires the ability to fit data with a vector that isn't at the default timestep @@ -67,7 +30,6 @@ struct Var_from scale::String name::Symbol mapping_function::Function - # mapping_data::Dict{NodeMTG, Vector{stuff}} end struct Var_to @@ -94,7 +56,7 @@ end Orchestrator2() = Orchestrator2(Day(1), Vector{ModelTimestepMapping}()) -# TODO parallelization, +# TODO parallelization function init_timestep_mapping_data(node_mtg::MultiScaleTreeGraph.Node, dependency_graph) traverse_dependency_graph!(x -> register_mtg_node_in_timestep_mapping(x, node_mtg), dependency_graph, visit_hard_dep=false) From d3ed73ce162ba4ba745e29f994f559e3570c2b49 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 24 Nov 2025 14:18:23 +0100 Subject: [PATCH 11/21] Fix case where children weren't being run properly with prior changes --- src/run.jl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/run.jl b/src/run.jl index 42ee7355..a84f9941 100644 --- a/src/run.jl +++ b/src/run.jl @@ -453,10 +453,14 @@ function run_node_multiscale!( node_ratio = node.timestep / Day(1) # Check if the model needs to run this timestep - if (1 + (i - 1) % node_ratio) != node_ratio - # TODO : This does prevent the node form being updated by two different parents but probably should be changed - if any([p.simulation_id[1] > node.simulation_id[1] for p in node.parent]) + skip = (1 + (i - 1) % node_ratio) != node_ratio + + if skip + + # TODO : This prevents a non-default timestep node form being updated by two different parents + # but probably should be changed, it is bug-prone + if isnothing(node.parent) || any([p.simulation_id[1] > node.simulation_id[1] for p in node.parent]) node.simulation_id[1] += 1 end else @@ -471,7 +475,7 @@ function run_node_multiscale!( node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale models_at_scale = models[node.scale] - # TODO : is empty check should be pre simulation + # TODO : is empty check should actually be checkde pre-simulation, it is incorrect behaviour if isnothing(node.timestep_mapping_data) || isempty(node.timestep_mapping_data) # Samuel : this is the happy path, no further timestep mapping checks needed @@ -481,6 +485,7 @@ function run_node_multiscale!( end node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already + # TODO remove this bit # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): for child in node.children #! check if we can run this safely in a @floop loop. I would say no, @@ -494,8 +499,8 @@ function run_node_multiscale!( for st in node_statuses run!(node.value, models_at_scale, st, meteo, constants, extra) - # TODO do the accumulation - # then write into the child's status if need be ? + + # Do the accumulation then write into the child's status if need be for tmst in node.timestep_mapping_data ratio = Int(tmst.node_to.timestep / node.timestep) # TODO assert etc. This is all assuming the ratio is an integer, whereas it can be, like 1/7 From 569aa05c2d1e8b84d1e805d9bdec8530828d1734 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 24 Nov 2025 15:19:17 +0100 Subject: [PATCH 12/21] Update a couple of comments --- src/Abstract_model_structs.jl | 4 +++- src/dependencies/dependency_graph.jl | 4 ++-- src/mtg/add_organ.jl | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Abstract_model_structs.jl b/src/Abstract_model_structs.jl index f8a079e0..68f91986 100644 --- a/src/Abstract_model_structs.jl +++ b/src/Abstract_model_structs.jl @@ -66,4 +66,6 @@ function is_timestep_in_range(tsr::TimestepRange, p::Period) return p >= tsr.lower_bound && p <= tsr.lower_bound end -# TODO should i set all timestep ranges to default and hope the modeler gets it right or should i force them to write something ? \ No newline at end of file +# TODO should i set all timestep ranges to default and hope the modeler gets it right or should i force them to write something ? + +# TODO properly test on models with default timestep \ No newline at end of file diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 59ec29f5..fe46f786 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -15,10 +15,10 @@ end mutable struct TimestepMapping variable_from::Symbol variable_to::Symbol - node_to # SoftDependencyNode causes a circular reference # TODO could it be a harddependencynode... ? + node_to # SoftDependencyNode causes a circular reference, removing it as a shortcut TODO mapping_function::Function mapping_data_template - mapping_data::Dict{Int, Any} # TODO Any's type is the variable's type, also, is Int good here ? Prob not + mapping_data::Dict{Int, Any} # TODO fix type stability : Int is the node id, Any is a vector of n elements of the variable's type, n being the # of required timesteps end # can hard dependency nodes also handle timestep mapped variables... ? diff --git a/src/mtg/add_organ.jl b/src/mtg/add_organ.jl index 58c71a9e..1147056a 100644 --- a/src/mtg/add_organ.jl +++ b/src/mtg/add_organ.jl @@ -33,8 +33,7 @@ function add_organ!(node::MultiScaleTreeGraph.Node, sim_object, link, symbol, sc new_node = MultiScaleTreeGraph.Node(id, node, MultiScaleTreeGraph.NodeMTG(link, symbol, index, scale), attributes) st = init_node_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.reverse_multiscale_mapping, sim_object.var_need_init, check=check) - # TODO add the node to the timestep mappings - # TODO initialise the MTG nodes in the timestep mappings + #TODO RefVector udpating with node insertion # NOTE : this isn't ideal, as it constrains the add_organ! function usage init_timestep_mapping_data(new_node, sim_object.dependency_graph) From 8cf445069233dd56d11e92c918d48bc8e06ce622 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Wed, 26 Nov 2025 13:18:09 +0100 Subject: [PATCH 13/21] Add a couple of extra tests for mutli timestep mapping --- test/test-multitimestep.jl | 683 +++++++++++++++++++++++++++++++++++-- 1 file changed, 657 insertions(+), 26 deletions(-) diff --git a/test/test-multitimestep.jl b/test/test-multitimestep.jl index 5adf4659..e9c890bd 100644 --- a/test/test-multitimestep.jl +++ b/test/test-multitimestep.jl @@ -1,5 +1,5 @@ - ########################### + # Simple test using an ad hoc connector model # Broken by subsequent changes, left just in case for now (TODO remove once prototyping is over) ########################### @@ -155,6 +155,7 @@ using MultiScaleTreeGraph using PlantSimEngine using PlantMeteo using PlantMeteo.Dates +using Test PlantSimEngine.@process "ToyDay" verbose = false @@ -173,7 +174,7 @@ struct MyToyWeekModel <: AbstractToyweekModel temperature_threshold::Float64 end -MyToyWeekModel() = MyToyWeekModel(30.0) +MyToyWeekModel() = MyToyWeekModel(28.0) function PlantSimEngine.inputs_(::MyToyWeekModel) (weekly_max_temperature=-Inf,) end @@ -213,10 +214,163 @@ orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +temps = [out["Default"][i].daily_temperature for i in 1:365] +temp_m = maximum(temps) + +# At least one week should have max temp > 28 +@test temp_m > 28 && unique!([out["Default2"][i].hot for i in 1:365]) == [0,1] + +########################### +# Test with three timesteps, multiscale ########################### + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay2" verbose = false + +struct MyToyDay2Model <: AbstractToyday2Model end + +PlantSimEngine.inputs_(m::MyToyDay2Model) = NamedTuple() +PlantSimEngine.outputs_(m::MyToyDay2Model) = (out_day=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day = meteo.data +end + +#=PlantSimEngine.@process "ToyDay3" verbose = false + +struct MyToyDay3Model <: AbstractToyday3Model end + +PlantSimEngine.inputs_(m::MyToyDay3Model) = (in_day=-Inf, in_day_summed_prev_timestep=-Inf) +PlantSimEngine.outputs_(m::MyToyDay3Model) = (out_day_summed=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay3Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day_summed = status.in_day + status.in_day_summed_prev_timestep +end + +PlantSimEngine.@process "ToyDay4" verbose = false + +struct MyToyDay4Model <: AbstractToyday4Model end + +PlantSimEngine.inputs_(m::MyToyDay4Model) = (in_day_summed=-Inf,) +PlantSimEngine.outputs_(m::MyToyDay4Model) = (out_day_summed_2= -Inf,) + +function PlantSimEngine.run!(m::MyToyDay4Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day_summed_2 = status.in_day_summed +end=# + +PlantSimEngine.@process "ToyWeek2" verbose = false + +struct MyToyWeek2Model <: AbstractToyweek2Model end + +PlantSimEngine.inputs_(::MyToyWeek2Model) = (in_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyWeek2Model) = (out_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_week = status.in_week +end + +PlantSimEngine.timestep_range_(m::MyToyWeek2Model) = TimestepRange(Week(1)) + + +PlantSimEngine.@process "ToyFourWeek2" verbose = false + +struct MyToyFourWeek2Model <: AbstractToyfourweek2Model end + +PlantSimEngine.inputs_(::MyToyFourWeek2Model) = (in_four_week_from_week=-Inf, in_four_week_from_day=-Inf,) +PlantSimEngine.outputs_(m::MyToyFourWeek2Model) = (inputs_agreement=false,) + +function PlantSimEngine.run!(m::MyToyFourWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.inputs_agreement = status.in_four_week_from_week == status.in_four_week_from_day +end + +PlantSimEngine.timestep_range_(m::MyToyFourWeek2Model) = TimestepRange(Week(4)) + + + +df = DataFrame(:data => [1 for i in 1:365], ) + + # TODO can make this optional if the timestep range is actually a single value + model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(from_d => to_w) + + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default2", :out_week, sum) + + dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) + + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + + +m_multiscale = Dict("Default" => ( + MyToyDay2Model(), + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[:in_week => "Default" => :out_day], + ), + ), + "Default3" => ( + MultiScaleModel(model=MyToyFourWeek2Model(), + mapped_variables=[ + :in_four_week_from_day => "Default" => :out_day, + :in_four_week_from_week => "Default2" => :out_week, + ], + ),), + #="Default4"=> ( + MultiScaleModel( + model=MyToyDay3Model(), + mapped_variables=[ + PlantSimEngine.PreviousTimeStep(:in_day_summed_prev_timestep) => "Default5" => :out_day_summed_2, + :in_day => "Default" => :out_day, + ]), + Status(in_day_summed_prev_timestep=0,) + ), + "Default5" => ( + MultiScaleModel(model=MyToyDay4Model(), + mapped_variables= [:in_day_summed => "Default4" => :out_day_summed], + ), + Status(in_day_summed=0,out_day_summed_2=0) + ),=# + ) + + +# TODO test with multiple nodes +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) +#mtg4 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default4", 1, 4)) +#mtg5 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default5", 1, 5)) + + #orch2 = PlantSimEngine.Orchestrator2() + +#out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) +out = run!(mtg, m_multiscale, df, orchestrator=orch2) + + + +using Test + @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [-Inf, 28.0] + @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [-Inf, 28.0] + + # Note : until the models actually run, inputs_agreement defaults to false, so it's only expected to be true + # from day 28 onwards + @test unique([out["Default3"][i].inputs_agreement for i in 28:length(out["Default3"])]) == [1] + + ########################### # Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting -# and explore alternatives # (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) +# and check mapping at the same scale ########################### m_singlescale = Dict("Default" => ( @@ -229,7 +383,7 @@ out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) to_w = PlantSimEngine.Var_to(:in_week) from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) - dict_to_from_w = Dict(to_w => from_d) + dict_to_from_w = Dict(from_d => to_w) to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) @@ -242,26 +396,51 @@ out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) -mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) - - - + mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) + out = run!(mtg_single, m_singlescale, df, orchestrator=orch2) + ########################### +# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting +# and explore alternatives +# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) +# Not plugged in together atm, the variable mapping doesn't work +########################### + m_singlescale = Dict("Default" => ( + MyToyDay2Model(), + MyToyWeek2Model(), + MyToyFourWeek2Model(), + ),) + model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(from_d => to_w) + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default", :out_week, sum) + dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) +mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) ########################### -# Test with three timesteps, multiscale +# Test with three timesteps, multiscale + previoustimestep ########################### +# note the daily models don't specify a timestep range +# if you copy-paste this elsewhere, bear in mind that you might need to specify it + using MultiScaleTreeGraph using PlantSimEngine using PlantMeteo @@ -313,6 +492,20 @@ end PlantSimEngine.timestep_range_(m::MyToyWeek2Model) = TimestepRange(Week(1)) +PlantSimEngine.@process "ToyPreviousWeek2" verbose = false + +struct MyToyPreviousWeek2Model <: AbstractToypreviousweek2Model end + +# TODO initialisation issue +PlantSimEngine.inputs_(::MyToyPreviousWeek2Model) = (in_last_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyPreviousWeek2Model) = (out_last_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyPreviousWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_last_week += status.in_last_week/7.0 +end + +PlantSimEngine.timestep_range_(m::MyToyPreviousWeek2Model) = TimestepRange(Week(1)) + PlantSimEngine.@process "ToyFourWeek2" verbose = false @@ -333,9 +526,6 @@ df = DataFrame(:data => [1 for i in 1:365], ) # TODO can make this optional if the timestep range is actually a single value model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) - tsm_d = PlantSimEngine.TimestepMapper(:out_day, Day(1), sum, nothing) - - tsm_w = PlantSimEngine.TimestepMapper(:out_week, Week(1), sum, nothing) to_w = PlantSimEngine.Var_to(:in_week) from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) @@ -353,7 +543,15 @@ df = DataFrame(:data => [1 for i in 1:365], ) orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) -m_multiscale = Dict("Default" => ( +m_multiscale = Dict( +"Default6" => + ( + MultiScaleModel(model=MyToyPreviousWeek2Model(), + mapped_variables=[PlantSimEngine.PreviousTimeStep(:in_last_week) => "Default2" => :out_week], + ), + Status(in_last_week=0.0, out_last_week=0.0) + ), +"Default" => ( MyToyDay2Model(), ), "Default2" => ( @@ -361,14 +559,14 @@ m_multiscale = Dict("Default" => ( mapped_variables=[:in_week => "Default" => :out_day], ), ), - "Default3" => ( + #="Default3" => ( MultiScaleModel(model=MyToyFourWeek2Model(), mapped_variables=[ :in_four_week_from_day => "Default" => :out_day, :in_four_week_from_week => "Default2" => :out_week, ], ),), - #="Default4"=> ( + "Default4"=> ( MultiScaleModel( model=MyToyDay3Model(), mapped_variables=[ @@ -383,29 +581,462 @@ m_multiscale = Dict("Default" => ( ), Status(in_day_summed=0,out_day_summed_2=0) ),=# + ) # TODO test with multiple nodes mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) -mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) +#mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) #mtg4 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default4", 1, 4)) #mtg5 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default5", 1, 5)) +mtg6 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default6", 1, 6)) - #orch2 = PlantSimEngine.Orchestrator2() +out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) +out = run!(mtg, m_multiscale, df, orchestrator=orch2) + +unique!([out["Default6"][i].out_last_week for i in 1:length(out["Default6"])]) + +# TODO : out_last_week is an output at the weekly scale, it isn't mapped +# Doesn't mesh well with current implementation : +# non_default_timestep_mapping requires input + output, so the model and variable aren't declared +# I can infer the timestep from the timestep_range in this situation, but not for a model with a wider range +# This means the model needs to be declared somewhere + + + + + + +########################### +# Test with one timestep, multiscale + previoustimestep +########################### + +# note the daily models don't specify a timestep range +# if you copy-paste this elsewhere, bear in mind that you might need to specify it + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay2" verbose = false + +struct MyToyDay2Model <: AbstractToyday2Model end + +PlantSimEngine.inputs_(m::MyToyDay2Model) = NamedTuple() +PlantSimEngine.outputs_(m::MyToyDay2Model) = (out_day=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day = meteo.data +end + +PlantSimEngine.@process "ToyWeek2" verbose = false + +struct MyToyWeek2Model <: AbstractToyweek2Model end + +PlantSimEngine.inputs_(::MyToyWeek2Model) = (in_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyWeek2Model) = (out_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_week = status.in_week +end + +PlantSimEngine.timestep_range_(m::MyToyWeek2Model) = TimestepRange(Day(1)) + +PlantSimEngine.@process "ToyPreviousWeek2" verbose = false + +struct MyToyPreviousWeek2Model <: AbstractToypreviousweek2Model end + +# TODO initialisation issue +PlantSimEngine.inputs_(::MyToyPreviousWeek2Model) = (in_last_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyPreviousWeek2Model) = (out_last_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyPreviousWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_last_week += status.in_last_week +end + +PlantSimEngine.timestep_range_(m::MyToyPreviousWeek2Model) = TimestepRange(Day(1)) + + +PlantSimEngine.@process "ToyFourWeek2" verbose = false + +struct MyToyFourWeek2Model <: AbstractToyfourweek2Model end + +PlantSimEngine.inputs_(::MyToyFourWeek2Model) = (in_four_week_from_week=-Inf, in_four_week_from_day=-Inf,) +PlantSimEngine.outputs_(m::MyToyFourWeek2Model) = (inputs_agreement=false,) + +function PlantSimEngine.run!(m::MyToyFourWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.inputs_agreement = status.in_four_week_from_week == status.in_four_week_from_day +end + +PlantSimEngine.timestep_range_(m::MyToyFourWeek2Model) = TimestepRange(Day(1)) + + + +df = DataFrame(:data => [1 for i in 1:365], ) + + # TODO can make this optional if the timestep range is actually a single value + #=model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) + + to_w = PlantSimEngine.Var_to(:in_week) + from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) + dict_to_from_w = Dict(from_d => to_w) + + to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) + to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) + from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default2", :out_week, sum) + + dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) + + orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4])=# +orch2 = PlantSimEngine.Orchestrator2() + +m_multiscale = Dict( +"Default6" => + ( + MultiScaleModel(model=MyToyPreviousWeek2Model(), + mapped_variables=[PlantSimEngine.PreviousTimeStep(:in_last_week) => "Default2" => :out_week], + ), + Status(in_last_week=0.0, out_last_week=0.0) + ), +"Default" => ( + MyToyDay2Model(), + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[:in_week => "Default" => :out_day], + ), + ), + + ) + + +# TODO test with multiple nodes +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) +mtg6 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default6", 1, 6)) out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) out = run!(mtg, m_multiscale, df, orchestrator=orch2) +unique!([out["Default6"][i].out_last_week for i in 1:length(out["Default6"])]) + + + +########################### +# Previous timestep debugging, not useful for testing timestep mapping atm +########################### +#= +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "current_timestep" verbose = false + +struct HelperCurrentTimestepModel <: AbstractCurrent_TimestepModel +end + +PlantSimEngine.inputs_(::HelperCurrentTimestepModel) = (next_timestep=1,) +PlantSimEngine.outputs_(m::HelperCurrentTimestepModel) = (current_timestep=1,) + +function PlantSimEngine.run!(m::HelperCurrentTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.current_timestep = status.next_timestep + end + + PlantSimEngine.@process "next_timestep" verbose = false + struct HelperNextTimestepModel <: AbstractNext_TimestepModel + end + + PlantSimEngine.inputs_(::HelperNextTimestepModel) = (current_timestep=1,) + PlantSimEngine.outputs_(m::HelperNextTimestepModel) = (next_timestep=1,) + + function PlantSimEngine.run!(m::HelperNextTimestepModel, models, status, meteo, constants=nothing, extra=nothing) + status.next_timestep = status.current_timestep + 1 + end + + df = DataFrame(:data => [1 for i in 1:365], ) + + m_ms = Dict( + "B" => ( + MultiScaleModel( + model=HelperCurrentTimestepModel(), + mapped_variables=[PreviousTimeStep(:next_timestep),], + ), + Status(next_timestep=2) + + ), + "A" => ( + HelperNextTimestepModel(), + Status(current_timestep=1) + ), + ) + +m_ss = Dict( + "A" => ( + HelperNextTimestepModel(), + MultiScaleModel( + model=HelperCurrentTimestepModel(), + mapped_variables=[PreviousTimeStep(:next_timestep),], + ), + Status(current_timestep=1, next_timestep=1)), +) + + mtg_ = Node(MultiScaleTreeGraph.NodeMTG("/", "A", 1, 1)) +#mtg2 = Node(mtg_, MultiScaleTreeGraph.NodeMTG("/", "B", 1, 2)) +out = @run run!(mtg_, m_ss, df) +=# + +########################### +# Test with a D -> W -> D configuration, with multiple variables mapped between timesteps +########################### + +# Currently, weekly_max_temperature will raise an error : +# It is mapped correctly, timestep-mapped correctly, but not detected as an output at its scale +# Since it doesn't appear explicitely as the output of a model +# Setting it as the output would cause issues in status creation and refs, as well as during mapping + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDayDWD" verbose = false + +struct MyToyDayDWDModel <: AbstractToydaydwdModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.data +end + +PlantSimEngine.@process "ToyWeekDWD" verbose = false + +struct MyToyWeekDWDModel <: AbstractToyweekdwdModel + temperature_threshold::Float64 +end + +MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekDWDModel) + (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) +end +PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) + +function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold + status.sum += status.weekly_sum_temperature +end + +PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) + +PlantSimEngine.@process "ToyDayDWDOut" verbose = false + +struct MyToyDayDWDOutModel <: AbstractToydaydwdoutModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDOutModel) = (weekly_max_temperature=-Inf,weekly_sum_temperature=-Inf,) +PlantSimEngine.outputs_(m::MyToyDayDWDOutModel) = (out=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDOutModel, models, status, meteo, constants=nothing, extra=nothing) + status.out = status.weekly_sum_temperature - 7.0*meteo.data +end + +df = DataFrame(:data => [1 for i in 1:365], ) + +m_dwd = Dict("Default" => ( + MyToyDayDWDModel(), + MultiScaleModel( + model=MyToyDayDWDOutModel(), + mapped_variables=[:weekly_max_temperature => "Default2", :weekly_sum_temperature => "Default2"] + ), + Status(a=1,out=0.0) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekDWDModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature, :weekly_sum_temperature => "Default" => :daily_temperature], + ), + Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum=0.0) + ), +) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + +to = PlantSimEngine.Var_to(:weekly_max_temperature) +from = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, maximum) + +to_sum = PlantSimEngine.Var_to(:weekly_sum_temperature) +from_sum = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, sum) + +dict_to_from = Dict(from => to, from_sum => to_sum) +mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1), dict_to_from) + +#dict_to_from_2 = Dict(from => to, from => to) +#mtsm2 = PlantSimEngine.ModelTimestepMapping(MyToyDayDWDOutModel, "Default", Day(1), dict_to_from2) + +orch_dwd = PlantSimEngine.Orchestrator2(Day(1), [mtsm_dwd,])#mtsm2]) + +out = run!(mtg, m_dwd, df, orchestrator=orch_dwd) + + +################################## +# Two variables mapped +################################## + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDayDWD" verbose = false + +struct MyToyDayDWDModel <: AbstractToydaydwdModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeekDWD" verbose = false + +struct MyToyWeekDWDModel <: AbstractToyweekdwdModel + temperature_threshold::Float64 +end + +MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekDWDModel) + (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) +end +PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) + +function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold + status.sum += status.weekly_sum_temperature +end + +PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) + +#df = DataFrame(:data => [1 for i in 1:365], ) +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_dwd = Dict("Default" => ( + MyToyDayDWDModel(), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekDWDModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature, :weekly_sum_temperature => "Default" => :daily_temperature], + ), + Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum =0.0) + ), +) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + +to = PlantSimEngine.Var_to(:weekly_max_temperature) +from = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, maximum) + +to_sum = PlantSimEngine.Var_to(:weekly_sum_temperature) +from_sum = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, sum) + +dict_to_from = Dict(from => to, from_sum => to_sum) +mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1), dict_to_from) + +orch_dwd = PlantSimEngine.Orchestrator2(Day(1), [mtsm_dwd,]) + +out = run!(mtg, m_dwd, meteo_day, orchestrator=orch_dwd) + + + +########################## +# Two models, D -> W, but D has two MTG nodes +# So simple test of a RefVector + timestep mapping combo +########################## + +# Currently errors due to refvectors not being handled properly + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=[-Inf],) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MyToyDayModel(), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) + +to = PlantSimEngine.Var_to(:weekly_max_temperature) +from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) + +dict_to_from = Dict(from => to) +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) + +#orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) +orch2 = PlantSimEngine.Orchestrator2() +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +#out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) -using Test -# TODO : fix these tests - @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [-Inf, 28.0] - @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [-Inf, 28.0] - @test unique([out["Default3"][i].inputs_agreement for i in 1:length(out["Default3"])]) == [1] - unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) - unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) - unique([out["Default3"][i].inputs_agreement for i in 1:length(out["Default3"])]) \ No newline at end of file +# alternate mtg with 2 nodes -> 2 nodes +#mtg_bis = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +#mtg_bis_1 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +#mtg_bis_2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) +#mtg_bis_3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) +#out = @enter run!(mtg_bis, m_multiscale, meteo_day, orchestrator=orch2) From ede94319f7b5c3d1559607d6ec6f8286c07e477a Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Wed, 26 Nov 2025 13:56:24 +0100 Subject: [PATCH 14/21] Fix doctest error --- src/mtg/initialisation.jl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index f738ee42..af1c7c57 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -23,7 +23,7 @@ a dictionary of variables that need to be initialised or computed by other model """ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())); type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator2()) # We compute the variables mapping for each scale: - mapped_vars = mapped_variables(mapping, dependency_graph, verbose=verbose) + mapped_vars = mapped_variables(mapping, dependency_graph, verbose=verbose,orchestrator=orchestrator) # Update the types of the variables as desired by the user: convert_vars!(mapped_vars, type_promotion) @@ -182,7 +182,7 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi end """ - status_from_template(d::Dict{Symbol,Any}) + status_from_template(d::Dict{Symbol,Any}, scale::String) Create a status from a template dictionary of variables and values. If the values are already RefValues or RefVectors, they are used as is, else they are converted to Refs. @@ -202,7 +202,7 @@ julia> using PlantSimEngine ``` ```jldoctest mylabel -julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); +julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0), "Dummy_Scale"); ``` ```jldoctest mylabel @@ -215,7 +215,7 @@ julia> b 2.0 ``` """ -function status_from_template(d::Dict{Symbol,T} where {T}, scale::String, orchestrator::Orchestrator2) +function status_from_template(d::Dict{Symbol,T} where {T}, scale::String, orchestrator::Orchestrator2=Orchestrator2()) # Sort vars to put the values that are of type PerStatusRef at the end (we need the pass on the other ones first): sorted_vars = Dict{Symbol,Any}(sort([pairs(d)...], by=v -> last(v) isa RefVariable ? 1 : 0)) # Note: PerStatusRef are used to reference variables in the same status for renaming. @@ -335,10 +335,17 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion # preliminary_check_timestep_data(mapping, orchestrator) + # soft_dep_graph_roots, hard_dep_dict = hard_dependencies(mapping; verbose=false, orchestrator=orchestrator) + # Get the status of each node by node type, pre-initialised considering multi-scale variables: + # statuses, status_templates, reverse_multiscale_mapping, vars_need_init = + # init_statuses(mtg, mapping, soft_dep_graph_roots; type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) + +# Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses, status_templates, reverse_multiscale_mapping, vars_need_init = init_statuses(mtg, mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=orchestrator)); type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) + # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") @@ -351,7 +358,8 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion outputs_index = Dict{String, Int}(s => 1 for s in keys(outputs)) - dependency_graph = dep(mapping, verbose=verbose, orchestrator=orchestrator) + dependency_graph = dep(mapping, verbose=verbose, orchestrator=orchestrator) + #dependency_graph = dep(mapping, soft_dep_graph_roots=soft_dep_graph_roots, hard_dep_dict=hard_dep_dict, orchestrator=orchestrator) # Samuel : Once the dependency graph is created, and the timestep mappings are added into it # We need to register the existing MTG nodes to initialize their individual data From b608126b30e9f9117ffd54d714e837d4e37dff75 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Mon, 1 Dec 2025 16:42:25 +0100 Subject: [PATCH 15/21] Departure from the previous approach : add timestep data to MultiScaleModels, which should enable better meshing with current mapping structures. More work needed --- src/PlantSimEngine.jl | 3 +- src/dependencies/dependencies.jl | 24 ----- src/dependencies/hard_dependencies.jl | 61 +++++++++++- src/dependencies/soft_dependencies.jl | 134 ++++++++++++++++++++------ src/mtg/GraphSimulation.jl | 16 +++ src/mtg/MultiScaleModel.jl | 61 ++++++++++-- src/mtg/initialisation.jl | 29 ++++-- src/mtg/mapping/compute_mapping.jl | 98 ++++++++++++++----- test/test-multitimestep.jl | 109 ++++++++++++++++++++- 9 files changed, 437 insertions(+), 98 deletions(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index f370a946..14c782cc 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -106,7 +106,8 @@ include("examples_import.jl") export PreviousTimeStep export AbstractModel -export ModelList, MultiScaleModel +export ModelList, MultiScaleModel, TimestepMappedVariable +export MultiScaleMapping export Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 27b9fcde..6672add8 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -99,27 +99,3 @@ end function dep(m::NamedTuple, nsteps=1; verbose::Bool=true) dep(nsteps; verbose=verbose, m...) end - -function dep(mapping::Dict{String,T}; verbose::Bool=true, orchestrator=Orchestrator2()) where {T} - # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want - # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they - # are independant. - soft_dep_graphs_roots, hard_dep_dict = hard_dependencies(mapping; verbose=verbose, orchestrator=Orchestrator2()) - # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the - # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the - # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children - # of the nodes that they depend on. - dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, mapping, hard_dep_dict, orchestrator=orchestrator) - # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, - # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale. - # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale. - - # Checking that the graph is acyclic: - iscyclic, cycle_vec = is_graph_cyclic(dep_graph; warn=false) - # Note: we could do that in `soft_dependencies_multiscale` but we prefer to keep the function as simple as possible, and - # usable on its own. - - iscyclic && error("Cyclic dependency detected in the graph. Cycle: \n $(print_cycle(cycle_vec)) \n You can break the cycle using the `PreviousTimeStep` variable in the mapping.") - # Third step, we identify which - return dep_graph -end diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 04fb16f4..78bb5f8a 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -156,11 +156,17 @@ function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orches end # TODO no idea how this meshes with refvector situations or previoustimestep +# if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true) +# push!(vars_, var => default) +# if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) +# organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) +# push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) +# end +# else if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true) push!(vars_, var => default) else - if haskey(vars_mapping[organ], var) organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var]) push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default)) @@ -180,6 +186,7 @@ function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orches end end end +# end return (; vars_...,) end end @@ -197,6 +204,16 @@ function _node_mapping(var_mapping) return organ_mapped, organ_mapped_var end +function extract_timestep_mapped_outputs(m::MultiScaleModel, organ::String, outputs_process, timestep_mapped_outputs_process) + if length(m.timestep_mapped_variables) > 0 + timestep_mapped_outputs_process[organ] = Dict{Symbol,Vector}() + key = process(m.model) + extra_outputs = timestep_mapped_outputs_(m) + #ind = findfirst(x -> first(x) == key, outputs_process[organ][key]) + timestep_mapped_outputs_process[organ][key] = (; extra_outputs...) #TODO + end +end + # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T} full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping) @@ -222,6 +239,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr #* dependency may not appear in its own scale, but this is treated in the soft-dependency computation inputs_process = Dict{String,Dict{Symbol,Vector}}() outputs_process = Dict{String,Dict{Symbol,Vector}}() + timestep_mapped_outputs_process = Dict{String,Dict{Symbol,NamedTuple}}() for (organ, model) in mapping # Get the status given by the user, that is used to set the default values of the variables in the mapping: st_scale_user = get_status(model) @@ -240,6 +258,32 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr inputs_process[organ] = Dict(key => [j.first => j.second.inputs for j in val] for (key, val) in status_scale) outputs_process[organ] = Dict(key => [j.first => j.second.outputs for j in val] for (key, val) in status_scale) + + # Samuel : This if else loop is a bit awkward + # None of the other code works this way, it uses the dependency grpah + # but the hard_dep graph loses the multiscale model information... + if isa(model, AbstractModel) + elseif isa(model, MultiScaleModel) + extract_timestep_mapped_outputs(model, organ, outputs_process, timestep_mapped_outputs_process) + else + for m in model + if isa(m, MultiScaleModel) + extract_timestep_mapped_outputs(m, organ, outputs_process, timestep_mapped_outputs_process) + end + end + end + + #=for m in model + if isa(m, MultiScaleModel) + if length(m.timestep_mapped_variables) > 0 + key = process(m.model) + extra_outputs = timestep_mapped_outputs_(m) + ind = findfirst(x -> first(x) == key, outputs_process[organ][key]) + outputs_process[organ][key][ind] = first(outputs_process[organ][key][ind]) => (; last(outputs_process[organ][key][ind])..., extra_outputs...) + + end + end + end=# end # If some models needed as hard-dependency are not found in their own scale, check the other scales: @@ -366,7 +410,18 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr ) for (process_, soft_dep_vars) in hard_deps[organ].roots # proc_ = :carbon_assimilation ; soft_dep_vars = hard_deps.roots[proc_] ) - + for (process_, soft_dep_vars) in hard_deps[organ].roots + # TODO this is not good enough for some model ranges, and doesn't check for inconsistencies errors for models that have a modeltimestepmapping + if timestep_range_(soft_dep_vars.value).lower_bound == timestep_range_(soft_dep_vars.value).upper_bound + timestep = timestep_range_(soft_dep_vars.value).lower_bound + + # if the model has infinite range, set it to the simulation timestep + if timestep == Second(0) + timestep = orchestrator.default_timestep + end + soft_dep_graph[process_].timestep = timestep + end + end # Update the parent node of the hard dependency nodes to be the new SoftDependencyNode instead of the old # HardDependencyNode. for (p, node) in soft_dep_graph @@ -375,7 +430,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr end end - soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ]) + soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ], timestep_mapped_outputs=haskey(timestep_mapped_outputs_process,organ) ? timestep_mapped_outputs_process[organ] : Dict{Symbol,NamedTuple}()) not_found = merge(not_found, hard_deps[organ].not_found) end diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 135e32b6..f156ef0e 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -71,7 +71,7 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, nothing, SoftDependencyNode[], fill(0, nsteps), - Day(1), # TODO + Day(1), # TODO should be orchestrator.default_timestep, nothing ) for (process_, soft_dep_vars) in d.roots @@ -140,27 +140,15 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, return DependencyGraph(independant_process_root, d.not_found) end -function timestep_mapped_variables(orchestrator) - -#=struct SimulationTimestepHandler#{W,V} - model_timesteps::Dict{Any, Period} # where {W <: AbstractModel} # if a model isn't in there, then it follows the default, todo check if the given timestep respects the model's range - timestep_variable_mapping::Dict{Any, TimestepMapper} #where {V} -end - non_default_timestep_data_per_scale::Dict{String, SimulationTimestepHandler} -=# -end - # For multiscale mapping: -function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{String,Any}}, mapping::Dict{String,A}, hard_dep_dict::Dict{Pair{Symbol,String},HardDependencyNode}; orchestrator::Orchestrator2=Orchestrator2()) where {A<:Any} - mapped_vars = mapped_variables(mapping, soft_dep_graphs_roots, verbose=false) - rev_mapping = reverse_mapping(mapped_vars, all=false) - +function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{String,Any}}, reverse_multiscale_mapping, hard_dep_dict::Dict{Pair{Symbol,String},HardDependencyNode}) independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() - for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs_roots.roots # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs_roots.roots[organ] + for (organ, (soft_dep_graph, ins, outs, timestep_mapped_outs)) in soft_dep_graphs_roots.roots # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs_roots.roots[organ] + for (proc, i) in soft_dep_graph # proc = :leaf_surface; i = soft_dep_graph[proc] # Search if the process has soft dependencies: - soft_deps = search_inputs_in_output(proc, ins, outs) + soft_deps = search_inputs_in_output(proc, ins, outs, timestep_mapped_outs) # Remove the hard dependencies from the soft dependencies: soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) @@ -170,7 +158,7 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency # Check if the process has soft dependencies at other scales: - soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs_roots.roots, rev_mapping, hard_dependencies_from_other_scale) + soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs_roots.roots, reverse_multiscale_mapping, hard_dependencies_from_other_scale) # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content # is computed by the process :soil_water at the scale "Soil". @@ -333,12 +321,21 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic end end end + + if haskey(timestep_mapped_outs, proc) + CreateTimeStepMapping(i, reverse_multiscale_mapping, organ, proc) + end + end end dep_graph = DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found) - traverse_dependency_graph!(x -> set_non_default_timestep_in_node(x, orchestrator), dep_graph, visit_hard_dep=false) - traverse_dependency_graph!(x -> add_timestep_data_to_node(x, orchestrator), dep_graph, visit_hard_dep=false) + + # TODO move these into the loop to directly act on the ndoe, use the provided timestep mappings + # Don't provide parents, and then change the run! function to apply from the node to itself + #traverse_dependency_graph!(x -> set_non_default_timestep_in_node(x, orchestrator), dependency_graph, visit_hard_dep=false) + #traverse_dependency_graph!(x -> add_timestep_data_to_node(x, orchestrator), dep_graph, visit_hard_dep=false) + return dep_graph end @@ -383,7 +380,7 @@ function add_timestep_data_to_node(soft_dependency_node, orchestrator::Orchestra end end -# TODO this is incorrect, there may be multiple variables mapped between the two nodes +# Create the structure to attach to the softdependencynode, (not to be confused with the user-declared timestep-mapping structure) function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependencyNode, var_to::Var_to, var_from::Var_from) @assert parent.timestep != 0 "Error : node timestep internally set to 0" @@ -404,10 +401,18 @@ function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependenc if isa(var, MappedVar) # check the source variable, because the sink one might be a vector...? # TODO multinode mapping - if var.source_variable == var_from.name - # This should be a fixed size array, ideally - var_type = eltype(mapped_default(var)) - break + if isa(var.source_organ, MultiNodeMapping) + if var.source_variable[1] == var_from.name + # This should be a fixed size array, ideally + var_type = eltype(eltype(mapped_default(var))) + break + end + else + if var.source_variable == var_from.name + # This should be a fixed size array, ideally + var_type = eltype(mapped_default(var)) + break + end end else if var.symbol == var_from.name @@ -419,10 +424,68 @@ function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependenc end end - mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio)) # TODO : type shouldn't be Any but Vector{var_type} return TimestepMapping(var_from.name, var_to.name, node, var_from.mapping_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG, Any}()) end + + +function CreateTimeStepMapping(soft_dependency_node, reverse_multiscale_mapping, scale, proc) + + for mapping_entry in reverse_multiscale_mapping[scale] + if isa(mapping_entry, MultiScaleModel) + if process(mapping_entry.model) == proc + for timestep_mapped_var_data in mapping_entry.timestep_mapped_variables + timestep_ratio = timestep_mapped_var_data.timestep_to / soft_dependency_node.timestep + if timestep_ratio > 1 # todo assert it's an int or a rational ? + @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio" + + tmvd = timestep_mapped_var_data + + var_type = DataType + + for (symbol, var_dump) in soft_dependency_node.outputs + for var in var_dump + if isa(var, MappedVar) + # check the source variable, because the sink one might be a vector...? + # TODO multinode mapping + if isa(var.source_organ, MultiNodeMapping) + if var.source_variable[1] == tmvd.name_from + # This should be a fixed size array, ideally + var_type = eltype(eltype(mapped_default(var))) + break + end + else + if var.source_variable == tmvd.name_from + # This should be a fixed size array, ideally + var_type = eltype(mapped_default(var)) + break + end + end + else + if var.symbol == tmvd.name_from + @assert "untested" + var_type = eltype(mapped_default(var)) + break + end + end + end + end + + mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio)) + tsm = TimestepMapping(tmvd.name_from, tmvd.name_to, soft_dependency_node, tmvd.aggregation_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG,Any}()) + + if isnothing(soft_dependency_node.timestep_mapping_data) + soft_dependency_node.timestep_mapping_data = Vector{TimestepMapping}() + end + push!(soft_dependency_node.timestep_mapping_data, tsm) + end + end + end + end +end +end + + """ drop_process(proc_vars, process) @@ -493,7 +556,7 @@ search_inputs_in_output(:process3, in_, out_) (process4 = (:var1, :var2),) ``` """ -function search_inputs_in_output(process, inputs, outputs) +function search_inputs_in_output(process, inputs, outputs, timestep_mapped_outputs=Dict{Symbol,NamedTuple}()) # proc, ins, outs # get the inputs of the node: vars_input = flatten_vars(inputs[process]) @@ -502,7 +565,12 @@ function search_inputs_in_output(process, inputs, outputs) for (proc_output, pairs_vars_output) in outputs # e.g. proc_output = :carbon_biomass; pairs_vars_output = outs[proc_output] if process != proc_output vars_output = flatten_vars(pairs_vars_output) - inputs_in_outputs = vars_in_variables(vars_input, vars_output) + vars_all_outputs = vars_output + if haskey(timestep_mapped_outputs, proc_output) + vars_all_outputs = (; vars_output..., flatten_vars(timestep_mapped_outputs[proc_output])...) + end + + inputs_in_outputs = vars_in_variables(vars_input, vars_all_outputs) if any(inputs_in_outputs) ins_in_outs = [vars_input...][inputs_in_outputs] @@ -608,11 +676,19 @@ end function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, organ_source, variable, value) + + timestep_mapped_outputs = soft_dep_graphs[organ_source][:timestep_mapped_outputs] for (proc_output, pairs_vars_output) in soft_dep_graphs[organ_source][:outputs] # e.g. proc_output = :maintenance_respiration; pairs_vars_output = soft_dep_graphs_roots.roots[organ_source][:outputs][proc_output] vars_output = flatten_vars(pairs_vars_output) + + vars_all_outputs = vars_output + + if haskey(timestep_mapped_outputs, proc_output) + vars_all_outputs = (; vars_output..., flatten_vars(timestep_mapped_outputs)...) + end # If the variable is found in the outputs of the process at the other scale: - if variable in keys(vars_output) + if variable in keys(vars_all_outputs) # The variable is found at another scale: if haskey(inputs_as_output_of_other_scale, organ_source) if haskey(inputs_as_output_of_other_scale[organ_source], proc_output) diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index efe17ec5..42b21975 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -147,4 +147,20 @@ end function convert_outputs(out::TimeStepTable{T} where T, sink) @assert Tables.istable(sink) "The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)" return sink(out) +end + + +struct MultiScaleMapping{T} + default_timestep::Date + mapping::Dict{String,T} +end + + +function MultiScaleMapping(mapping, mtg; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) + GraphSimulation(init_simulation(mtg, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose, orchestrator=orchestrator)...) +end + + +function dep(m::MultiScaleMapping) + return dep(m.graph) end \ No newline at end of file diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index 3243afc6..36fd8155 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -102,11 +102,26 @@ julia> PlantSimEngine.model_(multiscale_model) ToyCAllocationModel() ``` """ -struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Symbol},Vector{Pair{S,Symbol}}}}} where {A<:Union{Symbol,PreviousTimeStep},S<:AbstractString}} + +# timestep mapped variables : original variable name, aggregated variable name, timestep at which it operates, aggregation function +struct TimestepMappedVariable + name_from::Symbol + name_to::Symbol + timestep_to::Period + aggregation_function +end + +#=function TimestepMappedVariable(name::Symbol, name_to::Symbol, ts_to::Date, aggreg_fn) + TimestepMappedVariable(name, name_to, ts_to, aggreg_fn) +end=# + + +struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Symbol},Vector{Pair{S,Symbol}}}}} where {A<:Union{Symbol,PreviousTimeStep},S<:AbstractString}, W<:AbstractVector{TimestepMappedVariable}} model::T mapped_variables::V + timestep_mapped_variables::W - function MultiScaleModel{T}(model::T, mapped_variables) where {T<:AbstractModel} + function MultiScaleModel{T}(model::T, mapped_variables, timestep_mapped_variables) where {T<:AbstractModel} # Check that the variables in the mapping are variables of the model: model_variables = keys(variables(model)) for i in mapped_variables @@ -117,10 +132,28 @@ struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Sy var = isa(var, PreviousTimeStep) ? var.variable : var if !(var in model_variables) + # TODO check duplicates within the timestep_mapped_variables error("Mapping for model $model defines variable $var, but it is not a variable of the model.") end end + for i in timestep_mapped_variables + # TODO handle cases where name_from is a PreviousTimestep in the mapped variables + # var = isa(i, PreviousTimeStep) ? i.variable : first(i) + var = i.name_from + + # TODO ensure no conflicts with other mappings and refs on that variable... ? + if !(var in model_variables) + error("Timestep mapping for model $model requires variable $var, but it is not a variable of the model.") + end + + # Avoid name conflicts # TODO make sure no model at that scale causes name conflicts by having the same name (amongst its outputs), not just the model owning that variable + var_out = i.name_to + if var_out in model_variables + error("Timestep mapping for model $model defines an output variable $var, but that name is already used as a variable in the model.") + end + end + # If the name of the variable mapped from the other scale is not given, we add it as the same of the variable name in the model. Cases: # 1. `[:variable_name => "Plant"]` # We take one value from the Plant node # 2. `[:variable_name => ["Leaf"]]` # We take a vector of values from the Leaf nodes @@ -137,8 +170,12 @@ struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Sy push!(unfolded_mapping, _get_var(isa(i, PreviousTimeStep) ? i : Pair(i.first, i.second), process_)) # Note: We are using Pair(i.first, i.second) to make sure the Pair is specialized enough, because sometimes the vector in the mapping made the Pair not specialized enough e.g. [:v1 => "S" => :v2,:v3 => "S"] makes the pairs `Pair{Symbol, Any}`. end - - new{T,typeof(unfolded_mapping)}(model, unfolded_mapping) + + for i in timestep_mapped_variables + #TODO + end + + new{T,typeof(unfolded_mapping), typeof(timestep_mapped_variables)}(model, unfolded_mapping, timestep_mapped_variables) end end @@ -185,16 +222,24 @@ end -function MultiScaleModel(model::T, mapped_variables) where {T<:AbstractModel} - MultiScaleModel{T}(model, mapped_variables) +function MultiScaleModel(model::T, mapped_variables, timestep_mapped_variables) where {T<:AbstractModel} + MultiScaleModel{T}(model, mapped_variables, timestep_mapped_variables) end -MultiScaleModel(; model, mapped_variables) = MultiScaleModel(model, mapped_variables) + +MultiScaleModel(; model, mapped_variables, timestep_mapped_variables=TimestepMappedVariable[]) = MultiScaleModel(model, mapped_variables, timestep_mapped_variables) mapped_variables_(m::MultiScaleModel) = m.mapped_variables +#timestep_mapped_variables_(m::MultiScaleModel) = m.timestep_mapped_variables model_(m::MultiScaleModel) = m.model inputs_(m::MultiScaleModel) = inputs_(m.model) outputs_(m::MultiScaleModel) = outputs_(m.model) +function timestep_mapped_outputs_(m::MultiScaleModel) + # TODO this is not going to be correct beyond simple examples + (; [i.name_to => outputs_(m.model)[i.name_from] for i in m.timestep_mapped_variables]...) +end get_models(m::MultiScaleModel) = [model_(m)] # Get the models of a MultiScaleModel: # Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. get_status(m::MultiScaleModel) = nothing -get_mapped_variables(m::MultiScaleModel{T,S}) where {T,S} = mapped_variables_(m) \ No newline at end of file +function get_mapped_variables(m::MultiScaleModel{T,S}) where {T,S} + mapped_variables_(m) +end \ No newline at end of file diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index af1c7c57..57984efa 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -21,7 +21,7 @@ a dictionary of variables that need to be initialised or computed by other model `(;statuses, status_templates, reverse_multiscale_mapping, vars_need_init, nodes_with_models)` """ -function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())); type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator2()) +function init_statuses(mtg, mapping, dependency_graph#=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2()))=#; type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator2()) # We compute the variables mapping for each scale: mapped_vars = mapped_variables(mapping, dependency_graph, verbose=verbose,orchestrator=orchestrator) @@ -35,7 +35,7 @@ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(ma # Note 3: we do it before `convert_reference_values!` because we need the variables to be MappedVar{MultiNodeMapping} to get the reverse mapping. # Convert the MappedVar{SelfNodeMapping} or MappedVar{SingleNodeMapping} to RefValues, and MappedVar{MultiNodeMapping} to RefVectors: - convert_reference_values!(mapped_vars, orchestrator) + convert_reference_values!(mapped_vars)#, orchestrator) # Get the variables that are not initialised or computed by other models in the output: vars_need_init = Dict(org => filter(x -> isa(last(x), UninitializedVar), vars) |> keys |> collect for (org, vars) in mapped_vars) |> @@ -51,7 +51,7 @@ function init_statuses(mtg, mapping, dependency_graph=first(hard_dependencies(ma # TODO, *however*, this isn't the cleanest in its current state, # there may be some user initialisation issues that are hidden by this approach # Needs to be checked - filter_timestep_mapped_variables!(vars_need_init, orchestrator) + #filter_timestep_mapped_variables!(vars_need_init, orchestrator) # Note: these variables may be present in the MTG attributes, we check that below when traversing the MTG. @@ -335,7 +335,7 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion # preliminary_check_timestep_data(mapping, orchestrator) - # soft_dep_graph_roots, hard_dep_dict = hard_dependencies(mapping; verbose=false, orchestrator=orchestrator) + soft_dep_graph_roots, hard_dep_dict = hard_dependencies(mapping; verbose=false, orchestrator=orchestrator) # Get the status of each node by node type, pre-initialised considering multi-scale variables: # statuses, status_templates, reverse_multiscale_mapping, vars_need_init = @@ -343,7 +343,7 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion # Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses, status_templates, reverse_multiscale_mapping, vars_need_init = - init_statuses(mtg, mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=orchestrator)); type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) + init_statuses(mtg, mapping, soft_dep_graph_roots; type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) # Print an info if models are declared for nodes that don't exist in the MTG: @@ -358,9 +358,26 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion outputs_index = Dict{String, Int}(s => 1 for s in keys(outputs)) - dependency_graph = dep(mapping, verbose=verbose, orchestrator=orchestrator) + # dependency_graph = dep(mapping, verbose=verbose, orchestrator=orchestrator) #dependency_graph = dep(mapping, soft_dep_graph_roots=soft_dep_graph_roots, hard_dep_dict=hard_dep_dict, orchestrator=orchestrator) + # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the + # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the + # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children + # of the nodes that they depend on. + dependency_graph = soft_dependencies_multiscale(soft_dep_graph_roots, reverse_multiscale_mapping, hard_dep_dict) + # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, + # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale. + # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale. + + # Checking that the graph is acyclic: + iscyclic, cycle_vec = is_graph_cyclic(dependency_graph; warn=false) + # Note: we could do that in `soft_dependencies_multiscale` but we prefer to keep the function as simple as possible, and + # usable on its own. + + iscyclic && error("Cyclic dependency detected in the graph. Cycle: \n $(print_cycle(cycle_vec)) \n You can break the cycle using the `PreviousTimeStep` variable in the mapping.") + # Third step, we identify which + # Samuel : Once the dependency graph is created, and the timestep mappings are added into it # We need to register the existing MTG nodes to initialize their individual data # The current implementation is heavy, it may be quite slow for MTGs that already contain many nodes diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index 872fa67e..9036400d 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -11,7 +11,7 @@ However, models that are identified as hard-dependencies are not given individua nodes under other models. - `verbose::Bool`: whether to print the stacktrace of the search for the default value in the mapping. """ -function mapped_variables(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())); verbose=false) +function mapped_variables(mapping, dependency_graph; verbose=false, orchestrator=Orchestrator2()) # Initialise a dict that defines the multiscale variables for each organ type: mapped_vars = mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph) @@ -22,7 +22,7 @@ function mapped_variables(mapping, dependency_graph=first(hard_dependencies(mapp # Find variables that are inputs to other scales as a `SingleNodeMapping` and declare them as MappedVar from themselves in the source scale. # This helps us declare it as a reference when we create the template status objects. - transform_single_node_mapped_variables_as_self_node_output!(mapped_vars) + transform_single_node_mapped_variables_as_self_node_output!(mapped_vars, orchestrator) # We now merge inputs and outputs into a single dictionary: mapped_vars_per_organ = merge(merge, mapped_vars[:inputs], mapped_vars[:outputs]) @@ -54,10 +54,20 @@ This function returns a dictionary with the (multiscale-) inputs and outputs var Note that this function does not include the variables that are outputs from another scale and not computed by this scale, see `mapped_variables_with_outputs_as_inputs` for that. """ -function mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2()))) - nodes_insouts = Dict(organ => (inputs=ins, outputs=outs) for (organ, (soft_dep_graph, ins, outs)) in dependency_graph.roots) - ins = Dict{String,NamedTuple}(organ => flatten_vars(vcat(values(ins)...)) for (organ, (ins, outs)) in nodes_insouts) - outs = Dict{String,NamedTuple}(organ => flatten_vars(vcat(values(outs)...)) for (organ, (ins, outs)) in nodes_insouts) + +function combine_outputs(organ, outs, timestep_mapped_outputs_process) + if length(timestep_mapped_outputs_process) > 0 + timestep_mapped_organ = collect(values(timestep_mapped_outputs_process))[1] + (; flatten_vars(vcat(values(outs)...))..., timestep_mapped_organ...) + else + flatten_vars(vcat(values(outs)...)) + end +end + +function mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph) + nodes_insouts = Dict(organ => (inputs=ins, outputs=outs, timestep_outs=timestep_mapped_outs) for (organ, (soft_dep_graph, ins, outs, timestep_mapped_outs)) in dependency_graph.roots) + ins = Dict{String,NamedTuple}(organ => flatten_vars(vcat(values(ins)...)) for (organ, (ins, outs, timestep_mapped_outs)) in nodes_insouts) + outs = Dict{String,NamedTuple}(organ => combine_outputs(organ, outs, timestep_mapped_outs) for (organ, (ins, outs, timestep_mapped_outs)) in nodes_insouts) return Dict(:inputs => ins, :outputs => outs) end @@ -159,33 +169,69 @@ This helps us declare it as a reference when we create the template status objec These node are found in the mapping as `[:variable_name => "Plant"]` (notice that "Plant" is a scalar value). """ -function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars) +function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars, orchestrator=Orchestrator2()) for (organ, vars) in mapped_vars[:inputs] # e.g. organ = "Leaf"; vars = mapped_vars[:inputs][organ] for (var, mapped_var) in pairs(vars) # e.g. var = :carbon_biomass; mapped_var = vars[var] if isa(mapped_var, MappedVar{SingleNodeMapping}) source_organ = mapped_organ(mapped_var) source_organ == "" && continue # We skip the variables that are mapped to themselves (e.g. [PreviousTimeStep(:variable_name)], or just renaming a variable) + # TODO dirty prototyping to see how timestep mapped variables work at the same scale + # not good to special-case them + #source_organ == organ && continue @assert source_organ != organ "Variable `$var` is mapped to its own scale in organ $organ. This is not allowed." @assert haskey(mapped_vars[:outputs], source_organ) "Scale $source_organ not found in the mapping, but mapped to the $organ scale." - @assert haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * - "scale `$organ`, but is not computed by any model at `$source_organ` scale." - - # If the source variable was already defined as a `MappedVar{SelfNodeMapping}` by another scale, we skip it: - isa(mapped_vars[:outputs][source_organ][source_variable(mapped_var)], MappedVar{SelfNodeMapping}) && continue - # Note: this happens when a variable is mapped to several scales, e.g. soil_water_content computed at soil scale can be - # mapped at "Leaf" and "Internode" scale. - - # Transforming the variable into a MappedVar pointing to itself: - self_mapped_var = (; - source_variable(mapped_var) => - MappedVar( - SelfNodeMapping(), - source_variable(mapped_var), - source_variable(mapped_var), - mapped_vars[:outputs][source_organ][source_variable(mapped_var)], - ) - ) + + +#= if !haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) + + # Special case : Some variables computed by timestep mapping aren't necessarily 'true' outputs + # ie they are present in the *mapping*, but not as *outputs* of any model + # They should exist as *inputs* of a scale after computation, so map to that instead + # TODO ensure the models are consistent in terms of timestep, eg they correspond to the expected timestep mapping + # See example # TODO in the tests for a mapping requiring this code + # This might not be the ideal solution. It might be better to insert a proper SoftDependencyNode at that scale + # between the two models that handles the timestep mapping and has the variable as a proper input + if is_timestep_mapped(source_organ => source_variable(mapped_var), orchestrator) + # It isn't in the outputs, they should be in the inputs of the requested scale + # Assuming the user didn't mess up (TODO may be tricky to handle if so) + if haskey(mapped_vars[:inputs][source_organ], source_variable(mapped_var)) + self_mapped_var = (; + source_variable(mapped_var) => + MappedVar( + SelfNodeMapping(), + source_variable(mapped_var), + source_variable(mapped_var), + mapped_vars[:inputs][source_organ][source_variable(mapped_var)], + ) + ) + else + @assert false "Couldn't find a timestep-mapped variable at the expected scale" + end + else=# + # no timestep mapping fallback means this is a standard error + @assert haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * + "scale `$organ`, but is not computed by any model at `$source_organ` scale." + +# end +# else + + # If the source variable was already defined as a `MappedVar{SelfNodeMapping}` by another scale, we skip it: + isa(mapped_vars[:outputs][source_organ][source_variable(mapped_var)], MappedVar{SelfNodeMapping}) && continue + # Note: this happens when a variable is mapped to several scales, e.g. soil_water_content computed at soil scale can be + # mapped at "Leaf" and "Internode" scale. + + # Transforming the variable into a MappedVar pointing to itself: + self_mapped_var = (; + source_variable(mapped_var) => + MappedVar( + SelfNodeMapping(), + source_variable(mapped_var), + source_variable(mapped_var), + mapped_vars[:outputs][source_organ][source_variable(mapped_var)], + ) + ) +# end mapped_vars[:outputs][source_organ] = merge(mapped_vars[:outputs][source_organ], self_mapped_var) # Note: merge overwrites the LHS values with the RHS values if they have the same key. end @@ -298,7 +344,7 @@ Convert the variables that are `MappedVar{SelfNodeMapping}` or `MappedVar{Single common value for the variable; and convert `MappedVar{MultiNodeMapping}` to RefVectors that reference the values for the variable in the source organs. """ -function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}, orchestrator::Orchestrator2) +function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}})#, orchestrator::Orchestrator2) # For the variables that will be RefValues, i.e. referencing a value that exists for different scales, we need to first # create a common reference to the value that we use wherever we need this value. These values are created in the dict_mapped_vars # Dict, and then referenced from there every time we point to it. diff --git a/test/test-multitimestep.jl b/test/test-multitimestep.jl index e9c890bd..59879b49 100644 --- a/test/test-multitimestep.jl +++ b/test/test-multitimestep.jl @@ -212,7 +212,7 @@ mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) -out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) temps = [out["Default"][i].daily_temperature for i in 1:365] temp_m = maximum(temps) @@ -1040,3 +1040,110 @@ out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) #mtg_bis_2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) #mtg_bis_3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) #out = @enter run!(mtg_bis, m_multiscale, meteo_day, orchestrator=orch2) + + + + + + + + + + + + + + + + + + + +################################ +## API change : integrate timestep mapping into multiscalemodels +################################ + + +########################### +# Simple test with an orchestrator +########################### +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates +using Test + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(28.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=-Inf,) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MultiScaleModel(model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), max),] + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default" => :weekly_max_temperature], + timestep_mapped_variables=PlantSimEngine.TimestepMappedVariable[], #TODO avoid this + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + +#=to = PlantSimEngine.Var_to(:weekly_max_temperature) +from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) + +dict_to_from = Dict(from => to)=# +dict_to_from = Dict() +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) + +orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) + +out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + +temps = [out["Default"][i].daily_temperature for i in 1:365] +temp_m = maximum(temps) + +# At least one week should have max temp > 28 +@test temp_m > 28 && unique!([out["Default2"][i].hot for i in 1:365]) == [0,1] + + + m = @enter MultiScaleModel(model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), max),] + ) \ No newline at end of file From cba965cf56e3872afa2c3f7489a84d53cdd84389 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Tue, 2 Dec 2025 17:03:08 +0100 Subject: [PATCH 16/21] More changes, the new approach now works in some cases, more to investigate. More tests to implement, and other tests to fix, etc. --- src/PlantSimEngine.jl | 2 +- src/dependencies/dependency_graph.jl | 10 +- src/dependencies/hard_dependencies.jl | 62 ++-- src/dependencies/soft_dependencies.jl | 164 ++-------- src/mtg/GraphSimulation.jl | 6 +- src/mtg/MultiScaleModel.jl | 5 +- src/mtg/initialisation.jl | 20 +- src/mtg/mapping/compute_mapping.jl | 100 ++---- src/mtg/mapping/reverse_mapping.jl | 2 +- src/mtg/save_results.jl | 2 +- src/processes/model_initialisation.jl | 2 +- src/run.jl | 12 +- src/timestep/timestep_mapping.jl | 7 +- test/test-multitimestep.jl | 437 ++++++++++++++++++++++++-- 14 files changed, 521 insertions(+), 310 deletions(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 14c782cc..a4ff5c83 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -108,7 +108,7 @@ export PreviousTimeStep export AbstractModel export ModelList, MultiScaleModel, TimestepMappedVariable export MultiScaleMapping -export Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping +export Orchestrator, TimestepRange, ModelTimestepMapping export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index fe46f786..e31ea4aa 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -6,8 +6,8 @@ mutable struct HardDependencyNode{T} <: AbstractDependencyNode dependency::NamedTuple missing_dependency::Vector{Int} scale::String - inputs - outputs + #inputs + #outputs parent::Union{Nothing,<:AbstractDependencyNode} children::Vector{HardDependencyNode} end @@ -15,7 +15,7 @@ end mutable struct TimestepMapping variable_from::Symbol variable_to::Symbol - node_to # SoftDependencyNode causes a circular reference, removing it as a shortcut TODO + timestep_to::Period mapping_function::Function mapping_data_template mapping_data::Dict{Int, Any} # TODO fix type stability : Int is the node id, Any is a vector of n elements of the variable's type, n being the # of required timesteps @@ -26,8 +26,8 @@ mutable struct SoftDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol scale::String - inputs - outputs + #inputs + #outputs hard_dependency::Vector{HardDependencyNode} parent::Union{Nothing,Vector{SoftDependencyNode}} parent_vars::Union{Nothing,NamedTuple} diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 78bb5f8a..87bd3dd6 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -100,8 +100,8 @@ function initialise_all_as_hard_dependency_node(models, scale) NamedTuple(), Int[], scale, - inputs_(i), - outputs_(i), + #inputs_(i), + #outputs_(i), nothing, HardDependencyNode[] ) for (p, i) in pairs(models) @@ -134,7 +134,7 @@ Return a NamedTuple with the variables and their default values. The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is computed from the user mapping like so: """ -function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator2=Orchestrator2()) +function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator=Orchestrator()) node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,)) ins = node_vars.inputs ins_variables = keys(ins) @@ -154,36 +154,22 @@ function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orches # If the variable is an output, we use the default value given by the model: default = defaults[var] end - - # TODO no idea how this meshes with refvector situations or previoustimestep -# if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true) -# push!(vars_, var => default) -# if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) -# organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) -# push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) -# end -# else - if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true) - - push!(vars_, var => default) - else - if haskey(vars_mapping[organ], var) - organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var]) - push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default)) - #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current - #* values, and another one the previous values. - if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) - organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) - push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) - end - elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) - # If not found in the current time step, we check if the variable is mapped to the previous time step: + if haskey(vars_mapping[organ], var) + organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var]) + push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default)) + #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current + #* values, and another one the previous values. + if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) - else - # Else we take the default value: - push!(vars_, var => default) end + elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process)) + # If not found in the current time step, we check if the variable is mapped to the previous time step: + organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)]) + push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default)) + else + # Else we take the default value: + push!(vars_, var => default) end end # end @@ -210,12 +196,12 @@ function extract_timestep_mapped_outputs(m::MultiScaleModel, organ::String, outp key = process(m.model) extra_outputs = timestep_mapped_outputs_(m) #ind = findfirst(x -> first(x) == key, outputs_process[organ][key]) - timestep_mapped_outputs_process[organ][key] = (; extra_outputs...) #TODO + timestep_mapped_outputs_process[organ][key] = extra_outputs #TODO end end # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): -function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T} +function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator=Orchestrator()) where {T} full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping) soft_dep_graphs = Dict{String,Any}() not_found = Dict{Symbol,DataType}() @@ -239,7 +225,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr #* dependency may not appear in its own scale, but this is treated in the soft-dependency computation inputs_process = Dict{String,Dict{Symbol,Vector}}() outputs_process = Dict{String,Dict{Symbol,Vector}}() - timestep_mapped_outputs_process = Dict{String,Dict{Symbol,NamedTuple}}() + timestep_mapped_outputs_process = Dict{String,Dict{Symbol,Vector}}() for (organ, model) in mapping # Get the status given by the user, that is used to set the default values of the variables in the mapping: st_scale_user = get_status(model) @@ -324,8 +310,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr dep_node_model.dependency, dep_node_model.missing_dependency, dep_node_model.scale, - dep_node_model.inputs, - dep_node_model.outputs, + #dep_node_model.inputs, + #dep_node_model.outputs, parent_node, dep_node_model.children ) @@ -398,8 +384,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr soft_dep_vars.value, process_, # process name organ, # scale - inputs_process[organ][process_], # These are the inputs, potentially multiscale - outputs_process[organ][process_], # Same for outputs + #inputs_process[organ][process_], # These are the inputs, potentially multiscale + #outputs_process[organ][process_], # Same for outputs AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, @@ -430,7 +416,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr end end - soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ], timestep_mapped_outputs=haskey(timestep_mapped_outputs_process,organ) ? timestep_mapped_outputs_process[organ] : Dict{Symbol,NamedTuple}()) + soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ], timestep_mapped_outputs=haskey(timestep_mapped_outputs_process,organ) ? timestep_mapped_outputs_process[organ] : Dict{Symbol,Vector}()) not_found = merge(not_found, hard_deps[organ].not_found) end diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index f156ef0e..c282b789 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -64,8 +64,8 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, soft_dep_vars.value, process_, # process name "", - inputs_(soft_dep_vars.value), - outputs_(soft_dep_vars.value), + #inputs_(soft_dep_vars.value), + #outputs_(soft_dep_vars.value), AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, @@ -323,7 +323,7 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic end if haskey(timestep_mapped_outs, proc) - CreateTimeStepMapping(i, reverse_multiscale_mapping, organ, proc) + create_timestep_mapping(i, timestep_mapped_outs, proc) end end @@ -331,148 +331,24 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic dep_graph = DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found) - # TODO move these into the loop to directly act on the ndoe, use the provided timestep mappings - # Don't provide parents, and then change the run! function to apply from the node to itself - #traverse_dependency_graph!(x -> set_non_default_timestep_in_node(x, orchestrator), dependency_graph, visit_hard_dep=false) - #traverse_dependency_graph!(x -> add_timestep_data_to_node(x, orchestrator), dep_graph, visit_hard_dep=false) - - return dep_graph end -# # set the timestep for everyone first, else we might not use the correct timestep when looking at the parents later -function set_non_default_timestep_in_node(soft_dependency_node, orchestrator::Orchestrator2) - for mtsm in orchestrator.non_default_timestep_mapping - if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value)) - soft_dependency_node.timestep = mtsm.timestep - end - end -end - -function add_timestep_data_to_node(soft_dependency_node, orchestrator::Orchestrator2) - - # now we can create the mapping - for mtsm in orchestrator.non_default_timestep_mapping - if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value)) - for (var_from, var_to) in mtsm.var_to_var - if !isnothing(soft_dependency_node.parent) - parent = nothing - variable_mapping = nothing - for parent_node in soft_dependency_node.parent - if typeof(parent_node.value) == var_from.model && parent_node.scale == var_from.scale - parent = parent_node - variable_mapping = create_timestep_mapping(soft_dependency_node, parent, var_to, var_from) - break - end - end - if isnothing(parent) - #error - end - if isnothing(parent.timestep_mapping_data) - parent.timestep_mapping_data = Vector{TimestepMapping}() - end - push!(parent.timestep_mapping_data, variable_mapping) - else - # Error - end - end - end - end -end - -# Create the structure to attach to the softdependencynode, (not to be confused with the user-declared timestep-mapping structure) -function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependencyNode, var_to::Var_to, var_from::Var_from) - - @assert parent.timestep != 0 "Error : node timestep internally set to 0" - - timestep_ratio = node.timestep / parent.timestep - - # Keeping things simple for now, only integers allowed - @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio" - - # TODO ensure type compatibility between var_to and var_from - # Simplification probably possible by doing the check earlier - - # TODO test previoustimestep - var_type = DataType - - for (symbol, var_dump) in node.inputs - for var in var_dump - if isa(var, MappedVar) - # check the source variable, because the sink one might be a vector...? - # TODO multinode mapping - if isa(var.source_organ, MultiNodeMapping) - if var.source_variable[1] == var_from.name - # This should be a fixed size array, ideally - var_type = eltype(eltype(mapped_default(var))) - break - end - else - if var.source_variable == var_from.name - # This should be a fixed size array, ideally - var_type = eltype(mapped_default(var)) - break - end - end - else - if var.symbol == var_from.name - @assert "untested" - var_type = eltype(mapped_default(var)) - break - end - end - end - end - - # TODO : type shouldn't be Any but Vector{var_type} - return TimestepMapping(var_from.name, var_to.name, node, var_from.mapping_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG, Any}()) -end - - -function CreateTimeStepMapping(soft_dependency_node, reverse_multiscale_mapping, scale, proc) - - for mapping_entry in reverse_multiscale_mapping[scale] - if isa(mapping_entry, MultiScaleModel) - if process(mapping_entry.model) == proc - for timestep_mapped_var_data in mapping_entry.timestep_mapped_variables - timestep_ratio = timestep_mapped_var_data.timestep_to / soft_dependency_node.timestep - if timestep_ratio > 1 # todo assert it's an int or a rational ? - @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio" +function create_timestep_mapping(soft_dependency_node, timestep_mapped_outs, proc) - tmvd = timestep_mapped_var_data + for (proc_mapped, mapping_entries) in timestep_mapped_outs - var_type = DataType + if proc_mapped == proc + for (var_to, (timestep_mapped_var_data, default_value)) in mapping_entries + timestep_ratio = timestep_mapped_var_data.timestep_to / soft_dependency_node.timestep + if timestep_ratio > 1 # todo assert it's an int or a rational ? + @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio" - for (symbol, var_dump) in soft_dependency_node.outputs - for var in var_dump - if isa(var, MappedVar) - # check the source variable, because the sink one might be a vector...? - # TODO multinode mapping - if isa(var.source_organ, MultiNodeMapping) - if var.source_variable[1] == tmvd.name_from - # This should be a fixed size array, ideally - var_type = eltype(eltype(mapped_default(var))) - break - end - else - if var.source_variable == tmvd.name_from - # This should be a fixed size array, ideally - var_type = eltype(mapped_default(var)) - break - end - end - else - if var.symbol == tmvd.name_from - @assert "untested" - var_type = eltype(mapped_default(var)) - break - end - end - end - end + tmvd = timestep_mapped_var_data + var_type = typeof(default_value) - mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio)) - tsm = TimestepMapping(tmvd.name_from, tmvd.name_to, soft_dependency_node, tmvd.aggregation_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG,Any}()) + mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio)) + tsm = TimestepMapping(tmvd.name_from, tmvd.name_to, timestep_mapped_var_data.timestep_to, tmvd.aggregation_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG,Any}()) if isnothing(soft_dependency_node.timestep_mapping_data) soft_dependency_node.timestep_mapping_data = Vector{TimestepMapping}() @@ -483,10 +359,9 @@ function CreateTimeStepMapping(soft_dependency_node, reverse_multiscale_mapping, end end end -end -""" + """ drop_process(proc_vars, process) Return a new `NamedTuple` with the process `process` removed from the `NamedTuple` `proc_vars`. @@ -674,6 +549,13 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra return inputs_as_output_of_other_scale end +function extract_mapped_outputs(timestep_mapped_outputs) + extracted = Pair[] + for pair in timestep_mapped_outputs + push!(extracted, Pair(first(last(pair)).name_to, last(last(pair)))) + end + return extracted +end function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, organ_source, variable, value) @@ -684,7 +566,7 @@ function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, vars_all_outputs = vars_output if haskey(timestep_mapped_outputs, proc_output) - vars_all_outputs = (; vars_output..., flatten_vars(timestep_mapped_outputs)...) + vars_all_outputs = (; vars_output..., extract_mapped_outputs(timestep_mapped_outputs[proc_output])...) end # If the variable is found in the outputs of the process at the other scale: diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 42b21975..f0fb652b 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -27,11 +27,11 @@ struct GraphSimulation{T,S,U,O,V} models::Dict{String,U} outputs::Dict{String,O} outputs_index::Dict{String, Int} - orchestrator::Orchestrator2 + orchestrator::Orchestrator end -function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose, orchestrator=orchestrator)...) end @@ -156,7 +156,7 @@ struct MultiScaleMapping{T} end -function MultiScaleMapping(mapping, mtg; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) +function MultiScaleMapping(mapping, mtg; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) GraphSimulation(init_simulation(mtg, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose, orchestrator=orchestrator)...) end diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index 36fd8155..439ff53b 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -234,8 +234,9 @@ model_(m::MultiScaleModel) = m.model inputs_(m::MultiScaleModel) = inputs_(m.model) outputs_(m::MultiScaleModel) = outputs_(m.model) function timestep_mapped_outputs_(m::MultiScaleModel) - # TODO this is not going to be correct beyond simple examples - (; [i.name_to => outputs_(m.model)[i.name_from] for i in m.timestep_mapped_variables]...) + # TODO outputs_(m.model)[i.name_from] is the default value of the source variable + # this is not going to be correct beyond simple examples + [i.name_to => Pair(i, outputs_(m.model)[i.name_from]) for i in m.timestep_mapped_variables] end get_models(m::MultiScaleModel) = [model_(m)] # Get the models of a MultiScaleModel: # Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 57984efa..51de0dce 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -21,7 +21,7 @@ a dictionary of variables that need to be initialised or computed by other model `(;statuses, status_templates, reverse_multiscale_mapping, vars_need_init, nodes_with_models)` """ -function init_statuses(mtg, mapping, dependency_graph#=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2()))=#; type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator2()) +function init_statuses(mtg, mapping, dependency_graph#=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator()))=#; type_promotion=nothing, verbose=false, check=true, orchestrator=Orchestrator()) # We compute the variables mapping for each scale: mapped_vars = mapped_variables(mapping, dependency_graph, verbose=verbose,orchestrator=orchestrator) @@ -104,7 +104,7 @@ The `check` argument is a boolean indicating if variables initialisation should in the node attributes (using the variable name). If `true`, the function returns an error if the attribute is missing, otherwise it uses the default value from the model. """ -function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init=Dict{String,Any}(), type_promotion=nothing; check=true, attribute_name=:plantsimengine_status, orchestrator=Orchestrator2()) +function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init=Dict{String,Any}(), type_promotion=nothing; check=true, attribute_name=:plantsimengine_status, orchestrator=Orchestrator()) # Check if the node has a model defined for its symbol, if not, no need to compute symbol(node) ∉ collect(keys(mapped_vars)) && return @@ -215,23 +215,17 @@ julia> b 2.0 ``` """ -function status_from_template(d::Dict{Symbol,T} where {T}, scale::String, orchestrator::Orchestrator2=Orchestrator2()) +function status_from_template(d::Dict{Symbol,T} where {T}, scale::String, orchestrator::Orchestrator=Orchestrator()) # Sort vars to put the values that are of type PerStatusRef at the end (we need the pass on the other ones first): sorted_vars = Dict{Symbol,Any}(sort([pairs(d)...], by=v -> last(v) isa RefVariable ? 1 : 0)) # Note: PerStatusRef are used to reference variables in the same status for renaming. # We create the status with the right references for variables, and for PerStatusRef (we reference the reference variable): for (k, v) in sorted_vars - if is_timestep_mapped((scale => k), orchestrator, search_inputs_only=true) - # avoid referring to the original variable - sorted_vars[k] = ref_var(v) + if isa(v, RefVariable) + sorted_vars[k] = sorted_vars[v.reference_variable] else - - if isa(v, RefVariable) - sorted_vars[k] = sorted_vars[v.reference_variable] - else - sorted_vars[k] = ref_var(v) - end + sorted_vars[k] = ref_var(v) end end @@ -324,7 +318,7 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator2()) +function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=false, orchestrator=Orchestrator()) # Ensure the user called the model generation function to handle vectors passed into a status # before we keep going diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index 9036400d..94130a3b 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -11,7 +11,7 @@ However, models that are identified as hard-dependencies are not given individua nodes under other models. - `verbose::Bool`: whether to print the stacktrace of the search for the default value in the mapping. """ -function mapped_variables(mapping, dependency_graph; verbose=false, orchestrator=Orchestrator2()) +function mapped_variables(mapping, dependency_graph; verbose=false, orchestrator=Orchestrator()) # Initialise a dict that defines the multiscale variables for each organ type: mapped_vars = mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph) @@ -56,8 +56,12 @@ see `mapped_variables_with_outputs_as_inputs` for that. """ function combine_outputs(organ, outs, timestep_mapped_outputs_process) - if length(timestep_mapped_outputs_process) > 0 - timestep_mapped_organ = collect(values(timestep_mapped_outputs_process))[1] + if length(timestep_mapped_outputs_process) > 0 + timestep_mapped_organ = NamedTuple() + for (proc, data) in timestep_mapped_outputs_process + timestep_mapped_organ = (; timestep_mapped_organ..., extract_mapped_outputs(data)...) + end + #timestep_mapped_organ = collect(values(timestep_mapped_outputs_process))[1] (; flatten_vars(vcat(values(outs)...))..., timestep_mapped_organ...) else flatten_vars(vcat(values(outs)...)) @@ -169,7 +173,7 @@ This helps us declare it as a reference when we create the template status objec These node are found in the mapping as `[:variable_name => "Plant"]` (notice that "Plant" is a scalar value). """ -function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars, orchestrator=Orchestrator2()) +function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars, orchestrator=Orchestrator()) for (organ, vars) in mapped_vars[:inputs] # e.g. organ = "Leaf"; vars = mapped_vars[:inputs][organ] for (var, mapped_var) in pairs(vars) # e.g. var = :carbon_biomass; mapped_var = vars[var] if isa(mapped_var, MappedVar{SingleNodeMapping}) @@ -181,57 +185,25 @@ function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars @assert source_organ != organ "Variable `$var` is mapped to its own scale in organ $organ. This is not allowed." @assert haskey(mapped_vars[:outputs], source_organ) "Scale $source_organ not found in the mapping, but mapped to the $organ scale." - - -#= if !haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) - - # Special case : Some variables computed by timestep mapping aren't necessarily 'true' outputs - # ie they are present in the *mapping*, but not as *outputs* of any model - # They should exist as *inputs* of a scale after computation, so map to that instead - # TODO ensure the models are consistent in terms of timestep, eg they correspond to the expected timestep mapping - # See example # TODO in the tests for a mapping requiring this code - # This might not be the ideal solution. It might be better to insert a proper SoftDependencyNode at that scale - # between the two models that handles the timestep mapping and has the variable as a proper input - if is_timestep_mapped(source_organ => source_variable(mapped_var), orchestrator) - # It isn't in the outputs, they should be in the inputs of the requested scale - # Assuming the user didn't mess up (TODO may be tricky to handle if so) - if haskey(mapped_vars[:inputs][source_organ], source_variable(mapped_var)) - self_mapped_var = (; - source_variable(mapped_var) => - MappedVar( - SelfNodeMapping(), - source_variable(mapped_var), - source_variable(mapped_var), - mapped_vars[:inputs][source_organ][source_variable(mapped_var)], - ) - ) - else - @assert false "Couldn't find a timestep-mapped variable at the expected scale" - end - else=# - # no timestep mapping fallback means this is a standard error - @assert haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * - "scale `$organ`, but is not computed by any model at `$source_organ` scale." - -# end -# else - - # If the source variable was already defined as a `MappedVar{SelfNodeMapping}` by another scale, we skip it: - isa(mapped_vars[:outputs][source_organ][source_variable(mapped_var)], MappedVar{SelfNodeMapping}) && continue - # Note: this happens when a variable is mapped to several scales, e.g. soil_water_content computed at soil scale can be - # mapped at "Leaf" and "Internode" scale. - - # Transforming the variable into a MappedVar pointing to itself: - self_mapped_var = (; - source_variable(mapped_var) => - MappedVar( - SelfNodeMapping(), - source_variable(mapped_var), - source_variable(mapped_var), - mapped_vars[:outputs][source_organ][source_variable(mapped_var)], - ) - ) -# end + + @assert haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * + "scale `$organ`, but is not computed by any model at `$source_organ` scale." + # If the source variable was already defined as a `MappedVar{SelfNodeMapping}` by another scale, we skip it: + isa(mapped_vars[:outputs][source_organ][source_variable(mapped_var)], MappedVar{SelfNodeMapping}) && continue + # Note: this happens when a variable is mapped to several scales, e.g. soil_water_content computed at soil scale can be + # mapped at "Leaf" and "Internode" scale. + + # Transforming the variable into a MappedVar pointing to itself: + self_mapped_var = (; + source_variable(mapped_var) => + MappedVar( + SelfNodeMapping(), + source_variable(mapped_var), + source_variable(mapped_var), + mapped_vars[:outputs][source_organ][source_variable(mapped_var)], + ) + ) + # end mapped_vars[:outputs][source_organ] = merge(mapped_vars[:outputs][source_organ], self_mapped_var) # Note: merge overwrites the LHS values with the RHS values if they have the same key. end @@ -325,18 +297,6 @@ function default_variables_from_mapping(mapped_vars, verbose=true) end -function is_timestep_mapped(key, orchestrator::Orchestrator2; search_inputs_only::Bool=false) - for mtsm in orchestrator.non_default_timestep_mapping - for (var_from, var_to) in mtsm.var_to_var - if ((first(key) == mtsm.scale) && last(key) == var_to.name) || - (!search_inputs_only && (first(key) == var_from.scale && last(key) == var_from.name)) - return true - end - end - end - return false -end - """ convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}}) @@ -344,7 +304,7 @@ Convert the variables that are `MappedVar{SelfNodeMapping}` or `MappedVar{Single common value for the variable; and convert `MappedVar{MultiNodeMapping}` to RefVectors that reference the values for the variable in the source organs. """ -function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}})#, orchestrator::Orchestrator2) +function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}})#, orchestrator::Orchestrator) # For the variables that will be RefValues, i.e. referencing a value that exists for different scales, we need to first # create a common reference to the value that we use wherever we need this value. These values are created in the dict_mapped_vars # Dict, and then referenced from there every time we point to it. @@ -361,11 +321,7 @@ function convert_reference_values!(mapped_vars::Dict{String,Dict{Symbol,Any}})#, # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: if !haskey(dict_mapped_vars, key) -# if is_timestep_mapped(key, orchestrator) -# push!(dict_mapped_vars, key => (mapped_default(vars[k]))) -# else push!(dict_mapped_vars, key => Ref(mapped_default(vars[k]))) -# end end # Then we use the value for the particular variable to replace the MappedVar to a RefValue in the mapping: diff --git a/src/mtg/mapping/reverse_mapping.jl b/src/mtg/mapping/reverse_mapping.jl index 71ba46df..7b07135e 100644 --- a/src/mtg/mapping/reverse_mapping.jl +++ b/src/mtg/mapping/reverse_mapping.jl @@ -69,7 +69,7 @@ Dict{String, Dict{String, Dict{Symbol, Any}}} with 3 entries: """ function reverse_mapping(mapping::Dict{String,T}; all=true) where {T<:Any} # Method for the reverse mapping applied directly on the mapping (not used in the code base) - mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())), verbose=false) + mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator())), verbose=false) reverse_mapping(mapped_vars, all=all) end diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 52a94ba8..1845bcf1 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -110,7 +110,7 @@ julia> collect(keys(preallocated_vars["Leaf"])) ``` """ # TODO orchestrator prob shouldn't be a kwarg with a default -function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_mapping, vars_need_init, outs, nsteps; type_promotion=nothing, check=true, orchestrator=Orchestrator2()) +function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_mapping, vars_need_init, outs, nsteps; type_promotion=nothing, check=true, orchestrator=Orchestrator()) outs_ = Dict{String,Vector{Symbol}}() # default behaviour : track everything diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 04702072..d944b16c 100755 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -148,7 +148,7 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} end to_init = Dict(org => Symbol[] for org in keys(mapping)) - mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator2())), verbose=false) + mapped_vars = mapped_variables(mapping, first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator())), verbose=false) for (org, vars) in mapped_vars for (var, val) in vars if isa(val, UninitializedVar) && var ∉ vars_in_mtg diff --git a/src/run.jl b/src/run.jl index a84f9941..127fa323 100644 --- a/src/run.jl +++ b/src/run.jl @@ -369,7 +369,7 @@ function run!( tracked_outputs=nothing, check=true, executor=ThreadedEx(), - orchestrator::Orchestrator2=Orchestrator2(), + orchestrator::Orchestrator=Orchestrator(), ) isnothing(nsteps) && (nsteps = get_nsteps(meteo)) meteo_adjusted = adjust_weather_timesteps_to_given_length(nsteps, meteo) @@ -502,7 +502,7 @@ function run_node_multiscale!( # Do the accumulation then write into the child's status if need be for tmst in node.timestep_mapping_data - ratio = Int(tmst.node_to.timestep / node.timestep) + ratio = Int(tmst.timestep_to / node.timestep) # TODO assert etc. This is all assuming the ratio is an integer, whereas it can be, like 1/7 # do the accumulation for each variable index = Int(i*Day(1) / node.timestep) @@ -513,13 +513,13 @@ function run_node_multiscale!( # the mapped model # A full cycle isn't just the ratio to the parent, it's the ratio to the finest-grained timestep if accumulation_index == ratio - node_statuses_to = status(object)[tmst.node_to.scale] - + #node_statuses_to = status(object)[tmst.node_to.scale] + st[tmst.variable_to] = tmst.mapping_function(tmst.mapping_data[node_id(st.node)]) # TODO : INCORRECT in a scale with multiple mtg nodes - for st_to in node_statuses_to + #=for st_to in node_statuses_to # TODO might be able to catch mapping_function type incompatibility errors and make them clearer st_to[tmst.variable_to] = tmst.mapping_function(tmst.mapping_data[node_id(st.node)]) - end + end=# end end end diff --git a/src/timestep/timestep_mapping.jl b/src/timestep/timestep_mapping.jl index 85c7f4ea..d9ac170d 100644 --- a/src/timestep/timestep_mapping.jl +++ b/src/timestep/timestep_mapping.jl @@ -40,20 +40,19 @@ struct ModelTimestepMapping model scale::String timestep::Period - var_to_var::Dict{Var_from, Var_to} end -mutable struct Orchestrator2 +mutable struct Orchestrator default_timestep::Period non_default_timestep_mapping::Vector{ModelTimestepMapping} - function Orchestrator2(default::Period, non_default_timestep_mapping::Vector{ModelTimestepMapping}) + function Orchestrator(default::Period, non_default_timestep_mapping::Vector{ModelTimestepMapping}) @assert default >= Second(0) "The default_timestep should be greater than or equal to 0." return new(default, non_default_timestep_mapping) end end -Orchestrator2() = Orchestrator2(Day(1), Vector{ModelTimestepMapping}()) +Orchestrator() = Orchestrator(Day(1), Vector{ModelTimestepMapping}()) # TODO parallelization diff --git a/test/test-multitimestep.jl b/test/test-multitimestep.jl index 59879b49..0fe78bc6 100644 --- a/test/test-multitimestep.jl +++ b/test/test-multitimestep.jl @@ -210,7 +210,7 @@ from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, max dict_to_from = Dict(from => to) mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) -orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) @@ -309,7 +309,7 @@ df = DataFrame(:data => [1 for i in 1:365], ) mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) m_multiscale = Dict("Default" => ( @@ -352,7 +352,7 @@ mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) #mtg4 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default4", 1, 4)) #mtg5 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default5", 1, 5)) - #orch2 = PlantSimEngine.Orchestrator2() + #orch2 = PlantSimEngine.Orchestrator() #out = @run run!(mtg, m_multiscale, df, orchestrator=orch2) out = run!(mtg, m_multiscale, df, orchestrator=orch2) @@ -394,7 +394,7 @@ using Test mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) out = run!(mtg_single, m_singlescale, df, orchestrator=orch2) @@ -428,7 +428,7 @@ using Test mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) @@ -540,7 +540,7 @@ df = DataFrame(:data => [1 for i in 1:365], ) mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4]) + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) m_multiscale = Dict( @@ -693,8 +693,8 @@ df = DataFrame(:data => [1 for i in 1:365], ) mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1), dict_to_from_w) mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4), dict_to_from_w4) - orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm_w, mtsm_w4])=# -orch2 = PlantSimEngine.Orchestrator2() + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4])=# +orch2 = PlantSimEngine.Orchestrator() m_multiscale = Dict( "Default6" => @@ -882,7 +882,7 @@ mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", We #dict_to_from_2 = Dict(from => to, from => to) #mtsm2 = PlantSimEngine.ModelTimestepMapping(MyToyDayDWDOutModel, "Default", Day(1), dict_to_from2) -orch_dwd = PlantSimEngine.Orchestrator2(Day(1), [mtsm_dwd,])#mtsm2]) +orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,])#mtsm2]) out = run!(mtg, m_dwd, df, orchestrator=orch_dwd) @@ -955,7 +955,7 @@ from_sum = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperatu dict_to_from = Dict(from => to, from_sum => to_sum) mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1), dict_to_from) -orch_dwd = PlantSimEngine.Orchestrator2(Day(1), [mtsm_dwd,]) +orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,]) out = run!(mtg, m_dwd, meteo_day, orchestrator=orch_dwd) @@ -1027,8 +1027,8 @@ from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, max dict_to_from = Dict(from => to) mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) -#orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) -orch2 = PlantSimEngine.Orchestrator2() +#orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) +orch2 = PlantSimEngine.Orchestrator() out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) #out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) @@ -1090,7 +1090,7 @@ struct MyToyWeekModel <: AbstractToyweekModel temperature_threshold::Float64 end -MyToyWeekModel() = MyToyWeekModel(28.0) +MyToyWeekModel() = MyToyWeekModel(15.0) function PlantSimEngine.inputs_(::MyToyWeekModel) (weekly_max_temperature=-Inf,) end @@ -1108,7 +1108,7 @@ meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.cs m_multiscale = Dict("Default" => ( MultiScaleModel(model=MyToyDayModel(), mapped_variables=[], - timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), max),] + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), maximum),] ), Status(a=1,) ), @@ -1124,14 +1124,9 @@ m_multiscale = Dict("Default" => ( mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) -#=to = PlantSimEngine.Var_to(:weekly_max_temperature) -from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) - -dict_to_from = Dict(from => to)=# -dict_to_from = Dict() -mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) -orch2 = PlantSimEngine.Orchestrator2(Day(1), [mtsm,]) +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) @@ -1146,4 +1141,402 @@ temp_m = maximum(temps) m = @enter MultiScaleModel(model=MyToyDayModel(), mapped_variables=[], timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), max),] - ) \ No newline at end of file + ) + + + + + +########################### +# Test with three timesteps, multiscale +########################### + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay2" verbose = false + +struct MyToyDay2Model <: AbstractToyday2Model end + +PlantSimEngine.inputs_(m::MyToyDay2Model) = NamedTuple() +PlantSimEngine.outputs_(m::MyToyDay2Model) = (out_day=-Inf,) + +function PlantSimEngine.run!(m::MyToyDay2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_day = meteo.data +end + +PlantSimEngine.@process "ToyWeek2" verbose = false + +struct MyToyWeek2Model <: AbstractToyweek2Model end + +PlantSimEngine.inputs_(::MyToyWeek2Model) = (in_week=-Inf,) +PlantSimEngine.outputs_(m::MyToyWeek2Model) = (out_week=-Inf,) + +function PlantSimEngine.run!(m::MyToyWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.out_week = status.in_week +end + +PlantSimEngine.timestep_range_(m::MyToyWeek2Model) = TimestepRange(Week(1)) + + +PlantSimEngine.@process "ToyFourWeek2" verbose = false + +struct MyToyFourWeek2Model <: AbstractToyfourweek2Model end + +PlantSimEngine.inputs_(::MyToyFourWeek2Model) = (in_four_week_from_week=-Inf, in_four_week_from_day=-Inf,) +PlantSimEngine.outputs_(m::MyToyFourWeek2Model) = (inputs_agreement=false,) + +function PlantSimEngine.run!(m::MyToyFourWeek2Model, models, status, meteo, constants=nothing, extra=nothing) + status.inputs_agreement = status.in_four_week_from_week == status.in_four_week_from_day +end + +PlantSimEngine.timestep_range_(m::MyToyFourWeek2Model) = TimestepRange(Week(4)) + + + +df = DataFrame(:data => [1 for i in 1:365], ) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default2", Week(1)) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default3", Week(4)) + + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) + + +m_multiscale = Dict("Default" => ( + MultiScaleModel(model=MyToyDay2Model(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:out_day, :out_week_from_day, Week(1), sum), + TimestepMappedVariable(:out_day, :out_four_week_from_day, Week(4), sum),] + ), + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[:in_week => "Default" => :out_week_from_day], + timestep_mapped_variables=[TimestepMappedVariable(:out_week, :out_four_week_from_week, Week(4), sum),] + ), + ), + "Default3" => ( + MultiScaleModel(model=MyToyFourWeek2Model(), + mapped_variables=[ + :in_four_week_from_day => "Default" => :out_four_week_from_day, + :in_four_week_from_week => "Default2" => :out_four_week_from_week, + ], + ),), + ) + + +# TODO test with multiple nodes +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default3", 1, 3)) + +out = run!(mtg, m_multiscale, df, orchestrator=orch2) + + + +using Test + @test unique([out["Default3"][i].in_four_week_from_day for i in 1:length(out["Default3"])]) == [-Inf, 28.0] + @test unique([out["Default3"][i].in_four_week_from_week for i in 1:length(out["Default3"])]) == [-Inf, 28.0] + + # Note : until the models actually run, inputs_agreement defaults to false, so it's only expected to be true + # from day 28 onwards + @test unique([out["Default3"][i].inputs_agreement for i in 28:length(out["Default3"])]) == [1] + +########################### +# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting +# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) +# and check mapping at the same scale +########################### + +# This example has variable renaming at the same scale + + m_singlescale_mapped = Dict("Default" => ( + MultiScaleModel(model=MyToyDay2Model(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:out_day, :out_week_from_day, Week(1), sum), + TimestepMappedVariable(:out_day, :out_four_week_from_day, Week(4), sum),] + ), + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[:in_week => "Default" => :out_week_from_day], + timestep_mapped_variables=[TimestepMappedVariable(:out_week, :out_four_week_from_week, Week(4), sum),] + ), + MultiScaleModel(model=MyToyFourWeek2Model(), + mapped_variables=[ + :in_four_week_from_day => "Default" => :out_four_week_from_day, + :in_four_week_from_week => "Default" => :out_four_week_from_week, + ], + ),)) + + # This one resuses the variable names directly, so requires only timestep mapping + m_singlescale = Dict("Default" => ( + MultiScaleModel(model=MyToyDay2Model(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:out_day, :in_week, Week(1), sum), + TimestepMappedVariable(:out_day, :in_four_week_from_day, Week(4), sum),] + ), + MultiScaleModel(model=MyToyWeek2Model(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:out_week, :in_four_week_from_week, Week(4), sum),] + ), + MyToyFourWeek2Model(), + )) + + mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1)) + mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4)) + + orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) + +mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +out = run!(mtg_single, m_singlescale, df, orchestrator=orch2) +out = run!(mtg_single, m_singlescale_mapped, df, orchestrator=orch2) + + + +#TODO +########################## +# Two models, D -> W, but D has two MTG nodes +# So simple test of a RefVector + timestep mapping combo +########################## + +# Currently errors due to refvectors not being handled properly + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=[-Inf],) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MultiScaleModel( + model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_temperature, Week(1), maximum)], + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => ["Default" => :weekly_temperature]], # TODO test this + #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) + +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) + +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) + +out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +#out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + + + +# alternate mtg with 2 nodes -> 2 nodes +#mtg_bis = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +#mtg_bis_1 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +#mtg_bis_2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) +#mtg_bis_3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) +#out = @enter run!(mtg_bis, m_multiscale, meteo_day, orchestrator=orch2) + + + +########################### +# Test with a D -> W -> D configuration, with multiple variables mapped between timesteps +########################### + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDayDWD" verbose = false + +struct MyToyDayDWDModel <: AbstractToydaydwdModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.data +end + +PlantSimEngine.@process "ToyWeekDWD" verbose = false + +struct MyToyWeekDWDModel <: AbstractToyweekdwdModel + temperature_threshold::Float64 +end + +MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekDWDModel) + (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) +end +PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) + +function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold + status.sum += status.weekly_sum_temperature +end + +PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) + +PlantSimEngine.@process "ToyDayDWDOut" verbose = false + +struct MyToyDayDWDOutModel <: AbstractToydaydwdoutModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDOutModel) = (weekly_max_temperature=-Inf,weekly_sum_temperature=-Inf,) +PlantSimEngine.outputs_(m::MyToyDayDWDOutModel) = (out=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDOutModel, models, status, meteo, constants=nothing, extra=nothing) + status.out = status.weekly_sum_temperature - 7.0*meteo.data +end + +df = DataFrame(:data => [1 for i in 1:365], ) + +# TODO check that DWDOUT properly uses the variables from Default2 and not Default +m_dwd = Dict("Default" => ( + MultiScaleModel( + model=MyToyDayDWDModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), maximum), + TimestepMappedVariable(:daily_temperature, :weekly_sum_temperature, Week(1), sum), + ] ), + MultiScaleModel( + model=MyToyDayDWDOutModel(), + mapped_variables=[:weekly_max_temperature => "Default2", :weekly_sum_temperature => "Default2"] + ), + Status(a=1,out=0.0) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekDWDModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default", :weekly_sum_temperature => "Default"], + ), + Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum=0.0) + ), +) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + +mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1)) + +#dict_to_from_2 = Dict(from => to, from => to) +#mtsm2 = PlantSimEngine.ModelTimestepMapping(MyToyDayDWDOutModel, "Default", Day(1), dict_to_from2) + +orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,])#mtsm2]) + +out = @run run!(mtg, m_dwd, df, orchestrator=orch_dwd) + + +################################## +# Two variables mapped +################################## + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDayDWD" verbose = false + +struct MyToyDayDWDModel <: AbstractToydaydwdModel end + +PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeekDWD" verbose = false + +struct MyToyWeekDWDModel <: AbstractToyweekdwdModel + temperature_threshold::Float64 +end + +MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekDWDModel) + (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) +end +PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) + +function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold + status.sum += status.weekly_sum_temperature +end + +PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) + +#df = DataFrame(:data => [1 for i in 1:365], ) +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_dwd = Dict("Default" => ( + MultiScaleModel(model=MyToyDayDWDModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), maximum), + TimestepMappedVariable(:daily_temperature, :weekly_sum_temperature, Week(1), sum), + ] + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekDWDModel(), + #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this + mapped_variables=[:weekly_max_temperature => "Default", :weekly_sum_temperature => "Default" ], + ), + Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum =0.0) + ), +) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) + +mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1)) + +orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,]) + +out = run!(mtg, m_dwd, meteo_day, orchestrator=orch_dwd) + +# TODO previous timestep, timestep-mapping to the same variable name + + +#TODO should timestep mapped vars also be part of a model's outputs ? \ No newline at end of file From b044252b397f8bef940475234feca79daeb369e1 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Wed, 3 Dec 2025 14:16:15 +0100 Subject: [PATCH 17/21] Fix some timestep mapping -related bugs that are caught by more complex examples, and add/remove some tests --- src/dependencies/hard_dependencies.jl | 4 +- src/dependencies/soft_dependencies.jl | 20 +- test/test-multitimestep.jl | 769 ++++++++------------------ 3 files changed, 243 insertions(+), 550 deletions(-) diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 87bd3dd6..ae596d1e 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -192,7 +192,9 @@ end function extract_timestep_mapped_outputs(m::MultiScaleModel, organ::String, outputs_process, timestep_mapped_outputs_process) if length(m.timestep_mapped_variables) > 0 - timestep_mapped_outputs_process[organ] = Dict{Symbol,Vector}() + if !haskey(timestep_mapped_outputs_process, organ) + timestep_mapped_outputs_process[organ] = Dict{Symbol,Vector}() + end key = process(m.model) extra_outputs = timestep_mapped_outputs_(m) #ind = findfirst(x -> first(x) == key, outputs_process[organ][key]) diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index c282b789..12b98c92 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -431,6 +431,16 @@ search_inputs_in_output(:process3, in_, out_) (process4 = (:var1, :var2),) ``` """ + + +function extract_mapped_outputs(timestep_mapped_outputs) + extracted = Pair[] + for pair in timestep_mapped_outputs + push!(extracted, Pair(first(last(pair)).name_to, last(last(pair)))) + end + return extracted +end + function search_inputs_in_output(process, inputs, outputs, timestep_mapped_outputs=Dict{Symbol,NamedTuple}()) # proc, ins, outs # get the inputs of the node: @@ -442,7 +452,7 @@ function search_inputs_in_output(process, inputs, outputs, timestep_mapped_outpu vars_output = flatten_vars(pairs_vars_output) vars_all_outputs = vars_output if haskey(timestep_mapped_outputs, proc_output) - vars_all_outputs = (; vars_output..., flatten_vars(timestep_mapped_outputs[proc_output])...) + vars_all_outputs = (; vars_output..., extract_mapped_outputs(timestep_mapped_outputs[proc_output])...) end inputs_in_outputs = vars_in_variables(vars_input, vars_all_outputs) @@ -549,14 +559,6 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra return inputs_as_output_of_other_scale end -function extract_mapped_outputs(timestep_mapped_outputs) - extracted = Pair[] - for pair in timestep_mapped_outputs - push!(extracted, Pair(first(last(pair)).name_to, last(last(pair)))) - end - return extracted -end - function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, organ_source, variable, value) timestep_mapped_outputs = soft_dep_graphs[organ_source][:timestep_mapped_outputs] diff --git a/test/test-multitimestep.jl b/test/test-multitimestep.jl index 0fe78bc6..e4563527 100644 --- a/test/test-multitimestep.jl +++ b/test/test-multitimestep.jl @@ -1,224 +1,3 @@ -########################### - -# Simple test using an ad hoc connector model -# Broken by subsequent changes, left just in case for now (TODO remove once prototyping is over) -########################### - -#= -using PlantSimEngine -# Include the example dummy processes: -using PlantSimEngine.Examples -using Test, Aqua -using Tables, DataFrames, CSV -using MultiScaleTreeGraph -using PlantMeteo, Statistics -using Documenter # for doctests - -using PlantMeteo.Dates -include("helper-functions.jl") - - - -# These models might be worth exposing in the future ? -PlantSimEngine.@process "basic_current_timestep" verbose = false - -struct HelperCurrentTimestepModel <: AbstractBasic_Current_TimestepModel -end - -PlantSimEngine.inputs_(::HelperCurrentTimestepModel) = (next_timestep=1,) -PlantSimEngine.outputs_(m::HelperCurrentTimestepModel) = (current_timestep=1,) - -function PlantSimEngine.run!(m::HelperCurrentTimestepModel, models, status, meteo, constants=nothing, extra=nothing) - status.current_timestep = status.next_timestep - end - - PlantSimEngine.ObjectDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsObjectDependent() - PlantSimEngine.TimeStepDependencyTrait(::Type{<:HelperCurrentTimestepModel}) = PlantSimEngine.IsTimeStepDependent() - -PlantSimEngine.timestep_range_(m::HelperCurrentTimestepModel) = Day(1) - - - PlantSimEngine.@process "basic_next_timestep" verbose = false - struct HelperNextTimestepModel <: AbstractBasic_Next_TimestepModel - end - - PlantSimEngine.inputs_(::HelperNextTimestepModel) = (current_timestep=1,) - PlantSimEngine.outputs_(m::HelperNextTimestepModel) = (next_timestep=1,) - - function PlantSimEngine.run!(m::HelperNextTimestepModel, models, status, meteo, constants=nothing, extra=nothing) - status.next_timestep = status.current_timestep + 1 - end - -PlantSimEngine.timestep_range_(m::HelperNextTimestepModel) = Day(1) - - - - - -PlantSimEngine.@process "ToyDay" verbose = false - -struct MyToyDayModel <: AbstractToydayModel end - -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) - -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end - -PlantSimEngine.@process "ToyWeek" verbose = false - -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end - -MyToyWeekModel() = MyToyWeekModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=-Inf,) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) - -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end - -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = Week(1) - - - -PlantSimEngine.@process "DWConnector" verbose = false - -struct MyDwconnectorModel <: AbstractDwconnectorModel - T_daily::Array{Float64} -end - -MyDwconnectorModel() = MyDwconnectorModel(Array{Float64}(undef, 7)) - -function PlantSimEngine.inputs_(::MyDwconnectorModel) - (daily_temperature=-Inf, current_timestep=1,) -end -PlantSimEngine.outputs_(m::MyDwconnectorModel) = (weekly_max_temperature = 0.0,) - -function PlantSimEngine.run!(m::MyDwconnectorModel, models, status, meteo, constants=nothing, extra=nothing) - m.T_daily[1 + (status.current_timestep % 7)] = status.daily_temperature - - if(status.current_timestep % 7 == 1) - status.weekly_max_temperature = sum(m.T_daily)/7.0 - else - status.weekly_max_temperature = 0 - end -end - - PlantSimEngine.timestep_range_(m::MyDwconnectorModel) = Day(1) - - - - - -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - -m = Dict("Default" => ( - MyToyDayModel(), - MyToyWeekModel(), - MyDwconnectorModel(), - HelperNextTimestepModel(), - MultiScaleModel( - model=HelperCurrentTimestepModel(), - mapped_variables=[PreviousTimeStep(:next_timestep),], - ), - Status(a=1,))) - -to_initialize(m) - -models_timestep = Dict(MyToyDayModel=>1, MyDwconnectorModel => 1, MyToyWeekModel =>7, HelperNextTimestepModel => 1, HelperCurrentTimestepModel => 1) - -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) - -out = run!(mtg, m, meteo_day, default_timestep=1, model_timesteps=models_timestep) - - -# NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used -# otherwise there might be vector length conflicts with timesteps -sim = PlantSimEngine.GraphSimulation(mtg, m, nsteps=nothing, check=true, outputs=nothing, default_timestep=1, model_timesteps=models_timestep) - -=# - -# TODO could some mapping happen automatically for variables directly taken from weather data ? -# Does this happen often in a typical model ? - - - -########################### -# Simple test with an orchestrator -########################### -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates -using Test - -PlantSimEngine.@process "ToyDay" verbose = false - -struct MyToyDayModel <: AbstractToydayModel end - -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) - -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end - -PlantSimEngine.@process "ToyWeek" verbose = false - -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end - -MyToyWeekModel() = MyToyWeekModel(28.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=-Inf,) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) - -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end - -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) - - -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - -m_multiscale = Dict("Default" => ( - MyToyDayModel(), - Status(a=1,) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekModel(), - #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this - mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], - ), - ),) - - -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) - -to = PlantSimEngine.Var_to(:weekly_max_temperature) -from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) - -dict_to_from = Dict(from => to) -mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) - -orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) - -out = @enter run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) - -temps = [out["Default"][i].daily_temperature for i in 1:365] -temp_m = maximum(temps) - -# At least one week should have max temp > 28 -@test temp_m > 28 && unique!([out["Default2"][i].hot for i in 1:365]) == [0,1] ########################### # Test with three timesteps, multiscale @@ -400,39 +179,6 @@ using Test out = run!(mtg_single, m_singlescale, df, orchestrator=orch2) - ########################### -# Three timestep model that is single-scale, to circumvent refvector/refvalue overwriting -# and explore alternatives -# (eg filtering out timestep-mapped variables from vars_need_init and storing the values elsewhere) -# Not plugged in together atm, the variable mapping doesn't work -########################### - - m_singlescale = Dict("Default" => ( - MyToyDay2Model(), - MyToyWeek2Model(), - MyToyFourWeek2Model(), - ),) - - - model_timesteps_defaultscale = Dict(MyToyWeek2Model =>Week(1), MyToyFourWeek2Model =>Week(4), ) - to_w = PlantSimEngine.Var_to(:in_week) - from_d = PlantSimEngine.Var_from(MyToyDay2Model, "Default", :out_day, sum) - dict_to_from_w = Dict(from_d => to_w) - - to_w4_d = PlantSimEngine.Var_to(:in_four_week_from_day) - to_w4_w = PlantSimEngine.Var_to(:in_four_week_from_week) - from_w = PlantSimEngine.Var_from(MyToyWeek2Model, "Default", :out_week, sum) - - dict_to_from_w4 = Dict(from_d => to_w4_d, from_w => to_w4_w) - - mtsm_w = PlantSimEngine.ModelTimestepMapping(MyToyWeek2Model, "Default", Week(1), dict_to_from_w) - mtsm_w4 = PlantSimEngine.ModelTimestepMapping(MyToyFourWeek2Model, "Default", Week(4), dict_to_from_w4) - - orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm_w, mtsm_w4]) - -mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -out = @run run!(mtg_single, m_singlescale, df, orchestrator=orch2) - ########################### # Test with three timesteps, multiscale + previoustimestep @@ -605,10 +351,6 @@ unique!([out["Default6"][i].out_last_week for i in 1:length(out["Default6"])]) # This means the model needs to be declared somewhere - - - - ########################### # Test with one timestep, multiscale + previoustimestep ########################### @@ -726,8 +468,6 @@ out = run!(mtg, m_multiscale, df, orchestrator=orch2) unique!([out["Default6"][i].out_last_week for i in 1:length(out["Default6"])]) - - ########################### # Previous timestep debugging, not useful for testing timestep mapping atm ########################### @@ -792,254 +532,82 @@ m_ss = Dict( out = @run run!(mtg_, m_ss, df) =# -########################### -# Test with a D -> W -> D configuration, with multiple variables mapped between timesteps -########################### -# Currently, weekly_max_temperature will raise an error : -# It is mapped correctly, timestep-mapped correctly, but not detected as an output at its scale -# Since it doesn't appear explicitely as the output of a model -# Setting it as the output would cause issues in status creation and refs, as well as during mapping -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates -PlantSimEngine.@process "ToyDayDWD" verbose = false -struct MyToyDayDWDModel <: AbstractToydaydwdModel end -PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) -function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.data -end -PlantSimEngine.@process "ToyWeekDWD" verbose = false -struct MyToyWeekDWDModel <: AbstractToyweekdwdModel - temperature_threshold::Float64 -end -MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekDWDModel) - (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) -end -PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) -function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold - status.sum += status.weekly_sum_temperature -end -PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) -PlantSimEngine.@process "ToyDayDWDOut" verbose = false -struct MyToyDayDWDOutModel <: AbstractToydaydwdoutModel end -PlantSimEngine.inputs_(m::MyToyDayDWDOutModel) = (weekly_max_temperature=-Inf,weekly_sum_temperature=-Inf,) -PlantSimEngine.outputs_(m::MyToyDayDWDOutModel) = (out=-Inf,) -function PlantSimEngine.run!(m::MyToyDayDWDOutModel, models, status, meteo, constants=nothing, extra=nothing) - status.out = status.weekly_sum_temperature - 7.0*meteo.data -end -df = DataFrame(:data => [1 for i in 1:365], ) -m_dwd = Dict("Default" => ( - MyToyDayDWDModel(), - MultiScaleModel( - model=MyToyDayDWDOutModel(), - mapped_variables=[:weekly_max_temperature => "Default2", :weekly_sum_temperature => "Default2"] - ), - Status(a=1,out=0.0) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekDWDModel(), - #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this - mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature, :weekly_sum_temperature => "Default" => :daily_temperature], - ), - Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum=0.0) - ), -) -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) -to = PlantSimEngine.Var_to(:weekly_max_temperature) -from = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, maximum) -to_sum = PlantSimEngine.Var_to(:weekly_sum_temperature) -from_sum = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, sum) -dict_to_from = Dict(from => to, from_sum => to_sum) -mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1), dict_to_from) -#dict_to_from_2 = Dict(from => to, from => to) -#mtsm2 = PlantSimEngine.ModelTimestepMapping(MyToyDayDWDOutModel, "Default", Day(1), dict_to_from2) -orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,])#mtsm2]) -out = run!(mtg, m_dwd, df, orchestrator=orch_dwd) -################################## -# Two variables mapped -################################## -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates -PlantSimEngine.@process "ToyDayDWD" verbose = false -struct MyToyDayDWDModel <: AbstractToydaydwdModel end -PlantSimEngine.inputs_(m::MyToyDayDWDModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayDWDModel) = (daily_temperature=-Inf,) -function PlantSimEngine.run!(m::MyToyDayDWDModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end -PlantSimEngine.@process "ToyWeekDWD" verbose = false -struct MyToyWeekDWDModel <: AbstractToyweekdwdModel - temperature_threshold::Float64 -end -MyToyWeekDWDModel() = MyToyWeekDWDModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekDWDModel) - (weekly_max_temperature=-Inf, weekly_sum_temperature=-Inf) -end -PlantSimEngine.outputs_(m::MyToyWeekDWDModel) = (hot = false, sum=-Inf) -function PlantSimEngine.run!(m::MyToyWeekDWDModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold - status.sum += status.weekly_sum_temperature -end -PlantSimEngine.timestep_range_(m::MyToyWeekDWDModel) = TimestepRange(Week(1)) -#df = DataFrame(:data => [1 for i in 1:365], ) -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) -m_dwd = Dict("Default" => ( - MyToyDayDWDModel(), - Status(a=1,) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekDWDModel(), - #mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this - mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature, :weekly_sum_temperature => "Default" => :daily_temperature], - ), - Status(weekly_max_temperature=0.0, weekly_sum_temperature=0.0, sum =0.0) - ), -) -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) -to = PlantSimEngine.Var_to(:weekly_max_temperature) -from = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, maximum) -to_sum = PlantSimEngine.Var_to(:weekly_sum_temperature) -from_sum = PlantSimEngine.Var_from(MyToyDayDWDModel, "Default", :daily_temperature, sum) -dict_to_from = Dict(from => to, from_sum => to_sum) -mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1), dict_to_from) -orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,]) -out = run!(mtg, m_dwd, meteo_day, orchestrator=orch_dwd) -########################## -# Two models, D -> W, but D has two MTG nodes -# So simple test of a RefVector + timestep mapping combo -########################## -# Currently errors due to refvectors not being handled properly -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates -PlantSimEngine.@process "ToyDay" verbose = false -struct MyToyDayModel <: AbstractToydayModel end -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end -PlantSimEngine.@process "ToyWeek" verbose = false -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end -MyToyWeekModel() = MyToyWeekModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=[-Inf],) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) -m_multiscale = Dict("Default" => ( - MyToyDayModel(), - Status(a=1,) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekModel(), - mapped_variables=[:weekly_max_temperature => ["Default" => :daily_temperature]], # TODO test this - #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], - ), - ),) -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) -mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) -to = PlantSimEngine.Var_to(:weekly_max_temperature) -from = PlantSimEngine.Var_from(MyToyDayModel, "Default", :daily_temperature, maximum) -dict_to_from = Dict(from => to) -mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1), dict_to_from) -#orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) -orch2 = PlantSimEngine.Orchestrator() -out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) -#out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) -# alternate mtg with 2 nodes -> 2 nodes -#mtg_bis = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -#mtg_bis_1 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -#mtg_bis_2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) -#mtg_bis_3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) -#out = @enter run!(mtg_bis, m_multiscale, meteo_day, orchestrator=orch2) + + + + + @@ -1128,7 +696,7 @@ mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) -out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) +#out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) temps = [out["Default"][i].daily_temperature for i in 1:365] @@ -1138,15 +706,6 @@ temp_m = maximum(temps) @test temp_m > 28 && unique!([out["Default2"][i].hot for i in 1:365]) == [0,1] - m = @enter MultiScaleModel(model=MyToyDayModel(), - mapped_variables=[], - timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_max_temperature, Week(1), max),] - ) - - - - - ########################### # Test with three timesteps, multiscale ########################### @@ -1269,7 +828,7 @@ using Test ], ),)) - # This one resuses the variable names directly, so requires only timestep mapping + # This one reuses the variable names directly, so requires only timestep mapping m_singlescale = Dict("Default" => ( MultiScaleModel(model=MyToyDay2Model(), mapped_variables=[], @@ -1292,91 +851,6 @@ mtg_single = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) out = run!(mtg_single, m_singlescale, df, orchestrator=orch2) out = run!(mtg_single, m_singlescale_mapped, df, orchestrator=orch2) - - -#TODO -########################## -# Two models, D -> W, but D has two MTG nodes -# So simple test of a RefVector + timestep mapping combo -########################## - -# Currently errors due to refvectors not being handled properly - -using MultiScaleTreeGraph -using PlantSimEngine -using PlantMeteo -using PlantMeteo.Dates - -PlantSimEngine.@process "ToyDay" verbose = false - -struct MyToyDayModel <: AbstractToydayModel end - -PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) -PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) - -function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) - status.daily_temperature = meteo.T -end - -PlantSimEngine.@process "ToyWeek" verbose = false - -struct MyToyWeekModel <: AbstractToyweekModel - temperature_threshold::Float64 -end - -MyToyWeekModel() = MyToyWeekModel(30.0) -function PlantSimEngine.inputs_(::MyToyWeekModel) - (weekly_max_temperature=[-Inf],) -end -PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) - -function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) - status.hot = status.weekly_max_temperature > m.temperature_threshold -end - -PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) - - -meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - -m_multiscale = Dict("Default" => ( - MultiScaleModel( - model=MyToyDayModel(), - mapped_variables=[], - timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_temperature, Week(1), maximum)], - ), - Status(a=1,) - ), - "Default2" => ( - MultiScaleModel(model=MyToyWeekModel(), - mapped_variables=[:weekly_max_temperature => ["Default" => :weekly_temperature]], # TODO test this - #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], - ), - ),) - - -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) -mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) - -mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) - -orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) - -out = @run run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) -#out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) - - - -# alternate mtg with 2 nodes -> 2 nodes -#mtg_bis = Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -#mtg_bis_1 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default", 1, 1)) -#mtg_bis_2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) -#mtg_bis_3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default2", 1, 2)) -#out = @enter run!(mtg_bis, m_multiscale, meteo_day, orchestrator=orch2) - - - ########################### # Test with a D -> W -> D configuration, with multiple variables mapped between timesteps ########################### @@ -1420,11 +894,11 @@ PlantSimEngine.@process "ToyDayDWDOut" verbose = false struct MyToyDayDWDOutModel <: AbstractToydaydwdoutModel end -PlantSimEngine.inputs_(m::MyToyDayDWDOutModel) = (weekly_max_temperature=-Inf,weekly_sum_temperature=-Inf,) +PlantSimEngine.inputs_(m::MyToyDayDWDOutModel) = (sum=-Inf,weekly_sum_temperature=-Inf,) PlantSimEngine.outputs_(m::MyToyDayDWDOutModel) = (out=-Inf,) function PlantSimEngine.run!(m::MyToyDayDWDOutModel, models, status, meteo, constants=nothing, extra=nothing) - status.out = status.weekly_sum_temperature - 7.0*meteo.data + status.out = status.sum - status.weekly_sum_temperature end df = DataFrame(:data => [1 for i in 1:365], ) @@ -1439,8 +913,9 @@ m_dwd = Dict("Default" => ( ] ), MultiScaleModel( model=MyToyDayDWDOutModel(), - mapped_variables=[:weekly_max_temperature => "Default2", :weekly_sum_temperature => "Default2"] + mapped_variables=[:sum => "Default2",]# :weekly_sum_temperature => "Default2"] ), + #MyToyDayDWDOutModel(), Status(a=1,out=0.0) ), "Default2" => ( @@ -1458,12 +933,10 @@ mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 2)) mtsm_dwd = PlantSimEngine.ModelTimestepMapping(MyToyWeekDWDModel, "Default2", Week(1)) -#dict_to_from_2 = Dict(from => to, from => to) -#mtsm2 = PlantSimEngine.ModelTimestepMapping(MyToyDayDWDOutModel, "Default", Day(1), dict_to_from2) - orch_dwd = PlantSimEngine.Orchestrator(Day(1), [mtsm_dwd,])#mtsm2]) out = @run run!(mtg, m_dwd, df, orchestrator=orch_dwd) +out = run!(mtg, m_dwd, df, orchestrator=orch_dwd) ################################## @@ -1539,4 +1012,220 @@ out = run!(mtg, m_dwd, meteo_day, orchestrator=orch_dwd) # TODO previous timestep, timestep-mapping to the same variable name -#TODO should timestep mapped vars also be part of a model's outputs ? \ No newline at end of file +#TODO should timestep mapped vars also be part of a model's outputs ? + + + + +#TODO +########################## +# Two models, D -> W, but D has two MTG nodes +########################## + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=[-Inf],) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (hot = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.hot = status.weekly_max_temperature > m.temperature_threshold +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MultiScaleModel( + model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_temperature, Week(1), maximum)], + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => "Default" => :weekly_temperature], # TODO test this + #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 2)) + +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) + +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) + +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + + +########################## +# Two models, D -> W, but D has two MTG nodes, and we map as a refvector +########################## + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=[-Inf],) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (refvector = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.refvector = status.weekly_max_temperature[1] == status.weekly_max_temperature[2] +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MultiScaleModel( + model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_temperature, Week(1), maximum)], + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => ["Default" => :weekly_temperature]], # TODO test this + #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) +mtg3 = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 2)) + +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) + +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) + +# The RefVector will be in the outputs, so intermediate data is lost for such timestep-mapped variables, and it makes the outputs confusing +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + +using Test +#@test out["Default2"][1] + + +########################## +# Two models, D -> W, but both D and W have two MTG nodes +########################## + +using MultiScaleTreeGraph +using PlantSimEngine +using PlantMeteo +using PlantMeteo.Dates + +PlantSimEngine.@process "ToyDay" verbose = false + +struct MyToyDayModel <: AbstractToydayModel end + +PlantSimEngine.inputs_(m::MyToyDayModel) = (a=1,) +PlantSimEngine.outputs_(m::MyToyDayModel) = (daily_temperature=-Inf,) + +function PlantSimEngine.run!(m::MyToyDayModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_temperature = meteo.T + node_id(status.node) +end + +PlantSimEngine.@process "ToyWeek" verbose = false + +struct MyToyWeekModel <: AbstractToyweekModel + temperature_threshold::Float64 +end + +MyToyWeekModel() = MyToyWeekModel(30.0) +function PlantSimEngine.inputs_(::MyToyWeekModel) + (weekly_max_temperature=[-Inf],) +end +PlantSimEngine.outputs_(m::MyToyWeekModel) = (refvector = false,) + +function PlantSimEngine.run!(m::MyToyWeekModel, models, status, meteo, constants=nothing, extra=nothing) + status.refvector = status.weekly_max_temperature[1] + 1== status.weekly_max_temperature[2] +end + +PlantSimEngine.timestep_range_(m::MyToyWeekModel) = TimestepRange(Week(1)) + + +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + +m_multiscale = Dict("Default" => ( + MultiScaleModel( + model=MyToyDayModel(), + mapped_variables=[], + timestep_mapped_variables=[TimestepMappedVariable(:daily_temperature, :weekly_temperature, Week(1), maximum)], + ), + Status(a=1,) + ), + "Default2" => ( + MultiScaleModel(model=MyToyWeekModel(), + mapped_variables=[:weekly_max_temperature => ["Default" => :weekly_temperature]], # TODO test this + #mapped_variables=[:weekly_max_temperature => "Default" => :daily_temperature], + ), + ),) + + +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 1)) +mtg2 = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Default2", 1, 1)) +mtg3 = Node(mtg2, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 1)) +mtg4 = Node(mtg2, MultiScaleTreeGraph.NodeMTG("+", "Default", 1, 2)) + +mtsm = PlantSimEngine.ModelTimestepMapping(MyToyWeekModel, "Default2", Week(1)) + +orch2 = PlantSimEngine.Orchestrator(Day(1), [mtsm,]) + +# The RefVector will be in the outputs, so intermediate data is lost for such timestep-mapped variables +out = run!(mtg, m_multiscale, meteo_day, orchestrator=orch2) + +using Test +#@test out["Default2"][1] \ No newline at end of file From dc36bf0e4c9eb32321c8c470a309c11cc5f08324 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 4 Dec 2025 14:55:24 +0100 Subject: [PATCH 18/21] Fix a couple of tests --- test/test-mtg-multiscale-cyclic-dep.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test-mtg-multiscale-cyclic-dep.jl b/test/test-mtg-multiscale-cyclic-dep.jl index ca3258ea..3e7f6740 100644 --- a/test/test-mtg-multiscale-cyclic-dep.jl +++ b/test/test-mtg-multiscale-cyclic-dep.jl @@ -48,7 +48,10 @@ out_vars = Dict( @test_throws "Cyclic dependency detected in the graph. Cycle:" dep(mapping_cyclic) soft_dep_graphs_roots, hard_dep_dict = PlantSimEngine.hard_dependencies(mapping_cyclic) - dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, mapping_cyclic, hard_dep_dict) + + mapped_vars_cyclic = mapped_variables(mapping_cyclic, soft_dep_graphs_roots, verbose=false) + rev_mapping_cyclic = reverse_mapping(mapped_vars_cyclic, all=false) + dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, rev_mapping_cyclic, hard_dep_dict) iscyclic, cycle_vec = PlantSimEngine.is_graph_cyclic(dep_graph; warn=false) @test iscyclic @@ -97,7 +100,10 @@ end soft_dep_graphs_roots, hard_dep_dict = PlantSimEngine.hard_dependencies(mapping_nocyclic) # soft_dep_graphs_roots.roots["Leaf"].inputs - dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, mapping_nocyclic, hard_dep_dict) + + mapped_vars_nocyclic = mapped_variables(mapping_nocyclic, soft_dep_graphs_roots, verbose=false) + rev_mapping_nocyclic = reverse_mapping(mapped_vars_nocyclic, all=false) + dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, rev_mapping_nocyclic, hard_dep_dict) iscyclic, cycle_vec = PlantSimEngine.is_graph_cyclic(dep_graph; warn=false) @test !iscyclic From 13db61f8765bee3a45390cf8a3da40979d68887c Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 4 Dec 2025 16:29:25 +0100 Subject: [PATCH 19/21] Looks like I messed up the merge, oops, fixing it --- src/mtg/initialisation.jl | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 4a4adf7a..b9d65a8b 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -327,8 +327,6 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion @assert false "Error : Mapping status at $organ_with_vector level contains a vector. If this was intentional, call the function generate_models_from_status_vectors on your mapping before calling run!. And bear in mind this is not meant for production. If this wasn't intentional, then it's likely an issue on the mapping definition, or an unusual model." end - # preliminary_check_timestep_data(mapping, orchestrator) - soft_dep_graph_roots, hard_dep_dict = hard_dependencies(mapping; verbose=false, orchestrator=orchestrator) # Get the status of each node by node type, pre-initialised considering multi-scale variables: @@ -338,28 +336,6 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion # Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses, status_templates, reverse_multiscale_mapping, vars_need_init = init_statuses(mtg, mapping, soft_dep_graph_roots; type_promotion=type_promotion, verbose=verbose, check=check, orchestrator=orchestrator) - - - # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want - # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they - # are independant. - # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the - # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the - # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children - # of the nodes that they depend on. - dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, reverse_multiscale_mapping, hard_dep_dict) - # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, - # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale. - # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale. - - # Checking that the graph is acyclic: - iscyclic, cycle_vec = is_graph_cyclic(dep_graph; warn=false) - # Note: we could do that in `soft_dependencies_multiscale` but we prefer to keep the function as simple as possible, and - # usable on its own. - - iscyclic && error("Cyclic dependency detected in the graph. Cycle: \n $(print_cycle(cycle_vec)) \n You can break the cycle using the `PreviousTimeStep` variable in the mapping.") - # Third step, we identify which - # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") From d2ac65fcf34b4a459047ef5a71feb9987cbd15e0 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 4 Dec 2025 16:33:54 +0100 Subject: [PATCH 20/21] FIx up the remaining tests --- src/dependencies/dependency_graph.jl | 8 ++++---- src/dependencies/hard_dependencies.jl | 12 ++++++------ src/dependencies/soft_dependencies.jl | 4 ++-- src/mtg/mapping/compute_mapping.jl | 2 +- test/runtests.jl | 4 ++++ 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index e31ea4aa..f7abe996 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -6,8 +6,8 @@ mutable struct HardDependencyNode{T} <: AbstractDependencyNode dependency::NamedTuple missing_dependency::Vector{Int} scale::String - #inputs - #outputs + inputs + outputs parent::Union{Nothing,<:AbstractDependencyNode} children::Vector{HardDependencyNode} end @@ -26,8 +26,8 @@ mutable struct SoftDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol scale::String - #inputs - #outputs + inputs + outputs hard_dependency::Vector{HardDependencyNode} parent::Union{Nothing,Vector{SoftDependencyNode}} parent_vars::Union{Nothing,NamedTuple} diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index ae596d1e..4da58a97 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -100,8 +100,8 @@ function initialise_all_as_hard_dependency_node(models, scale) NamedTuple(), Int[], scale, - #inputs_(i), - #outputs_(i), + inputs_(i), + outputs_(i), nothing, HardDependencyNode[] ) for (p, i) in pairs(models) @@ -312,8 +312,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr dep_node_model.dependency, dep_node_model.missing_dependency, dep_node_model.scale, - #dep_node_model.inputs, - #dep_node_model.outputs, + dep_node_model.inputs, + dep_node_model.outputs, parent_node, dep_node_model.children ) @@ -386,8 +386,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr soft_dep_vars.value, process_, # process name organ, # scale - #inputs_process[organ][process_], # These are the inputs, potentially multiscale - #outputs_process[organ][process_], # Same for outputs + inputs_process[organ][process_], # These are the inputs, potentially multiscale + outputs_process[organ][process_], # Same for outputs AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 12b98c92..bf2941ab 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -64,8 +64,8 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, soft_dep_vars.value, process_, # process name "", - #inputs_(soft_dep_vars.value), - #outputs_(soft_dep_vars.value), + inputs_(soft_dep_vars.value), + outputs_(soft_dep_vars.value), AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index 94130a3b..8be4d4b2 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -11,7 +11,7 @@ However, models that are identified as hard-dependencies are not given individua nodes under other models. - `verbose::Bool`: whether to print the stacktrace of the search for the default value in the mapping. """ -function mapped_variables(mapping, dependency_graph; verbose=false, orchestrator=Orchestrator()) +function mapped_variables(mapping, dependency_graph=first(hard_dependencies(mapping; verbose=false, orchestrator=Orchestrator())); verbose=false, orchestrator=Orchestrator()) # Initialise a dict that defines the multiscale variables for each organ type: mapped_vars = mapped_variables_no_outputs_from_other_scale(mapping, dependency_graph) diff --git a/test/runtests.jl b/test/runtests.jl index 6cf550f7..42baaa2c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -72,6 +72,10 @@ include("helper-functions.jl") include("test-performance.jl") end + @testset "Multi timestep" begin + #include("test-multitimestep.jl") + end + if VERSION >= v"1.10" # Some formating changed in Julia 1.10, e.g. @NamedTuple instead of NamedTuple. @testset "Doctests" begin From 28f76f2c84b600296523b5b5c15d402784b4da54 Mon Sep 17 00:00:00 2001 From: Samuel-AMAP Date: Thu, 4 Dec 2025 16:58:04 +0100 Subject: [PATCH 21/21] Fix test that must have been broken a while ago (made use of the mapping from another test). Could be avoided by sandboxing testsuites completely. --- test/test-mtg-multiscale-cyclic-dep.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test-mtg-multiscale-cyclic-dep.jl b/test/test-mtg-multiscale-cyclic-dep.jl index 3e7f6740..f5752854 100644 --- a/test/test-mtg-multiscale-cyclic-dep.jl +++ b/test/test-mtg-multiscale-cyclic-dep.jl @@ -49,8 +49,8 @@ out_vars = Dict( soft_dep_graphs_roots, hard_dep_dict = PlantSimEngine.hard_dependencies(mapping_cyclic) - mapped_vars_cyclic = mapped_variables(mapping_cyclic, soft_dep_graphs_roots, verbose=false) - rev_mapping_cyclic = reverse_mapping(mapped_vars_cyclic, all=false) + mapped_vars_cyclic = PlantSimEngine.mapped_variables(mapping_cyclic, soft_dep_graphs_roots, verbose=false) + rev_mapping_cyclic = PlantSimEngine.reverse_mapping(mapped_vars_cyclic, all=false) dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, rev_mapping_cyclic, hard_dep_dict) iscyclic, cycle_vec = PlantSimEngine.is_graph_cyclic(dep_graph; warn=false) @@ -101,8 +101,8 @@ end soft_dep_graphs_roots, hard_dep_dict = PlantSimEngine.hard_dependencies(mapping_nocyclic) # soft_dep_graphs_roots.roots["Leaf"].inputs - mapped_vars_nocyclic = mapped_variables(mapping_nocyclic, soft_dep_graphs_roots, verbose=false) - rev_mapping_nocyclic = reverse_mapping(mapped_vars_nocyclic, all=false) + mapped_vars_nocyclic = PlantSimEngine.mapped_variables(mapping_nocyclic, soft_dep_graphs_roots, verbose=false) + rev_mapping_nocyclic = PlantSimEngine.reverse_mapping(mapped_vars_nocyclic, all=false) dep_graph = PlantSimEngine.soft_dependencies_multiscale(soft_dep_graphs_roots, rev_mapping_nocyclic, hard_dep_dict) iscyclic, cycle_vec = PlantSimEngine.is_graph_cyclic(dep_graph; warn=false) @@ -113,7 +113,7 @@ end #out = @test_nowarn run!(mtg, mapping_nocyclic, meteo, tracked_outputs=out_vars, executor=SequentialEx()) nsteps = PlantSimEngine.get_nsteps(meteo) - sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) + sim = PlantSimEngine.GraphSimulation(mtg, mapping_nocyclic, nsteps=nsteps, check=true, outputs=out_vars) out = @test_nowarn run!(sim,meteo) st = status(sim)