#138 outputs conversion failing when organ is not created but requested as output

src/Abstract_model_structs.jl

@@ -25,4 +25,47 @@ model_(m::AbstractModel) = m
 get_models(m::AbstractModel) = [model_(m)] # Get the models of an AbstractModel
 # Note: it is returning a vector of models, because in this case the user provided a single model instead of a vector of.
 get_status(m::AbstractModel) = nothing
-get_mapped_variables(m::AbstractModel) = Pair{Symbol,String}[]
+get_mapped_variables(m::AbstractModel) = Pair{Symbol,String}[]
+
+
+#using Dates
+struct TimestepRange 
+    lower_bound::Period
+    upper_bound::Period
+end
+
+# Default, no specified range, meaning the model either doesn't depend on time or uses the simulation's default (eg smallest) timestep
+TimestepRange() = TimestepRange(Second(0), Second(0))
+# Only a single timestep type possible
+TimestepRange(p::Period) = TimestepRange(p, p)
+
+"""
+    timestep_range_(tsr::TimestepRange)
+
+Return the model's valid range for timesteps (which corresponds to the simulation base timestep in the default case).
+"""
+function timestep_range_(model::AbstractModel)
+    return TimestepRange()
+end
+
+"""
+    timestep_valid(tsr::TimestepRange)
+"""
+timestep_valid(tsr::TimestepRange) = tsr.lower_bound <= tsr.upper_bound
+
+function is_timestep_in_range(tsr::TimestepRange, p::Period) 
+    if !timestep_valid(tsr)
+        return false
+    end
+
+    # 0 means any timestep is valid, no timestep constraints
+    if tsr.upper_bound == Second(0)
+        return true
+    end
+
+    return p >= tsr.lower_bound && p <= tsr.lower_bound
+end
+
+# TODO should i set all timestep ranges to default and hope the modeler gets it right or should i force them to write something ?
+
+# TODO properly test on models with default timestep

src/PlantSimEngine.jl

@@ -26,6 +26,7 @@ import Statistics
 import SHA: sha1
 
 using PlantMeteo
+using PlantMeteo.Dates
 
 # UninitializedVar + PreviousTimeStep:
 include("variables_wrappers.jl")
@@ -56,6 +57,9 @@ include("component_models/get_status.jl")
 # Transform into a dataframe:
 include("dataframe.jl")
 
+# Timesteps. : 
+include("timestep/timestep_mapping.jl")
+
 # Computing model dependencies:
 include("dependencies/soft_dependencies.jl")
 include("dependencies/hard_dependencies.jl")
@@ -103,6 +107,7 @@ include("examples_import.jl")
 export PreviousTimeStep
 export AbstractModel
 export ModelList, MultiScaleModel
+export Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping
 export RMSE, NRMSE, EF, dr
 export Status, TimeStepTable, status
 export init_status!

src/dependencies/dependencies.jl

@@ -100,16 +100,16 @@ function dep(m::NamedTuple, nsteps=1; verbose::Bool=true)
     dep(nsteps; verbose=verbose, m...)
 end
 
-function dep(mapping::Dict{String,T}; verbose::Bool=true) where {T}
+function dep(mapping::Dict{String,T}; verbose::Bool=true, orchestrator=Orchestrator2()) where {T}
     # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want 
     # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they
     # are independant.
-    soft_dep_graphs_roots, hard_dep_dict = hard_dependencies(mapping; verbose=verbose)
+    soft_dep_graphs_roots, hard_dep_dict = hard_dependencies(mapping; verbose=verbose, orchestrator=Orchestrator2())
     # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the 
     # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the
     # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children
     # of the nodes that they depend on.
-    dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, mapping, hard_dep_dict)
+    dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, mapping, hard_dep_dict, orchestrator=orchestrator)
     # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, 
     # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale.
     # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale.

src/dependencies/dependency_graph.jl

@@ -12,6 +12,16 @@ mutable struct HardDependencyNode{T} <: AbstractDependencyNode
     children::Vector{HardDependencyNode}
 end
 
+mutable struct TimestepMapping
+    variable_from::Symbol
+    variable_to::Symbol
+    node_to # SoftDependencyNode causes a circular reference, removing it as a shortcut TODO
+    mapping_function::Function
+    mapping_data_template
+    mapping_data::Dict{Int, Any} # TODO fix type stability : Int is the node id, Any is a vector of n elements of the variable's type, n being the # of required timesteps
+end
+
+# can hard dependency nodes also handle timestep mapped variables... ?
 mutable struct SoftDependencyNode{T} <: AbstractDependencyNode
     value::T
     process::Symbol
@@ -23,6 +33,8 @@ mutable struct SoftDependencyNode{T} <: AbstractDependencyNode
     parent_vars::Union{Nothing,NamedTuple}
     children::Vector{SoftDependencyNode}
     simulation_id::Vector{Int} # id of the simulation
+    timestep::Period
+    timestep_mapping_data::Union{Nothing, Vector{TimestepMapping}} # TODO : this approach might not play too well with parallelisation over MTG nodes
 end
 
 # Add methods to check if a node is parallelizable:
@@ -77,82 +89,4 @@ function Base.show(io::IO, ::MIME"text/plain", t::DependencyGraph)
     else
         draw_dependency_graph(io, t)
     end
-end
-
-"""
-    variables_multiscale(node, organ, mapping, st=NamedTuple())
-
-Get the variables of a HardDependencyNode, taking into account the multiscale mapping, *i.e.*
-defining variables as `MappedVar` if they are mapped to another scale. The default values are 
-taken from the model if not given by the user (`st`), and are marked as `UninitializedVar` if 
-they are inputs of the node.
-
-Return a NamedTuple with the variables and their default values.
-
-# Arguments
-
-- `node::HardDependencyNode`: the node to get the variables from.
-- `organ::String`: the organ type, *e.g.* "Leaf".
-- `vars_mapping::Dict{String,T}`: the mapping of the models (see details below).
-- `st::NamedTuple`: an optional named tuple with default values for the variables.
-
-# Details
-
-The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is 
-computed from the user mapping like so:
-"""
-function variables_multiscale(node, organ, vars_mapping, st=NamedTuple())
-    node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,))
-    ins = node_vars.inputs
-    ins_variables = keys(ins)
-    outs_variables = keys(node_vars.outputs)
-    defaults = merge(node_vars...)
-    map((inputs=ins_variables, outputs=outs_variables)) do vars # Map over vars from :inputs and vars from :outputs
-        vars_ = Vector{Pair{Symbol,Any}}()
-        for var in vars # e.g. var = :carbon_biomass
-            if var in keys(st)
-                #If the user has given a status, we use it as default value.
-                default = st[var]
-            elseif var in ins_variables
-                # Otherwise, we use the default value given by the model:
-                # If the variable is an input, we mark it as uninitialized:
-                default = UninitializedVar(var, defaults[var])
-            else
-                # If the variable is an output, we use the default value given by the model:
-                default = defaults[var]
-            end
-
-            if haskey(vars_mapping[organ], var)
-                organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var])
-                push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default))
-                #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current 
-                #* values, and another one the previous values.
-                if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
-                    organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
-                    push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
-                end
-            elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
-                # If not found in the current time step, we check if the variable is mapped to the previous time step:
-                organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
-                push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
-            else
-                # Else we take the default value:
-                push!(vars_, var => default)
-            end
-        end
-        return (; vars_...,)
-    end
-end
-
-function _node_mapping(var_mapping::Pair{String,Symbol})
-    # One organ is mapped to the variable:
-    return SingleNodeMapping(first(var_mapping)), last(var_mapping)
-end
-
-function _node_mapping(var_mapping)
-    # Several organs are mapped to the variable:
-    organ_mapped = MultiNodeMapping([first(i) for i in var_mapping])
-    organ_mapped_var = [last(i) for i in var_mapping]
-
-    return organ_mapped, organ_mapped_var
 end

src/dependencies/hard_dependencies.jl

@@ -110,9 +110,95 @@ function initialise_all_as_hard_dependency_node(models, scale)
     return dep_graph
 end
 
+# Samuel : this requires the orchestrator, which requires the dependency graph
+# Leaving it in dependency_graph.jl causes forward declaration issues, moving it here as a quick protoyping hack, it might not be the ideal spot
+"""
+    variables_multiscale(node, organ, mapping, st=NamedTuple())
+
+Get the variables of a HardDependencyNode, taking into account the multiscale mapping, *i.e.*
+defining variables as `MappedVar` if they are mapped to another scale. The default values are 
+taken from the model if not given by the user (`st`), and are marked as `UninitializedVar` if 
+they are inputs of the node.
+
+Return a NamedTuple with the variables and their default values.
+
+# Arguments
+
+- `node::HardDependencyNode`: the node to get the variables from.
+- `organ::String`: the organ type, *e.g.* "Leaf".
+- `vars_mapping::Dict{String,T}`: the mapping of the models (see details below).
+- `st::NamedTuple`: an optional named tuple with default values for the variables.
+
+# Details
+
+The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is 
+computed from the user mapping like so:
+"""
+function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator2=Orchestrator2())
+    node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,))
+    ins = node_vars.inputs
+    ins_variables = keys(ins)
+    outs_variables = keys(node_vars.outputs)
+    defaults = merge(node_vars...)
+    map((inputs=ins_variables, outputs=outs_variables)) do vars # Map over vars from :inputs and vars from :outputs
+        vars_ = Vector{Pair{Symbol,Any}}()
+        for var in vars # e.g. var = :carbon_biomass
+            if var in keys(st)
+                #If the user has given a status, we use it as default value.
+                default = st[var]
+            elseif var in ins_variables
+                # Otherwise, we use the default value given by the model:
+                # If the variable is an input, we mark it as uninitialized:
+                default = UninitializedVar(var, defaults[var])
+            else
+                # If the variable is an output, we use the default value given by the model:
+                default = defaults[var]
+            end
+
+            # TODO no idea how this meshes with refvector situations or previoustimestep
+            if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true)
+
+                push!(vars_, var => default)
+            else
+
+                if haskey(vars_mapping[organ], var)
+                    organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var])
+                    push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default))
+                    #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current 
+                    #* values, and another one the previous values.
+                    if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
+                        organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
+                        push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
+                    end
+                elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
+                    # If not found in the current time step, we check if the variable is mapped to the previous time step:
+                    organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
+                    push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
+                else
+                    # Else we take the default value:
+                    push!(vars_, var => default)
+                end
+            end
+        end
+        return (; vars_...,)
+    end
+end
+
+function _node_mapping(var_mapping::Pair{String,Symbol})
+    # One organ is mapped to the variable:
+    return SingleNodeMapping(first(var_mapping)), last(var_mapping)
+end
+
+function _node_mapping(var_mapping)
+    # Several organs are mapped to the variable:
+    organ_mapped = MultiNodeMapping([first(i) for i in var_mapping])
+    organ_mapped_var = [last(i) for i in var_mapping]
+
+    return organ_mapped, organ_mapped_var
+end
 
 # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children):
-function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T}
+function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T}
     full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping)
     soft_dep_graphs = Dict{String,Any}()
     not_found = Dict{Symbol,DataType}()
@@ -148,7 +234,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T
         status_scale = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}()
         for (procname, node) in hard_deps[organ].roots # procname = :leaf_surface ; node = hard_deps.roots[procname]
             var = Pair{Symbol,NamedTuple}[]
-            traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_vars_mapping, st_scale_user), var)
+            traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_vars_mapping, st_scale_user, orchestrator), var)
             push!(status_scale, procname => var)
         end
 
@@ -226,6 +312,42 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T
         end
     end
 
+#=
+ # TODO check whether this is a bit late in the game
+                # maybe the timestep mapping should be done before we enter this function
+                if length(orchestrator.non_default_timestep_data_per_scale) > 0
+                    if haskey(orchestrator.non_default_timestep_data_per_scale, symbol(node))
+                        tvm = orchestrator.non_default_timestep_data_per_scale[symbol(node)].timestep_variable_mapping
+                        if haskey(twm, var)
+
+                        end
+                    end
+                end
+                error("Variable `$(var)` is not computed by any model, not mapped from a different scale or timestep not initialised by the user in the status, and not found in the MTG at scale $(symbol(node)) (checked for MTG node $(node_id(node))).")
+=#
+
+
+    # Once multiscale mapping has been dealt with, check if any variable has a timestep mapping
+    # Which will add potential new dependencies
+    #=if !isempty(orchestrator.non_default_timestep_data_per_scale)
+        # TODO the user can get away with not declaring the model, only the scale if necessary
+        # a prepass that recomputes everything might simplify code here and make the simulation require less variable digging
+        for (scale, tsh) in non_default_timestep_data_per_scale
+            # TODO find which model the variable is pulled from
+            # TODO check the variable exists
+            for (model, timestep) in tsh.model_timesteps
+            # TODO check the timestep is within the model's accepted timestep range
+            # TODO recover the right variables
+            end
+
+            for (variable, tvm) in tsh.timestep_variable_mapping
+                # TODO check the variable isn't already mapped
+                # If it is, ensure there are no name conflicts 
+                # and the model of the variable it is taken from has the expected timestep
+                # If it isn't, create a new link
+            end
+        end
+    end=#
     for (organ, model) in mapping
         soft_dep_graph = Dict(
             process_ => SoftDependencyNode(
@@ -238,7 +360,9 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T
                 nothing,
                 nothing,
                 SoftDependencyNode[],
-                [0] # Vector of zeros of length = number of time-steps
+                [0], # Vector of zeros of length = number of time-steps
+                orchestrator.default_timestep,
+                nothing
             )
             for (process_, soft_dep_vars) in hard_deps[organ].roots # proc_ = :carbon_assimilation ; soft_dep_vars = hard_deps.roots[proc_]
         )