text:
# This file holds the definition of the functions pertaining to the
#   ManpowerSimulation type.

export  isSimulationFresh,
        isSimulationConsistent,
        verifySimulation!,
        setSimulationKey!,
        setSimulationName!,
        addSimulationAttribute!,
        removeSimulationAttribute!,
        clearSimulationAttributes!,
        setSimulationAttributes!,
        addSimulationBaseNode!,
        removeSimulationBaseNode!,
        clearSimulationBaseNodes!,
        setSimulationBaseNodes!,
        addSimulationCompoundNode!,
        removeSimulationCompoundNode!,
        clearSimulationCompoundNodes!,
        setSimulationCompoundNodes!,
        addSimulationRecruitment!,
        clearSimulationRecruitment!,
        setSimulationRecruitment!,
        addSimulationTransition!,
        clearSimulationTransitions!,
        setSimulationTransitions!,
        addSimulationTransitionTypeOrder!,
        removeSimulationTransitionTypeOrder!,
        clearSimulationTransitionTypeOrder!,
        setSimulationTransitionTypeOrder!,
        addSimulationBaseNodeOrder!,
        removeSimulationBaseNodeOrder!,
        clearSimulationBaseNodeOrder!,
        setSimulationBaseNodeOrder!,
        setSimulationRetirement!,
        removeSimulationRetirement!,
        clearSimulationRetirement!,
        addSimulationAttrition!,
        removeSimulationAttrition!,
        clearSimulationAttrition!,
        setSimulationAttrition!,
        setSimulationLength!,
        setSimulationPersonnelTarget!,
        setSimulationDatabaseName!,
        setSimulationDatabase!,
        setSimulationShowInfo!

"""
```
isSimulationFresh( mpSim::MPsim )
```
This function returns the freshness state of the configuration of the manpower simulation `mpSim`.
"""
isSimulationFresh( mpSim::MPsim )::Bool = !mpSim.isStale

"""
```
isSimulationConsistent( mpSim::MPsim )
```
This function returns the consistency state of the configuration of the manpower simulation `mpSim`. If the simulation is not fresh, it is automatically assumed to be inconsistent.
"""
isSimulationConsistent( mpSim::MPsim )::Bool =
    isSimulationFresh( mpSim ) && mpSim.isConsistent

"""
```
verifySimulation!(
    mpSim::MPsim,
    forceCheck::Bool = false )
```
This function verifies if the configuration of the manpower simulation `mpSim` is consistent, always doing a complete check if `forceCheck` is `true`. This means that:
1. all attributes and attribute values used in the base nodes,
2. all attrition schemes attached to the base nodes,
3. all source/target nodes of the transitions,
4. all attributes and attribute values used in the transitions,
5. all base nodes part of compound nodes, and
6. all target nodes of the recruitmentment schemes must be defined in the simulation.

The `isStale` flag will be set to `false`, and the `isConsistent` flag will be set to the result of the test.

If the simulation's configuration is fresh, and `forceCheck` is false, no complete check happens.

If a complete check happens, and there are issues, the function issues the appropriate warnings.

The function returns the value of the `isConsistent` flag.
"""
function verifySimulation!( mpSim::MPsim, forceCheck::Bool = false )::Bool

    if isSimulationFresh( mpSim ) && !forceCheck
        return isSimulationConsistent( mpSim )
    end  # if isSimulationFresh( mpSim ) && ...

    mpSim.isConsistent = true
    verifyBaseNodeAttributes!( mpSim )
    verifyBaseNodeAttrition!( mpSim )
    verifyTransitionBaseNodes!( mpSim )
    verifyTransitionConditions!( mpSim )
    verifyTransitionChanges!( mpSim )
    verifyCompoundNodeComponents!( mpSim )
    verifyRecruitmentTargets!( mpSim )
    mpSim.isStale = false
    return isSimulationConsistent( mpSim )

end  # verifySimulation!( mpSim, forceCheck )

"""
```
setSimulationKey!(
    mpSim::MPsim,
    idKey::KeyType = "id" )
```
This function sets the personnel identifier key in the manpower simulation `mpSim` to `idKey`. Note that the function does not perform any updates of the database.

The function returns `true`, indicating that the identifier key has been successfully set.
"""
function setSimulationKey!( mpSim::MPsim, idKey::KeyType = "id" )::Bool

    mpSim.idKey = string( idKey )
    return true

end  # setSimulationKey!( mpSim, idKey )

"""
```
setSimulationName!(
    mpSim::MPsim,
    simName::String )
```
This function sets the name of the manpower simulation `mpSim` to `simName` and adjusts the names of the results database tables.

This function returns `true`, indicating that the name of the simulation has been successfully set.
"""
function setSimulationName!( mpSim::MPsim, simName::String )::Bool

    mpSim.simName = simName
    mpSim.persDBname = string( "Personnel_", simName )
    mpSim.histDBname = string( "History_", simName )
    mpSim.transDBname = string( "Transitions_", simName )
    return true

end  # setSimulationName!( mpSim, simName )

"""
```
addSimulationAttribute!(
    mpSim::MPsim,
    attributes::Attribute... )
```
This function adds the attributes in `attributes` to the manpower simulation `mpSim`. This function makes the simulation configuration stale.

If there are multiple attributes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the attributes have been successfully added, and `false` if multiple attributes had the same name.
"""
addSimulationAttribute!( mpSim::MPsim, attributes::Attribute... )::Bool =
    setSimAttributes!( mpSim, collect( attributes ), false )

"""
```
removeSimulationAttribute!(
    mpSim::MPsim,
    attributes::String... )
```
This function removes the attributes with names in `attributes` from the manpower simulation `mpSim`.

This function returns `true` if any attributes were successfully removed, and `false` if the simulation doesn't have any attributes matching the given names.
"""
function removeSimulationAttribute!( mpSim::MPsim, attributes::String... )::Bool

    if !any( attribute -> haskey( mpSim.attributeList, attribute ), attributes )
        return false
    end  # if !any( attribute -> ...

    for attribute in attributes
        delete!( mpSim.attributeList, attribute )
    end  # for attribute in attributes

    mpSim.isStale = true
    return true

end  # removeSimulationAttribute!( mpSim, attributes )

"""
```
clearSimulationAttributes!( mpSim::MPsim )
```
This function clears all the attributes from the manpower simulation `mpSim`.

This function returns `true`, indicating the attributes have been successfully cleared.
"""
function clearSimulationAttributes!( mpSim::MPsim )::Bool

    empty!( mpSim.attributeList )
    mpSim.isStale = true
    return true

end  # clearSimulationAttributes!( mpSim )

"""
```
setSimulationAttributes!(
    mpSim::MPsim,
    attributes::Vector{Attribute} )
```
This function sets the attributes of the manpower simulation `mpSim` to the list in `attributes`. This function makes the simulation configuration stale.

If there are multiple attributes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the attributes have been successfully set, and `false` if multiple attributes had the same name.
"""
setSimulationAttributes!( mpSim::MPsim, attributes::Vector{Attribute} )::Bool =
    setSimAttributes!( mpSim, attributes, true )

"""
```
addSimulationBaseNode!(
    mpSim::MPsim,
    nodes::BaseNode... )
```
This function adds the base nodes in `nodes` to the manpower simulation `mpSim`. This function makes the simulation configuration stale.

If there are multiple nodes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the base nodes have been successfully added, and `false` if multiple nodes had the same name.
"""
addSimulationBaseNode!( mpSim::MPsim, nodes::BaseNode... )::Bool =
    setSimBaseNodes!( mpSim, collect( nodes ), false )

"""
```
removeSimulationBaseNode!(
    mpSim::MPsim,
    nodes::String... )
```
This function removes the base nodes with names in `nodes` from the manpower simulation `mpSim`.

This function returns `true` if any nodes were successfully removed, and `false` if the simulation doesn't have any base nodes matching the given names.
"""
function removeSimulationBaseNode!( mpSim::MPsim, nodes::String... )::Bool

    if !any( node -> haskey( mpSim.baseNodeList, node ), nodes )
        return false
    end  # if !any( node -> ...

    for node in nodes
        delete!( mpSim.baseNodeList, node )
    end  # for node in nodes

    mpSim.isStale = true
    return true

end  # removeSimulationBaseNode!( mpSim, nodes )

"""
```
clearSimulationBaseNodes!( mpSim::MPsim )
```
This function clears all the base nodes from the manpower simulation `mpSim`.

This function returns `true`, indicating the base nodes have been successfully cleared.
"""
function clearSimulationBaseNodes!( mpSim::MPsim )::Bool

    empty!( mpSim.baseNodeList )
    mpSim.isStale = true
    return true

end  # clearSimulationBaseNodes!( mpSim )

"""
```
setSimulationBaseNodes!(
    mpSim::MPsim,
    nodes::Vector{BaseNode} )
```
This function sets the base nodes of the manpower simulation `mpSim` to the list in `nodes`. This function makes the simulation configuration stale.

If there are multiple base nodes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the base nodes have been successfully set, and `false` if multiple nodes had the same name.
"""
setSimulationBaseNodes!( mpSim::MPsim, nodes::Vector{BaseNode} )::Bool =
    setSimBaseNodes!( mpSim, nodes, true )

"""
```
addSimulationCompoundNode!(
    mpSim::MPsim,
    nodes::CompoundNode... )
```
This function adds the compound nodes in `nodes` to the manpower simulation `mpSim`. This function makes the simulation configuration stale.

If there are multiple nodes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the compound nodes have been successfully added, and `false` if multiple nodes had the same name.
"""
function addSimulationCompoundNode!( mpSim::MPsim,
    nodes::CompoundNode... )::Bool

    if isempty( nodes )
        return false
    end  # if isempty( nodes )

    return setSimCompoundNodes!( mpSim, collect( nodes ), false )

end  # addSimulationCompoundNode!( mpSim, nodes )

"""
```
addSimulationCompoundNode!(
    mpSim::MPsim,
    nodeName::String,
    nodeTarget::Integer,
    baseNodeList::String... )
```
This function adds a compound node with name `nodeName` to the manpower simulation `mpSim`. This compound node has a target population of `nodeTarget`, and consists of the base nodes in `baseNodeList`.

The function returns `true`, indicating that the compound node has been successfully added or replaced.
"""
function addSimulationCompoundNode!( mpSim::MPsim, nodeName::String, nodeTarget::Integer, baseNodeList::String... )::Bool

    newCompoundNode = CompoundNode( nodeName )
    addCompoundNodeComponent!( newCompoundNode, baseNodeList... )
    setCompoundNodeTarget!( newCompoundNode, nodeTarget )
    return addSimulationCompoundNode!( mpSim, newCompoundNode )

end  # addSimulationCompoundNode!( mpSim, nodeName, nodeTarget, baseNodeList )

"""
```
removeSimulationCompoundNode!(
    mpSim::MPsim,
    nodes::String... )
```
This function removes the compound nodes with names in `nodes` from the manpower simulation `mpSim`.

This function returns `true` if any nodes were successfully removed, and `false` if the simulation doesn't have any compound nodes matching the given names.
"""
function removeSimulationCompoundNode!( mpSim::MPsim, nodes::String... )::Bool

    if !any( node -> haskey( mpSim.compoundNodeList, node ), nodes )
        return false
    end  # if !any( node -> ...

    for node in nodes
        delete!( mpSim.compoundNodeList, node )
    end  # for node in nodes

    mpSim.isStale = true
    return true

end  # removeSimulationCompoundNode!( mpSim, nodes )

"""
```
clearSimulationCompoundNodes!( mpSim::MPsim )
```
This function clears all the compound nodes from the manpower simulation `mpSim`.

This function returns `true`, indicating the compound nodes have been successfully cleared.
"""
function clearSimulationCompoundNodes!( mpSim::MPsim )::Bool

    empty!( mpSim.compoundNodeList )
    mpSim.isStale = true
    return true

end  # clearSimulationCompoundNodes!( mpSim )

"""
```
setSimulationCompoundNodes!(
    mpSim::MPsim,
    nodes::Vector{CompoundNode} )
```
This function sets the compound nodes of the manpower simulation `mpSim` to the list in `nodes`. This function makes the simulation configuration stale.

If there are multiple compound nodes with the same name, this function issues a warning and makes no changes.

This function returns `true` if the compound nodes have been successfully set, and `false` if multiple nodes had the same name.
"""
setSimulationCompoundNodes!( mpSim::MPsim, nodes::Vector{CompoundNode} )::Bool =
    setSimCompoundNodes!( mpSim, nodes, true )

"""
```
addSimulationRecruitment!(
    mpSim::MPsim,
    recruitmentList::Recruitment... )
```
This function adds the recruitment schemes in `recruitmentList` to the manpower simulation `mpSim`. Recruitment schemes with a target node left blank, or set to `"dummy"` will not be added. This function makes the simulation configuration stale.

This function returns `true` if any recruitment schemes have been successfully added, and `false` if all recruitment schemes had a blank or dummy target node.
"""
addSimulationRecruitment!( mpSim::MPsim,
    recruitmentList::Recruitment... )::Bool = setSimRecruitment!( mpSim,
    collect( recruitmentList ), false )

"""
```
clearSimulationRecruitment!( mpSim )
```
This function clears all recruitment schemes from the manpower simulation `mpSim`. Recruitment schemes with a target node left blank, or set to `"dummy"` will not be added. This function makes the simulation configuration stale.

This function returns `true`, indicating the recruitment schemes have been successfully cleared.
"""
function clearSimulationRecruitment!( mpSim::MPsim )::Bool

    empty!( mpSim.recruitmentByName )
    empty!( mpSim.recruitmentByTarget )
    mpSim.isStale = true
    return true

end  # clearSimulationRecruitment!( mpSim::MPsim )

"""
```
setSimulationRecruitment!(
    mpSim::MPsim,
    recruitmentList::Vector{Recruitment} )
```
This function sets the recruitment schemes of the manpower simulation `mpSim` to the list in `nodes`. This function makes the simulation configuration stale.

This function returns `true` if any recruitment schemes have been successfully added, and `false` if all recruitment schemes had a blank or dummy target node.
"""
setSimulationRecruitment!( mpSim::MPsim,
    recruitmentList::Vector{Recruitment} )::Bool = setSimRecruitment!( mpSim, recruitmentList, true )

"""
```
addSimulationTransition!(
    mpSim::MPsim,
    transitions::Transition... )
```
This function adds the transitions in `transitions` to the manpower simulation `mpSim`. Transitions with a source or target node left blank, or set to `"dummy"` will not be added (only source is checked for OUT transitions). This function makes the simulation configuration stale.

This function returns `true` if any transitions have been successfully added, and `false` if all transitions had a blank or dummy source or target node.
"""
addSimulationTransition!( mpSim::MPsim, transitions::Transition... )::Bool =
    setSimTransitions!( mpSim, collect( transitions ), false )

"""
```
clearSimulationTransitions!( mpSim::MPsim )
```
This function clears all the transitions from the manpower simulation `mpSim`.

The function returns `true`, indicating that the transitions have been successfully cleared.
"""
function clearSimulationTransitions!( mpSim::MPsim )::Bool

    empty!( mpSim.transitionsByName )
    empty!( mpSim.transitionsBySource )
    mpSim.transitionsByTarget = Dict( "OUT" => Vector{Transition}() )
    mpSim.isStale = true
    return true

end  # clearSimulationTransitions!( mpSim )

"""
```
setSimulationTransitions!(
    mpSim::MPsim,
    transitions::Vector{Transition} )
```
This function sets the transitions of the manpower simulation `mpSim` to the list of transitions in `transitions`. Transitions with a source or target node left blank, or set to `"dummy"` will not be added (only source is checked for OUT transitions). This function makes the simulation configuration stale.

This function returns `true` if any transitions have been successfully set, and `false` if all transitions had a blank or dummy source or target node.
"""
setSimulationTransitions!( mpSim::MPsim,
    transitions::Vector{Transition} )::Bool =
    setSimTransitions!( mpSim, collect( transitions ), true )

"""
```
addSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transOrder::Dict{String,Int} )
```
This function adds the transition type priorities in the list `transOrder` to the manpower simulation `mpSim`. The transition type priorities are given as a dictionary with the transition type as key, and the priority as the key's value, where lower numbers mean transitions of that type are handled first. Transition types that already have a priority, have their priority overwritten.

This function returns `true` if any transition type has a priority added or changed, and `false` otherwise.
"""
addSimulationTransitionTypeOrder!( mpSim::MPsim,
    transOrder::Dict{String,Int} )::Bool = setSimTransitionTypeOrder!( mpSim,
    transOrder, false )

"""
```
addSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transOrder::Tuple{String,Int}... )
```
This function adds the transition type priorities in the list `transOrder` to the manpower simulation `mpSim`. The transition type priorities are given as a list of transition type/priority, where lower numbers for the priority mean transitions of that type are handled first. Transition types that already have a priority, have their priority overwritten.

If this list contains multiple entries for the same transition type, this function issues a warning and makes no changes.

This function returns `true` if any transition type has a priority added or changed, and `false` otherwise.
"""
function addSimulationTransitionTypeOrder!( mpSim::MPsim,
    transOrder::Tuple{String,Int}... )::Bool

    transTypes = map( orderPair -> orderPair[1], transOrder )

    if length( transTypes ) != length( unique( transTypes ) )
        @warn "Duplicate entries in the transition type/priority list, not making any changes."
        return false
    end  # if length( transTypes ) != length( unique( transTypes ) )

    transOrderDict = Dict{String,Int}()

    for orderPair in transOrder
        transOrderDict[orderPair[1]] = orderPair[2]
    end  # for orderPair in transOrder

    return setSimTransitionTypeOrder!( mpSim, transOrderDict, false )

end  # addSimulationTransitionTypeOrder!( mpSim, transOrder )

"""
```
addSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transType::String,
    transPrio::Int )
```
This function adds the transition type `transType` to the manpower simulation `mpSim`, giving it priority `transPrio`, where a lower number for the priority means transitions of that type are handled earlier. If the transition type already has a priority, its priority gets overwritten.

This function returns `true`, indicating the transition type's priority has been added/changed successfully.
"""
addSimulationTransitionTypeOrder!( mpSim::MPsim, transType::String,
    transPrio::Int )::Bool = setSimTransitionTypeOrder!( mpSim,
    Dict( transType => transPrio ), false )

"""
```
removeSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transTypes::String... )
```
This function removes the transition type priorities in the manpower simulation `mpSim` for all transitions in `transTypes`.

This function returns `true` if any transition type priorities have been removes, and `false` otherwise.
"""
function removeSimulationTransitionTypeOrder!( mpSim::MPsim,
    transTypes::String... )::Bool

    if !any( haskey.( Ref( mpSim.transitionTypeOrder ), transTypes ) )
        return false
    end  # if !any( haskey.( Ref( mpSim.transitionTypeOrder ), transTypes ) )

    delete!.( Ref( mpSim.transitionTypeOrder ), transTypes )
    return true

end  # removeSimulationTransitionTypeOrder!( mpSim, transTypes )

"""
```
clearSimulationTransitionTypeOrder!( mpSim::MPsim )
```
This function clears all transition type priorities from the manpower simulation `mpSim`.

This function returns `true`, indicating the transition type priorities have been successfully cleared.
"""
function clearSimulationTransitionTypeOrder!( mpSim::MPsim )::Bool

    empty!( mpSim.transitionTypeOrder )
    return true

end  # clearSimulationTransitionTypeOrder!( mpSim )

"""
```
setSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transOrder::Dict{String,Int} )
```
This function sets the transition type priorities in the list `transOrder` to the manpower simulation `mpSim`. The transition type priorities are given as a dictionary with the transition type as key, and the priority as the key's value, where lower numbers mean transitions of that type are handled first.

This function returns `true` if the transition type priorities have been set successfully, and `false` otherwise.
"""
setSimulationTransitionTypeOrder!( mpSim::MPsim,
    transOrder::Dict{String,Int} )::Bool = setSimTransitionTypeOrder!( mpSim,
    transOrder, true )

"""
```
setSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transOrder::Tuple{String,Int}... )
```
This function sets the transition type priorities in the list `transOrder` to the manpower simulation `mpSim`. The transition type priorities are given as a list of transition type/priority, where lower numbers for the priority mean transitions of that type are handled first.

If this list contains multiple entries for the same transition type, this function issues a warning and makes no changes.

This function returns `true` if the transition type priorities have been set successfully, and `false` otherwise.
"""
function setSimulationTransitionTypeOrder!( mpSim::MPsim,
    transOrder::Tuple{String,Int}... )::Bool

    transTypes = map( orderPair -> orderPair[1], transOrder )

    if length( transTypes ) != length( unique( transTypes ) )
        @warn "Duplicate entries in the transition type/priority list, not making any changes."
        return false
    end  # if length( transTypes ) != length( unique( transTypes ) )

    transOrderDict = Dict{String,Int}()

    for orderPair in transOrder
        transOrderDict[orderPair[1]] = orderPair[2]
    end  # for orderPair in transOrder

    return setSimTransitionTypeOrder!( mpSim, transOrderDict, true )

end  # setSimulationTransitionTypeOrder!( mpSim, transOrder )

"""
```
setSimulationTransitionTypeOrder!(
    mpSim::MPsim,
    transTypes::Vector{String},
    transPrios::Vector{Int} )
```
This function sets the transition type priorities in the lists `transTypes` and `transPrios` to the manpower simulation `mpSim`. The vector `transTypes` has the transition types, and the vector `transPrios` has the corresponding priorities, where lower numbers for the priority mean transitions of that type are handled first.

If the vectors are of unequal length, the function issues a warning and makes no changes.

If this list contains multiple entries for the same transition type, the function issues a warning and makes no changes.

This function returns `true` if the transition type priorities have been set successfully, and `false` otherwise.
"""
function setSimulationTransitionTypeOrder!( mpSim::MPsim,
    transTypes::Vector{String}, transPrios::Vector{Int} )::Bool

    if length( transTypes ) != length( transPrios )
        @warn "Mismatched lengths of vector of transition types and priorities, not making any changes."
        return false
    end  # if length( transTypes ) != length( transPrios )

    if length( transTypes ) != length( unique( transTypes ) )
        @warn "Duplicate entries in the transition type/priority list, not making any changes."
        return false
    end  # if length( transTypes ) != length( unique( transTypes ) )

    transOrderDict = Dict{String,Int}()

    for ii in eachindex( transTypes )
        transOrderDict[transTypes[ii]] = transPrios[ii]
    end  # for ii in eachindex( transTypes )

    return setSimTransitionTypeOrder!( mpSim, transOrderDict, true )

end  # setSimulationTransitionTypeOrder!( mpSim, transTypes, transPrios )

"""
```
addSimulationBaseNodeOrder!(
    mpSim::MPsim,
    nodeOrder::Dict{String,Int} )
```
This function adds the base node priorities in the list `nodeOrder` to the manpower simulation `mpSim`. The base node priorities are given as a dictionary with the base node names as key, and the priority as the key's value, where lower numbers mean transitions to/from that node are handled first. Base nodes that already have a priority, have their priority overwritten.

This function returns `true` if any base node has a priority added or changed, and `false` otherwise.
"""
addSimulationBaseNodeOrder!( mpSim::MPsim,
    nodeOrder::Dict{String,Int} )::Bool = setSimBaseNodeOrder!( mpSim,
    nodeOrder, false )

"""
```
addSimulationBaseNodeOrder!(
    mpSim::MPsim,
    nodeOrder::Tuple{String,Int}... )
```
This function adds the base node priorities in the list `nodeOrder` to the manpower simulation `mpSim`. The base node priorities are given as a list of base node/priority, where lower numbers for the priority mean transitions to/from that node are handled first. Base nodes that already have a priority, have their priority overwritten.

If this list contains multiple entries for the same base node, this function issues a warning and makes no changes.

This function returns `true` if any base node has a priority added or changed, and `false` otherwise.
"""
function addSimulationBaseNodeOrder!( mpSim::MPsim,
    nodeOrder::Tuple{String,Int}... )::Bool

    baseNodes = map( orderPair -> orderPair[1], nodeOrder )

    if length( baseNodes ) != length( unique( baseNodes ) )
        @warn "Duplicate entries in the base node/priority list, not making any changes."
        return false
    end  # if length( baseNodes ) != length( unique( baseNodes ) )

    nodeOrderDict = Dict{String,Int}()

    for orderPair in nodeOrder
        nodeOrderDict[orderPair[1]] = orderPair[2]
    end  # for orderPair in nodeOrder

    return setSimBaseNodeOrder!( mpSim, nodeOrderDict, false )

end  # addSimulationBaseNodeOrder!( mpSim, nodeOrder )

"""
```
addSimulationBaseNodeOrder!(
    mpSim::MPsim,
    baseNode::String,
    nodePrio::Int )
```
This function adds the base node `baseNode` to the manpower simulation `mpSim`, giving it priority `nodePrio`, where a lower number for the priority means transitions to/from that node are handled earlier. If the base node already has a priority, its priority gets overwritten.

This function returns `true`, indicating the transition type's priority has been added/changed successfully.
"""
addSimulationBaseNodeOrder!( mpSim::MPsim, baseNode::String,
    nodePrio::Int )::Bool = setSimBaseNodeOrder!( mpSim,
    Dict( baseNode => nodePrio ), false )

"""
```
removeSimulationBaseNodeOrder!(
    mpSim::MPsim,
    baseNodes::String... )
```
This function removes the base node priorities in the manpower simulation `mpSim` for all nodes in `baseNodes`.

This function returns `true` if any base node priorities have been removes, and `false` otherwise.
"""
function removeSimulationBaseNodeOrder!( mpSim::MPsim,
    baseNodes::String... )::Bool

    if !any( haskey.( Ref( mpSim.baseNodeOrder ), baseNodes ) )
        return false
    end  # if !any( haskey.( Ref( mpSim.baseNodeOrder ), baseNodes ) )

    delete!.( Ref( mpSim.baseNodeOrder ), baseNodes )
    return true

end  # removeSimulationBaseNodeOrder!( mpSim, baseNodes )

"""
```
clearSimulationBaseNodeOrder!( mpSim::MPsim )
```
This function clears all base node priorities from the manpower simulation `mpSim`.

This function returns `true`, indicating the base node priorities have been successfully cleared.
"""
function clearSimulationBaseNodeOrder!( mpSim::MPsim )::Bool

    empty!( mpSim.baseNodeOrder )
    return true

end  # clearSimulationBaseNodeOrder!( mpSim )

"""
```
setSimulationBaseNodeOrder!(
    mpSim::MPsim,
    nodeOrder::Dict{String,Int} )
```
This function sets the transition type priorities in the list `nodeOrder` to the manpower simulation `mpSim`. The transition type priorities are given as a dictionary with the transition type as key, and the priority as the key's value, where lower numbers mean transitions of that type are handled first.

This function returns `true` if the transition type priorities have been set successfully, and `false` otherwise.
"""
setSimulationBaseNodeOrder!( mpSim::MPsim,
    nodeOrder::Dict{String,Int} )::Bool = setSimBaseNodeOrder!( mpSim,
    nodeOrder, true )

"""
```
setSimulationBaseNodeOrder!(
    mpSim::MPsim,
    nodeOrder::Tuple{String,Int}... )
```
This function sets the base node priorities in the list `nodeOrder` to the manpower simulation `mpSim`. The base node priorities are given as a list of base node/priority, where lower numbers for the priority mean transitions to/from that node are handled first.

If this list contains multiple entries for the same base node, this function issues a warning and makes no changes.

This function returns `true` if the base node priorities have been set successfully, and `false` otherwise.
"""
function setSimulationBaseNodeOrder!( mpSim::MPsim,
    nodeOrder::Tuple{String,Int}... )::Bool

    baseNodes = map( orderPair -> orderPair[1], nodeOrder )

    if length( baseNodes ) != length( unique( baseNodes ) )
        @warn "Duplicate entries in the transition type/priority list, not making any changes."
        return false
    end  # if length( baseNodes ) != length( unique( baseNodes ) )

    nodeOrderDict = Dict{String,Int}()

    for orderPair in nodeOrder
        nodeOrderDict[orderPair[1]] = orderPair[2]
    end  # for orderPair in nodeOrder

    return setSimBaseNodeOrder!( mpSim, nodeOrderDict, true )

end  # setSimulationBaseNodeOrder!( mpSim, nodeOrder )

"""
```
setSimulationBaseNodeOrder!(
    mpSim::MPsim,
    baseNodes::Vector{String},
    nodePrios::Vector{Int} )
```
This function sets the base node priorities in the lists `baseNodes` and `nodePrios` to the manpower simulation `mpSim`. The vector `baseNodes` has the base nodes, and the vector `nodePrios` has the corresponding priorities, where lower numbers for the priority mean transitions to/from that node are handled first.

If the vectors are of unequal length, the function issues a warning and makes no changes.

If this list contains multiple entries for the same base node, the function issues a warning and makes no changes.

This function returns `true` if the base node priorities have been set successfully, and `false` otherwise.
"""
function setSimulationBaseNodeOrder!( mpSim::MPsim,
    baseNodes::Vector{String}, nodePrios::Vector{Int} )::Bool

    if length( baseNodes ) != length( nodePrios )
        @warn "Mismatched lengths of vector of base nodes and priorities, not making any changes."
        return false
    end  # if length( baseNodes ) != length( nodePrios )

    if length( baseNodes ) != length( unique( baseNodes ) )
        @warn "Duplicate entries in the base node/priority list, not making any changes."
        return false
    end  # if length( baseNodes ) != length( unique( baseNodes ) )

    transOrderDict = Dict{String,Int}()

    for ii in eachindex( baseNodes )
        transOrderDict[baseNodes[ii]] = nodePrios[ii]
    end  # for ii in eachindex( baseNodes )

    return setSimBaseNodeOrder!( mpSim, transOrderDict, true )

end  # setSimulationBaseNodeOrder!( mpSim, baseNodes, nodePrios )

"""
```
setSimulationRetirement!(
    mpSim::MPsim,
    retirement::Retirement,
    node::String = "default";
    name::String = "retirement",
    overwrite::Bool = true )
```
This function sets the retirement scheme of the base node `node` in the manpower simulation to `retirement`, giving it the name `name`. If the node is `"default"`, this becomes the fallback retirement scheme for any personnel members who do not have a specific retirement schedule for their node. If the flag `overwrite` is `true`, this function also clears the existing retirement schedule(s) for the node.

This function returns `true` if the retirement schedule has been successfully set, or `false` if the target node did not exist, or the given retirement schedule is a non-retirement (no max age or tenure defined).
"""
function setSimulationRetirement!( mpSim::MPsim, retirement::Retirement, node::String = "default"; name::String = "retirement", overwrite::Bool = true )::Bool

    if ( lowercase( node ) != "default" ) && !haskey( mpSim.baseNodeList, node )
        return false
    end  # if ( lowercase( node ) != "default" ) && ...

    # If the node is "default", change the fallback retirement scheme.
    if lowercase( node ) == "default"
        mpSim.retirement = deepcopy( retirement )
        return true
    end  # if lowercase( node ) == "default"

    # No change if there's no real retirement happening.
    if ( retirement.maxCareerLength == 0.0 ) &&
        ( retirement.retirementAge == 0.0 )
        return false
    end  # if ( retirement.maxCareerLength == 0.0 ) && ...

    # Find transitions which already cover retirement from this node and remove
    #   them.
    if overwrite
        removeSimulationRetirement!( mpSim, node )
    end  # if overwrite

    # Both retirement conditions apply.
    if !retirement.isEither
        transition = Transition( name, node )
        setTransitionSchedule!( transition, retirement.freq, retirement.offset )

        if retirement.maxCareerLength > 0.0
            addTransitionCondition!( transition,
                MPcondition( "tenure", >=, retirement.maxCareerLength ) )
        end  # if retirement.maxCareerLength > 0.0

        if retirement.retirementAge > 0.0
            addTransitionCondition!( transition,
                MPcondition( "age", >=, retirement.retirementAge ) )
        end  # if retirement.retirementAge > 0.0

        addSimulationTransition!( mpSim, transition )
        return true
    end  # if !retirement.isEither

    # Either retirement condition applies.
    if retirement.maxCareerLength > 0.0
        transition = Transition( name, node )
        setTransitionSchedule!( transition, retirement.freq, retirement.offset )
        addTransitionCondition!( transition,
            MPcondition( "tenure", >=, retirement.maxCareerLength ) )
        addSimulationTransition!( mpSim, transition )
    end  # if retirement.maxCareerLength > 0.0

    if retirement.retirementAge > 0.0
        transition = Transition( name, node )
        setTransitionSchedule!( transition, retirement.freq, retirement.offset )
        addTransitionCondition!( transition,
            MPcondition( "age", >=, retirement.retirementAge ) )
        addSimulationTransition!( mpSim, transition )
    end  # if retirement.retirementAge > 0.0

    return true

end  # setSimulationRetirement!( mpSim, retirement, node, name, overwrite )

"""
```
removeSimulationRetirement!(
    mpSim::MPsim,
    node::String )
```
This function removes all OUT transitions from the base node `node` from the manpower simulation `mpSim`.

The function returns `true` if the retirement schemes have been successfully removed, and `false` if the node didn't exist in the simulation.
"""
function removeSimulationRetirement!( mpSim::MPsim, node::String )::Bool

    if !haskey( mpSim.baseNodeList, node )
        return false
    end  # if haskey( mpSim.baseNodeList, node )

    if haskey( mpSim.transitionsBySource, node )
        for transition in filter( transition -> transition.isOutTransition,
            mpSim.transitionsBySource[node] )
            nameInd = findfirst( map( trans -> transition === trans,
                mpSim.transitionsByName[transition.name] ) )
            sourceInd = findfirst( map( trans -> transition === trans,
                mpSim.transitionsBySource[node] ) )
            targetInd = findfirst( map( trans -> transition === trans,
                mpSim.transitionsByTarget["OUT"] ) )

            deleteat!( mpSim.transitionsByName[transition.name], nameInd )
            deleteat!( mpSim.transitionsBySource[node], sourceInd )
            deleteat!( mpSim.transitionsByTarget["OUT"], targetInd )
        end  # for transition in filter( ... )
    end  # if haskey( mpSim.transitionsBySource, node )

    return true

end  # removeSimulationRetirement!( mpSim, node )

"""
```
clearSimulationRetirement!( mpSim::MPsim )
```
This function clears all OUT transitions from the manpower simulation `mpSim`, and sets the default retirement scheme to "no retirement".

This function returns `true`, indicating that all OUT transitions have been successfully cleared.
"""
function clearSimulationRetirement!( mpSim::MPsim )::Bool

    for transition in mpSim.transitionsByTarget["OUT"]
        nameInd = findfirst( map( trans -> transition === trans,
            mpSim.transitionsByName[transition.name] ) )
        sourceInd = findfirst( map( trans -> transition === trans,
            mpSim.transitionsBySource[transition.sourceNode] ) )

        deleteat!( mpSim.transitionsByName[transition.name], nameInd )
        deleteat!( mpSim.transitionsBySource[transition.sourceNode],
            sourceInd )
    end  # for transition in mpSim.transitionsByTarget["OUT"]

    empty!( mpSim.transitionsByTarget["OUT"] )
    mpSim.retirement = Retirement()
    return true

end  # function clearSimulationRetirement!( mpSim )

"""
```
addSimulationAttrition!(
    mpSim::MPsim,
    attritionList::Attrition... )
```
This function adds the attrition schemes in `attritionList` to the manpower simulation `mpSim`.

If there are multiple attrition schemes with the same name, the function issues a warning and makes no changes. For the default attrition scheme, `""` and any capitalisation of `"default"` are considered the same name.

This function returns `true` if the attrition schemes have been successfully added, and `false` if multiple attrition schemes have the same name.
"""
addSimulationAttrition!( mpSim::MPsim, attritionList::Attrition... )::Bool =
    setSimAttrition!( mpSim, collect( attritionList ), false )

"""
```
removeSimulationAttrition!(
    mpSim::MPsim,
    attritionList::String... )
```
This function removes the attrition schemes with names in `attritionList` form the manpower simulation `mpSim`. For the default attrition scheme, `""` and any capitalisation of `"default"` are considered the same name, and the default attrition scheme is set to a no-attrition scheme (flat zero attrition rate).

This function returns `true` if any attrition scheme has been successfully removed, and `false` if no names in the list referred to attrition schemes in the simulation.
"""
function removeSimulationAttrition!( mpSim::MPsim,
    attritionList::String... )::Bool

    tmpAttritionNames = collect( attritionList )
    tmpAttritionNames[map( attrition -> lowercase( attrition ) ∈
        ["", "default"], tmpAttritionNames )] .= "default"

    if !any( haskey.( Ref( mpSim.attritionSchemes ), tmpAttritionNames ) )
        return false
    end  # if !any( ... ) )

    delete!.( Ref( mpSim.attritionSchemes ), tmpAttritionNames )

    if "default" ∈ tmpAttritionNames
        mpSim.attritionSchemes["default"] = Attrition()
    end  # if "default" ∈ tmpAttritionNames

    mpSim.isStale = true
    return true

end  # removeSimulationAttrition!( mpSim, attritionList )

"""
```
clearSimulationAttrition!( mpSim::MPsim )
```
This function clears all attrition schemes from the manpower simulation `mpSim`, and sets the default attrition scheme to a no-attrition scheme (flat zero attrition rate).

This funciton returns `true`, indicatin ght at the attrition schemes have been successfully cleared.
"""
function clearSimulationAttrition!( mpSim::MPsim )::Bool

    mpSim.attritionSchemes = Dict( "default" => Attrition() )
    mpSim.isStale = true
    return true

end  # clearSimulationAttrition!( mpSim )

"""
```
setSimulationAttrition!(
    mpSim::MPsim,
    attritionList::Vector{Attrition} )
```
This function set the attrition schemes of the manpower simulation `mpSim` to the ones in `attritionList`.

If there are multiple attrition schemes with the same name, the function issues a warning and makes no changes. For the default attrition scheme, `""` and any capitalisation of `"default"` are considered the same name.

This function returns `true` if the attrition schemes have been successfully set, and `false` if multiple attrition schemes have the same name.
"""
setSimulationAttrition!( mpSim::MPsim,
    attritionList::Vector{Attrition} )::Bool = setSimAttrition!( mpSim,
    attritionList, true )

"""
```
setSimulationLength!(
    mpSim::MPsim,
    simLength::Real )
```
This function sets the length of the simulation `mpSim` to `simLength`.

If a length < 0 is entered, the function issues a warning and makes no changes.

This function returns `true` if the simulation length is successfully set, and `false` if the entered length is < 0.
"""
function setSimulationLength!( mpSim::MPsim, simLength::Real )::Bool

    if simLength < 0
        @warn "Simulation length must be ⩾ 0, not making any changes."
        return false
    end

    mpSim.simLength = simLength
    return true

end  # setSimulationLength!( mpSim, simLength )

"""
```
setSimulationPersonnelTarget!(
    mpSim::MPsim,
    personnelTarget::Integer )
```
This function sets the personnel target of the manpower simulation `mpSim` to `personnelTarget`. Any value ⩽ 0 is treated as "no target".

This function returns `true`, indicating that the personnel target is successfully set.
"""
function setSimulationPersonnelTarget!( mpSim::MPsim,
    personnelTarget::Integer )::Bool

    mpSim.personnelTarget = max( 0, personnelTarget )
    return true

end  # setSimulationPersonnelTarget!( mpSim, personnelTarget )

"""
```
setSimulationDatabaseName!(
    mpSim::MPsim,
    dbName::String = "" )
```
This function sets the name of the results database of the manpower simulation `mpSim` to `dbName`, adding the extension `.sqlite` if needed. If the name is the empty string `""`, the database will be stored in memory instead.

This function returns `true`, indicating that the database name is successfully set.
"""
function setSimulationDatabaseName!( mpSim::MPsim, dbName::String = "" )::Bool

    mpSim.dbName = endswith( dbName, ".sqlite" ) || isempty( dbName ) ? dbName :
        string( dbName, ".sqlite" )
    mpSim.reportFolder = dirname( dbName )
    return true

end  # setSimulationDatabaseName!( mpSim, dbName )

"""
```
setSimulationDatabase!(
    mpSim::MPsim,
    dbName::String = "" )
```
This function sets the results database of the manpower simulation `mpSim` to a database with name `dbName`, adding the extension `.sqlite` if needed. If the name is the empty string `""`, the database will be stored in memory instead.

If the database is an existing database, the function also checks if it's a new or old style database, and will give a deprecation warning in the latter case.

This function returns `true`, indicating that the database is successfully set.
"""
function setSimulationDatabase!( mpSim::MPsim, dbName::String="" )::Bool
    setSimulationDatabaseName!( mpSim, dbName )
    dbpath = dirname(mpSim.dbName)
    !ispath(dbpath) && mkpath(dbpath)
    mpSim.simDB = SQLite.DB(mpSim.dbName)
    validateDatabaseAge!(mpSim)
    true
end  # setSimulationDatabase!( mpSim, dbName )

"""
```
setShowInfo!(
    mpSim::MPsim,
    showInfo::Bool )
```
This function sets the flag controlling whether timing and other information about the manpower simulation (`mpSim`) run is displayed to `showInfo`.

This function returns `true`, indicating that the flag is successfully set.
"""
function setSimulationShowInfo!( mpSim::MPsim, showInfo::Bool )::Bool

    mpSim.showInfo = showInfo
    return true

end  # setSimulationShowInfo!( mpSim, showInfo )

"""
```
Dates.now( mpSim::MPsim )
```
This function returns the current simulation time in the manpower simulation `mpSim`.
"""
Dates.now( mpSim::MPsim ) = min( now( mpSim.sim ), mpSim.simLength )

include( joinpath( privPath, "manpowersimulation.jl" ) )
include( joinpath( simPath, "manpowersimulation.jl" ) )


text:
import Base: checkbounds

# -----------------------------------------------------------------------------
# Types

# TODO : Cache metadata such as extname, extver, image size and data type?
#        This might allow faster access for size(ImageHDU) and ndim(ImageHDU)
abstract HDU
type ImageHDU <: HDU
    fitsfile::FITSFile
    ext::Int
end

type TableHDU <: HDU
    fitsfile::FITSFile
    ext::Int
end

type AsciiHDU <: HDU
    fitsfile::FITSFile
    ext::Int
end

# FITS is analagous to FITSFile, but holds a reference to all of its
# HDU objects. This is so that only a single HDU object is created for
# each extension in the file. It also allows a FITS object to tell
# previously created HDUs about events that happen to the file, such
# as deleting extensions. This could be done by, e.g., setting ext=-1 in
# the HDU object.
type FITS
    fitsfile::FITSFile
    filename::String
    mode::String
    hdus::Dict{Int, HDU}

    function FITS(filename::String, mode::String="r")
        f = (mode == "r"                     ? fits_open_file(filename, 0)   :
             mode == "r+" && isfile(filename)? fits_open_file(filename, 1)   :
             mode == "r+"                    ? fits_create_file(filename)    :
             mode == "w"                     ? fits_create_file("!"*filename):
             error("invalid open mode: $mode"))

        new(f, filename, mode, Dict{Int, HDU}())
    end
end

# FITSHeader stores the (key, value, comment) information for each card in
# a header. We could almost just use an OrderedDict for this, but we need
# to store comments so they are not lost when reading from disk,
# then writing back to disk with modifications.
type FITSHeader
    keys::Vector{ASCIIString}
    values::Vector{Any}
    comments::Vector{ASCIIString}
    map::Dict{ASCIIString, Int}

    function FITSHeader(keys::Vector{ASCIIString}, values::Vector{Any},
                        comments::Vector{ASCIIString})
        if ((length(keys) != length(values)) ||
            (length(keys) != length(comments)))
            error("keys, values, comments must be same length")
        end
        map = [keys[i]=>i for i=1:length(keys)]
        new(keys, values, comments, map)
    end
end

# -----------------------------------------------------------------------------
# FITS methods

function length(f::FITS)
    fits_assert_open(f.fitsfile)
    @compat Int(fits_get_num_hdus(f.fitsfile))
end

endof(f::FITS) = length(f)

function show(io::IO, f::FITS)
    fits_assert_open(f.fitsfile)

    print(io, "file: ", f.filename, "\n")
    print(io, "mode: ", f.mode, "\n")
    print(io, "extnum exttype         extname\n")

    for i = 1:length(f)
        hdutype = fits_movabs_hdu(f.fitsfile, i)
        extname = ""
        try
            extname = fits_read_keyword(f.fitsfile, "EXTNAME")[1]
        catch
            try
                extname = fits_read_keyword(f.fitsfile, "HDUNAME")[1]
            catch
            end
        end
        @printf io "%-6d %-15s %s\n" i hdutype extname
    end
end

# Returns HDU object based on extension number
function getindex(f::FITS, i::Integer)
    fits_assert_open(f.fitsfile)

    if haskey(f.hdus, i)
        return f.hdus[i]
    end

    if i > length(f)
        error("index out of bounds")
    end
    hdutype = fits_movabs_hdu(f.fitsfile, i)
    f.hdus[i] = (hdutype == :image_hdu ? ImageHDU(f.fitsfile, i) :
                 hdutype == :binary_table ? TableHDU(f.fitsfile, i) :
                 hdutype == :ascii_table ? AsciiHDU(f.fitsfile, i) :
                 error("bad HDU type"))
    return f.hdus[i]
end

# Returns HDU based on hduname, version
function getindex(f::FITS, name::String, ver::Int=0)
    fits_assert_open(f.fitsfile)
    fits_movnam_hdu(f.fitsfile, name, ver)
    i = fits_get_hdu_num(f.fitsfile)

    if haskey(f.hdus, i)
        return f.hdus[i]
    end

    hdutype = fits_get_hdu_type(f.fitsfile)
    f.hdus[i] = (hdutype == :image_hdu ? ImageHDU(f.fitsfile, i) :
                 hdutype == :binary_table ? TableHDU(f.fitsfile, i) :
                 hdutype == :ascii_table ? AsciiHDU(f.fitsfile, i) :
                 error("bad HDU type"))
    return f.hdus[i]
end

function close(f::FITS)
    fits_assert_open(f.fitsfile)
    fits_close_file(f.fitsfile)
    f.filename = ""
    f.mode = ""
    empty!(f.hdus)
    nothing
end

# -----------------------------------------------------------------------------
# Header methods

# returns one of: ASCIIString, Bool, Int, Float64, nothing
function parse_header_val(val::ASCIIString)
    len = length(val)
    if len < 1
        return nothing
    end
    c = val[1]
    if len == 1 && c == 'T'
        return true
    elseif len == 1 && c == 'F'
        return false
    elseif len >= 2 && c == '\'' && val[end] == '\''
        # This is a character string; according to FITS standard, trailing
        # spaces are insignificant, thus we remove them as well as the
        # surrounding quotes.
        return rstrip(val[2:end-1])
    else
        try
            return @compat parse(Int, val)
        catch
            try
                return @compat parse(Float64, val)
            catch
            end
        end
    end
    return val  # The value (probably) doesn't comply with the FITS
                # standard. Give up and return the unparsed string.
end

function readkey(hdu::HDU, key::Integer)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    keyout, value, comment = fits_read_keyn(hdu.fitsfile, key)
    keyout, parse_header_val(value), comment
end

function readkey(hdu::HDU, key::ASCIIString)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    value, comment = fits_read_keyword(hdu.fitsfile, key)
    parse_header_val(value), comment
end

function readheader(hdu::HDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)

    # Below, we use a direct call to ffgkyn so that we can keep reusing the
    # same buffers.
    key = Array(Uint8, 81)
    value = Array(Uint8, 81)
    comment = Array(Uint8, 81)
    status = Cint[0]

    nkeys, morekeys = fits_get_hdrspace(hdu.fitsfile)

    # Initialize output arrays
    keys = Array(ASCIIString, nkeys)
    values = Array(Any, nkeys)
    comments = Array(ASCIIString, nkeys)
    for i=1:nkeys
        ccall((:ffgkyn,libcfitsio), Cint,
              (Ptr{Void},Cint,Ptr{Uint8},Ptr{Uint8},Ptr{Uint8},Ptr{Cint}),
              hdu.fitsfile.ptr, i, key, value, comment, status)
        keys[i] = bytestring(pointer(key))
        values[i] = parse_header_val(bytestring(pointer(value)))
        comments[i] = bytestring(pointer(comment))
    end
    fits_assert_ok(status[1])
    FITSHeader(keys, values, comments)
end

length(hdr::FITSHeader) = length(hdr.keys)
haskey(hdr::FITSHeader, key::ASCIIString) = in(key, hdr.keys)
keys(hdr::FITSHeader) = hdr.keys
values(hdr::FITSHeader) = hdr.values
getindex(hdr::FITSHeader, key::ASCIIString) = hdr.values[hdr.map[key]]
getindex(hdr::FITSHeader, i::Integer) = hdr.values[i]

function setindex!(hdr::FITSHeader, value::Any, key::ASCIIString)
    if in(key, hdr.keys)
        hdr.values[hdr.map[key]] = value
    else
        push!(hdr.keys, key)
        push!(hdr.values, value)
        push!(hdr.comments, "")
        hdr.map[key] = length(hdr.keys)
    end
end

function setindex!(hdr::FITSHeader, value::Any, i::Integer)
    hdr.values[i] = value
end

# Comments
getcomment(hdr::FITSHeader, key::ASCIIString) = hdr.comments[hdr.map[key]]
getcomment(hdr::FITSHeader, i::Integer) = hdr.comments[i]
function setcomment!(hdr::FITSHeader, key::ASCIIString, comment::ASCIIString)
    hdr.comments[hdr.map[key]] = comment
end
function setcomment!(hdr::FITSHeader, i::Integer, comment::ASCIIString)
    hdr.comments[i] = comment
end

# Display the header
hdrval_to_str(val::Bool) = val ? "T" : "F"
hdrval_to_str(val::ASCIIString) = @sprintf "'%s'" val
hdrval_to_str(val::Union(FloatingPoint, Integer)) = string(val)
function show(io::IO, hdr::FITSHeader)
    for i=1:length(hdr)
        cl = length(hdr.comments[i])
        if hdr.keys[i] == "COMMENT" || hdr.keys[i] == "HISTORY"
            if cl > 71
                @printf io "%s %s\n" hdr.keys[i] hdr.comments[i][1:71]
            else
                @printf io "%s %s\n" hdr.keys[i] hdr.comments[i]
            end
        else
            @printf io "%-8s" hdr.keys[i]
            if hdr.values[i] === nothing
                print(io, "                      ")
                rc = 50  # remaining characters on line
            else
                val = hdrval_to_str(hdr.values[i])
                @printf io "= %20s" val
                rc = length(val) <= 20 ? 50 : 70 - length(val)
            end

            if cl > 0
                if cl > rc - 3
                    @printf io " / %s" hdr.comments[i][1:rc-3]
                else
                    @printf io " / %s" hdr.comments[i]
                end
            end
            print(io, "\n")
        end
    end
end

const RESERVED_KEYS = ["SIMPLE","EXTEND","XTENSION","BITPIX","PCOUNT","GCOUNT",
                       "THEAP","EXTNAME","BUNIT","BSCALE","BZERO","BLANK",
                       "ZQUANTIZ","ZDITHER0","ZIMAGE","ZCMPTYPE","ZSIMPLE",
                       "ZTENSION","ZPCOUNT","ZGCOUNT","ZBITPIX","ZEXTEND",
                       "CHECKSUM","DATASUM"]

# This is more complex than you would think because some reserved keys
# are only reserved when other keys are present. Also, in general a key
# may appear more than once in a header.
function reserved_key_indicies(hdr::FITSHeader)
    nhdr = length(hdr)
    indicies = Int[]
    for i=1:nhdr
        if in(hdr.keys[i], RESERVED_KEYS)
            push!(indicies, i)
        end
    end

    # Note that this removes anything matching NAXIS\d regardless of # of axes.
    if in("NAXIS", hdr.keys)
        for i=1:nhdr
            if ismatch(r"^NAXIS\d*$", hdr.keys[i])
                push!(indicies, i)
            end
        end
    end

    if in("ZNAXIS", hdr.keys)
        for i=1:nhdr
            if (ismatch(r"^ZNAXIS\d*$", hdr.keys[i]) ||
                ismatch(r"^ZTILE\d*$", hdr.keys[i]) ||
                ismatch(r"^ZNAME\d*$", hdr.keys[i]) ||
                ismatch(r"^ZVAL\d*$", hdr.keys[i]))
                push!(indicies, i)
            end
        end
    end

    if in("TFIELDS", hdr.keys)
        for i=1:nhdr
            for re in [r"^TFORM\d*$", r"^TTYPE\d*$", r"^TDIM\d*$",
                       r"^TUNIT\d*$", r"^TSCAL\d*$", r"^TZERO\d*$",
                       r"^TNULL\d*$", r"^TDISP\d*$", r"^TDMIN\d*$",
                       r"^TDMAX\d*$", r"^TDESC\d*$", r"^TROTA\d*$",
                       r"^TRPIX\d*$", r"^TRVAL\d*$", r"^TDELT\d*$",
                       r"^TCUNI\d*$", r"^TFIELDS$"]
                if ismatch(re, hdr.keys[i])
                    push!(indicies, i)
                end
            end
        end
    end

    return indicies
end

# Header writing: "low-level" (works directly on FITSFile)
# if `clean` is true, skip writing reserved header keywords
function write_header(f::FITSFile, hdr::FITSHeader, clean::Bool=true)
    indicies = clean? reserved_key_indicies(hdr): Int[]
    for i=1:length(hdr)
        if clean && in(i, indicies)
            continue
        end
        if hdr.keys[i] == "COMMENT"
            fits_write_comment(f, hdr.comments[i])
        elseif hdr.keys[i] == "HISTORY"
            fits_write_history(f, hdr.comments[i])
        elseif hdr.comments[i] == ""
            fits_update_key(f, hdr.keys[i], hdr.values[i])
        else
            fits_update_key(f, hdr.keys[i], hdr.values[i], hdr.comments[i])
        end
    end
end

# -----------------------------------------------------------------------------
# ImageHDU methods

# Display the image datatype and dimensions
function show(io::IO, hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    bitpix = fits_get_img_equivtype(hdu.fitsfile)
    sz = fits_get_img_size(hdu.fitsfile)
    @printf io "file: %s\nextension: %d\ntype: IMAGE\nimage info:\n  bitpix: %d\n  size: %s" fits_file_name(hdu.fitsfile) hdu.ext bitpix tuple(sz...)
end

# Get image dimensions
function ndims(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    fits_get_img_dim(hdu.fitsfile)
end

# Get image size
function size(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    tuple(sz...)
end

function size(hdu::ImageHDU, i::Integer)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    sz[i]
end

# `endof` is needed so that hdu[:] can throw DimensionMismatch
# when ndim != 1, rather than no method.
length(hdu::ImageHDU) = prod(size(hdu))
endof(hdu::ImageHDU) = length(hdu::ImageHDU)

# Read a full image from an HDU
function read(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    bitpix = fits_get_img_equivtype(hdu.fitsfile)
    data = Array(bitpix_to_type[bitpix], sz...)
    fits_read_pix(hdu.fitsfile, data)
    data
end

# `trailingsize` is used to compute last index in checkbounds.
# sz: array size (tuple), n: starting index.
# Same as Base.trailingsize but takes size tuple rather than array.
function trailingsize(sz, n)
    s = 1
    for i=n:length(sz)
        s *= sz[i]
    end
    return s
end

# `checkbounds` method, used for ImageHDU. (ImageHDU is not a subtype
# of AbstractArray, so we can't just use methods in Base.) Note that
# this takes a size tuple whereas Methods in Base take array A and use
# size(A,n). For ImageHDU, size(hdu,n) gets size(hdu) as in
# intermediary, so it is better to just get that tuple once.
function checkbounds(sz::NTuple, I::Union(Int, Range{Int})...)
    n = length(I)
    if n > 0
        for dim = 1:(n-1)
            checkbounds(sz[dim], I[dim])
        end
        checkbounds(trailingsize(sz,n), I[n])
    end
end

# Read a subset of an ImageHDU
function _getindex(hdu::ImageHDU, I::Union(Range{Int},Int)...)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)

    # check number of indicies and bounds.
    # Note that N_indicies != ndim is not supported, differing from Array.
    # It could be supported in the future with care taken in constructing
    # first, last, step arrays passed to cfitsio.
    sz = tuple(fits_get_img_size(hdu.fitsfile)...)
    if length(I) != length(sz)
        throw(DimensionMismatch("number of indicies must match dimensions"))
    end
    checkbounds(sz, I...)

    # construct first, last and step vectors
    firsts = Clong[first(i) for i in I]
    lasts = Clong[last(i) for i in I]
    steps = Clong[isa(i, Int)? 1: step(i) for i in I]

    # construct output array
    bitpix = fits_get_img_equivtype(hdu.fitsfile)
    data = Array(bitpix_to_type[bitpix], Base.index_shape(I...))

    fits_read_subset(hdu.fitsfile, firsts, lasts, steps, data)
    data
end

# general method and version that returns a single value rather than 0-d array
getindex(hdu::ImageHDU, I::Union(Range{Int},Int)...) = _getindex(hdu, I...)
getindex(hdu::ImageHDU, I::Int...) = _getindex(hdu, I...)[1]

# Add a new ImageHDU to a FITS object
# The following Julia data types are supported for writing images by cfitsio:
# Uint8, Int8, Uint16, Int16, Uint32, Int32, Int64, Float32, Float64
function write{T}(f::FITS, data::Array{T};
                  header::Union(Nothing, FITSHeader)=nothing)
    fits_assert_open(f.fitsfile)
    s = size(data)
    fits_create_img(f.fitsfile, T, [s...])
    if isa(header, FITSHeader)
        write_header(f.fitsfile, header, true)
    end
    fits_write_pix(f.fitsfile, ones(Int, length(s)), length(data), data)
    nothing
end

# Copy a rectangular section of an image and write it to a new FITS
# primary image or image extension. The new image HDU is appended to
# the end of the output file; all the keywords in the input image will
# be copied to the output image. The common WCS keywords will be
# updated if necessary to correspond to the coordinates of the section.

# TODO: Change Range types once v0.2 is no longer supported.

range2fits_str(r::Range1) = @sprintf "%d:%d" first(r) last(r)
range2fits_str(r::Range) = @sprintf "%d:%d:%d" first(r) last(r) step(r)
fits_copy_image_section(fin::FITSFile, fout::FITSFile, r...) =
    fits_copy_image_section(fin, fout, join([range2str(ri) for ri in r], ','))
function copy_section(hdu::ImageHDU, destination::FITS, r::Range...)
    fits_assert_open(hdu.fitsfile)
    fits_assert_open(destination.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    fits_copy_image_section(hdu.fitsfile, destination.fitsfile,
                            join([range2fits_str(ri) for ri in r], ','))
end

ext: jl


text:
using SparseDiffTools, CUDA, Test, LinearAlgebra
using ArrayInterfaceCore: allowed_getindex, allowed_setindex!
using SparseArrays
using ArrayInterfaceGPUArrays

function f(dx,x)
    dx[2:end-1] = x[1:end-2] - 2x[2:end-1] + x[3:end]
    allowed_setindex!(dx,-2allowed_getindex(x,1) + allowed_getindex(x,2),1)
    allowed_setindex!(dx,-2allowed_getindex(x,30) + allowed_getindex(x,29),30)
    nothing
end
x = rand(4)
_J1 = similar(rand(30,30))
_denseJ1 = cu(collect(_J1))
x = cu(rand(30))
CUDA.allowscalar(false)
_J2 = sparse(forwarddiff_color_jacobian!(_denseJ1, f, x))
out = copy(_J2)
forwarddiff_color_jacobian!(out, f, x, colorvec = repeat(1:3,10), sparsity = _J2)

@test_broken forwarddiff_color_jacobian!(_denseJ1, f, x, sparsity = cu(_J1)) isa Nothing
@test_broken forwarddiff_color_jacobian!(_denseJ1, f, x, colorvec = repeat(1:3,10), sparsity = cu(_J1)) isa Nothing
_Jt = similar(Tridiagonal(_J1))
@test_broken forwarddiff_color_jacobian!(_denseJ1, f, x, colorvec = repeat(1:3,10), sparsity = _Jt) isa Nothing
_Jt2 = similar(Tridiagonal(cu(_J1)))
@test_broken forwarddiff_color_jacobian!(_denseJ1, f, x, colorvec = repeat(1:3,10), sparsity = _Jt2) isa Nothing


text:
# --------------------------------------------------------------------------
# ACE1.jl: Julia implementation of the Atomic Cluster Expansion
# Copyright (c) 2019 Christoph Ortner <christophortner0@gmail.com>
# Licensed under ASL - see ASL.md for terms and conditions.
# --------------------------------------------------------------------------

@testset "PolyPairPot" begin

@info("--------------- PolyPairPot Implementation ---------------")

##

using ACE1
using Printf, Test, LinearAlgebra, JuLIP, JuLIP.Testing
using JuLIP: evaluate, evaluate_d
using JuLIP.Potentials: i2z, numz
using JuLIP.MLIPs: combine

randr() = 1.0 + rand()
randcoeffs(B) = rand(length(B)) .* (1:length(B)).^(-2)

##

maxdeg = 8
r0 = 1.0
rcut = 3.0

trans = PolyTransform(1, r0)
Pr = transformed_jacobi(maxdeg, trans, rcut; pcut = 2)
pB = ACE1.PairPotentials.PolyPairBasis(Pr, :W)
coeffs = randcoeffs(pB)
V = combine(pB, coeffs)

##

at = bulk(:W, cubic=true) * 3
rattle!(at, 0.03)
r0 = rnn(:W)
X = copy(positions(at))
energy(V, at)

@info("Testing correctness of `PolyPairPot` against `PolyPairBasis`")
@info("    test `combine`")
coeffs = randcoeffs(pB)
V = combine(pB, coeffs)
println(@test energy(V, at) ≈ sum(V.coeffs .*  energy(pB, at)))

##
@info("   test (de-)dictionisation")
println(@test all(JuLIP.Testing.test_fio(V)))

@info("      check that PolyPairBasis ≈ PolyPairPot")
for ntest = 1:10
   rattle!(at, 0.01)
   coeffs = randcoeffs(pB)
   V = combine(pB, coeffs)

   E_V = energy(V, at)
   E_b = dot(energy(pB, at), coeffs)
   print_tf(@test E_V ≈ E_b)

   F_V = forces(V, at)
   F_b = sum(coeffs .* forces(pB, at))
   print_tf(@test F_V ≈ F_b)

   V_V = virial(V, at)
   V_b = sum(coeffs .* virial(pB, at))
   print_tf(@test V_V ≈ V_b)
end
println()

##

@info("      Standard JuLIP Force Consistency Test")
variablecell!(at)
rattle!(at, 0.03)
println(@test JuLIP.Testing.fdtest(V, at))

##
end


text:
@testset "testing input output functions" begin
    package_path = pathof(NighttimeLights)
    path_len = length(package_path)
    assets_path = package_path[1:path_len-22] * "assets"
    map_path = assets_path * "/mumbai_map/mumbai_districts.shp"
    radiance_jld_path = assets_path * "/mumbai_ntl/datacube/mumbai_radiance.jld"
    radiance_img_path = assets_path * "/mumbai_ntl/img/april2012.tif"
    @test length(size(load_datacube(radiance_jld_path))) == 3
    @test length(size(load_img(radiance_img_path))) == 2
    @test length(size(load_img(radiance_img_path ,[10, 10], [50, 50]))) == 2
    @test sizeof(load_shapefile(map_path)) > 0
end


text:
# reduce.jl

export TransTens, DecompMMTDg, print, reduce_transTens, reconstruct_transTens;

struct TransTens
    A::Array
    order::Int
    function TransTens(AA::Array{T}; sum1tol=1.0e-6) where T <: Real
        ord = ndims(AA) - 1
        d = size(AA)
        all( d.==d[1]) || error("All dimensions of transition tensor must have same length.")
        all( AA .>= 0.0 ) || error("Transition tensor must have non-negative entries.")
        maximum(abs.( sum(AA, dims=1) .- 1.0 )) < sum1tol || error("Transition tensor must add to unity along first dimension.")
        return new(AA, ord)
    end
end

struct DecompMMTDg
    Omega::Union{TransTens, Array{T}} where T <: Real
    order::Int
    totalweight_tol::T where T <: Real
    levels::Array{T,2} where T <: Real
    configs::Vector{Vector{Int}}
    Q::Vector{Array{T}} where T <: Real
    involv::Vector{T} where T <: Real
    level_lam_total::Vector{T} where T <: Real

    function DecompMMTDg(Omega::Union{TransTens, Array{T}}, totalweight_tol::T, levels::Array{T,2},
                         configs::Vector{Vector{Int}}, Q::Vector{Array{T}}, involv::Vector{T}, level_lam_total::Vector{T}) where T <: Real
        sum(levels[:,2]) <= 1.0 + 1.0e-9 || error("Total mixture weight cannot exceed 1.0. Was $(sum(levels[:,2]))")
        if typeof(Omega) == TransTens
            ord = Omega.order
        else
            ord = ndims(Omega) - 1
        end
        return new(Omega, ord, totalweight_tol, levels, configs, Q, involv, level_lam_total)
    end
end

function Base.print(x::DecompMMTDg, io::IO=stdout, digits=5)
    y = "\n======================================================\n\n"
    y = string(y, "MMTDg decomposition of order $(x.order) transition distribution\n\n")
    y = string(y, "------------------------------------------------------\n\n")
    nrow = size(x.levels, 1)
    for i = 1:nrow
        y = string(y, "Sub-order: $(Int(x.levels[i,1])), \t Weight: $(round(x.levels[i,2], digits=digits)), \t Lags: $(x.configs[i])\n")
    end

    y = string(y, "\nTotal weight: $(round(sum(x.levels[:,2]), digits=digits))\n")
    y = string(y, "Weight tolerance: $(x.totalweight_tol)\n\n")

    y = string(y, "------------------------------------------------------\n")

    y = string(y, "\nTotal weight associated with each lag\n\n")
    for i = 1:x.order
    y = string(y, "Lag $(i): $(round(x.involv[i], digits=digits))\n")
    end

    y = string(y, "\n======================================================\n")

    print(io, y)
end

function reduce_intercept(Omeg::TransTens, leftoversum_tol=1.0e-16)

    d = Omeg.order + 1
    d > 1 || error("Argument must be a multidimensional array.")

    a0 = vec(minimum(Omeg.A, dims=2:d))
    lam = sum(a0)
    if lam > 0.0
        a = a0 ./ lam
    else
        a = a0 .* 0.0
    end

    leftover = Omeg.A .- a0
    leftover_sums = vec(sum(leftover, dims=1))
    whatsleft = 1.0 - lam  # assumes intercept is extracted first
    maximum(abs.( leftover_sums .- whatsleft )) < leftoversum_tol || error("All leftover marginal sums should be within the specified tolerance (set with leftoversum_tol).")

    return (lam, a, leftover, whatsleft)
end

function reduce_gen(A::Array{T}, sub_order::Int, leftoversum_tol=1.0e-16, random=false) where T <: Real

    order = ndims(A) - 1
    d = size(A)
    K = d[1]

    whatsleft_old = sum(A, dims=1)[1]

    sub_order > 0 && sub_order < order || order("Sub-order cannot be higher than tensor order.")
    all( d.==d[1]) || error("All dimensions of tensor must have same length.")
    # all( A .>= 0.0 ) || error("Tensor must have non-negative entries.")

    lam_indx = build_λ_indx(order, sub_order)
    n_cand = lam_indx.lens[sub_order]
    configs = lam_indx.indxs[sub_order]

    lam_cand = Vector{T}(undef, n_cand)
    # take_cand = Array{T}(undef, vcat(fill(K, sub_order+1), n_cand)...)
    take_cand = Vector{Array{T}}(undef, n_cand)

    for ℓ = 1:n_cand
        Mins = minimum(A, dims= 1 .+ setdiff(1:order, configs[ℓ]))
        sums_Mins = sum(Mins, dims=1)
        lam_cand[ℓ] = minimum(sums_Mins)
        take_cand[ℓ] =  Mins ./ sums_Mins
    end

    if random
        lag_sel_indx = sample(StatsBase.Weights(lam_cand))
        lam_sel = deepcopy(lam_cand[lag_sel_indx])
    else
        lam_sel, lag_sel_indx = findmax(lam_cand) # Currently follows the lam_indx ordering
    end

    if abs(lam_sel) < 1.0e-6
        lam_sel = 0.0
        leftover = deepcopy(A)
        whatsleft = deepcopy(whatsleft_old)
        take = zeros(T, 1 + sub_order)
        lag_sel_indx = 0
        config_out = nothing
    else
        leftover = A .- lam_sel .* take_cand[lag_sel_indx] # do not reshpae yet; automatically broadcasts (should set up test in case broadcasting behavior changes someday)
        whatsleft = whatsleft_old - lam_sel
        take = reshape(take_cand[lag_sel_indx], fill(K, 1+sub_order)...)
        config_out = configs[lag_sel_indx]
    end

    leftover_sums = vec(sum(leftover, dims=1))
    maximum(abs.( leftover_sums .- whatsleft )) < leftoversum_tol || error("All leftover marginal sums should be within specified tolerance; set with leftoversum_tol.")

    return lam_sel, (lag_sel_indx, config_out), take, leftover, whatsleft
end

"""
    reduce_transTens(Omeg, [totalweight_thresh=0.001, maxiter=100, leftoversum_tol=1.0e-9, random=false])

Returns an MMTDg decomposition from provided transition probability tensor Omega. The algorithm continues until either 1.0 - totalweight_thresh has been assigned, or maxiter is reached.
At each step of decomposition, all distributions remaining must sum to within leftoversum_tol of one another.

If random=true, lag configurations will be selected for extraction according to a discrete distribution with probabilities proportional to the potential weights.

### Example
```julia
Omeg = reshape([0.7, 0.3, 0.7, 0.3, 0.4, 0.6, 0.4, 0.6], (2,2,2)) # only lag 2 matters
r = reduce_transTens(TransTens(Omeg), leftoversum_tol=1.0e-6)
print(r)
```
"""
function reduce_transTens(Omeg::Union{TransTens, Array{T}}; totalweight_thresh=0.001, maxiter=100, leftoversum_tol=1.0e-9, random=false) where T <: Real
    if typeof(Omeg) == TransTens
        ord = Omeg.order
    else
        ord = ndims(Omeg) - 1
        d = size(Omeg)
        all( d.==d[1]) || error("All dimensions of transition tensor must have same length.")
        all( Omeg .>= 0.0 ) || error("Transition tensor must have non-negative entries.")
        maximum(abs.( sum(Omeg, dims=1) .- 1.0 )) < leftoversum_tol || error("Transition tensor must add to unity along first dimension.")
    end

    lam = [0.0]
    Q = Vector{Array{Float64}}(undef, 1)
    configs = Vector{Vector{Int}}(undef, 1)
    configs[1] = [0]
    level = [0]
    total_lag_involvement = zeros(ord)
    level_lam_total = zeros(ord + 1)

    lam_now, Q_now, A, whatsleft = reduce_intercept(Omeg, leftoversum_tol)

    lam[1] = copy(lam_now)
    level_lam_total[1] += lam_now
    Q[1] = deepcopy(Q_now)

    iter = 1
    sub_ord_now = 1

    while sub_ord_now < ord && whatsleft >= totalweight_thresh && iter < maxiter

        lam_now, lagsel_now, Q_now, A, whatsleft = reduce_gen(A, sub_ord_now, leftoversum_tol, random)

        if lam_now > 0.0
            push!(lam, lam_now)
            push!(configs, lagsel_now[2])
            push!(level, sub_ord_now)
            push!(Q, Q_now)
            total_lag_involvement[lagsel_now[2]] .+= lam_now
            level_lam_total[sub_ord_now+1] += lam_now
        else
            sub_ord_now += 1
        end

        iter += 1

    end

    if iter >= maxiter
        error("Maximum iterations reached.")
    end

    if sub_ord_now == ord # means there was no more lower orders could do
        push!(lam, whatsleft)
        push!(configs, collect(1:ord))
        push!(level, sub_ord_now)
        push!(Q, A ./ whatsleft)
        total_lag_involvement .+= whatsleft
        level_lam_total[ord+1] += whatsleft
    end

    return DecompMMTDg( Omeg, totalweight_thresh, hcat(level, lam), configs, Q, total_lag_involvement, level_lam_total)

end

"""
    reconstruct_transTens(dcp::DecompMMTDg)

Reconstructs a transition probability tensor from its decomposition.

### Example
```julia
Omeg = reshape([0.7, 0.3, 0.7, 0.3, 0.4, 0.6, 0.4, 0.6], (2,2,2)) # only lag 2 matters
r = reduce_transTens(TransTens(Omeg), leftoversum_tol=1.0e-6)
print(r)
Omeg2 = reconstruct_transTens(r)
maximum(abs.(Omeg .- Omeg2))
```
"""
function reconstruct_transTens(dcp::DecompMMTDg)
        if typeof(dcp.Omega) == TransTens
            ord = dcp.Omega.order
        else
            ord = ndims(dcp.Omega) - 1
        end
        K = length(dcp.Q[1])
        out = zeros(fill(K, ord+1)...)

        for ℓ in 1:length(dcp.configs)

            dims = [K, fill(1, ord)...]
            for c in dcp.configs[ℓ]
                dims[c+1] = K
            end

            Qbdcst = reshape(vec(dcp.Q[ℓ]), dims...)

            out .+= dcp.levels[ℓ,2] .* Qbdcst

        end

        return out
end

"""
    TankReduction(Λ::Vector{Float64}, λ::Vector{Vector{Float64}},
        Q0::Vector{Float64}, Q::Vector{<:Array{Float64}}, λ_indx::λindxMMTD) (intercept only)

    Computes Tank Reduction of the MMTD (intercept only).

    ### Example
    ```julia
    Λ = [0.1, 0.5, 0.4]
    λ = [[0.4, 0.3, 0.3], [0.2, 0.5, 0.3]]
    Q0 = [0.5, 0.5]
    Q = [ reshape([0.8, 0.2, 0.3, 0.7],2,2), reshape([0.4, 0.6, 0.7, 0.3, 0.9, 0.1, 0.2, 0.8], 2,2,2) ]
    λ_indx = build_λ_indx(3, 2)

    tr = TankReduction(Λ, λ, Q0, Q, λ_indx)
    ```
"""
function TankReduction(Λ::Vector{Float64}, λ::Vector{Vector{Float64}},
    Q0::Vector{Float64}, Q::Vector{<:Array{Float64}}, λ_indx::λindxMMTD) # currently intercept only

    # here each Q is organized with tos as rows and froms as columns

    K = length(Q0)
    R = length(Λ) - 1

    Q_mats = [ reshape(Q[r], K, K^r) for r in 1:R ]

    Z0 = Λ[1] .* Q0
    Z = [ [ Matrix{Float64}(undef, K, K^r) for j = 1:λ_indx.lens[r] ] for r = 1:R ]
    a = [ [ Vector{Float64}(undef, K) for j = 1:λ_indx.lens[r] ] for r = 1:R ]

    for r = 1:R
        for j = 1:λ_indx.lens[r]
            Z[r][j] = Q_mats[r] .* Λ[r+1] .* λ[r][j]
            a[r][j] = [ minimum( Z[r][j][k,:] ) for k = 1:K ]
        end
    end

    global ZZ0 = deepcopy(Z0)
    global ZZ = deepcopy(Z)

    for r = 1:R
        for j = 1:λ_indx.lens[r]
            global ZZ0 += a[r][j]
            for k = 1:K
                global ZZ[r][j][k,:] .-= a[r][j][k]
            end
        end
    end

    global λtilde = deepcopy(λ)
    for r = 1:R
        for j = 1:λ_indx.lens[r]
            global λtilde[r][j] = sum(ZZ[r][j][:,1])
        end
    end

    Λr = vcat( sum(ZZ0), [ sum(λtilde[r]) for r = 1:R ] )
    global λr = deepcopy(λtilde)
    for r = 1:R
        if Λr[r+1] > 0.0
            global λr[r] ./= Λr[r+1]
        end
    end
    global Q0r = deepcopy(ZZ0)
    global Qr = [ zeros(Float64, fill(K, r+1)...) for r in 1:R ]

    if Λr[1] > 0.0
        global Q0r ./= Λr[1]
    end

    for r = 1:R
        maxval, indx = findmax(λtilde[r])
        if λr[r][indx] > 0.0
            global Qr[r] = reshape(ZZ[r][indx], fill(K, r+1)...) ./ maxval
        end
    end

    Λr, λr, Q0r, Qr
end

max_stars_repo_name: mheiner/MTD

max_issues_repo_name: mheiner/MTD

max_forks_repo_name: mheiner/MTD


text:
using Flux
using Zygote
using MAT
#using Plots
using Statistics
using Distributions
using LinearAlgebra
using Random
using BSON
using JLD
using PyPlot
using CUDA
#using Base.Iterators: partition

# load variables
#=
mf = matopen("/home/naxos2-raid1/maxiao/Projects/pTXDL/JuliaPrograms/ImageBasedDLForpTX.jl/Data/Data_processed.mat")
Xtrain = read(mf, "Xtrain")
Ytrain = read(mf, "Ytrain")
close(mf)

Xtrain = Xtrain./maximum(Xtrain)
Ytrain = Ytrain./maximum(Ytrain)

Xall = Xtrain
Yall = Ytrain

Xtrain = Xall[:,:,:,:,1:4]
Ytrain = Yall[:,:,:,:,1:4]

Sall = size(Xtrain)
Xtrain = reshape(Xtrain,Sall[1],Sall[2],1,Sall[3]*Sall[4]*Sall[5])

Sall = size(Ytrain)
Ytrain = reshape(Ytrain,Sall[1],Sall[2],1,Sall[3]*Sall[4]*Sall[5])

XValid = Xall[:,:,:,:,5]
YValid = Yall[:,:,:,:,5]

Xall = nothing;
Yall = nothing;

mf = matopen("/home/naxos2-raid1/maxiao/Projects/pTXDL/JuliaPrograms/ImageBasedDLForpTX.jl/Data/T1wBrain_processed.mat")
T1wBrain_resize_ds = read(mf, "T1wBrain_resize_ds")
close(mf)

T1wBrain_resize_ds = T1wBrain_resize_ds./maximum(T1wBrain_resize_ds)

Xtrain_new = zeros(Float32,size(Xtrain,1),size(Xtrain,1),2,size(Xtrain,4))
Xtrain_new[:,:,1,:] = Xtrain
Xtrain_new[:,:,2,:] = reshape(repeat(T1wBrain_resize_ds[:,:,:,1:4],1,1,7,1),size(Xtrain,1),size(Xtrain,1),1,size(Xtrain,4))

Xtrain = Xtrain_new
Xtrain_new=nothing

Sall = size(XValid)
XValid = reshape(XValid,Sall[1],Sall[2],1,Sall[3]*Sall[4])
YValid = reshape(YValid,Sall[1],Sall[2],1,Sall[3]*Sall[4])

XValid_new = zeros(Float32,size(XValid,1),size(XValid,1),2,size(XValid,4))
XValid_new[:,:,1,:] = XValid
XValid_new[:,:,2,:] = reshape(repeat(T1wBrain_resize_ds[:,:,:,5],1,1,7),size(XValid,1),size(XValid,1),1,size(XValid,4))

XValid = XValid_new
XValid_new=nothing

JLD.save("Data/XYtrain_XYvalid_processed_T1w.jld","Xtrain",Xtrain,"Ytrain",Ytrain,"XValid",XValid,"YValid",YValid)

=#

Xtrain = JLD.load("Data/XYtrain_XYvalid_processed_T1w.jld","Xtrain")
Ytrain = JLD.load("Data/XYtrain_XYvalid_processed_T1w.jld","Ytrain")
XValid = JLD.load("Data/XYtrain_XYvalid_processed_T1w.jld","XValid")
YValid = JLD.load("Data/XYtrain_XYvalid_processed_T1w.jld","YValid")

#=
fig, ax = plt.subplots()
PyPlot.gray()
ax.imshow(abs.(Xtrain[:,:,1,1]))
fig, ax = plt.subplots()
PyPlot.gray()
ax.imshow(abs.(Xtrain[:,:,2,1]))
fig2, ax2 = plt.subplots()
PyPlot.gray()
ax2.imshow(abs.(Ytrain[:,:,1,1]))
#display(fig)
=#

idx_shuffle = shuffle(collect(1:size(Xtrain,4)))
Xtrain = Xtrain[:,:,:,idx_shuffle]
Ytrain = Ytrain[:,:,:,idx_shuffle]

Xtrain = gpu(Xtrain)
Ytrain = gpu(Ytrain)

batch_size = 10

train_loader = Flux.Data.DataLoader(Xtrain,Ytrain, batchsize=batch_size,shuffle=true)

XValid = gpu(XValid)
YValid = gpu(YValid)

channels_in=2
channels_out=1

# Unet define
function _random_normal(shape...)
  return Float32.(rand(Normal(0.f0,0.02f0),shape...))
end

expand_dims(x,n::Int) = reshape(x,ones(Int64,n)...,size(x)...)
function squeeze(x)
    if size(x)[end] != 1
        return dropdims(x, dims = tuple(findall(size(x) .== 1)...))
    else
        # For the case BATCH_SIZE = 1
        int_val = dropdims(x, dims = tuple(findall(size(x) .== 1)...))
        return reshape(int_val,size(int_val)...,1)
    end
end

BatchNormWrap(out_ch) = BatchNorm(out_ch)

UNetConvBlock(in_chs, out_chs, kernel = (3, 3)) =
    Chain(Conv(kernel, in_chs=>out_chs,pad = (1, 1);init=_random_normal),
	BatchNormWrap(out_chs),
	x->leakyrelu.(x,0.2f0))

ConvDown(in_chs,out_chs,kernel = (4,4)) =
  Chain(Conv(kernel,in_chs=>out_chs,pad=(1,1),stride=(2,2);init=_random_normal),
	BatchNormWrap(out_chs),
	x->leakyrelu.(x,0.2f0))

UNetUpBlock(in_chs, out_chs; kernel = (3, 3), p = 0.5f0) =
  Chain(x->leakyrelu.(x,0.2f0),
  	ConvTranspose((2, 2), in_chs=>out_chs,stride=(2, 2);init=_random_normal),
	BatchNormWrap(out_chs),
	Dropout(p))

u = Chain(
       UNetConvBlock(channels_in, 64),
       UNetConvBlock(64, 64), #op
       SkipConnection(Chain(ConvDown(64,64),
                            UNetConvBlock(64, 128),
                            UNetConvBlock(128, 128),#x1
                            SkipConnection(Chain(ConvDown(128,128),
		                                 UNetConvBlock(128, 256),
		                                 UNetConvBlock(256, 256),#x2
		                                 SkipConnection(Chain(ConvDown(256,256),
				                                      UNetConvBlock(256, 512),
				                                      UNetConvBlock(512, 512),#x3
				                                      SkipConnection(Chain(ConvDown(512,512),
					                                                   UNetConvBlock(512, 1024),#x4
					                                                   UNetConvBlock(1024, 1024),#up_x4
					                                                   UNetUpBlock(1024,512)),
					                                                   (mx,x)->cat(mx,x,dims=3)),
				                                      UNetConvBlock(1024, 512),
				                                      UNetConvBlock(512, 512),#up_x1
				                                      UNetUpBlock(512,256)),
				                                      (mx,x)->cat(mx,x,dims=3)),
                                                 UNetConvBlock(512, 256),
                                                 UNetConvBlock(256, 256),#up_x2
                                                 UNetUpBlock(256,128)),
                                                 (mx,x)->cat(mx,x,dims=3)),
                            UNetConvBlock(256, 128),
                            UNetConvBlock(128, 128),#up_x3
                            UNetUpBlock(128,64)),
                            (mx,x)->cat(mx,x,dims=3)),
       UNetConvBlock(128, 64),
       UNetConvBlock(64, 64),#up_x5
       Chain(x->leakyrelu.(x,0.2f0),Conv((1, 1), 64=>channels_out;init=_random_normal))
)

u = gpu(u)

loss(x,y) = Flux.mse(u(x),y)
#loss(x,y) = mean((u(x).-y).^2)
validationloss() = mean(Flux.mse(u(XValid[:,:,:,(idx-1)*100+1:idx*100]),YValid[:,:,:,(idx-1)*100+1:idx*100]) for idx=1:round(Int,size(XValid,4)/100))
#validationloss() = mean(Flux.mse(u(XValid[:,:,:,idx]),YValid[:,:,:,idx]) for idx=1:size(XValid,4))

#CuArrays.@cufunc loss_cu(x,y) = mean((u(x).-y).^2)

#=
θ = params(u)
gs = Flux.gradient(θ) do
  training_loss = loss(XValid[:,:,:,1:2],YValid[:,:,:,1:2])
end
#=

#cb = Flux.throttle(() -> @show(validationloss()), 10)
#@time Flux.train!(loss, θ, dataset, opt, cb = cb)
opt = ADAM(0.001)

function my_custom_train!(loss, ps, data, opt)
    # training_loss is declared local so it will be available for logging outside the gradient calculation.
    local training_loss
    ps = Params(ps)
    for d in data
      if d isa AbstractArray{<:Number}
        gs = Flux.gradient(ps) do
          training_loss = loss(d)
          return training_loss
        end
      else
        gs = Flux.gradient(ps) do
          training_loss = loss(d...)
          return training_loss
        end
      end
      # Insert whatever code you want here that needs training_loss, e.g. logging.
      # logging_callback(training_loss)
      # Insert what ever code you want here that needs gradient.
      # E.g. logging with TensorBoardLogger.jl as histogram so you can see if it is becoming huge.
      #println("training_loss = ",training_loss,"; gradient[1] = ",gs[ps[1]],"; W1 = ",ps.order.data[1][1],"; b1 = ",ps.order.data[2][1])
      println("training_loss = ",training_loss)
      Flux.update!(opt, ps, gs)
      # Here you might like to check validation set accuracy, and break out to do early stopping.
    end
  end

loss_valid = zeros(10)
for epoch in 1:10
  #@time Flux.train!(loss, Flux.params(u), train_loader, opt, cb = cb)
  @time my_custom_train!(loss, Flux.params(u), train_loader, opt)
  #@time Flux.train!(loss, Flux.params(u), train_set, opt)
  model = cpu(u)
  BSON.@save string("Results/model_unet_ptx_processed_7dir_T1w_gpu_epoch",string(epoch),".bson") model
  u = gpu(model)
  loss_valid[epoch] = validationloss()
end

#test
#=
XValid = JLD.load("XYtrain_XYvalid_processed.jld","XValid")
YValid = JLD.load("XYtrain_XYvalid_processed.jld","YValid")
=#
Xtest = zeros(Float32,size(XValid,1),size(XValid,2),2,1)
Ytest = zeros(Float32,size(XValid,1),size(XValid,2),1,1)

idx_dir_test = 2
idx_slc_test = 15
idx_test = idx_slc_test + (idx_dir_test-1)*100
Xtest[:,:,:,1] = XValid[:,:,:,idx_test]
Ytest[:,:,:,1] = YValid[:,:,:,idx_test]
Xtest_output = u(gpu(Xtest))

fig, ax = plt.subplots()
PyPlot.gray()
ax.imshow(abs.(Xtest[:,:,1,1]))

fig2, ax2 = plt.subplots()
PyPlot.gray()
#savefig("Xttrain_40.png",fig)
ax2.imshow(abs.(Xtest_output[:,:,1,1]))

fig3, ax3 = plt.subplots()
PyPlot.gray()
ax3.imshow(abs.(Ytest[:,:,1,1]))

BSON.@load string("Results/model_unet_ptx_processed_7dir_T1w_gpu_epoch",string(10),".bson") model
slc_list = collect(1:100)
dir_list = collect(1:7)
XtestOutput4write = zeros(size(XValid,1),size(XValid,2),length(slc_list),length(dir_list))

Xtest = zeros(Float32,size(XValid,1),size(XValid,2),2,1)
Ytest = zeros(Float32,size(XValid,1),size(XValid,2),1,1)
for idx_dir_test in 1:length(dir_list)
  for idx_slc_test in 1:length(slc_list)
    idx_test = idx_slc_test + (idx_dir_test-1)*100
    Xtest[:,:,:,1] = XValid[:,:,:,idx_test]
    #Ytest[:,:,1,1] = YValid[:,:,idx_slc_test,idx_dir_test]
    XtestOutput4write[:,:,idx_slc_test,idx_dir_test] = model(Xtest)
  end
end

slc_list = [10 25]
dir_list = [1 2]
XtestOutput4write_slc10_25_dir1_2 = zeros(Float32,size(XValid,1),size(XValid,2),length(slc_list),length(dir_list),10)
for idx_epoch in 1:10
  BSON.@load string("Results/model_unet_ptx_processed_7dir_T1w_gpu_epoch",string(idx_epoch),".bson") model
  Xtest = zeros(Float32,size(XValid,1),size(XValid,2),2,1)
  Ytest = zeros(Float32,size(XValid,1),size(XValid,2),1,1)
  for idx_dir_test in 1:length(dir_list)
    for idx_slc_test in 1:length(slc_list)
      idx_test = idx_slc_test + (idx_dir_test-1)*100
      Xtest[:,:,:,1] = XValid[:,:,:,idx_test]
      XtestOutput4write_slc10_25_dir1_2[:,:,idx_slc_test,idx_dir_test,idx_epoch] = model(Xtest)
    end
  end
end

file = matopen("Results/XValidOutput_subj5_processed_7dir_T1w_gpu_epoch10.mat", "w")
write(file, "XtestOutput4write", XtestOutput4write)
write(file, "XtestOutput4write_slc10_25_dir1_2", XtestOutput4write_slc10_25_dir1_2)
close(file)

# testing on training data
BSON.@load string("model_unet_ptx_processed_7dir_epoch",string(10),".bson") u
XValid = Xtrain[:,:,:,1:700]
XValid = reshape(XValid,size(XValid,1),size(XValid,2),100,7)
XtestOutput4write = zeros(size(XValid))

Xtest = zeros(Float32,size(XValid,1),size(XValid,2),1,1)
Ytest = zeros(Float32,size(XValid,1),size(XValid,2),1,1)
for idx_dir_test in 1:size(XValid,4)
  for idx_slc_test in 1:size(XValid,3)
    Xtest[:,:,1,1] = XValid[:,:,idx_slc_test,idx_dir_test]
    #Ytest[:,:,1,1] = YValid[:,:,idx_slc_test,idx_dir_test]
    XtestOutput4write[:,:,idx_slc_test,idx_dir_test] = u(Xtest)
  end
end

file = matopen("XValidOutput_traingsubj1_processed_7dir_epoch10.mat", "w")
write(file, "XtestOutput4write", XtestOutput4write)
close(file)


text:
"""
    load_design!(design::Matrix{Any}, ata_model::AbstractModel)

# Description

Load a 0-1 `IxT` (or `n_fsxT` if the items are grouped by friend sets) design matrix into the ATA model.
Useful for loading a custom starting design before the `assemble!` step or to print/plot the features of the tests produced by a custom design before running `print_results`

# Arguments
- **`design::Matrix{Any}`** : Required. The `IxT` design matrix.
- **`ata_model::AbstractModel`** : Required. The model built with `start_ata()`, settings loaded by [`start_ata`](#ATA.start_ata).
"""
function load_design!(design::Matrix{Any}, ata_model::AbstractModel)

    try
        ata_model.output.design = Float64.(design)
        success!(ata_model, string("Starting design loaded."))
    catch e
        error!(ata_model, string(sprint(showerror, e)))

    end
    return nothing
end


text:
## R code 8.2
m8.1 <- quap(
    alist(
        log_gdp_std ~ dnorm( mu , sigma ) ,
        mu <- a + b*( rugged_std - 0.215 ) ,
        a ~ dnorm( 1 , 1 ) ,
        b ~ dnorm( 0 , 1 ) ,
        sigma ~ dexp( 1 )
    ) , data=dd )

## R code 8.5
m8.1 <- quap(
    alist(
        log_gdp_std ~ dnorm( mu , sigma ) ,
        mu <- a + b*( rugged_std - 0.215 ) ,
        a ~ dnorm( 1 , 0.1 ) ,
        b ~ dnorm( 0 , 0.3 ) ,
        sigma ~ dexp(1)
    ) , data=dd )

## R code 8.8
m8.2 <- quap(
    alist(
        log_gdp_std ~ dnorm( mu , sigma ) ,
        mu <- a[cid] + b*( rugged_std - 0.215 ) ,
        a[cid] ~ dnorm( 1 , 0.1 ) ,
        b ~ dnorm( 0 , 0.3 ) ,
        sigma ~ dexp( 1 )
    ) , data=dd )

## R code 8.13
m8.3 <- quap(
    alist(
        log_gdp_std ~ dnorm( mu , sigma ) ,
        mu <- a[cid] + b[cid]*( rugged_std - 0.215 ) ,
        a[cid] ~ dnorm( 1 , 0.1 ) ,
        b[cid] ~ dnorm( 0 , 0.3 ) ,
        sigma ~ dexp( 1 )
    ) , data=dd )

## R code 8.23
m8.4 <- quap(
    alist(
        blooms_std ~ dnorm( mu , sigma ) ,
        mu <- a + bw*water_cent + bs*shade_cent ,
        a ~ dnorm( 0.5 , 0.25 ) ,
        bw ~ dnorm( 0 , 0.25 ) ,
        bs ~ dnorm( 0 , 0.25 ) ,
        sigma ~ dexp( 1 )
    ) , data=d )

## R code 8.24
m8.5 <- quap(
    alist(
        blooms_std ~ dnorm( mu , sigma ) ,
        mu <- a + bw*water_cent + bs*shade_cent + bws*water_cent*shade_cent ,
        a ~ dnorm( 0.5 , 0.25 ) ,
        bw ~ dnorm( 0 , 0.25 ) ,
        bs ~ dnorm( 0 , 0.25 ) ,
        bws ~ dnorm( 0 , 0.25 ) ,
        sigma ~ dexp( 1 )
    ) , data=d )


text:
set(obj, ::typeof(@lens last(_)), val) = @set obj[lastindex(obj)] = val
set(obj, ::typeof(@lens first(_)), val) = @set obj[firstindex(obj)] = val

################################################################################
##### eltype
################################################################################
function set(obj, ::typeof(@lens eltype(_)), ::Type{T}) where {T}
    return set_eltype(obj, T)
end

set_eltype(obj::Array,  T::Type) = collect(T, obj)
set_eltype(obj::Number, T::Type) = T(obj)
set_eltype(::Type{<:Number}, ::Type{T}) where {T} = T
set_eltype(::Type{<:Array{<:Any, N}}, ::Type{T}) where {N, T} = Array{T, N}
set_eltype(::Type{<:Dict}, ::Type{Pair{K, V}}) where {K, V} = Dict{K, V}
set_eltype(obj::Dict, ::Type{T}) where {T} = set_eltype(typeof(obj), T)(obj)

set(obj::Dict, l::Union{typeof(@lens keytype(_)), typeof(@lens valtype(_))},
    T::Type) = set(typeof(obj), l, T)(obj)
set(::Type{<:Dict{<:Any,V}}, ::typeof(@lens keytype(_)), ::Type{K}) where {K, V} =
    Dict{K, V}
set(::Type{<:Dict{K}}, ::typeof(@lens valtype(_)), ::Type{V}) where {K, V} =
    Dict{K, V}


text:
# This file was generated, do not modify it. # hide
hot_mach = fit!(machine(OneHotEncoder(), X1), verbosity=0)
X = transform(hot_mach, X1);

max_stars_repo_name: ven-k/MLJTutorials

max_issues_repo_name: ven-k/MLJTutorials

max_forks_repo_name: ven-k/MLJTutorials


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

mutable struct ConnectionMonitorHttpConfiguration <: SwaggerModel
    port::Any # spec type: Union{ Nothing, Int32 } # spec name: port
    method::Any # spec type: Union{ Nothing, String } # spec name: method
    path::Any # spec type: Union{ Nothing, String } # spec name: path
    requestHeaders::Any # spec type: Union{ Nothing, Vector{HTTPHeader} } # spec name: requestHeaders
    validStatusCodeRanges::Any # spec type: Union{ Nothing, Vector{String} } # spec name: validStatusCodeRanges
    preferHTTPS::Any # spec type: Union{ Nothing, Bool } # spec name: preferHTTPS

    function ConnectionMonitorHttpConfiguration(;port=nothing, method=nothing, path=nothing, requestHeaders=nothing, validStatusCodeRanges=nothing, preferHTTPS=nothing)
        o = new()
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("port"), port)
        setfield!(o, Symbol("port"), port)
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("method"), method)
        setfield!(o, Symbol("method"), method)
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("path"), path)
        setfield!(o, Symbol("path"), path)
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("requestHeaders"), requestHeaders)
        setfield!(o, Symbol("requestHeaders"), requestHeaders)
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("validStatusCodeRanges"), validStatusCodeRanges)
        setfield!(o, Symbol("validStatusCodeRanges"), validStatusCodeRanges)
        validate_property(ConnectionMonitorHttpConfiguration, Symbol("preferHTTPS"), preferHTTPS)
        setfield!(o, Symbol("preferHTTPS"), preferHTTPS)
        o
    end
end # type ConnectionMonitorHttpConfiguration

const _property_map_ConnectionMonitorHttpConfiguration = Dict{Symbol,Symbol}(Symbol("port")=>Symbol("port"), Symbol("method")=>Symbol("method"), Symbol("path")=>Symbol("path"), Symbol("requestHeaders")=>Symbol("requestHeaders"), Symbol("validStatusCodeRanges")=>Symbol("validStatusCodeRanges"), Symbol("preferHTTPS")=>Symbol("preferHTTPS"))
const _property_types_ConnectionMonitorHttpConfiguration = Dict{Symbol,String}(Symbol("port")=>"Int32", Symbol("method")=>"String", Symbol("path")=>"String", Symbol("requestHeaders")=>"Vector{HTTPHeader}", Symbol("validStatusCodeRanges")=>"Vector{String}", Symbol("preferHTTPS")=>"Bool")
Base.propertynames(::Type{ ConnectionMonitorHttpConfiguration }) = collect(keys(_property_map_ConnectionMonitorHttpConfiguration))
Swagger.property_type(::Type{ ConnectionMonitorHttpConfiguration }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_ConnectionMonitorHttpConfiguration[name]))}
Swagger.field_name(::Type{ ConnectionMonitorHttpConfiguration }, property_name::Symbol) =  _property_map_ConnectionMonitorHttpConfiguration[property_name]

const _allowed_ConnectionMonitorHttpConfiguration_method = ["Get", "Post"]

function check_required(o::ConnectionMonitorHttpConfiguration)
    true
end

function validate_property(::Type{ ConnectionMonitorHttpConfiguration }, name::Symbol, val)
    if name === Symbol("method")
        Swagger.validate_param(name, "ConnectionMonitorHttpConfiguration", :enum, val, _allowed_ConnectionMonitorHttpConfiguration_method)
    end
end


text:
module DashHtmlComponents
    function __init__()
        @warn "The `DashHtmlComponents` package is deprecated. All DashHtmlComponents functions have been moved to the Dash package. Please switch to `using Dash` instead."
    end
end

max_stars_repo_name: plotly/dash-html-components

max_issues_repo_name: plotly/dash-html-components

max_forks_repo_name: plotly/dash-html-components


text:
function ExperienceReplay(;π,
                          S,
                          𝒫=(;),
                          A=action_space(π),
                          N_experience_replay,
                          ER_frac = 0.5,
                          solver=TD3,
                          required_columns=Symbol[],
                          c_opt=(;),
                          a_opt=(;),
                          replay_store_weight = (D)->1f0,
                          kwargs...)

    buffer_er = ExperienceBuffer(S, A, N_experience_replay, [:weight, :value, required_columns...])
    𝒫 = (;buffer_er, 𝒫...)
    function cb(D; 𝒮, info=Dict())
        if 𝒮.agent.π isa LatentConditionedNetwork
            D[:value] = mean(value(critic(𝒮.agent.π), vcat(D[:s], D[:z]), D[:a]))
        else
            D[:value] = mean(value(𝒮.agent.π, D[:s], D[:a]))
        end
        D[:weight] .= replay_store_weight(D)
        push_reservoir!(buffer_er, D, weighted=true)
        info["Experience_size"] = length(buffer_er)
    end
    creg = action_value_regularization
    areg = action_regularization
    if π isa LatentConditionedNetwork
        creg = TIER_action_value_regularization
        areg = TIER_action_regularization
    end

    solver(;  π=π,
              S=S,
              𝒫=𝒫,
              required_columns=unique([:weight, required_columns...]),
              post_sample_callback=cb,
              c_opt=(;regularizer=BatchRegularizer(buffers=[buffer_er], batch_size=128, λ=0.5f0, loss=creg), c_opt...),
              a_opt=(;regularizer=BatchRegularizer(buffers=[buffer_er], batch_size=128, λ=0.5f0, loss=areg), a_opt...),
              extra_buffers=[buffer_er],
              buffer_fractions=[1.0-ER_frac, ER_frac],
              kwargs...
              )
end


text:
module GoogleMaps

    import JSON, Requests, ImageMagick

    include("utils.jl")
    include("directions.jl")
    include("geocode.jl")
    include("nearby.jl")
    include("search.jl")
    include("details.jl")
    include("photos.jl")
    include("matchroads.jl")

end