text:
"""
Kucoin.get_ws_token([api_data]) -> JSON3.Object

Apply for a websocket connection token. See Kucoin official
[docs](https://docs.kucoin.com/#apply-connect-token) for more details.

# Arguments
- `api_data::UserApiData`: [Optional] holds API data susch as api key, secret, passphrase,
etc. If absent returns a token that can be used only with public channels.

# Returns
- `JSON3.Object`
```json
{
  "instanceServers": [
    {
      "endpoint": "wss://push1-v2.kucoin.com/endpoint",
      "protocol": "websocket",
      "encrypt": true,
      "pingInterval": 50000,
      "pingTimeout": 10000
    }
  ],
  "token": "vYNlCtbz4XNJ1QncwWilJnBtmmfe4geLQDUA62kK=="
}
```
"""
function get_ws_token end

function get_ws_token(api_data::ApiData)::JSON3.Object
    return _handle(_kucoin_request(api_data, _HTTP_METHOD_POST, "/api/v1/bullet-private"))
end

function get_ws_token()::JSON3.Object
    return _handle(_kucoin_request(_HTTP_METHOD_POST, "/api/v1/bullet-public"))
end

struct WebSocketClient
    endpoint::String
    ping_interval::Integer
    is_private::Bool
    message_channel::Channel
    ws::Ref{WebSocket}
    write_lock::ReentrantLock
end

function WebSocketClient(api_data::ApiData, connection_id::String=string(uuid1()))
    return WebSocketClient(get_ws_token(api_data), connection_id, true)
end

function WebSocketClient(connection_id::String=string(uuid1()))
    return WebSocketClient(get_ws_token(), connection_id, false)
end

function WebSocketClient(ws_token::JSON3.Object, connection_id::String, is_private::Bool)
    endpoint = ws_token["instanceServers"][1]["endpoint"]
    token = ws_token["token"]
    ws_endpoint = "$endpoint?token=$token&connectId=$connection_id"
    ping_interval = ws_token["instanceServers"][1]["pingInterval"] ÷ 1000
    ws_client = WebSocketClient(
        ws_endpoint, ping_interval, is_private, Channel(), Ref{WebSocket}(), ReentrantLock()
    )
    _start_read_loop(ws_client)
    return ws_client
end

take!(ws_client::WebSocketClient) = take!(ws_client.message_channel)
fetch(ws_client::WebSocketClient) = fetch(ws_client.message_channel)
isready(ws_client::WebSocketClient) = isready(ws_client.message_channel)

"""
Kucoin.subscribe(ws_client, topic[, id[, is_private_channel[, return_response]]]) -> nothing

Send a subscription message to the server.  See Kucoin official
[docs](https://docs.kucoin.com/#subscribe) for more details.

# Arguments
- `ws_client::WebSocketClient`: web socket client
- `topic::String`: the topic you want to subscribe to
- `id::String`: [Optional] is unique string to mark the request which is same as id property
 of ack.
- `is_private_channel::Bool`: [Optional] for some specific topics (e.g. `/market/level2`),
`is_private_channel` is available. If the `is_private_channel` is set to true, the user will
 only receive messages related to himself on the topic.
- `return_response::Bool`: [Optional] if is set as true, kucoin will return the
acknowledgment message after successful subscription.
"""
function subscribe(
    ws_client::WebSocketClient,
    topic::String,
    id::String=string(uuid1()),
    is_private_channel::Bool=ws_client.is_private,
    return_response::Bool=true,
)
    lock(ws_client.write_lock) do
        write(
            ws_client.ws[],
            """{"id": "$id", "type": "subscribe", "topic": "$topic", "privateChannel": $is_private_channel, "response": $return_response}""",
        )
    end
end

function unsubscribe(
    ws_client::WebSocketClient,
    topic::String,
    id::String=string(uuid1()),
    is_private_channel::Bool=ws_client.is_private,
    return_response::Bool=true,
)
    lock(ws_client.write_lock) do
        write(
            ws_client.ws[],
            """{"id": "$id", "type": "unsubscribe", "topic": "$topic", "privateChannel": $is_private_channel, "response": $return_response}""",
        )
    end
end

function _start_read_loop(ws_client::WebSocketClient)
    ref_assigned_event = Base.Event()
    task = @async WebSockets.open(ws_client.endpoint) do ws
        ws_client.ws[] = ws
        notify(ref_assigned_event)
        _start_ping_loop(ws_client)
        while !eof(ws)
            msg = JSON3.read(readavailable(ws))
            put!(ws_client.message_channel, msg)
        end
    end
    errormonitor(task)
    wait(ref_assigned_event)
    return nothing
end

function _start_ping_loop(ws_client::WebSocketClient)
    task = @async while true
        lock(ws_client.write_lock) do
            write(ws_client.ws[], """{"id": "$(uuid1())", "type": "ping"}""")
        end
        sleep(ws_client.ping_interval)
    end
    errormonitor(task)
    return nothing
end


text:
export Import
export sourceMap, targetMap, distributor, isLocallyComplete
export permuteToLIDs, permuteFromLIDs, exportLIDs, remoteLIDs, remotePIDs, numSameIDs

"""
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distribution.
"""
struct Import{GID <: Integer, PID <:Integer, LID <: Integer}
    importData::ImportExportData{GID, PID, LID}

    #default constructor appeared to accept a pair of BlockMaps
    function Import{GID, PID, LID}(
            importData::ImportExportData{GID, PID, LID}) where {
                GID <: Integer, PID <: Integer, LID <: Integer}
        new(importData)
    end
end

## Constructors ##

function Import(source::BlockMap{GID, PID, LID}, target::BlockMap{GID, PID, LID},
        userRemotePIDs::AbstractArray{PID}, remoteGIDs::AbstractArray{GID},
        userExportLIDs::AbstractArray{LID}, userExportPIDs::AbstractArray{PID},
        useRemotePIDGID::Bool
        ) where {GID <: Integer, PID <: Integer, LID <: Integer}

    importData = ImportExportData(source, target)

    remoteLIDs = JuliaPetra.remoteLIDs(importData)

    if !userRemotePIDGID
        empty!(remoteGIDs)
        empty!(remoteLIDs)
    end

    getIDSource(data, remoteGIDs, !userRemotePIDGID)

    if length(remoteGIDs) > 0 && !isDistributed(source)
        throw(InvalidArgumentError("Target has remote LIDs but source is not distributed globally"))
    end

    (remotePIDs, _) = remoteIDList(source, remoteGIDs)

    remoteProcIDs = (useRemotePIDGID) ? userRemotePIDs : remotePIDs

    if !(length(remoteProcIDs) == length(remoteGIDs) && length(remoteGIDs) == length(remoteLIDs))
        throw(InvalidArgumentError("Size miss match on remoteProcIDs, remoteGIDs and remoteLIDs"))
    end

    # ensure remoteProcIDs[i], remoteGIDs[i] and remoteLIDs[i] refer to the same thing
    order = sortperm(remoteProcIDs)
    permute!(remoteProcIDs, order)
    permute!(remoteGIDs, order)
    permute!(remoteLIDs, order)

    exportPIDs = Array{PID, 1}(undef, length(userExportPIDs))
    exportLIDs = Array{PID, 1}(undef, length(userExportPIDs))

    #need the funcitons with these names, not the variables
    JuliaPetra.remoteLIDs(importData, remoteLIDs)
    JuliaPetra.exportPIDs(importData, exportPIDs)
    JuliaPetra.exportLIDs(importData, exportLIDs)

    locallyComplete = true
    for i = 1:length(userExportPIDs)
        if userExportPIDs[i] == 0
            locallyComplete = false
        end

        exportPIDs[i] = userExportPIDs[i]
        exportLIDs[i] = userExportLIDs[i]
    end

    isLocallyComplete(importData, locallyComplete)
    #TODO create and upgrade to createFromSendsAndRecvs
    #createFromSendsAndRecvs(distributor(importData), exportPIDs, remoteProcIDs)
    createFromRecvs(distributor(importData), remoteGIDs, remotePIDs)

    Import(importData)
end

function Import(source::BlockMap{GID, PID, LID}, target::BlockMap{GID, PID, LID}, remotePIDs::Union{AbstractArray{PID}, Nothing}=nothing) where {GID <: Integer, PID <: Integer, LID <: Integer}

    impor = Import{GID, PID, LID}(ImportExportData(source, target))

    remoteGIDs = setupSamePermuteRemote(impor)

    if distributedGlobal(source)
        setupExport(impor, remoteGIDs, remotePIDs)
    end

    impor
end

## internal construction methods ##

function setupSamePermuteRemote(impor::Import{GID, PID, LID}) where {GID <: Integer, PID <: Integer, LID <: Integer}

    data = impor.importData

    remoteGIDs = Array{GID, 1}(undef, 0)

    getIDSources(data, remoteGIDs)

    if length(remoteLIDs(data)) != 0 && !distributedGlobal(sourceMap(impor))
        isLocallyComplete(data, false)

        warn("Target has remote LIDs but source is not distributed globally.  " *
            "Importing a submap of the target map")
    end

    remoteGIDs
end

function getIDSources(data, remoteGIDs, useRemotes=true)
    source = sourceMap(data)
    target = targetMap(data)

    sourceGIDs = myGlobalElements(source)
    targetGIDs = myGlobalElements(target)

    numSrcGIDs = length(sourceGIDs)
    numTgtGIDs = length(targetGIDs)
    numGIDs = min(numSrcGIDs, numTgtGIDs)

    numSameGIDs = 1
    while numSameGIDs <= numGIDs && sourceGIDs[numSameGIDs] == targetGIDs[numSameGIDs]
        numSameGIDs += 1
    end
    numSameGIDs -= 1
    numSameIDs(data, numSameGIDs)

    permuteToLIDs = JuliaPetra.permuteToLIDs(data)
    permuteFromLIDs = JuliaPetra.permuteFromLIDs(data)
    remoteLIDs = JuliaPetra.remoteLIDs(data)

    for tgtLID = (numSameGIDs+1):numTgtGIDs
        curTargetGID = targetGIDs[tgtLID]
        srcLID = lid(source, curTargetGID)
        if srcLID != 0
            push!(permuteToLIDs, tgtLID)
            push!(permuteFromLIDs, srcLID)
        elseif useRemotes
            push!(remoteGIDs, curTargetGID)
            push!(remoteLIDs, tgtLID)
        end
    end
end

function setupExport(impor::Import{GID, PID, LID}, remoteGIDs::AbstractArray{GID}, userRemotePIDs::Union{AbstractArray{PID}, Nothing}) where {GID <: Integer, PID <: Integer, LID <: Integer}
    data = impor.importData
    source = sourceMap(impor)

    useRemotePIDs = userRemotePIDs != nothing

    # Sanity Checks
    if useRemotePIDs && length(userRemotePIDs) != length(remoteGIDs)
        throw(InvalidArgumentError("remotePIDs must either be null " *
                "or match the size of remoteGIDs."))
    end

    missingGID = 0

    if !useRemotePIDs
        newRemotePIDs = Array{PID, 1}(undef, length(remoteGIDs))
        (remoteProcIDs, remoteLIDs) = remoteIDList(source, remoteGIDs)
        for e in remoteLIDs
            if e == 0
                missingGID += 1
            end
        end
    else
        remoteProcIDs = userRemotePIDs
    end

    #line 688

    if missingGID != 0
        isLocallyComplete(data, false)

        warn("Source map was un-able to figure out which process owns one " *
            "or more of the GIDs in the list of remote GIDs.  This probably " *
            "means that there is at least one GID owned by some process in " *
            "the target map which is not owned by any process in the source " *
            "Map.  (That is, the source and target maps do not contain the " *
            "same set of GIDs globally")

        #ignore remote GIDs that aren't owned by any process in the source Map
        numInvalidRemote = missingGID
        totalNumRemote = length(remoteGIDs)
        if numInvalidRemote == totalNumRemote
            #if all remotes are invalid, can delete them all
            empty!(remoteProcIDs)
            empty!(remoteGIDs)
            empty!(JuliaPetra.remoteLIDs(data))
        else
            numValidRemote = 1

            remoteLIDs = JuliaPetra.remoteLIDs(data)

            for r = 1:totalNumRemote
                if remoteProcIds[r] != 0
                    remoteProcIds[numValidRemote] = remoteProcIDs[r]
                    remoteGIDs[numValidRemote] = remoteGIDs[r]
                    remoteLIDs[numValidRemote] = remoteLIDs[r]
                    numValidRemote += 1
                end
            end
            numValidRemote -= 1

            if numValidRemote != totalNumRemote - numInvalidRemote
                throw(InvalidStateError("numValidRemote = $numValidRemote " *
                        "!= totalNumRemote - numInvalidRemote " *
                        "= $(totalNumRemote - numInvalidRemote)"))
            end

            resize!(remoteProcIDs, numValidRemote)
            resize!(remoteGIDs, numValidRemote)
            resize!(remoteLIDs, numValidRemote)
        end
    end

    order = sortperm(remoteProcIDs)
    permute!(remoteProcIDs, order)
    permute!(remoteGIDs, order)
    permute!(remoteLIDs, order)

    (exportGIDs, exportPIDs) = createFromRecvs(distributor(data), remoteGIDs, remoteProcIDs)

    JuliaPetra.exportPIDs(data, exportPIDs)

    numExportIDs = length(exportGIDs)

    if numExportIDs > 0
        exportLIDs = JuliaPetra.exportLIDs(data)
        resize!(exportLIDs, numExportIDs)
        for k in 1:numExportIDs
            exportLIDs[k] = lid(source, exportGIDs[k])
        end
    end
end

## Getters ##

"""
Get the source map for the given ImportExportData
"""
function sourceMap(impor::Import{GID, PID, LID})::BlockMap{GID, PID, LID} where GID <: Integer where PID <:Integer where LID <: Integer
    impor.importData.source
end

"""
Get the target map for the given ImportExportData
"""
function targetMap(impor::Import{GID, PID, LID})::BlockMap{GID, PID, LID} where GID <: Integer where PID <:Integer where LID <: Integer
    impor.importData.target
end

"""
List of elements in the target map that are permuted.
"""
function permuteToLIDs(impor::Import{GID, PID, LID})::Array{LID} where GID <: Integer where PID <:Integer where LID <: Integer
    impor.importData.permuteToLIDs
end

"""
List of elements in the source map that are permuted.
"""
function permuteFromLIDs(impor::Import{GID, PID, LID})::Array{LID} where GID <: Integer where PID <:Integer where LID <: Integer
    impor.importData.permuteFromLIDs
end

"""
List of elements in the target map that are coming from other processors
"""
function remoteLIDs(impor::Import{GID, PID, LID})::Array{LID} where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.remoteLIDs
end

"""
List of elements that will be sent to other processors
"""
function exportLIDs(impor::Import{GID, PID, LID})::Array{LID} where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.exportLIDs
end

"""
List of processors to which elements will be sent `exportLID[i]` will be sent to processor `exportPIDs[i]`
"""
function exportPIDs(impor::Import{GID, PID, LID})::Array{PID} where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.exportPIDs
end

"""
Returns the number of elements that are identical between the source and target maps, up to the first different ID
"""
function numSameIDs(impor::Import{GID, PID, LID})::LID where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.numSameIDs
end

"""
Returns the distributor being used
"""
function distributor(impor::Import{GID, PID, LID})::Distributor{GID, PID, LID} where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.distributor
end

"""
Returns whether the import or export is locally complete
"""
function isLocallyComplete(impor::Import{GID, PID, LID})::Bool where GID <: Integer where PID <: Integer where LID <: Integer
    impor.importData.isLocallyComplete
end

ext: jl


text:
import RDates: RDates, CalendarManager, RDate, Calendar, apply, NullCalendarManager
import Dates: Dates, Date

abstract type CalendarRule end
includehols(::CalendarRule, from::Date, to::Date) = repeat([false], (to-from).value + 1)
excludehols(::CalendarRule, from::Date, to::Date) = repeat([false], (to-from).value + 1)

"""
    WindowCalendarRule(rule; from::Union{Nothing,Date}=nothing, to::Union{Nothing,Date}=nothing)

Restrict another calendar rule to within a window of [from, to].
"""
struct WindowCalendarRule <: CalendarRule
    rule::CalendarRule
    from::Union{Nothing,Date}
    to::Union{Nothing,Date}

    WindowCalendarRule(rule; from=nothing, to=nothing) = new(rule, from, to)
end

function includehols(rule::WindowCalendarRule, from::Date, to::Date)
    rule.from != nothing && rule.from > to && return repeat([false], (to-from).value + 1)
    rule.to != nothing && rule.to < from && return repeat([false], (to-from).value + 1)
    hols = includehols(rule.rule, from, to)
    r1 = rule.from != nothing ? (1:(rule.from-from).value) : []
    r2 = rule.to != nothing ? ((rule.to-from).value+2:(to-from).vaue+1) : []
    for i in Iterators.flatten([r1,r2])
        @inbounds hols[i] = false
    end
    hols
end

function excludehols(rule::WindowCalendarRule, from::Date, to::Date)
    rule.from != nothing && rule.from > to && return repeat([false], (to-from).value + 1)
    rule.to != nothing && rule.to < from && return repeat([false], (to-from).value + 1)
    hols = excludehols(rule.rule, from, to)
    r1 = rule.from != nothing ? (1:(rule.from-from).value) : []
    r2 = rule.to != nothing ? ((rule.to-from).value+2:(to-from).vaue+1) : []
    for i in Iterators.flatten([r1,r2])
        @inbounds hols[i] = false
    end
    hols
end

"""
    PeriodicCalendarRule(period::RDate, increment::RDate; include::Bool=true, cal_mgr::CalendarManager=NullCalendarManager())

A calendar rule which is generated by iterating on a `period` and then applying an `increment`. To mark
the calendar rule is an exclusion then can set the `include` parameter to false.

```julia-repl
julia> PeriodicCalendarRule(rd"1y", rd"1MAY+Last MON");
```
"""
struct PeriodicCalendarRule <: CalendarRule
    period::RDate
    increment::RDate
    include::Bool
    cal_mgr::CalendarManager

    PeriodicCalendarRule(period::RDate, increment::RDate; include=true, cal_mgr=NullCalendarManager()) = new(period, increment, include, cal_mgr)
end

function holidays(rule::PeriodicCalendarRule, from::Date, to::Date)
    hols = repeat([false], (to-from).value + 1)
    for d in RDates.range(from, to, rule.period; cal_mgr=rule.cal_mgr)
        d1 = apply(rule.increment, d, rule.cal_mgr)
        if from <= d1 <= to
            @inbounds hols[(d1-from).value+1] = true
        end
    end
    hols
end
includehols(rule::PeriodicCalendarRule, from::Date, to::Date) = rule.include ? holidays(rule, from, to) : repeat([false], (to-from).value + 1)
excludehols(rule::PeriodicCalendarRule, from::Date, to::Date) = rule.include ? repeat([false], (to-from).value + 1) : holidays(rule, from, to)

"""
    ExplicitDateRule(;inclusions::Vector{Date}=[], exclusions::Vector{Date}=[])

A calendar rule to mark an explicit set of dates as holiday or not holidays.
"""
struct ExplicitDateRule <: CalendarRule
    inclusions::Vector{Date}
    exclusions::Vector{Date}
    ExplicitDateRule(;inclusions=Vector{Date}(), exclusions=Vector{Date}()) = new(inclusions, exclusions)
end

function includehols(rule::ExplicitDateRule, from::Date, to::Date)
    hols = repeat([false], (to-from).value + 1)
    for d in rule.inclusions
        if from <= d <= to
            @inbounds hols[(d-from).value+1] = true
        end
    end
    hols
end

function excludehols(rule::ExplicitDateRule, from::Date, to::Date)
    hols = repeat([false], (to-from).value + 1)
    for d in rule.exclusions
        if from <= d <= to
            @inbounds hols[(d-from).value+1] = true
        end
    end
    hols
end

"""
    RuleBasedCalendar(rules::Vector{CalendarRule})

A calendar system which is calculated by applying a set of of rules in order.
"""
struct RuleBasedCalendar <: Calendar
    rules::Vector{CalendarRule}
end

function RDates.holidays(cal::RuleBasedCalendar, from::Date, to::Date)
    hols = repeat([false], (to-from).value + 1)
    for rule in cal.rules
        hols = hols .| includehols(rule, from, to)
        hols = hols .& (.! excludehols(rule, from, to))
    end
    hols
end

function RDates.is_holiday(cal::RuleBasedCalendar, date::Date)
    year = Dates.year(date); from = Date(year,1,1); to = Date(year,12,31)
    holidays(cal, from, to)[(date-from).value + 1]
end


text:
@testset "Handlers" begin
    FMT_STR = "[{level}]:{name} - {msg}"

    LEVELS = Dict(
        "not_set" => 0,
        "debug" => 10,
        "info" => 20,
        "warn" => 30,
        "error" => 40,
        "fubar" => 50
    )

    @testset "Custom Handlers" begin
        @testset "SimplestHandler" begin
            io = IOBuffer()

            try
                handler = SimplestHandler(io)
                logger = Logger(
                    "SimplestHandler",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test getlevel(handler) == "not_set"
                @test isempty(getfilters(handler))

                msg = "It works!"
                Memento.info(logger, msg)
                @test occursin(msg, String(take!(io)))

                Memento.debug(logger, "This shouldn't get logged")
                @test isempty(String(take!(io)))
            finally
                close(io)
            end
        end

        @testset "FilterHandler" begin
            io = IOBuffer()

            try
                handler = FilterHandler(io)
                handler_filter = Memento.Filter(handler)
                push!(handler, handler_filter)
                logger = Logger(
                    "FilterHandler",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test getlevel(handler) == "not_set"
                @test getfilters(handler) == [handler_filter]

                msg = "It works!"
                Memento.info(logger, msg)
                @test occursin(msg, String(take!(io)))

                Memento.debug(logger, "This shouldn't get logged")
                @test isempty(String(take!(io)))
            finally
                close(io)
            end
        end

        @testset "AsyncHandler" begin
            io = IOBuffer()

            function wait_for_empty(c::Channel)
                while isready(c)
                    sleep(1)
                end
            end

            try
                handler = AsyncHandler(io)
                logger = Logger(
                    "AsyncHandler",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test getlevel(handler) == "not_set"
                @test isempty(getfilters(handler))

                msg = "It works!"
                Memento.info(logger, msg)
                wait_for_empty(handler.channel)
                @test occursin(msg, String(take!(io)))

                Memento.debug(logger, "This shouldn't get logged")
                wait_for_empty(handler.channel)
                @test isempty(String(take!(io)))

                # Asynchronous handlers can cause issues when loggers are used across
                # multiple processes. We'll emulate the following issue:
                #=
                using Distributed, Memento
                const LOGGER = getlogger()
                addprocs(1)
                @everywhere using Memento
                # Note: Define AsyncHandler on all processes
                @everywhere push!(getlogger(), AsyncHandler(IOBuffer()))
                pmap(1:3) do i
                    Memento.info(LOGGER, i)  # ERROR: cannot serialize a running Task
                end
                =#
                try
                    serialize(io, handler)
                    @test false
                catch e
                    @test e isa ErrorException
                    @test e.msg == "cannot serialize a running Task"
                end

                @test_throws LoggerSerializationError serialize(io, logger)
            finally
                close(io)
            end
        end
    end

    @testset "DefaultHandler" begin
        @testset "Sample Usage w/ IO" begin
            io = IOBuffer()

            try
                handler = DefaultHandler(io, DefaultFormatter(FMT_STR))

                logger = Logger(
                    "DefaultHandler.sample_io",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test logger.name == "DefaultHandler.sample_io"
                @test logger.level == "info"

                msg = "It works!"
                Memento.info(logger, msg)
                @test occursin("[info]:$(logger.name) - $msg", String(take!(io)))

                Memento.debug(logger, "This shouldn't get logged")
                @test isempty(String(take!(io)))

                msg = "Something went very wrong"
                log(logger, "fubar", msg)
                @test occursin("[fubar]:$(logger.name) - $msg", String(take!(io)))
            finally
                close(io)
            end
        end

        @testset "Sample Usage w/ File" begin
            filename = tempname()
            @info("Path to log file: $filename")

            handler = DefaultHandler(filename)

            logger = Logger(
                "DefaultHandler.sample_file",
                Dict("File" => handler),
                "info",
                LEVELS,
                DefaultRecord,
                true
            )

            @test logger.name == "DefaultHandler.sample_file"
            @test logger.level == "info"

            Memento.info(logger, "It works!")
            Memento.debug(logger, "This shouldn't get logged")

            log(logger, "fubar", "Something went very wrong")

            @test isfile(filename)
            @test Sys.iswindows() ? true : success(`rm $filename`)
        end

        @testset "IO Construction" begin
            io = IOBuffer()
            try
                handler1 = DefaultHandler(io)

                @test handler1.fmt.fmt_str == Memento.DEFAULT_FMT_STRING
                @test isempty(String(take!(handler1.io)))
                @test !(handler1.opts[:is_colorized])

                handler2 = DefaultHandler(
                    io,
                    DefaultFormatter(FMT_STR),
                    Dict{Symbol, Any}(:is_colorized => true)
                )

                @test handler2.fmt.fmt_str == FMT_STR
                @test handler2.opts[:is_colorized]
                @test haskey(handler2.opts, :colors)
                @test haskey(handler2.opts[:colors], "debug")
                @test handler2.opts[:colors]["debug"] == :blue
            finally
                close(io)
            end
        end

        @testset "Filename Construction" begin
            filename = tempname()
            @info("Path to log file: $filename")
            handler1 = DefaultHandler(filename)

            @test handler1.fmt.fmt_str == Memento.DEFAULT_FMT_STRING
            @test !(handler1.opts[:is_colorized])

            handler2 = DefaultHandler(
                filename,
                DefaultFormatter(FMT_STR),
                Dict{Symbol, Any}(:is_colorized => true)
            )

            @test handler2.fmt.fmt_str == FMT_STR
            @test handler2.opts[:is_colorized]
            @test haskey(handler2.opts, :colors)
            @test haskey(handler2.opts[:colors], "debug")
            @test handler2.opts[:colors]["debug"] == :blue
        end

        @testset "Colors" begin
            io = IOBuffer()

            try
                handler = DefaultHandler(
                    io, DefaultFormatter(FMT_STR),
                    Dict{Symbol, Any}(
                        :is_colorized => true,
                        :colors => Dict{AbstractString, Symbol}(
                            "debug" => :black,
                            "info" => :blue,
                            "notice" => :green,
                            "warn" => :cyan,
                            "error" => :magenta,
                            "critical" => :red,
                            "alert" => :yellow,
                            "emergency" => :white,
                        )
                    )
                )

                logger = Logger(
                    "DefaultHandler.sample_io",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test logger.name == "DefaultHandler.sample_io"
                @test logger.level == "info"

                msg = "It works!"
                Memento.info(logger, msg)
                @test occursin("[info]:$(logger.name) - $msg", String(take!(io)))

                Memento.debug(logger, "This shouldn't get logged")
                @test isempty(String(take!(io)))

                msg = "Something went very wrong"
                log(logger, "fubar", msg)
                @test occursin("[fubar]:$(logger.name) - $msg", String(take!(io)))
            finally
                close(io)
            end
        end

        @testset "Level Filter" begin
            io = IOBuffer()

            try
                handler = DefaultHandler(io, DefaultFormatter(FMT_STR))

                logger = Logger(
                    "DefaultHandler.sample_io",
                    Dict("Buffer" => handler),
                    "info",
                    LEVELS,
                    DefaultRecord,
                    true
                )

                @test logger.name == "DefaultHandler.sample_io"
                @test logger.level == "info"

                msg = "It works!"
                Memento.info(logger, msg)
                @test occursin("[info]:$(logger.name) - $msg", String(take!(io)))

                # Filter out log messages < LEVELS["warn"]
                setlevel!(handler, "warn")
                # add_filter(
                #     handler,
                #     Filter((rec) -> rec.levelnum >= LEVELS["warn"])
                # )

                Memento.info(logger, "This shouldn't get logged")
                @test isempty(String(take!(io)))
            finally
                close(io)
            end
        end
    end

    @testset "Escalator" begin
        @testset "Basic" begin
            # Using default behaviour to convert warnings to errors
            handler = Escalator()
            logger = Logger(
                "Escalator.basic",
                Dict("Escalator" => handler),
                "info",
                LEVELS,
                DefaultRecord,
                true,
            )

            # Info messages should work fine as no-ops
            info(logger, "Hello World!")

            # Test that we throw an escalation error for warnings
            @test_throws Memento.EscalationError warn(logger, "Goodbye World!")
        end

        @testset "info" begin
            # Using default behaviour to convert notice and warnings to errors
            handler = Escalator()
            setlevel!(handler, "info")
            logger = Logger(
                "Escalator.basic",
                Dict("Escalator" => handler),
                "info",
                LEVELS,
                DefaultRecord,
                true,
            )

            # Test that we throw an escalation error for info messages
            @test_throws Memento.EscalationError info(logger, "Goodbye World!")

            # Test that we throw an escalation error for warnings
            @test_throws Memento.EscalationError warn(logger, "Goodbye World!")
        end

        @testset "stdlib" begin
            # Test escalating base logging records
            orig_logger = Base.CoreLogging.global_logger()
            base_logger = Memento.BaseLogger(min_enabled_level(global_logger()))
            logger = getlogger(string(@__MODULE__))

            try
                global_logger(base_logger)
                push!(logger, Escalator())
                @test_throws Memento.EscalationError @warn("Goodbye World!")
            finally
                # Remove the logger and replace the core logging
                delete!(Memento._loggers, string(@__MODULE__))
                global_logger(orig_logger)
            end
        end
    end
end


text:
include("../imports.jl")

@testset ExtendedTestSet "`offsetcropindices`" begin

    @test offsetcropindices((50, 50), (1:100, 1:100), (0., 0.)) == (1:50, 1:50)
    @test offsetcropindices((50, 50), (1:100, 1:100), (0.5, 0.5)) == (26:75, 26:75)
    @test offsetcropindices((50, 50, 50), (1:100, 1:100, 1:100), (1., 1., 1.)) == (51:100, 51:100, 51:100)

end

@testset ExtendedTestSet "CroppedProjectiveTransform" begin

    image = Image(rand(100, 100))
    tfm = Project(Translation(20, 20))
    crop = CenterCrop((50, 50))
    @test tfm |> crop isa CroppedProjectiveTransform

    cropped = tfm |> crop
    @test_nowarn apply(cropped, image)
end


text:
# Exercise 8-1
function backward(str)
    index = lastindex(str)
    while index >= firstindex(str)
        letter = str[index]
        println(letter)
        index = prevind(str, index)
    end
end

# backward("abcdef")
# Alternatively loop on reverse(str)

# Exercise 8-2
prefixes = "JKLMNOPQ"
suffix = "ack"

for letter in prefixes
    if letter == 'O' || letter == 'Q'
        letter *= 'u'
    end
    println(letter * suffix)
end

# Exercise 8-3
# str[:] creates a copy of the original string str.

# Exercise 8-4
function find(word, letter, start)
    index = start
    while index <= sizeof(word)
        if word[index] == letter
            return index
        end
        index = nextind(word, index)
    end
    return -1
end

# println(find("abcdefabc", 'b', 4))
# println(find("abcdefabc", 'd', 5))

# Exercise 8-5
function count(word, l)
    counter = 0
    for letter in word
        if letter == l
            counter = counter + 1
        end
    end
    println(counter)
end

function count(word, l)
    index = find(word, l, 1)
    count = 0
    while index != -1
        count += 1
        index = find(word, l, index + 1)
        println
    end
    println(count)
end

# count("abbccccdef", 'c')
# count("abbccccdef", 'b')

# Exercise 8-6
function isreverse(word1, word2)
    if length(word1) != length(word2)
        return false
    end
    i = firstindex(word1)
    j = lastindex(word2)
    while j >= 0 && i <= lastindex(word1)
        # @show i j
        if word1[i] != word2[j]
            return false
        end
        i = nextind(word1, i)
        j = prevind(word2, j)
    end
    true
end

println(isreverse("pots", "stop"))

# Exercise 8-7
println(strip("    a b c    "))
println(strip("{3, 5}\n", ['{', '}', '\n']))

println(replace("There is a large tree in this forest of trees.", "tree" => "monkey"))
println(replace("The fast human is a fast runner.", "fast " => ""))
println(replace("The fast human is a fast runner.", "fast " => "", count=1))

# Exercise 8-8
println(count(i -> i == 'a', "banana"))

# Exercise 8-9
function ispalindrome(word)
    reversed = word[end:-1:begin]
    return word == reversed
end

println(ispalindrome("abccba"))
println(ispalindrome("abcdef"))

# Exercise 8-10
# This checks if the first character is lowercase.
function anylowercase1(s)
    for c in s
        if islowercase(c)
            return true
        else
            return false
        end
    end
end

# This checks if the character 'c' is lowercase, and will always return the string "true".
function anylowercase2(s)
    for c in s
        if islowercase('c')
            return "true"
        else
            return "false"
        end
    end
end

# This checks if the last character is lowercase.
function anylowercase3(s)
    for c in s
        flag = islowercase(c)
    end
    flag
end

# This correctly checks if any character contains a lowercase.
function anylowercase4(s)
    flag = false
    for c in s
        flag = flag || islowercase(c)
    end
    flag
end

# This returns false the moment it finds the first lower case.
function anylowercase5(s)
    for c in s
        if !islowercase(c)
            return false
        end
    end
    true
end

# Exercise 8-11
function rotatechar(c, i)
    if i < 0
        i = mod(i, 26)
    end
    if islowercase(c)
        return Char((c - 'a' + i) % 26 + 'a')
    else
        return Char((c - 'A' + i) % 26 + 'A')
    end
end

# println(rotatechar('a', 1))
# println(rotatechar('Z', 5))

function rotateword(str, i)
    result = ""
    for char in str
        result *= rotatechar(char, i)
    end
    return result
end

println(rotateword("cheer", 7))
println(rotateword("melon", -10))
println(rotateword("IBM", -1))


text:
function VDLTrack(Depth,vdl;DepthFrom=minimum(Depth),DepthTo=maximum(Depth),Dtick=false)

    x=1:size(vdl,2)
    X, Y =meshgrid(x,Depth)

    vdlax=gca()
    vdlax.contour(X,Y,vdl, cmap="Greys")
    vdlax.set_ylim([DepthTo,DepthFrom])
    vdlax.set_xlabel("VDL")
    vdlax.set_yticks(range(DepthFrom, stop=DepthTo, length=51),minor=true)
    vdlax.set_yticks(range(DepthFrom, stop=DepthTo, length=11))
    vdlax.xaxis.tick_top()
    vdlax.xaxis.set_label_position("top")
    setp(vdlax.get_yticklabels(),visible=Dtick)
end


text:
@testset "Reading of various example file formats" begin
    swscale_options = (sws_flags = "accurate_rnd+full_chroma_inp+full_chroma_int",)
    for testvid in values(VideoIO.TestVideos.videofiles)
        name = testvid.name
        test_frameno = testvid.testframe
        @testset "Reading $(testvid.name)" begin
            testvid_path = joinpath(@__DIR__, "../videos", name)
            comparison_frame = make_comparison_frame_png(load, testvid_path, test_frameno)
            f = VideoIO.testvideo(testvid_path)
            v = VideoIO.openvideo(f; swscale_options = swscale_options)
            try
                time_seconds = VideoIO.gettime(v)
                @test time_seconds == 0
                width, height = VideoIO.out_frame_size(v)
                @test VideoIO.width(v) == width
                @test VideoIO.height(v) == height
                @test VideoIO.out_frame_eltype(v) == RGB{N0f8}
                if size(comparison_frame, 1) > height
                    trimmed_comparison_frame =
                        comparison_frame[1+size(comparison_frame,1)-height:end,:]
                else
                    trimmed_comparison_frame = comparison_frame
                end

                @test VideoIO.framerate(v) != 0

                # Find the first non-trivial image
                first_img = read(v)
                first_time = VideoIO.gettime(v)
                seekstart(v)
                img = read(v)
                @test VideoIO.gettime(v) == first_time
                @test img == first_img
                @test size(img) == VideoIO.out_frame_size(v)[[2, 1]]
                # no scaling currently
                @test VideoIO.out_frame_size(v) == VideoIO.raw_frame_size(v)
                @test VideoIO.raw_pixel_format(v) == 0 # true for current test videos
                i=1
                while i < test_frameno
                    read!(v, img)
                    i += 1
                end
                test_compare_frames(img, trimmed_comparison_frame, required_accuracy)
                test_time = VideoIO.gettime(v)
                seek(v, test_time)
                raw_img = parent(img)
                read!(v, raw_img) # VideoReader should accept scanline-major images
                test_compare_frames(img, trimmed_comparison_frame, required_accuracy)
                @test VideoIO.gettime(v) == test_time
                if size(img, 1) != size(img, 2)
                    # Passing an arrray that is not scanline-major does not work
                    @test_throws ArgumentError read!(v, similar(img))
                    @test VideoIO.gettime(v) == test_time
                end
                @test_throws(ArgumentError,
                             read!(v, similar(raw_img, size(raw_img) .- 1)))
                @test_throws MethodError read!(v, similar(raw_img, Rational{Int}))
                @test_throws ArgumentError read!(v, similar(raw_img, Gray{N0f8}))
                @test VideoIO.gettime(v) == test_time

                seekstart(v)
                for i in 1:50
                    read!(v,img)
                end
                fiftieth_frame = copy(img)
                fiftytime = VideoIO.gettime(v)

                while !eof(v)
                    read!(v, img)
                end

                seek(v,fiftytime)
                read!(v,img)

                @test img == fiftieth_frame

                seekstart(v)
                start_t = VideoIO.gettime(v)
                @test start_t <= 0
                buff, align = VideoIO.read_raw(v, 1)
                @test VideoIO.out_bytes_size(v) == length(buff)
                @test align == 1
                buff_bak = copy(buff)
                seekstart(v)
                VideoIO.read_raw!(v, buff, 1)
                last_time = VideoIO.gettime(v)
                @test buff == buff_bak
                @test_throws(ArgumentError,
                             VideoIO.read_raw!(v, similar(buff, size(buff) .- 1)))
                @test_throws MethodError VideoIO.read_raw!(v, similar(buff, Int))
                @test VideoIO.gettime(v) == last_time
                notranscode_buff = VideoIO.openvideo(read, testvid_path,
                                                     transcode = false)
                @test notranscode_buff == buff_bak

                # read first frames again, and compare
                read_frameno!(img, v, test_frameno)
                test_compare_frames(img, trimmed_comparison_frame, required_accuracy)

                # make sure read! works with both PermutedDimsArray and Array
                # The above tests already use read! for PermutedDimsArray,
                # so just test the type of img
                @test typeof(img) <: PermutedDimsArray

                img_p = parent(img)
                @assert typeof(img_p) <: Array
                # img is a view of img_p, so calling read! on img_p should alter img
                #
                # first, zero img out to be sure we get the desired result from
                # calls to read on img_p!
                fill!(img, zero(eltype(img)))
                # Then get the first frame, which uses read!
                read_frameno!(img_p, v, test_frameno)
                # Finally compare the result to make sure it's right
                test_compare_frames(img, trimmed_comparison_frame, required_accuracy)

                # Skipping & frame counting
                VideoIO.seekstart(v)
                VideoIO.skipframe(v)
                VideoIO.skipframes(v, 10)
                @test VideoIO.counttotalframes(v) ==
                    VideoIO.TestVideos.videofiles[name].numframes
            finally
                close(f)
            end

            if occursin("annie_oakley", name)
                framestack = VideoIO.load(testvid_path)
                @test length(framestack) == VideoIO.TestVideos.videofiles[name].numframes
                if VERSION < v"1.6.3"
                    @test_broken Base.summarysize(framestack) == VideoIO.TestVideos.videofiles[name].summarysize
                else
                    @test Base.summarysize(framestack) == VideoIO.TestVideos.videofiles[name].summarysize
                end
                f = File{DataFormat{:OGG}}(testvid_path)
                framestack = VideoIO.fileio_load(f)
                @test length(framestack) == VideoIO.TestVideos.videofiles[name].numframes
                if VERSION < v"1.6.3"
                    @test_broken Base.summarysize(framestack) == VideoIO.TestVideos.videofiles[name].summarysize
                else
                    @test Base.summarysize(framestack) == VideoIO.TestVideos.videofiles[name].summarysize
                end
                path, io = mktemp()
                f = File{DataFormat{:MP4}}(path * ".mp4")
                VideoIO.fileio_save(f, framestack)
                @test isfile(path * ".mp4")
                @test stat(path * ".mp4").size > 0
                framestack = nothing
                GC.gc()
            end
        end
    end
end

@testset "Reading monochrome videos" begin
    testvid_path = joinpath(@__DIR__, "../videos", "annie_oakley.ogg")
    # Test that limited range YCbCr values are translated to "full range"
    minp, maxp = VideoIO.openvideo(get_video_extrema, testvid_path,
                                   target_format = VideoIO.AV_PIX_FMT_GRAY8)
    @test typeof(minp) == Gray{N0f8}
    @test minp.val.i < 16
    @test maxp.val.i > 235
    # Disable automatic rescaling
    minp, maxp = VideoIO.openvideo(get_video_extrema, testvid_path,
                                   target_format = VideoIO.AV_PIX_FMT_GRAY8,
                                   target_colorspace_details =
                                   VideoIO.VioColorspaceDetails())
    @test minp.val.i >= 16
    @test maxp.val.i <= 235
end

@testset "Reading RGB video as monochrome" begin
    @testset "Iterative" begin
        io = VideoIO.testvideo("ladybird")
        f = VideoIO.openvideo(io, target_format = VideoIO.AV_PIX_FMT_GRAY8)
        img = read(f)
        for i in 1:10
            read!(f, img)
        end
        @test eltype(img) == Gray{N0f8}
        close(f)
    end
    @testset "Full load" begin
        testvid_path = joinpath(@__DIR__, "../videos", "ladybird.mp4")
        vid = VideoIO.load(testvid_path, target_format = VideoIO.AV_PIX_FMT_GRAY8)
        @test eltype(first(vid)) == Gray{N0f8}
    end
end

@testset "IO reading of various example file formats" begin
    swscale_options = (sws_flags = "accurate_rnd+full_chroma_inp+full_chroma_int",)
    for testvid in values(VideoIO.TestVideos.videofiles)
        name = testvid.name
        test_frameno = testvid.testframe
        # TODO: fix me?
        (startswith(name, "ladybird") || startswith(name, "NPS")) && continue
        @testset "Testing $name" begin
            testvid_path = joinpath(@__DIR__, "../videos", name)
            comparison_frame = make_comparison_frame_png(load, testvid_path, test_frameno)
            filename = joinpath(videodir, name)
            v = VideoIO.openvideo(filename; swscale_options = swscale_options)
            try
                width, height = VideoIO.out_frame_size(v)
                if size(comparison_frame, 1) > height
                    trimmed_comparison_frame =
                        comparison_frame[1+size(comparison_frame,1)-height:end,:]
                else
                    trimmed_comparison_frame = comparison_frame
                end
                img = read(v)
                i=1
                while i < test_frameno
                    read!(v, img)
                    i += 1
                end
                test_compare_frames(img, trimmed_comparison_frame,
                                    required_accuracy)
                while !eof(v)
                    read!(v, img)
                end

                # Iterator interface
                VT = typeof(v)
                @test Base.IteratorSize(VT) === Base.SizeUnknown()
                @test Base.IteratorEltype(VT) === Base.EltypeUnknown()

                VideoIO.seekstart(v)
                i = 0
                local first_frame
                local last_frame
                for frame in v
                    i += 1
                    if i == 1
                        first_frame = frame
                    end
                    last_frame = frame
                end
                @test i == VideoIO.TestVideos.videofiles[name].numframes
                # test that the frames returned by the iterator have distinct storage
                if i > 1
                    @test first_frame !== last_frame
                end

                ## Test that iterator is mutable, and continues where iteration last
                ## stopped.
                @test iterate(v) === nothing
            finally
                close(v)
            end
        end
    end

    VideoIO.testvideo("ladybird") # coverage testing
    @test_throws ErrorException VideoIO.testvideo("rickroll")
    @test_throws ErrorException VideoIO.testvideo("")
end

@testset "Reading video metadata" begin
    @testset "Reading Storage Aspect Ratio: SAR" begin
        # currently, the SAR of all the test videos is 1, we should get another video with a valid SAR that is not equal to 1
        vids = Dict("ladybird.mp4" => 1, "black_hole.webm" => 1, "crescent-moon.ogv" => 1, "annie_oakley.ogg" => 1)
        @test all(VideoIO.aspect_ratio(VideoIO.openvideo(joinpath(videodir, k))) == v for (k,v) in vids)
    end
    @testset "Reading video duration, start date, and duration" begin
        # tesing the duration and date & time functions:
        file = joinpath(videodir, "annie_oakley.ogg")
        @test VideoIO.get_duration(file) == 24224200/1e6
        @test VideoIO.get_start_time(file) == DateTime(1970, 1, 1)
        @test VideoIO.get_time_duration(file) == (DateTime(1970, 1, 1), 24224200/1e6)
        @test VideoIO.get_number_frames(file) === nothing
    end
    @testset "Reading the number of frames from container" begin
        file = joinpath(videodir, "ladybird.mp4")
        @test VideoIO.get_number_frames(file) == 398
        @test VideoIO.get_number_frames(file, 0) == 398
        @test_throws ArgumentError VideoIO.get_number_frames(file, -1)
        @test_throws ErrorException VideoIO.get_number_frames("Not_a_file")
    end
end


text:
# This file was generated, do not modify it. # hide
Σ2= V * V'

max_stars_repo_name: cscherrer/InformativePrior

max_issues_repo_name: cscherrer/InformativePrior

max_forks_repo_name: cscherrer/InformativePrior


text:
export @step, breakpoint

macro step(args...)
  :(Atom.Debugger.@step($(map(esc, args)...)))
end

breakpoint(args...) = Atom.Debugger.breakpoint(args...)


text:
# This file is a part of LegendDataTypes.jl, licensed under the MIT License (MIT).

import Test
Test.@testset "Package LegendDataTypes" begin

include("test_radware_sigcompress.jl")

end # testset


text:
#! format: off

abstract type AbstractAGCFormulation <: AbstractServiceFormulation end
struct PIDSmoothACE <: AbstractAGCFormulation end

"""
Steady State deviation of the frequency
"""
function add_variables!(optimization_container::OptimizationContainer, ::Type{SteadyStateFrequencyDeviation})
    variable_name = make_variable_name(SteadyStateFrequencyDeviation)
    time_steps = model_time_steps(optimization_container)
    variable = add_var_container!(optimization_container, variable_name, time_steps)
    for t in time_steps
        variable[t] = JuMP.@variable(optimization_container.JuMPmodel, base_name = "ΔF_{$(t)}")
    end
end

get_variable_sign(_, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = NaN
########################## ActivePowerVariable, Area ###########################

get_variable_binary(::ActivePowerVariable, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = false

########################## SmoothACE, AggregationTopology ###########################

get_variable_binary(::SmoothACE, ::Type{<:PSY.AggregationTopology}, ::AbstractAGCFormulation) = false

########################## DeltaActivePowerUpVariable, Area ###########################

get_variable_binary(::DeltaActivePowerUpVariable, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = false
get_variable_lower_bound(::DeltaActivePowerUpVariable, ::PSY.Area, ::AbstractAGCFormulation) = 0.0

########################## DeltaActivePowerDownVariable, Area ###########################

get_variable_binary(::DeltaActivePowerDownVariable, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = false
get_variable_lower_bound(::DeltaActivePowerDownVariable, ::PSY.Area, ::AbstractAGCFormulation) = 0.0

########################## AdditionalDeltaPowerUpVariable, Area ###########################

# Commented out since not in use. These functions will substitute balancing_auxiliary_variables!
# """
# This function add the upwards scheduled regulation variables for power generation output to the model
# """
# function AddVariableSpec(
#     ::Type{AdditionalDeltaActivePowerUpVariable},
#     ::Type{U},
#     optimization_container::OptimizationContainer,
# ) where {U <: PSY.Area}
#     return AddVariableSpec(;
#         variable_name = make_variable_name(AdditionalDeltaActivePowerUpVariable, U),
#         binary = false,
#         lb_value_func = x -> 0.0,
#         expression_name = :emergency_up,
#     )
# end
#
# """
# This function add the variables for power generation output to the model
# """
# function AddVariableSpec(
#     ::Type{AdditionalDeltaActivePowerDownVariable},
#     ::Type{U},
#     optimization_container::OptimizationContainer,
# ) where {U <: PSY.Area}
#     return AddVariableSpec(;
#         variable_name = make_variable_name(AdditionalDeltaActivePowerDownVariable, U),
#         binary = false,
#         lb_value_func = x -> 0.0,
#         expression_name = :emergency_dn,
#     )
# end

########################## AreaMismatchVariable, Area ###########################

make_variable_name(::Type{AreaMismatchVariable}, _) = make_variable_name(AreaMismatchVariable)
get_variable_binary(::AreaMismatchVariable, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = false

########################## LiftVariable, Area ###########################

make_variable_name(::Type{LiftVariable}, _) = make_variable_name(LiftVariable)
get_variable_binary(::LiftVariable, ::Type{<:PSY.Area}, ::AbstractAGCFormulation) = false
get_variable_lower_bound(::LiftVariable, ::PSY.Area, ::AbstractAGCFormulation) = 0.0

#! format: off

########################## Initial Condition ###########################
function area_control_initial_condition!(
    optimization_container::OptimizationContainer,
    services::Vector{PSY.AGC},
    ::D,
) where {D <: AbstractAGCFormulation}
    key = ICKey(AreaControlError, PSY.AGC)
    _make_initial_conditions!(
        optimization_container,
        services,
        D(),
        nothing,
        key,
        _make_initial_condition_area_control,
        _get_variable_initial_value,
        # Doesn't require Cache
    )

    return
end

function _get_variable_initial_value(
    d::PSY.Component,
    key::ICKey,
    ::AbstractAGCFormulation,
    ::Nothing,
)
    return _get_ace_error(d, key)
end

########################## , ###########################

function balancing_auxiliary_variables!(optimization_container, sys)
    area_names = [PSY.get_name(a) for a in PSY.get_components(PSY.Area, sys)]
    time_steps = model_time_steps(optimization_container)
    R_up_emergency = JuMPVariableArray(undef, area_names, time_steps)
    R_dn_emergency = JuMPVariableArray(undef, area_names, time_steps)
    assign_variable!(
        optimization_container,
        make_variable_name(AdditionalDeltaActivePowerUpVariable, PSY.Area),
        R_up_emergency,
    )
    assign_variable!(
        optimization_container,
        make_variable_name(AdditionalDeltaActivePowerDownVariable, PSY.Area),
        R_dn_emergency,
    )
    emergency_up =
        add_expression_container!(optimization_container, :emergency_up, area_names, time_steps)
    emergency_dn =
        add_expression_container!(optimization_container, :emergency_dn, area_names, time_steps)
    for t in time_steps, a in area_names
        R_up_emergency[a, t] = JuMP.@variable(
            optimization_container.JuMPmodel,
            base_name = "Re_up_{$(a),$(t)}",
            lower_bound = 0.0
        )
        emergency_up[a, t] = R_up_emergency[a, t] + 0.0
        R_dn_emergency[a, t] = JuMP.@variable(
            optimization_container.JuMPmodel,
            base_name = "Re_dn_{$(a),$(t)}",
            lower_bound = 0.0
        )
        emergency_dn[a, t] = R_dn_emergency[a, t] + 0.0
    end

    return
end

function absolute_value_lift(optimization_container::OptimizationContainer, areas)
    time_steps = model_time_steps(optimization_container)
    area_names = [PSY.get_name(a) for a in areas]
    container_lb = JuMPConstraintArray(undef, area_names, time_steps)
    assign_constraint!(optimization_container, "absolute_value_lb", container_lb)
    container_ub = JuMPConstraintArray(undef, area_names, time_steps)
    assign_constraint!(optimization_container, "absolute_value_ub", container_ub)
    mismatch = get_variable(optimization_container, :area_mismatch)
    z = get_variable(optimization_container, :lift)

    for t in time_steps, a in area_names
        container_lb[a, t] =
            JuMP.@constraint(optimization_container.JuMPmodel, mismatch[a, t] <= z[a, t])
        container_ub[a, t] =
            JuMP.@constraint(optimization_container.JuMPmodel, -1 * mismatch[a, t] <= z[a, t])
    end

    JuMP.add_to_expression!(
        optimization_container.cost_function,
        sum(z[a, t] for t in time_steps, a in area_names) * SERVICES_SLACK_COST,
    )
    return
end

"""
Expression for the power deviation given deviation in the frequency. This expression allows updating the response of the frequency depending on commitment decisions
"""
function frequency_response_constraint!(optimization_container::OptimizationContainer, sys::PSY.System)
    time_steps = model_time_steps(optimization_container)
    services = PSY.get_components(PSY.AGC, sys)
    area_names = [PSY.get_name(PSY.get_area(s)) for s in services]
    frequency_response = 0.0
    for area in PSY.get_components(PSY.Area, sys)
        frequency_response += PSY.get_load_response(area)
    end
    for g in PSY.get_components(PSY.RegulationDevice, sys, x -> PSY.get_available(x))
        d = PSY.get_droop(g)
        response = 1 / d
        frequency_response += response
    end

    @assert frequency_response >= 0.0
    # This value is the one updated later in simulation based on the UC result
    inv_frequency_reponse = 1 / frequency_response
    area_balance = get_variable(optimization_container, ActivePowerVariable, PSY.Area)
    frequency = get_variable(optimization_container, "Δf")
    R_up = get_variable(optimization_container, DeltaActivePowerUpVariable, PSY.Area)
    R_dn = get_variable(optimization_container, DeltaActivePowerDownVariable, PSY.Area)
    R_up_emergency =
        get_variable(optimization_container, AdditionalDeltaActivePowerUpVariable, PSY.Area)
    R_dn_emergency =
        get_variable(optimization_container, AdditionalDeltaActivePowerUpVariable, PSY.Area)

    container = JuMPConstraintArray(undef, time_steps)
    assign_constraint!(optimization_container, "frequency_response", container)

    for s in services, t in time_steps
        system_balance = sum(area_balance.data[:, t])
        total_reg = JuMP.AffExpr(0.0)
        for a in area_names
            JuMP.add_to_expression!(total_reg, R_up[a, t])
            JuMP.add_to_expression!(total_reg, -1 * R_dn[a, t])
            JuMP.add_to_expression!(total_reg, R_up_emergency[a, t])
            JuMP.add_to_expression!(total_reg, -1 * R_dn_emergency[a, t])
        end
        container[t] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            frequency[t] == -inv_frequency_reponse * (system_balance + total_reg)
        )
    end
    return
end

function smooth_ace_pid!(optimization_container::OptimizationContainer, services::Vector{PSY.AGC})
    time_steps = model_time_steps(optimization_container)
    area_names = [PSY.get_name(PSY.get_area(s)) for s in services]
    RAW_ACE = add_expression_container!(optimization_container, :RAW_ACE, area_names, time_steps)
    SACE = get_variable(optimization_container, SmoothACE, PSY.Area)
    SACE_pid = JuMPConstraintArray(undef, area_names, time_steps)
    assign_constraint!(optimization_container, "SACE_pid", SACE_pid)

    Δf = get_variable(optimization_container, make_variable_name("Δf"))

    for (ix, service) in enumerate(services)
        kp = PSY.get_K_p(service)
        ki = PSY.get_K_i(service)
        kd = PSY.get_K_d(service)
        B = PSY.get_bias(service)
        Δt = convert(Dates.Second, optimization_container.resolution).value
        a = PSY.get_name(PSY.get_area(service))
        for t in time_steps
            # Todo: Add initial Frequency Deviation
            RAW_ACE[a, t] = -10 * B * Δf[t] + 0.0
            SACE[a, t] =
                JuMP.@variable(optimization_container.JuMPmodel, base_name = "SACE_{$(a),$(t)}")
            if t == 1
                SACE_ini =
                    get_initial_conditions(optimization_container, ICKey(AreaControlError, PSY.AGC))[ix]
                sace_exp = SACE_ini.value + kp * ((1 + Δt / (kp / ki)) * (RAW_ACE[a, t]))
                SACE_pid[a, t] =
                    JuMP.@constraint(optimization_container.JuMPmodel, SACE[a, t] == sace_exp)
                continue
            end
            SACE_pid[a, t] = JuMP.@constraint(
                optimization_container.JuMPmodel,
                SACE[a, t] ==
                SACE[a, t - 1] +
                kp * (
                    (1 + Δt / (kp / ki) + (kd / kp) / Δt) * (RAW_ACE[a, t]) +
                    (-1 - 2 * (kd / kp) / Δt) * (RAW_ACE[a, t - 1])
                )
            )
        end
    end
    return
end

function aux_constraints!(optimization_container::OptimizationContainer, sys::PSY.System)
    time_steps = model_time_steps(optimization_container)
    area_names = [PSY.get_name(a) for a in PSY.get_components(PSY.Area, sys)]
    aux_equation = JuMPConstraintArray(undef, area_names, time_steps)
    assign_constraint!(optimization_container, "balance_aux", aux_equation)
    area_mismatch = get_variable(optimization_container, :area_mismatch)
    SACE = get_variable(optimization_container, SmoothACE, PSY.Area)
    R_up = get_variable(optimization_container, DeltaActivePowerUpVariable, PSY.Area)
    R_dn = get_variable(optimization_container, DeltaActivePowerDownVariable, PSY.Area)
    R_up_emergency =
        get_variable(optimization_container, AdditionalDeltaActivePowerUpVariable, PSY.Area)
    R_dn_emergency =
        get_variable(optimization_container, AdditionalDeltaActivePowerUpVariable, PSY.Area)

    for t in time_steps, a in area_names
        aux_equation[a, t] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            -1 * SACE[a, t] ==
            (R_up[a, t] - R_dn[a, t]) +
            (R_up_emergency[a, t] - R_dn_emergency[a, t]) +
            area_mismatch[a, t]
        )
    end
    return
end


text:
module OpenAPI

using Configurations: Maybe
using ..KungIChi: @schema

# TODO:
# implement customize lowering
# to generate the key $ref
@schema struct Reference
    ref::String
end

@schema struct Contact
    name::Maybe{String} = nothing
    url::Maybe{String} = nothing
    email::Maybe{String} = nothing
end

@schema struct License
    name::String
    url::Maybe{String} = nothing
end

@schema struct Info
    title::String
    description::Maybe{String} = nothing
    termsOfService::Maybe{String} = nothing
    contact::Maybe{Contact} = nothing
    license::Maybe{License} = nothing
    version::String
end

@schema struct ServerVariable
    enum::Maybe{Vector{String}} = nothing
    default::String
    description::String

    # @check !isempty(enum)
end

@schema struct Schema
end

@schema struct Response
end

@schema struct Parameter
end

@schema struct Example
end

@schema struct RequestBody
end

@schema struct Header
end

@schema struct SecurityScheme
end

@schema struct Link
end

@schema struct Callback
end

@schema struct Operation
end

@schema struct Server
    url::Maybe{String} = nothing
    description::Maybe{String} = nothing
    variables::Maybe{Dict{String, ServerVariable}} = nothing
end

@schema struct PathItem
    ref::String
    summary::Maybe{String} = nothing
    description::Maybe{String} = nothing
    get::Operation
    put::Operation
    post::Operation
    delete::Operation
    options::Operation
    head::Operation
    patch::Operation
    trace::Operation
    servers::Vector{Server}
    parameters::Vector{Parameter}
end

@schema struct Components
    schemas::Maybe{Dict{String, Schema}} = nothing
    responses::Maybe{Dict{String, Response}} = nothing
    parameters::Maybe{Dict{String, Parameter}} = nothing
    examples::Maybe{Dict{String, Example}} = nothing
    requestBodies::Maybe{Dict{String, RequestBody}} = nothing
    headers::Maybe{Dict{String, Header}} = nothing
    securitySchemes::Maybe{Dict{String, SecurityScheme}} = nothing
    links::Maybe{Dict{String, Link}} = nothing
    callbacks::Maybe{Dict{String, Callback}} = nothing
end

@schema struct Security
end

@schema struct Tag
end

@schema struct ExternalDocumentation
end

# TODO:
# custom lowering/parsing
@schema struct Paths
    path::String
    item::PathItem
end

@schema struct OpenAPIObject
    openapi::VersionNumber
    info::Info
    servers::Vector{Server}
    paths::Vector{Paths}
    components::Components
    security::Security
    tags::Vector{Tag}
    externalDocs::Vector{ExternalDocumentation}
end

end


text:
#=

  Magic hexagon in ConstraintSolver.jl

  Prob023: Magic Hexagon
  http://www.comp.rgu.ac.uk/staff/ha/ZCSP/prob023/prob023.pdf
  http://www.cse.unsw.edu.au/~tw/csplib/prob/prob023/

  Model created by Hakan Kjellerstrand, hakank@gmail.com
  See also my Julia page: http://www.hakank.org/julia/

=#

using ConstraintSolver, JuMP
using Cbc, GLPK, Ipopt
const CS = ConstraintSolver
include("constraints_utils.jl")

function magic_hexagon(print_solutions=true,all_solutions=true,timeout=6)

    cbc_optimizer = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0)
    glpk_optimizer = optimizer_with_attributes(GLPK.Optimizer)
    ipopt_optimizer = optimizer_with_attributes(Ipopt.Optimizer)

    model = Model(optimizer_with_attributes(CS.Optimizer,   "all_solutions"=> all_solutions,
                                                            # "all_optimal_solutions"=>all_solutions,
                                                            "logging"=>[],

                                                            "traverse_strategy"=>:BFS,
                                                            # "traverse_strategy"=>:DFS,
                                                            # "traverse_strategy"=>:DBFS,

                                                            # "branch_split"=>:Smallest,
                                                            # "branch_split"=>:Biggest,
                                                            "branch_split"=>:InHalf,

                                                            # https://wikunia.github.io/ConstraintSolver.jl/stable/options/#branch_strategy-(:Auto)
                                                            "branch_strategy" => :IMPS, # default
                                                            # "branch_strategy" => :ABS, # Activity Based Search
                                                            # "activity.decay" => 0.999, # default 0.999
                                                            # "activity.max_probes" => 10, # default, 10
                                                            # "activity.max_confidence_deviation" => 20, # default 20

                                                            # "simplify"=>false,
                                                            # "simplify"=>true, # default

                                                            "time_limit"=>timeout,

                                                            # "backtrack" => false, # default true
                                                            # "backtrack_sorting" => false, # default true

                                                            # "lp_optimizer" => cbc_optimizer,
                                                            # "lp_optimizer" => glpk_optimizer,
                                                            # "lp_optimizer" => ipopt_optimizer,
                                        ))

    n = 19
    @variable(model, 1 <= x[1:n] <= n, Int)
    A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S = x

    @constraint(model, x in CS.AllDifferent())

    @constraint(model, A + B + C ==  38)
    @constraint(model, D + E + F + G ==  38)
    @constraint(model, H + I + J + K + L ==  38,)
    @constraint(model, M + N + O + P ==  38,)
    @constraint(model, Q + R + S ==  38,)
    @constraint(model, A + D + H ==  38,)
    @constraint(model, B + E + I + M ==  38,)
    @constraint(model, C + F + J + N + Q ==  38,)
    @constraint(model, G + K + O + R ==  38,)
    @constraint(model, L + P + S ==  38,)
    @constraint(model, C + G + L ==  38,)
    @constraint(model, B + F + K + P ==  38,)
    @constraint(model, A + E + J + O + S ==  38,)
    @constraint(model, D + I + N + R ==  38,)
    @constraint(model, H + M + Q ==  38,)

    @constraint(model, A < H)
    @constraint(model, A < C)
    @constraint(model, A < L)
    @constraint(model, A < Q)
    @constraint(model, A < S)
    @constraint(model, C < H)

    # @objective(model,Max,z)

    # Solve the problem
    optimize!(model)

    status = JuMP.termination_status(model)
    # println("status:$status")
    num_sols = 0
    if status == MOI.OPTIMAL
        num_sols = MOI.get(model, MOI.ResultCount())
        println("num_sols:$num_sols\n")
        if print_solutions
            for sol in 1:num_sols
                println("solution #$sol")
                x_val = convert.(Integer,JuMP.value.(x; result=sol))
                println("x:$x_val")

            end
        end
    else
        println("status:$status")
    end

    return status, num_sols
end

@time magic_hexagon()

max_stars_repo_name: tias/hakank

max_issues_repo_name: tias/hakank

max_forks_repo_name: tias/hakank


text:
using Compat
using LightXML

# define placeholder types
type AWSEnv end

include("../src/codegen.jl")

const DIR = dirname(@__FILE__)
const SRCDIR = joinpath(DIR, "..", "src")

const SPEC_NAMES = [
    "AutoScaling",
    "SQS",
    "EC2"
]

function do_gen(spec_name::String, includes::Vector{String}, out_types_file::String, out_operations_file::String)
    map(include, includes)

    info("generating ", spec_name, " types...")
    generate_types(out_types_file, "$(spec_name)TypeDict", eval(Symbol("$(spec_name)Types")))

    info("generating ", spec_name, " apis...")
    generate_apis(out_operations_file, "$(spec_name)TypeDict", eval(Symbol("$(spec_name)Api")))
end

function gen_all()
    for spec_name in SPEC_NAMES
        lower_spec_name = lowercase(spec_name)
        spec_file = joinpath("..", "src", lower_spec_name * "_spec.jl")
        placeholder_types_file = lower_spec_name * "_gen_types.jl"
        out_types_file = joinpath(SRCDIR, "$(lower_spec_name)_types.jl")
        out_operations_file = joinpath(SRCDIR, "$(lower_spec_name)_operations.jl")

        pid = addprocs(1)[1]
        remotecall_fetch(include, pid, @__FILE__)
        remotecall_fetch(do_gen, pid, spec_name, [placeholder_types_file, spec_file], out_types_file, out_operations_file)
        rmprocs(pid)
    end
end

(length(ARGS) > 0) && (ARGS[1] == "gen_all") && gen_all()


text:
# Autogenerated wrapper script for Atomsk_jll for i686-w64-mingw32-libgfortran5
export atomsk

using CompilerSupportLibraries_jll
using LAPACK_jll
JLLWrappers.@generate_wrapper_header("Atomsk")
JLLWrappers.@declare_executable_product(atomsk)
function __init__()
    JLLWrappers.@generate_init_header(CompilerSupportLibraries_jll, LAPACK_jll)
    JLLWrappers.@init_executable_product(
        atomsk,
        "bin\\atomsk.exe",
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
@testset "Hydrus1d write atmosph" begin
	data = DataFrame(Prec=[0.01,0.1,0.2,0.3],rSoil=[0.0,0.1,0.2,0.9],
		   datetime=[DateTime(2010,1,1,0),DateTime(2010,1,1,1),
			   DateTime(2010,1,1,2),DateTime(2010,1,1,4)],
			   hCritA=[1,1,1,1]);
	writeatmosph(data,joinpath(dirname(@__DIR__),"test","output","atmosph_data.in"),
				decimal=[1],
				hCritS=1000.);
	# Read header
	open(joinpath(dirname(@__DIR__),"test","output","atmosph_data.in")) do fid
		row = " ";
		for i in 1:4
			row = readline(fid);
		end
		@test Meta.parse(row) == 4
		for i in 1:3
			row = readline(fid);
		end
		hcrits = fid |> readline |> Meta.parse
		@test hcrits == 1000
	end
	data_read = readdlm(joinpath(dirname(@__DIR__),"test","output","atmosph_data.in"),skipstart=9);
	@test size(data_read) == (5,8)
	@test data_read[1:4,1:8] == [1 0   0   0 1 0 0 0;
							     2 0.1 0.1 0 1 0 0 0;
							     3 0.2 0.2 0 1 0 0 0;
							     4 0.3 0.9 0 1 0 0 0];
end

@testset "Hydrus1d write profile" begin
	# write data
	soilinfo = DataFrame(start=[0.0,4.2],stop=[4.0,6],res=[0.5,0.2],h=[100,200],Mat=[1,2],Lay=[1,1])
	output_file = joinpath(dirname(@__DIR__),"test","output","profile1d_data.dat");
	print_nodes = [5,7,9];
	writeprofile1d(soilinfo,output_file,iObs=print_nodes);
	# check file
	t = readdlm(output_file);
	for i in 5:23
		@test t[i,1] == i-4
		@test t[i,5:9] == [1,0.0,1.0,1.0,1.0]
		if t[i,2]>-soilinfo[:start][2]
			@test t[i,3] == soilinfo[:h][1]
			@test t[i,4] == soilinfo[:Mat][1]
		else
			@test t[i,3] == soilinfo[:h][2]
			@test t[i,4] == soilinfo[:Mat][2]
		end
	end
	@test t[24,1] == length(print_nodes)
	@test t[end,1:3] == print_nodes
end

@testset "Hydrus1d read obs notedes" begin
	input_file = joinpath(dirname(@__DIR__),"test","input","hydrus1d_Obs_Node.out")
	# Theta/soil moisture
	ttest = readhydrus1d_obsnode(input_file,paramout=:theta)
	@test ttest[:time] == collect(1:1:12)
	@test size(ttest) == (12,11)
	@test ttest[:theta1] == [0.1676,0.1759,0.1809,0.1852,0.1882,0.191,0.1935,0.1955,0.1973,0.1991,0.2008,0.2024];
	@test ttest[:theta30] == [0.136,0.136,0.136,0.136,0.136,0.136,0.1361,0.1365,0.1372,0.1386,0.1407,0.1437];
	@test ttest[:theta1000] == [0.136,0.136,0.136,0.136,0.136,0.136,0.136,0.136,0.136,0.136,0.136,0.136];

	# h
	ttest = readhydrus1d_obsnode(input_file,paramout=:h)
	@test ttest[:time] == collect(1:1:12)
	@test size(ttest) == (12,11)
	@test ttest[:h1] == [-16.82,-13.47,-11.87,-10.73,-9.98,-9.37,-8.9,-8.51,-8.16,-7.84,-7.57,-7.35];
	@test ttest[:h30] == [-50,-50,-50,-50,-49.98,-49.9,-49.67,-49.07,-47.7,-45.12,-41.29,-36.7];
	@test ttest[:h1000] == [-50,-50,-50,-50,-50,-50,-50,-50,-50,-50,-50,-50];

	# flux
	ttest = readhydrus1d_obsnode(input_file,paramout=:Flux)
	@test ttest[:time] == collect(1:1:12)
	@test size(ttest) == (12,11)
	@test ttest[:Flux1] == [-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03,-1.00E-03];
	@test ttest[:Flux30] == [-1.51E-07,-1.51E-07,-1.55E-07,-1.80E-07,-2.94E-07,-8.74E-07,-2.47E-06,-7.15E-06,-1.78E-05,-3.82E-05,-6.93E-05,-1.12E-04];
	@test ttest[:Flux1000] == [-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07,-1.51E-07];

end

@testset "Hydrus1d read node info" begin
	input_file = joinpath(dirname(@__DIR__),"test","input","hydrus1d_Nod_Inf.out");
	# soil moisture only
	ttest = readhydrus1d_nodinf(input_file,paramout=:theta)
	@test ttest[:time] == collect(0.:1.:3)
	@test size(ttest) == (4,11)
	@test ttest[:node1] == [0.1360,0.1676,0.1759,0.1809]
	@test ttest[:node3] == [0.1360,0.1569,0.1684,0.1751]
	@test ttest[:node10] == [0.1360,0.1362,0.1400,0.1486]

	# all
	moisture,k,h = readhydrus1d_nodinf(input_file)
	@test moisture[:time] == collect(0.:1.:3)
	@test k[:time] == collect(0.:1.:3)
	@test h[:time] == collect(0.:1.:3)
	@test size(moisture) == (4,11)
	@test size(k) == (4,11)
	@test size(h) == (4,11)
	@test moisture[:node1] == [0.1360,0.1676,0.1759,0.1809]
	@test moisture[:node3] == [0.1360,0.1569,0.1684,0.1751]
	@test moisture[:node10] == [0.1360,0.1362,0.1400,0.1486]
	@test k[:node1] == [0.1511E-06,0.4351E-05,0.8321E-05,0.1234E-04]
	@test k[:node3] == [0.1511E-06,0.1621E-05,0.4484E-05,0.8037E-05]
	@test k[:node10] == [0.1511E-06,0.1570E-06,0.2626E-06,0.6815E-06]
	@test h[:node1] == [-50.000,-16.820,-13.468,-11.867]
	@test h[:node3] == [-50.000,-23.178,-16.446,-13.780]
	@test h[:node10] == [-50.000,-49.480,-42.537,-30.490]
end

@testset "Hydrus1d read atmosph" begin
	input_file = joinpath(dirname(@__DIR__),"test","input","hydrus1d_atmosph.in");
	ttest = readatmosph(input_file)
	@test ttest[:time] == collect(1.:1.:11)
	@test size(ttest) == (11,9)
	@test ttest[:Prec] == [0.,0.,0.,0.,0.02,0.,0.,0.,0.,0.,0.]
	@test ttest[:rSoil] == [0.,0.,0.,0.001,0.,0.,0.,0.,0.,0.,0.]
	@test ttest[:ht] == zeros(11)
	@test all(isnan.(ttest[:RootDepth]))
end


text:
#!/usr/bin/env julia --project

include("../shared_source/boilerplate.jl")
include("../three_dimensional/ThreeDimensionalCompressibleNavierStokesEquations.jl")
include("sphere_helper_functions.jl")

ClimateMachine.init()

#! format: off
refVals = (
[
 [ "state",     "ρ",   9.99999999999998778755e-01,  1.00000000000000111022e+00,  1.00000000000000000000e+00,  4.03070839356576998483e-16 ],
 [ "state", "ρu[1]",  -6.93674606129580362185e-18,  1.19513727285696382782e-17,  9.77991462267996185035e-19,  2.78015892816367900233e-18 ],
 [ "state", "ρu[2]",  -1.21340560458612968582e-17,  1.22630936333135748291e-17, -1.30065979768613433360e-21,  2.78919703417897612909e-18 ],
 [ "state", "ρu[3]",  -1.00545092917566730233e-17,  1.00043577527427175213e-17,  3.76131929192945135398e-19,  2.47195369518944690265e-18 ],
 [ "state",    "ρθ",  -3.00959038541689510859e-02, -3.00957518135526284897e-02, -3.00958465374152675520e-02,  3.12498428220800724718e-08 ],
],
[
 [ "state",     "ρ",    12,    12,    12,     0 ],
 [ "state", "ρu[1]",     0,     0,     0,     0 ],
 [ "state", "ρu[2]",     0,     0,     0,     0 ],
 [ "state", "ρu[3]",     0,     0,     0,     0 ],
 [ "state",    "ρθ",    12,    12,    12,     0 ],
],
)
#! format: on

#################
# RUN THE TESTS #
#################
@testset "$(@__FILE__)" begin

    ########
    # Setup physical and numerical domains
    ########
    Ω = AtmosDomain(radius = 1, height = 0.2)

    grid = DiscretizedDomain(
        Ω;
        elements = (vertical = 1, horizontal = 5),
        polynomial_order = (vertical = 1 + 0, horizontal = 3 + 0),
        overintegration_order = (vertical = 1, horizontal = 1),
    )

    ########
    # Define physical parameters and parameterizations
    ########
    parameters = (
        ρₒ = 1,        # reference density
        cₛ = 1e-2,     # sound speed
        R = Ω.radius, # [m]
        ω = 0.0,      # [s⁻¹]
        K = 7.848e-6, # [s⁻¹]
        n = 4,        # dimensionless
        Ω = 0.0,      # [s⁻¹] 2π/86400
    )

    physics = FluidPhysics(;
        orientation = SphericalOrientation(),
        advection = NonLinearAdvectionTerm(),
        dissipation = ConstantViscosity{Float64}(μ = 0, ν = 0.0, κ = 1e-3),
        coriolis = nothing,
        buoyancy = Buoyancy{Float64}(α = 2e-4, g = 0),
    )

    ########
    # Define timestepping parameters
    ########
    Δt = 0.1 * min_node_distance(grid.numerical)^2 / physics.dissipation.κ
    start_time = 0
    end_time = 20.0 * Δt

    method = SSPRK22Heuns

    timestepper = TimeStepper(method = method, timestep = Δt)

    callbacks = (Info(), StateCheck(10))

    ########
    # Define boundary conditions (west east are the ones that are enforced for a sphere)
    ########
    ρu_bcs = (bottom = Impenetrable(FreeSlip()), top = Impenetrable(FreeSlip()))
    ρθ_bcs = (
        bottom = TemperatureFlux(
            flux = (p, state, aux, t) -> p.Q,
            params = (Q = 1e-3,), # positive means removing heat from system
        ),
        top = TemperatureFlux(
            flux = (p, state, aux, t) -> p.Q,
            params = (Q = 1e-3,), # positive means removing heat from system
        ),
    )
    BC = (ρθ = ρθ_bcs, ρu = ρu_bcs)

    ########
    # Define initial conditions
    ########

    # Earth Spherical Representation
    # longitude: λ ∈ [-π, π), λ = 0 is the Greenwich meridian
    # latitude:  ϕ ∈ [-π/2, π/2], ϕ = 0 is the equator
    # radius:    r ∈ [Rₑ - hᵐⁱⁿ, Rₑ + hᵐᵃˣ]
    # Rₑ = Radius of sphere; hᵐⁱⁿ, hᵐᵃˣ ≥ 0

    ρ₀(p, λ, ϕ, r) = p.ρₒ
    ρuʳᵃᵈ(p, λ, ϕ, r) = 0.0
    ρuˡᵃᵗ(p, λ, ϕ, r) = 0.0
    ρuˡᵒⁿ(p, λ, ϕ, r) = 0.0
    ρθ₀(p, λ, ϕ, r) = 0.0

    # Cartesian Representation (boiler plate really)
    ρ₀ᶜᵃʳᵗ(p, x...) = ρ₀(p, lon(x...), lat(x...), rad(x...))
    ρu⃗₀ᶜᵃʳᵗ(p, x...) = (
        ρuʳᵃᵈ(p, lon(x...), lat(x...), rad(x...)) * r̂(x...) +
        ρuˡᵃᵗ(p, lon(x...), lat(x...), rad(x...)) * ϕ̂(x...) +
        ρuˡᵒⁿ(p, lon(x...), lat(x...), rad(x...)) * λ̂(x...)
    )
    ρθ₀ᶜᵃʳᵗ(p, x...) = ρθ₀(p, lon(x...), lat(x...), rad(x...))

    initial_conditions = (ρ = ρ₀ᶜᵃʳᵗ, ρu = ρu⃗₀ᶜᵃʳᵗ, ρθ = ρθ₀ᶜᵃʳᵗ)

    ########
    # Create the things
    ########

    model = SpatialModel(
        balance_law = Fluid3D(),
        physics = physics,
        numerics = (flux = RoeNumericalFlux(),),
        grid = grid,
        boundary_conditions = BC,
        parameters = parameters,
    )

    simulation = Simulation(
        model = model,
        initial_conditions = initial_conditions,
        timestepper = timestepper,
        callbacks = callbacks,
        time = (; start = start_time, finish = end_time),
    )

    ########
    # Run the model
    ########
    tic = Base.time()

    @testset "State Check" begin
        evolve!(simulation, model, refDat = refVals)
    end

    toc = Base.time()
    time = toc - tic
    println(time)

    ## Check the budget
    θ̄ = weightedsum(simulation.state, 5)

    @testset "Exact Geometry" begin
        # Exact (with exact geometry)
        R = grid.domain.radius
        H = grid.domain.height

        θ̄ᴱ = 0
        θ̄ᴱ -= 4π * R^2 * end_time * BC.ρθ.bottom.params.Q
        θ̄ᴱ -= 4π * (R + H)^2 * end_time * BC.ρθ.top.params.Q

        δθ̄ᴱ = abs(θ̄ - θ̄ᴱ) / abs(θ̄ᴱ)
        println("The relative error w.r.t. exact geometry ", δθ̄ᴱ)

        @test isapprox(0, δθ̄ᴱ; atol = 1e-9)
    end

    # TODO: make Approx Geom test MPI safe
    #=
    @testset "Approx Geometry" begin
        # Exact (with approximated geometry)
        n_e = convention(grid.resolution.elements, Val(3))
        n_ijk = convention(grid.resolution.polynomial_order, Val(3))
        n_ijk =
            n_ijk .+ convention(grid.resolution.overintegration_order, Val(3))
        n_ijk = n_ijk .+ (1, 1, 1)

        Mᴴ = reshape(
            grid.numerical.vgeo[:, grid.numerical.MHid, :],
            (n_ijk..., n_e[3], 6 * n_e[1] * n_e[2]),
        )
        Aᴮᵒᵗ = sum(Mᴴ[:, :, 1, 1, :])
        Aᵀᵒᵖ = sum(Mᴴ[:, :, end, end, :])

        θ̄ᴬ = 0
        θ̄ᴬ -= Aᴮᵒᵗ * end_time * BC.ρθ.bottom.params.Q
        θ̄ᴬ -= Aᵀᵒᵖ * end_time * BC.ρθ.top.params.Q

        δθ̄ᴬ = abs(θ̄ - θ̄ᴬ) / abs(θ̄ᴬ)
        println("The relative error w.r.t. approximated geometry ", δθ̄ᴬ)

        @test isapprox(0, δθ̄ᴬ; atol = 1e-15)
    end
    =#

end


text:
using Documenter, DocumenterTools
include("DocumenterShowcase.jl")

# The DOCSARGS environment variable can be used to pass additional arguments to make.jl.
# This is useful on CI, if you need to change the behavior of the build slightly but you
# can not change the .travis.yml or make.jl scripts any more (e.g. for a tag build).
if haskey(ENV, "DOCSARGS")
    for arg in split(ENV["DOCSARGS"])
        (arg in ARGS) || push!(ARGS, arg)
    end
end

makedocs(
    modules = [Documenter, DocumenterTools, DocumenterShowcase],
    format = Documenter.HTML(
        # Use clean URLs, unless built as a "local" build
        prettyurls = !("local" in ARGS),
        canonical = "https://juliadocs.github.io/Documenter.jl/stable/",
        assets = ["assets/favicon.ico"],
        analytics = "UA-136089579-2",
        highlights = ["yaml"],
    ),
    clean = false,
    sitename = "Documenter.jl",
    authors = "Michael Hatherly, Morten Piibeleht, and contributors.",
    linkcheck = "linkcheck" in ARGS,
    linkcheck_ignore = [
        # timelessrepo.com seems to 404 on any CURL request...
        "http://timelessrepo.com/json-isnt-a-javascript-subset",
        # We'll ignore links that point to GitHub's edit pages, as they redirect to the
        # login screen and cause a warning:
        r"https://github.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)/edit(.*)",
    ] ∪ (get(ENV, "GITHUB_ACTIONS", nothing)  == "true" ? [
        # Extra ones we ignore only on CI.
        #
        # It seems that CTAN blocks GitHub Actions?
        "https://ctan.org/pkg/minted",
    ] : []),
    pages = [
        "Home" => "index.md",
        "Manual" => Any[
            "Guide" => "man/guide.md",
            "man/examples.md",
            "man/syntax.md",
            "man/doctests.md",
            "man/latex.md",
            hide("man/hosting.md", [
                "man/hosting/walkthrough.md"
            ]),
            "man/other-formats.md",
        ],
        "showcase.md",
        "Library" => Any[
            "Public" => "lib/public.md",
            "Internals" => map(
                s -> "lib/internals/$(s)",
                sort(readdir(joinpath(@__DIR__, "src/lib/internals")))
            ),
        ],
        "contributing.md",
    ],
    strict = !("strict=false" in ARGS),
    doctest = ("doctest=only" in ARGS) ? :only : true,
)

deploydocs(
    repo = "github.com/JuliaDocs/Documenter.jl.git",
    target = "build",
    push_preview = true,
)


text:
function serialize{T<:ArbFloat}(ser::AbstractSerializer, a::T)
    serialize_type(ser, T)
    write(ser.io, stringall(a))
end

function deserialize{T<:ArbFloat}(ser::AbstractSerializer, ::Type{T})
    T( read(ser.io, String) )
end


text:
module CassetteSpaceJam

using ..Ahorn, Maple

@mapdef Entity "brokemiahelper/cassetteDreamBlock" CassetteDreamBlock(x::Integer, y::Integer, width::Integer=8, height::Integer=8, fastMoving::Bool=false, oneUse::Bool=false, below::Bool=false, index::Integer=0, tempo::Number=1)

colorNames = Dict{String, Int}(
    "Blue" => 0,
    "Rose" => 1,
    "Bright Sun" => 2,
    "Malachite" => 3
)

const placements = Ahorn.PlacementDict(
    "Cassette Space Jam ($index - $color)" => Ahorn.EntityPlacement(
        CassetteDreamBlock,
        "rectangle",
        Dict{String, Any}(
            "index" => index,
        )
    ) for (color, index) in colorNames
)

Ahorn.nodeLimits(entity::CassetteDreamBlock) = 0, 1

Ahorn.minimumSize(entity::CassetteDreamBlock) = 8, 8
Ahorn.resizable(entity::CassetteDreamBlock) = true, true

Ahorn.editingOptions(entity::CassetteDreamBlock) = Dict{String, Any}(
    "index" => colorNames
)

#function Ahorn.render(ctx::Ahorn.Cairo.CairoContext, entity::CassetteCar, room::Maple.Room)
#     # x, y = Ahorn.position(entity)

#     # width = 64
#     # height = 18

#     # # index = Int(get(entity.data, "index", 0))
#     # # color = get(colors, index, defaultColor)

#     # Ahorn.drawRectangle(ctx, 0, 0, width, height, defaultColor, defaultColor)
#     x = Int(get(entity.data, "x", 0))
#     y = Int(get(entity.data, "y", 0))

#     width = Int(get(entity.data, "width", 32))
#     height = Int(get(entity.data, "height", 32))

     #Ahorn.drawRectangle(ctx, 0, 0, 64, 18, (0.0, 0.0, 1.0, 0.4), (0.0, 0.0, 1.0, 1.0))
#end

function Ahorn.selection(entity::CassetteDreamBlock)
    x, y = Ahorn.position(entity)

    width = Int(get(entity.data, "width", 8))
    height = Int(get(entity.data, "height", 8))

    nodes = get(entity.data, "nodes", ())
    if isempty(nodes)
        return Ahorn.Rectangle(x, y, width, height)

    else
        nx, ny = Int.(nodes[1])
        return [Ahorn.Rectangle(x, y, width, height), Ahorn.Rectangle(nx, ny, width, height)]
    end
end

colors = Dict{Integer, Ahorn.colorTupleType}(
    1 => (240, 73, 190, 255) ./ 255,
	2 => (252, 220, 58, 255) ./ 255,
	3 => (56, 224, 78, 255) ./ 255,
)

defaultColor = (73, 170, 240, 255) ./ 255

function renderCassetteSpaceJam(ctx::Ahorn.Cairo.CairoContext, x::Number, y::Number, width::Number, height::Number, index::Number)
    Ahorn.Cairo.save(ctx)

    Ahorn.set_antialias(ctx, 1)
    Ahorn.set_line_width(ctx, 1)

    color = get(colors, index, defaultColor)
    Ahorn.drawRectangle(ctx, x, y, width, height, (0.0, 0.0, 0.0, 0.4), color)

    Ahorn.restore(ctx)
end

function Ahorn.renderSelectedAbs(ctx::Ahorn.Cairo.CairoContext, entity::CassetteDreamBlock)
    x, y = Ahorn.position(entity)
    nodes = get(entity.data, "nodes", ())

    width = Int(get(entity.data, "width", 8))
    height = Int(get(entity.data, "height", 8))

    if !isempty(nodes)
        nx, ny = Int.(nodes[1])

        cox, coy = floor(Int, width / 2), floor(Int, height / 2)

        renderCassetteSpaceJam(ctx, nx, ny, width, height, Int(get(entity.data, "index", 0)))
        Ahorn.drawArrow(ctx, x + cox, y + coy, nx + cox, ny + coy, Ahorn.colors.selection_selected_fc, headLength=6)
    end
end

function Ahorn.render(ctx::Ahorn.Cairo.CairoContext, entity::CassetteDreamBlock, room::Maple.Room)
    x = Int(get(entity.data, "x", 0))
    y = Int(get(entity.data, "y", 0))

    width = Int(get(entity.data, "width", 32))
    height = Int(get(entity.data, "height", 32))

    renderCassetteSpaceJam(ctx, 0, 0, width, height, Int(get(entity.data, "index", 0)))
end

borderMultiplier = (0.9, 0.9, 0.9, 1)

end

max_stars_repo_name: Brokemia/BrokemiaHelper

max_issues_repo_name: Brokemia/BrokemiaHelper

max_forks_repo_name: Brokemia/BrokemiaHelper


text:
# This file contains the game configs!
# In debug builds, this file is reloaded in real-time -- every
# `config_reload_time_s` seconds, so you test changes on the fly just by
# modifying these constants.
# In release builds, this file is only included once, during compilation, so
# these values are all defined once as globals.

defaultFontName = "assets/fonts/FiraCode/ttf/FiraCode-Regular.ttf"
defaultFontSize = 26

minWinWidth = Int32(20)  # basically 0.
minWinHeight = Int32(425)  # Prevent getting any smaller than this.

# Menu constants
kPauseSceneTitleFontSize = 40
kPauseSceneSubtitleFontSize = 30

kMenuButtonFontSize = 20
kKeyButtonFontSize = 16
kMenuButtonColor = SDL2.Color(80, 80, 200, 255)
kKeySettingButtonColor = SDL2.Color(185, 55, 48, 255)

kControlsHeaderFontSize = 15

kProgrammedWithJuliaText = ["Programmed with ", " ", " using Julialang ", ""]
kCopyrightNotices = ["Copyright (c) 2018 @nhdalyMadeThis, LLC.",
                     "Theme music copyright http://www.freesfx.co.uk"]

# Game Rendering
kBackgroundColor = SDL2.Color(210,210,210,255)
kBallColor = SDL2.Color(58, 95, 204, 255)
kPaddleColor = SDL2.Color(203, 60, 51, 255)
kPaddleRenderH = 15

# Game constants
paddleSpeed = 1000
paddleTimeToMaxSpeed = 0.15
paddleTimeToDecelerate = 0.05
paddleAccel = paddleSpeed/paddleTimeToMaxSpeed
paddleDeccel = paddleSpeed/paddleTimeToDecelerate
ballSpeed = 350
ballWidth=10

winningScore = 11

# meta config
config_reload_time_s = 1  # seconds

max_stars_repo_name: NHDaly/pongClone

max_issues_repo_name: NHDaly/pongClone

max_forks_repo_name: NHDaly/pongClone


text:
using Distributed
using JLD2

@time @everywhere include("src/required.jl")

df = CSV.read("../data/ebd/processed/ebd_warblers_cut.csv"; header=true, delim="\t")

## Data preparation

# Prepare data
df = prepare_ebd_data(df)
# Select specific columns
select!(
    df,
    [
        :species,
        :groupIdentifier,
        :observerId,
        :samplingEventIdentifier,
        :observationCount,
        :year,
        :latitude,
        :longitude,
    ],
)

# Remove missing groups (unimportant for investigation)
df_gr = dropmissing(df, :groupIdentifier)
# Sort by group & species
df_gp = sort(df_gr, [:groupIdentifier, :species])

#### Investigate differences in group observations

# 1. Groups & observation counts
# 2. Groups & Dates

## 1. Groups & observation counts

# Get unique groups & species combinations
unique_groups = unique(df_gr, [:species, :groupIdentifier])
# Get unique groups, species & counts combinattions
unique_counts = unique(df_gr, [:species, :groupIdentifier, :observationCount])
# Compare number of observations
n_unique_groups = nrow(unique_groups)
n_unique_counts = nrow(unique_counts)
# unique_counts > unique groups, meaning groups don't always have same counts

# Get unique species & groups combination within unique_counts
unique_counts2 = unique(unique_counts, [:species, :groupIdentifier])
n_unique_counts2 = nrow(unique_counts2)
# Compare number of observations
unique_counts2 == unique_groups
# equal, confirms it's only groups that don't have the same count

# Get non unique observations
nonu = unique_counts[nonunique(unique_counts, [:species, :groupIdentifier]), :]
sort!(nonu, [:groupIdentifier, :species])
# Doesn't work!!!
# nonunique() returns rows equal to one of the previous rows, hence not the first duplicate

# Alternative way: using groupby for similar observations
grouped = groupby(unique_counts, [:groupIdentifier, :species])
# Get non unique observations as groups with > 1 observation
nonu_grouped = grouped[nrow.(grouped) .> 1]

nonu_grouped[1]
nonu_grouped[2]
nonu_grouped[3]
nonu_grouped[4]
nonu_grouped[100]
# Confirms observation counts are NOT the same per observer
# Also, observerId can come twice within a group obs.
# I guess one is the master observation and the 2nd second was edited to be the observer's personal count
# Hence, right thing to do seems to be: remove duplicates BUT select observation with highest count

## 2. Groups & dates

# Get unique groups, species & dates combinations
unique_dates = unique(df_gr, [:species, :groupIdentifier, :observationDate])
n_unique_dates = nrow(n_unique_dates)
# Compare number of observations
n_unique_dates - n_unique_groups

# Get non unique observations (groups-species-dates)
grouped_dates1 = groupby(unique_dates, [:groupIdentifier, :species])
nonu_grouped_dates1 = grouped_dates1[nrow.(grouped_dates1) .> 1]
# Reformat to dataframe & select columns
dates_test1 = vcat(nonu_grouped_dates1...)[[
    :species, :groupIdentifier, :observerId, :observationDate, :observationCount
]]
# Check results
nrow(dates_test1) # 7895 observations, 3940 groups
show(first(dates_test1, 100); allrows=true)
# Most differences seem to be minor or typos (± a few days), but sometimes the difference is a few years
# If the year is ever important for the analyses, those should be removed

# Get non unique dates from dataset of unique observation counts (less important, but was my initial intent)
grouped_dates = map(x -> unique(x, :observationDate), nonu_grouped)
nonu_grouped_dates = grouped_dates[nrow.(grouped_dates) .> 1]
nonu_grouped_dates[1] |>
DataFrame |>
x -> select(x, [:species, :groupIdentifier, :observerId, :observationDate])
nonu_grouped_dates[2] |>
DataFrame |>
x -> select(x, [:species, :groupIdentifier, :observerId, :observationDate])
nonu_grouped_dates[3] |>
DataFrame |>
x -> select(x, [:species, :groupIdentifier, :observerId, :observationDate])
nonu_grouped_dates[4] |>
DataFrame |>
x -> select(x, [:species, :groupIdentifier, :observerId, :observationDate])
test = vcat(nonu_grouped_dates...)[[
    :species, :groupIdentifier, :observerId, :observationDate
]]
nrow(test) # 146 observations, 73 groups
show(test; allrows=true)

max_stars_repo_name: gabrieldansereau/betadiversity-hotspots

max_issues_repo_name: gabrieldansereau/betadiversity-hotspots

max_forks_repo_name: gabrieldansereau/betadiversity-hotspots


text:
using Flux:onecold
using Test

@testset "onecold" begin
  a = [1, 2, 5, 3.]
  A = [1 20 5; 2 7 6; 3 9 10; 2 1 14]
  labels = ['A', 'B', 'C', 'D']

  @test onecold(a) == 3
  @test onecold(A) == [3, 1, 4]
  @test onecold(a, labels) == 'C'
  @test onecold(A, labels) == ['C', 'A', 'D']
end

@testset "onehotbatch indexing" begin
  y = Flux.onehotbatch(ones(3), 1:10)
  @test y[:,1] isa Flux.OneHotVector
  @test y[:,:] isa Flux.OneHotMatrix
end