text:
# TODO: cleanup type params. key type of partition must equal K
struct BoxFun{P<:AbstractBoxPartition,K,V}
    partition::P
    dict::Dict{K,V}
end

Base.length(fun::BoxFun) = length(fun.dict)

function Base.sum(f, boxfun::BoxFun{K,V}) where {K,V}
    return sum(let partition = boxfun.partition
        pair -> begin
            key, value = pair
            box = key_to_box(partition, key)
            return volume(box) * f(value)
        end
    end, boxfun.dict)
end

LinearAlgebra.norm(boxfun::BoxFun) = sqrt(sum(abs2, boxfun))

function LinearAlgebra.normalize!(boxfun::BoxFun)
    λ = inv(norm(boxfun))
    map!(x -> λ*x, values(boxfun.dict))
    return boxfun
end

import Base: ∘

function ∘(f, boxfun::BoxFun{P,K,V}) where {P,K,V}
    fV = typeof(f(first(values(boxfun.dict))))
    dict = Dict{K,fV}()
    sizehint!(dict, length(boxfun.dict))

    for (key, value) in boxfun.dict
        dict[key] = f(value)
    end

    return BoxFun(boxfun.partition, dict)
end


text:
using Clang
using LibExpat
using MacroTools

const workdir = @__DIR__
const srcdir = joinpath(workdir, "../src")
include(joinpath(workdir, "doc.jl"))

const gdalpath = "/usr/local/include"
const includedirs = [gdalpath]

const headerfiles = [
    "cpl_conv.h",
    "cpl_error.h",
    "cpl_minixml.h",
    "cpl_port.h",
    "cpl_progress.h",
    "cpl_virtualmem.h",
    "cpl_vsi.h",
    "gdal.h",
    "gdal_alg.h",
    "gdalwarper.h",
    "gdal_vrt.h",
    "gdal_utils.h",
    "ogr_api.h",
    "ogr_core.h",
    "ogr_srs_api.h"
]

const headerpaths = [joinpath(gdalpath, h) for h in headerfiles]

# skip these; right hand side is undefined
# usually because of Skipping MacroDefinition
const skip_expr = [
    :(const CPL_LSBPTR16 = x),
    :(const CPL_LSBPTR32 = x),
    :(const CPL_LSBPTR64 = x),
    :(const CPL_WARN_DEPRECATED_IF_GDAL_COMPILATION = x),
    :(const CPL_STATIC_ASSERT_IF_AVAILABLE = x),
    :(const CPLAssert = expr),
    :(const EMULATED_BOOL = bool),
    :(const GINT64_MIN = GINTBIG_MIN),
    :(const GINT64_MAX = GINTBIG_MAX),
    :(const GUINT64_MAX = GUINTBIG_MAX),
    :(const VSI_L_OFFSET_MAX = GUINTBIG_MAX),
    # function aliases give bootstrapping problems
    # since common.jl is loaded before the functions
    :(const CPLReadDir = VSIReadDir),
    :(const CPLFree = VSIFree),
    # common.jl internal bootstrapping problems
    :(const VALIDATE_POINTER_ERR = CE_Failure)
]

const skip_func = [:CPLErrorV] # problem with va_list ihnorton/Clang.jl#17

"indicates if the pair should be wrapped"
function header_wrapped(headerfile, cursorname)
    # don't wrap included files
    headerfile == cursorname
end

"skip certain expressions"
function minimal_rewriter(ex::Expr)
    # comment out predefined expressions
    if in(ex, skip_expr)
        return string("# ", ex)
    end

    # if it is a function
    if ex.head == :function
        funcname = ex.args[1].args[1]
        @show funcname
        # skip predefined functions
        if funcname in skip_func
            return ""
        end
        # add docstring
        fnode_vec = functionnode(et, string(funcname))
        docstr = if isempty(fnode_vec)
            # this happens for instance when Doxygen is suppressed such as
            # for this function OGR_G_Intersect from ogr_api.h
            # /*! @cond Doxygen_Suppress */
            # /* backward compatibility (non-standard methods) */
            # int    CPL_DLL OGR_G_Intersect( OGRGeometryH, OGRGeometryH ) CPL_WARN_DEPRECATED("Non standard method. Use OGR_G_Intersects() instead");
            ""
        else
            fnode = fnode_vec[1]
            build_docstring(fnode)
        end

        # turn it into a tuple, special cased in visr/Clang.jl
        # done because a block Expr gets a begin/end block when printing
        ex = (docstr, ex)
    elseif ex.head == :type
        # detect singletons: "mutable struct A end"
        # and rewrite them to "const A = Ptr{Void}"
        # without this rewrite they are not usable:
        # ERROR: ccall: the type of argument 1 doesn't correspond to a C type
        # see https://github.com/JuliaGeo/GDAL.jl/pull/41#discussion_r143345433
        # these singletons are meant to be opaque types, but in C are not of
        # type `typedef void *` but instead something like
        # `typedef struct OGRGeomFieldDefnHS *` (for OGRGeomFieldDefnH)

        # detect singletons by checking if any of the block args
        # are of type Symbol, which is the type of fieldnames
        # has_fields = any(el isa Symbol for el in ex.args[3].args)
        # if !has_fields
        #     typename = ex.args[2]::Symbol
        #     ex = :(const $typename = Ptr{Void})
        # end
        @capture(ex, mutable struct T_ fields__ end)
        if isempty(fields)
            ex = :(const $T = Ptr{Void})
        end
    end
    ex
end

minimal_rewriter(s::AbstractString) = s # keep comments
minimal_rewriter(A::Vector) = [minimal_rewriter(a) for a in A]

context=wrap_c.init(headers=headerpaths,
                    output_dir=joinpath(srcdir, "C"),
                    common_file="common.jl",
                    clang_includes=includedirs,
                    header_wrapped=header_wrapped,
                    header_library=x->"libgdal",
                    rewriter=minimal_rewriter)

run(context)

rm(joinpath(srcdir, "C/cpl_port.jl")) # delete empty file

ext: jl


text:
import Base: @deprecate, @deprecate_binding, depwarn

###
### v0.16 deprecations
###

### Changed in PR #844
@deprecate silence_errors(f::Function) f()

for name in names(API; all=true)
    if name ∉ names(HDF5; all=true) && startswith(uppercase(String(name)), "H")
        depmsg = ", use HDF5.API.$name instead."
        @eval Base.@deprecate_binding $name API.$name false $depmsg
    end
end

### Changed in PR #847
import Base: getindex, setindex!
@deprecate getindex(p::Properties, name::Symbol) Base.getproperty(p, name)
@deprecate setindex!(p::Properties, val, name::Symbol) Base.setproperty!(p, name, val)

# UTF8_LINK_PROPERTIES etc. used to be Refs, so required UTF8_LINK_PROPERTIES[] to set.
@deprecate getindex(p::Properties) p

function create_property(class; kwargs...)
    (oldname, newtype) =
        class == HDF5.API.H5P_OBJECT_CREATE    ? (:H5P_OBJECT_CREATE, ObjectCreateProperties) :
        class == HDF5.API.H5P_FILE_CREATE      ? (:H5P_FILE_CREATE, FileCreateProperties) :
        class == HDF5.API.H5P_FILE_ACCESS      ? (:H5P_FILE_ACCESS, FileAccessProperties) :
        class == HDF5.API.H5P_DATASET_CREATE   ? (:H5P_DATASET_CREATE, DatasetCreateProperties) :
        class == HDF5.API.H5P_DATASET_ACCESS   ? (:H5P_DATASET_ACCESS, DatasetAccessProperties) :
        class == HDF5.API.H5P_DATASET_XFER     ? (:H5P_DATASET_XFER, DatasetTransferProperties) :
        class == HDF5.API.H5P_FILE_MOUNT       ? (:H5P_FILE_MOUNT, FileMountProperties) :
        class == HDF5.API.H5P_GROUP_CREATE     ? (:H5P_GROUP_CREATE, GroupCreateProperties) :
        class == HDF5.API.H5P_GROUP_ACCESS     ? (:H5P_GROUP_ACCESS, GroupAccessProperties) :
        class == HDF5.API.H5P_DATATYPE_CREATE  ? (:H5P_DATATYPE_CREATE, DatatypeCreateProperties) :
        class == HDF5.API.H5P_DATATYPE_ACCESS  ? (:H5P_DATATYPE_ACCESS, DatatypeAccessProperties) :
        class == HDF5.API.H5P_STRING_CREATE    ? (:H5P_STRING_CREATE, StringCreateProperties) :
        class == HDF5.API.H5P_ATTRIBUTE_CREATE ? (:H5P_ATTRIBUTE_CREATE, AttributeCreateProperties) :
        class == HDF5.API.H5P_OBJECT_COPY      ? (:H5P_OBJECT_COPY, ObjectCopyProperties) :
        class == HDF5.API.H5P_LINK_CREATE      ? (:H5P_LINK_CREATE, LinkCreateProperties) :
        class == HDF5.API.H5P_LINK_ACCESS      ? (:H5P_LINK_ACCESS, LinkAccessProperties) :
        error("invalid class")
    Base.depwarn("`create_property(HDF5.$oldname; kwargs...)` has been deprecated, use `$newtype(;kwargs...)` instead.", :create_property)
    init!(newtype(;kwargs...))
end

@deprecate set_chunk(p::Properties, dims...) set_chunk!(p, dims) false
@deprecate get_userblock(p::Properties) p.userblock false

function set_shuffle!(p::Properties, ::Tuple{})
    depwarn("`shuffle=()` option is deprecated, use `shuffle=true`", :set_shuffle!)
    set_shuffle!(p, true)
end

# see src/properties.jl for the following deprecated keywords
# :compress
# :fapl_mpio
# :track_times

### Changed in PR #887
# see src/properties.jl for the following deprecated keyword
# :filter

### Changed in PR #902
import Base: append!, push!
import .Filters: UnknownFilter
@deprecate append!(filters::Filters.FilterPipeline, extra::NTuple{N, Integer}) where N append!(filters, [UnknownFilter(extra...)])
@deprecate push!(p::Filters.FilterPipeline, f::NTuple{N, Integer}) where N push!(p, UnknownFilter(f...))
@deprecate UnknownFilter(t::Tuple) UnknownFilter(t...) false


text:
using Revise; using BenchmarkTools; using FunctionWrappers:FunctionWrapper

function expgen(terms)
  unpacker = ""
  for i in 0:(2*terms)
    unpacker = string(unpacker, "a", i, ", ")
  end
  unpacker = string(unpacker, "= params")
  unpacker_expression = Meta.parse(unpacker)

  expsum = "a0"
  for i in 1:2:(2*terms)
    expsum = string(expsum, " + a", i, "*exp(-ti/a", i + 1, ")")
  end
  expsum_expression = Meta.parse(expsum)

  @eval return((t, params) -> ($unpacker_expression; [$expsum_expression for ti in t]) |> FunctionWrapper{Vector{Float64}, Tuple{Vector{Float64}, Vector{Float64}}})
end

macro expgen(terms::Int)
  unpacker = ""
  for i in 0:(2*terms)
    unpacker = string(unpacker, "a", i, ", ")
  end
  unpacker = string(unpacker, "= params")
  unpacker_expression = Meta.parse(unpacker)

  expsum = "a0"
  for i in 1:2:(2*terms)
    expsum = string(expsum, " + a", string(i), "*exp(-ti/a", string(i + 1), ")")
  end
  expsum_expression = Meta.parse(expsum)

  return quote (t, params) -> ($unpacker_expression; [$expsum_expression for ti in t]) |> FunctionWrapper{Vector{Float64}, Tuple{Vector{Float64}, Vector{Float64}}} end
end

fgen = expgen(1)
mgen = @expgen 1

times = Vector(0.0:0.1:10.0);
params = [1.0 for i in 1:21]

println("Benching individual functions")
println("Using @eval function")
@btime fgen($times, $params)
println("Using a macro")
@btime mgen($times, $params)
println("\nResults: exactly the same.")

function fexpgenLooped()
  for i in 1:10
    expgen(10)
  end
end

function mexpgenLooped()
  for z in 1:10
    @expgen 10
  end
end

println("\nBenching looped function generations")
println("Using @eval function")
@btime fexpgenLooped()
println("Using a macro")
@btime mexpgenLooped()
println("Macro much faster at generating the function, work is moved to compile time?")

max_stars_repo_name: JuliaRheology/RheoBenchAndTest

max_issues_repo_name: JuliaRheology/RheoBenchAndTest

max_forks_repo_name: JuliaRheology/RheoBenchAndTest


text:
"""
Parse MTG section

# Arguments
- `f::IOStream`: A buffered IO stream to the mtg file, *e.g.* `f = open(file, "r")`.
- `header::Array{String,1}`: A string defining the expected header for the class.
- `section::String`: The section name.
- `line::Array{Int64,1}`: The line number at which f is at the start of the funtion (mutated).
- `l::Array{String,1}`: the current line

# Note

The buffered IO stream (`f`) should start at the line of the section.

# Returns

The parsed section of the MTG

# Examples

```julia
file = joinpath(dirname(dirname(pathof(MultiScaleTreeGraph))),"test","files","simple_plant.mtg")
f = open(file, "r")
line = [0] ; l = [""]; l[1] = MultiScaleTreeGraph.next_line!(f,line)

while MultiScaleTreeGraph.issection(l[1]) || MultiScaleTreeGraph.issection(l[1],"CLASSES")
    l[1] = MultiScaleTreeGraph.next_line!(f,line)
end

classes = MultiScaleTreeGraph.parse_section!(f,["SYMBOL","SCALE","DECOMPOSITION","INDEXATION","DEFINITION"],"CLASSES",line,l)

close(f)
```
"""
function parse_section!(f, header, section, line, l;allow_empty = false)

    l[1] = next_line!(f, line)

    if length(l[1]) == 0
        allow_empty && return
        error("No header was found for MTG section `",section,"`. did you put an empty line in-between ",
        "the section name and its header?")
    end
    l_header = split(l[1], "\t")

    if l_header != header
        error("The header of the MTG `", section, "` section is different than:\n", join(header, "\t"))
    end

    l[1] = next_line!(f, line)

    if length(l[1]) == 0
        allow_empty && return
        error("Data not found in MTG section `",section,"`. Did you put empty lines between the header",
        " of the section and its header? If so, remove them before proceeding.")
    end

    # Return if the line is a new section
    issection(l[1]) && return

    section_l = strip.(split(l[1], "\t"))
    classes = [section_l]

    while (length(section_l) == length(header)) && !issection(l[1]) && !eof(f)
        l[1] = next_line!(f, line)

        # Read next line if the line is empty:
        if length(l[1]) == 0
            continue
        end

        # Break if line is a new section
        issection(l[1]) && break

        section_l = strip.(split(l[1], "\t"))
        append!(classes, [section_l])
    end

    DataFrame([classes[x][y] for x = 1:length(classes), y = 1:length(l_header)], l_header)
end


text:
struct WikiCorpus{S}
    path::S
end
WikiCorpus() = WikiCorpus(datadep"English WikiCorpus v1.0")

MultiResolutionIterators.levelname_map(::Type{WikiCorpus}) = [
    :doc=>1, :document=>1,
    :section=>2,
    :para=>3, :line=>3, :paragraph=>3,
    :sent=>4, :sentence=>4,
    :word=>5, :token=>5,
    :char=>6, :character=>6
    ]

"""
    push_nonempty!(xss, xs)

Helper method for pushing iterables to iterables of iterables.
Does a push! only if the second has some content.
"""
function push_nonempty!(xss, xs)
    if length(xs)>0
        push!(xss,xs)
    end
nothing
end

function load(corpus::WikiCorpus)
    UltimateType = Document{@NestedVector(String, 4), String}

    Channel(ctype=UltimateType, csize=4) do docs
        for file in readdir(glob"*Text*", corpus.path)
            open(file, enc"latin1") do fh
                local cur_doc_title
                cur_doc_content=@NestedVector(String, 4)() # content is for a given section
                no_cur_doc = true
                while(!eof(fh))
                    cur_section_content=@NestedVector(String, 3)()
                    section_text = strip(readuntil(fh, "\n\n"))
                    for line in split(section_text, "\n")
                        #@show line
                        # Line is sometimes paragraph, sometimes list entry. For now we are not capturing that detail
                        if length(line)==0 || line=="ENDOFARTICLE." || line=="</doc>"
                        #    println("*")
                            continue
                        end

                        # deal with maybe this starts a new doc
                        doc_title_match = match(r"<doc id=.* title=\"(.*)\" nonfiltered.*>", line,1)
                        if doc_title_match != nothing
                            # Output current doc
                            if !no_cur_doc
                                push_nonempty!(cur_doc_content, cur_section_content)
                                put!(docs, Document(cur_doc_title, cur_doc_content))
                            end

                            # Start a new Doc
                            # this means splitting a section's content
                            cur_doc_title = String(first(doc_title_match.captures))
                            cur_content = @NestedVector(String, 4)()
                            cur_section_content=@NestedVector(String, 3)()
                            no_cur_doc=false
                            continue # Nothing more to do with this line
                        end

                        # an actual paragraph (or whatever)
                        paragraph_of_sents = [intern.(tokenize(sent)) for sent in split_sentences(line)]
                        push_nonempty!(cur_section_content, paragraph_of_sents)
                    end # foreach line in section
                    push_nonempty!(cur_doc_content, cur_section_content)
                end # while(!eof)
                put!(docs, Document(cur_doc_title, cur_content))
            end # open
        end # for each file
    end # Channel
end


text:
function rollout!(solver::iLQRSolver2{T,Q,n}, α) where {T,Q,n}
    Z = solver.Z; Z̄ = solver.Z̄
    K = solver.K; d = solver.d;

    Z̄[1].z = [solver.x0; control(Z[1])]

    temp = 0.0
	δx = solver.S[end].x
	δu = solver.S[end].u

    for k = 1:solver.N-1
        δx .= state_diff(solver.model, state(Z̄[k]), state(Z[k]))
		δu .= d[k] .* α
		mul!(δu, K[k], δx, 1.0, 1.0)
        ū = control(Z[k]) + δu
        set_control!(Z̄[k], ū)

        # Z̄[k].z = [state(Z̄[k]); control(Z[k]) + δu]
        Z̄[k+1].z = [discrete_dynamics(Q, solver.model, Z̄[k]);
            control(Z[k+1])]

        temp = norm(Z̄[k+1].z)
        if temp > solver.opts.max_state_value
            return false
        end
    end
    return true
end

"Simulate the forward the dynamics open-loop"
function rollout!(solver::iLQRSolver)
    rollout!(solver.model, solver.Z, solver.x0)
    for k in eachindex(solver.Z)
        solver.Z̄[k].t = solver.Z[k].t
    end
end

function rollout!(model::AbstractModel, Z::Traj, x0)
    Z[1].z = [x0; control(Z[1])]
    for k = 2:length(Z)
        propagate_dynamics(DEFAULT_Q, model, Z[k], Z[k-1])
    end
end


text:
using JSON
using Printf
using ArgParse

function parse_commandline()
    s = ArgParseSettings()
    @add_arg_table! s begin
        "--str-func"
            help     = "function on string: choose \"split\" (word-based) or \"identity\" (character-based)"
            arg_type = String
            default  = "identity"
        "prediction"
            help = "the .json file containing the prediction"
            required = true
        "groundTruth"
            help = "the .json file containing the ground truth"
            required = true
    end
    return parse_args(s)
end

const MetricType0 = NamedTuple{(:tp, :fp, :fn), Tuple{Float64, Float64, Float64}}
const MetricType1 = Tuple{Union{T, Vector{T}}, MetricType0} where {T <: AbstractString}
const MetricType2 = Tuple{Vector, MetricType0}
const MetricType3 = Tuple{T, MetricType0} where {T <: AbstractDict{String,Any}}

function countTpFpFn(predicted::T, actual::T, strFunc)::MetricType3 where {T <: AbstractDict{String,Any}}

    # 1) this function is the entry point

    emptyDict   = T()
    emptyVector = Vector{T}()
    emptyString = strFunc == split ? SubString{String}[] : ""
    xs = countTpFpFn(predicted, actual, strFunc, emptyDict, emptyVector, emptyString)
    xx = xs |> first
    tp, fp, fn = xs |> last

    return xx, (tp=tp, fp=fp, fn=fn)
end

function countTpFpFn(predicted::T, actual::T, strFunc, emptyDict::T, emptyVector::Vector{T}, emptyString::Union{S, Vector{S}})::MetricType3 where {T <: AbstractDict{String,Any}, S <: AbstractString}

    # 2) for comparing between Dictionaries,

    # keys in both predicted and actual
    commonKeys = intersect(keys(predicted), keys(actual))

    # keys in predicted only
    predictedOnly = setdiff(keys(predicted), commonKeys)

    # keys in actual only
    actualOnly = setdiff(keys(actual), commonKeys)

    tp = 0
    fp = 0
    fn = 0

    xs = T()
    for k in commonKeys
        if typeof(predicted[k]) <: AbstractString
            if actual[k] == nothing
                actual[k] = emptyString
            end
            xs[k] = countTpFpFn(strFunc(predicted[k]), strFunc(actual[k]), strFunc, emptyDict, emptyVector, emptyString)
        else
            if actual[k] == nothing
                actual[k] = T[]
            else
                actual[k] = convert(Vector{T}, actual[k])
            end
            xs[k] = countTpFpFn(convert(Vector{T}, predicted[k]), actual[k], strFunc, emptyDict, emptyVector, emptyString)
        end

        tp_, fp_, fn_ = xs[k] |> last
        tp += tp_
        fp += fp_
        fn += fn_
    end

    for k in predictedOnly
        if typeof(predicted[k]) <: AbstractString
            xs[k] = countTpFpFn(strFunc(predicted[k]), emptyString, strFunc, emptyDict, emptyVector, emptyString)
        else
            xs[k] = countTpFpFn(convert(Vector{T}, predicted[k]), emptyVector, strFunc, emptyDict, emptyVector, emptyString)
        end
        tp_, fp_, fn_ = xs[k] |> last
        @assert tp_ == 0
        @assert fn_ == 0
        fp += fp_
    end

    for k in actualOnly
        if actual[k] == nothing
            continue
        end

        if typeof(actual[k]) <: Vector{T} where {T <: Any}
            xs[k] = countTpFpFn(emptyVector, convert(Vector{T}, actual[k]), strFunc, emptyDict, emptyVector, emptyString)
        else
            xs[k] = countTpFpFn(emptyString, strFunc(actual[k]), strFunc, emptyDict, emptyVector, emptyString)
        end

        tp_, fp_, fn_ = xs[k] |> last
        @assert tp_ == 0
        @assert fp_ == 0
        fn += fn_
    end

    return xs, (tp=tp, fp=fp, fn=fn)
end

function countTpFpFn(predicted::Vector{T}, actual::Vector{T}, strFunc, emptyDict::T, emptyVector::Vector{T}, emptyString::Union{S, Vector{S}})::MetricType2 where {T <: AbstractDict{String,Any}, S <: AbstractString}

    # 3) entry function for comparing between line items
    memo = Dict{NTuple{2,Int}, MetricType2}()
    xs = countTpFpFn(predicted, actual, strFunc, emptyDict, emptyVector, emptyString, 1, 1, memo)
    return xs
end

function countTpFpFn(predicted::Vector{T}, actual::Vector{T}, strFunc, emptyDict::T, emptyVector::Vector{T}, emptyString::Union{S, Vector{S}}, i::Int, j::Int, memo::Dict{NTuple{2,Int}, MetricType2})::MetricType2 where {T <: AbstractDict{String,Any}, S <: AbstractString}

    # 4) line item edit distance

    if haskey(memo, (i,j))
        return memo[i,j]
    end

    xs = Vector() # a vector of tuple
    tp, fp, fn = 0, 0, 0
    # the base cases
    if i > length(predicted)
        # calculate the false negatives for remaining actual
        for k=j:length(actual)
            xx = countTpFpFn(emptyDict, actual[k], strFunc, emptyDict, emptyVector, emptyString)
            tp_, fp_, fn_ = xx |> last
            @assert tp_ == 0
            @assert fp_ == 0
            fn += fn_
            push!(xs, xx)
        end

        memo[i,j] = (xs, (tp=tp, fp=fp, fn=fn))
        return memo[i,j]
    end

    if j > length(actual)
        for k=i:length(predicted)
            xx = countTpFpFn(predicted[k], emptyDict, strFunc, emptyDict, emptyVector, emptyString)
            tp_, fp_, fn_ = xx |> last
            @assert tp_ == 0
            @assert fn_ == 0
            fp += fp_
            push!(xs, xx)
        end
        memo[i,j] = (xs, (tp=tp, fp=fp, fn=fn))
        return memo[i,j]
    end

    # choose to do 3 things
    # 1) compare predicted[i] and actual[j]
    xx = countTpFpFn(predicted[i], actual[j], strFunc, emptyDict, emptyVector, emptyString) # MetricType3
    yy = countTpFpFn(predicted, actual, strFunc, emptyDict, emptyVector, emptyString, i+1, j+1, memo) # MetricType2

    tp_xx, fp_xx, fn_xx = xx |> last
    tp_yy, fp_yy, fn_yy = yy |> last
    dist1 = fp_xx + fp_yy + fn_xx + fn_yy

    # 2) delete predicted[i], a.k.a. increment i
    x_ = countTpFpFn(predicted[i], emptyDict, strFunc, emptyDict, emptyVector, emptyString)
    xy = countTpFpFn(predicted, actual, strFunc, emptyDict, emptyVector, emptyString, i+1, j, memo) # MetricType2

    tp_x_, fp_x_, fn_x_ = x_ |> last
    @assert tp_x_ == 0
    @assert fn_x_ == 0
    tp_xy, fp_xy, fn_xy = xy |> last
    dist2 = fp_x_ + fp_xy + fn_x_ + fn_xy

    # 3) delete actual[j], a.k.a. increment j
    _x = countTpFpFn(emptyDict, actual[j], strFunc, emptyDict, emptyVector, emptyString)
    yx = countTpFpFn(predicted, actual, strFunc, emptyDict, emptyVector, emptyString, i, j+1, memo) # MetricType2

    tp__x, fp__x, fn__x = _x |> last
    @assert tp__x == 0
    @assert fp__x == 0
    tp_yx, fp_yx, fn_yx = yx |> last
    dist3 = fp__x + fp_yx + fn__x + fn_yx

    # choose the minimum path
    if dist1 <= dist2 && dist1 <= dist3
        # choose this
        tp = tp_xx + tp_yy
        fp = fp_xx + fp_yy
        fn = fn_xx + fn_yy

        push!(xs, xx)
        if length(yy[1]) > 0
            append!(xs, yy[1])
        end
    elseif dist2 <= dist1 && dist2 <= dist3
        tp = tp_x_ + tp_xy
        fp = fp_x_ + fp_xy
        fn = fn_x_ + fn_xy
        push!(xs, x_)
        if length(xy[1]) > 0
            append!(xs, xy[1])
        end
    else
        @assert dist3 <= dist1 && dist3 <= dist2
        tp = tp__x + tp_yx
        fp = fp__x + fp_yx
        fn = fn__x + fn_yx
        push!(xs, _x)
        if length(yx[1]) > 0
            append!(xs, yx[1])
        end
    end

    memo[i,j] = (xs, (tp=tp, fp=fp, fn=fn))
    return memo[i,j]
end

function countTpFpFn(predicted::Union{S, Vector{S}}, actual::Union{S, Vector{S}}, strFunc,
    emptyDict::T, emptyVector::Vector{T}, emptyString::Union{S, Vector{S}})::MetricType1 where {S <: AbstractString, T <: AbstractDict{String,Any}}

    # 5) for comparing between primitives

    memo = Dict{NTuple{2,Int}, NTuple{3,Int}}()
    equality = (x, y)->x==y
    tp, fp, fn = countTpFpFn(predicted, actual, firstindex(predicted), firstindex(actual), lastindex(predicted), lastindex(actual), memo; equality=equality)

    return (predicted, (tp=tp, fp=fp, fn=fn))
end

function countTpFpFn(predicted::T, actual::T, i::Int, j::Int, I::Int, J::Int,
    memo::Dict{NTuple{2,Int}, NTuple{3,Int}}; equality=(x, y) -> x==y)::NTuple{3,Int} where T
    # TP: number of characters correct
    # FP: number of characters deleted in predicted
    # FN: number of characters inserted in predicted
    # return (TP, FP, FN)

    # 6) Field-level edit distance

    if haskey(memo, (i,j))
        return memo[i,j]
    end

    if i > I
        # delete the rest of the characters in actual
        # equivalent to insertion of characters in predicted
        return 0, 0, length(actual[j:end])
    end

    if j > J
        # delete the rest of the characters in predicted
        return 0, length(predicted[i:end]), 0
    end

    if equality(predicted[i], actual[j]) # this needs to be defined for the type
        tp, fp, fn = countTpFpFn(predicted, actual, nextind(predicted, i), nextind(actual, j), I, J, memo; equality=equality)
        tp += 1
        memo[i,j] = (tp, fp, fn)
    else
        # delete i-th character in predicted
        tp₁, fp₁, fn₁ = countTpFpFn(predicted, actual, nextind(predicted, i), j, I, J, memo; equality=equality)
        fp₁ += 1

        # insert character at i-th for predicted,
        # equivalent to deletion of jth at actual
        tp₂, fp₂, fn₂ = countTpFpFn(predicted, actual, i, nextind(actual, j), I, J, memo; equality=equality)
        fn₂ += 1

        # choose the smaller of the two
        if (fp₁ + fn₁) < (fp₂ + fn₂)
            memo[i,j] = (tp₁, fp₁, fn₁)
        else
            memo[i,j] = (tp₂, fp₂, fn₂)
        end
    end
    return memo[i,j]
end

function main()
    args = parse_commandline()
    if args["str-func"] == "identity"
        strFunc = identity
    elseif args["str-func"] == "split"
        strFunc = split
    else
        throw("--str-func is neither split nor identity")
    end

    predictionJs  = JSON.parsefile(args["prediction"])
    groundTruthJs = JSON.parsefile(args["groundTruth"])

    tpFpFn = countTpFpFn(predictionJs, groundTruthJs, strFunc)

    JSON.print(tpFpFn, 4)

    tp, fp, fn = tpFpFn |> last

    precision = tp/(tp + fp)
    recall = tp/(tp + fn)
    f1 = 2 * precision * recall / (precision + recall)
    @printf("TP = %d, FP = %d, FN = %d, Precision = %.4g, Recall = %.4g, F₁ = %.4g\n", tp, fp, fn, precision, recall, f1)
end

main()

max_stars_repo_name: andreajparker/datasets

max_issues_repo_name: andreajparker/datasets

max_forks_repo_name: andreajparker/datasets


text:
using POLO
using Test
using LinearAlgebra

@info "Loading loss functions..."
loss_functions = [("Least squares", Loss.LeastSquares(100,10)),
                  ("Log loss", Loss.LogLoss(100,10))]

solvers = [("Gradient descent", POLO.GradientDescent()),
           ("Momentum descent", POLO.MomentumDescent()),
           ("Nesterov descent", POLO.NesterovDescent()),
           ("Adagrad", POLO.AdaGrad()),
           ("Adadelta", POLO.AdaDelta()),
           ("BB", POLO.BB())]

@info "Loss functions loaded. Starting test sequence."
@testset "$sname Solver on loss function $name" for (sname,solver) in solvers, (name,loss) in loss_functions
    solver(rand(Loss.nfeatures(loss)), loss, Utility.GradientNorm(1e-6), Utility.None())
    g = copy(getx!(solver))
    Loss.loss!(loss, getx(solver), g)
    @test norm(g) <= 1e-6
end

macro TestPOLOSolver(alg)
    serial = Symbol(alg,:_s)
    algname = Symbol(titlecase(string(alg)))
    quote
        solvername = string($(Meta.quot(algname)))
        @testset "POLO Solvers: $solvername on loss function $name" for (name, loss) in loss_functions
            solver = POLOSolvers.$algname()
            solver(rand(Loss.nfeatures(loss)), loss, Utility.GradientNorm(1e-6), Utility.None())
            g = copy(solver.x)
            Loss.loss!(loss, solver.x, g)
            @test norm(g) <= 1e-6
        end
    end
end

@TestPOLOSolver(gradient)
@TestPOLOSolver(momentum)
@TestPOLOSolver(nesterov)
@TestPOLOSolver(adagrad)
@TestPOLOSolver(adadelta)

include("/usr/share/julia/test/testenv.jl")
push!(test_exeflags.exec,"--color=yes")
cmd = `env JULIA_NUM_THREADS=4 $test_exename $test_exeflags run_shm_tests.jl`

if !success(pipeline(cmd; stdout=stdout, stderr=stderr)) && ccall(:jl_running_on_valgrind,Cint,()) == 0
    error("Distributed test failed, cmd : $cmd")
end


text:
using StructJuMP, JuMP
using StructJuMPSolverInterface

include("select_solver.jl")

#############
# A sample model
#############
scen = 2
m = StructuredModel(num_scenarios=scen)
@variable(m, x[1:2])
@NLobjective(m, Min, x[1]^2+x[2]^2)
@NLconstraint(m, x[1] * x[2] <= 100)

for i in 1:scen
    bl = StructuredModel(parent=m)
    @variable(bl, y)
    @NLconstraint(bl, x[2]*x[1] + x[1]*y <= 10)
    @NLobjective(bl, Min, y^2)
end

structJuMPSolve(m)

getVarValue(m)

#verification
# x[1]^2+x[2]^2 + y1^2 + y2^2

# x[1]*x[2]

# x[2]*x[1] + x[1]*y1

# x[2]*x[1] + x[1]*y2


text:
function monitor(net::Network, variables, n_ofsteps::Int64;
				 every = 1)
	res = Dict()
	pointers = Dict()
	ranges = Dict()
	for (key, variable, range) in variables
		res[key] = []
		pointers[key] = variable
		ranges[key] = range
	end
	for t in 1:n_ofsteps
		updatenet!(net)
		if t % every == 0
			for (key, value) in res
				push!(value, pointers[key][ranges[key]])
			end
		end
	end
	for (key, value) in res
		res[key] = hcat(value...)'
	end
	res
end
export monitor

max_stars_repo_name: jbrea/NetSim

max_issues_repo_name: jbrea/NetSim

max_forks_repo_name: jbrea/NetSim


text:
# Julia wrapper for header: /Volumes/BOOTCAMP/programs/compa_libs/iup/include/iuptuio.h
# Automatically generated using Clang.jl wrap_c, version 0.0.0

function IupTuioOpen()
  ccall( (:IupTuioOpen, iup), Cint, (), )
end
function IupTuioClient()
  ccall( (:IupTuioClient, iup), Ptr{Cint}, (), )
end


text:
using PSCOPF

using Test
using Dates
using DataStructures

@testset verbose=true "example_usecase" begin

    #=
    S: [S1,S2]
    TS: [11h, 11h30]
                      FO
          EM   EM   EM |
    ECH: [7h, 7h30, 10h, ...]
                            bus 1                   bus 2
                               |                      |
                               |                      |
           TS1         TS2     |                      |       TS1         TS2
                               |                      |
        load-------------------|                      |----load
    S1:   ,   ,   |   ,   ,    |                      |S1:   ,   ,   |   ,   ,
    S2:   ,   ,   |   ,   ,    |                      |S2:   ,   ,   |   ,   ,
                               |                      |
                               |                      |
        wind_1-----------------|                      |----wind_2
             0, 200            |                      |         0, 200
             0, 10             |                      |         0, 11
        15mins, 15mins         |                      |    15mins, 15mins
    S1:   ,   ,   |   ,   ,    |                      |S1:   ,   ,   |   ,   ,
    S2:   ,   ,   |   ,   ,    |                      |S2:   ,   ,   |   ,   ,
                               |                      |
                               |                      |
        ccg_1------------------|                      |----ccg_2
           150, 600            |                      |       100, 600
           45k, 30             |                      |       50k, 20
            4h, 15mins         |                      |        4h, 15mins
                               |                      |
                               |                      |
        tac_1------------------|                      |
            10, 300            |                      |
           12k, 100            |                      |
        30mins, 15mins         |                      |
                               |                      |

    =#
    @testset "example_small_usecase" begin
        out_path = joinpath(@__DIR__, "..", "..", "default_out", "example_small_usecase")
        rm(out_path, recursive=true, force=true)

        ECH = [DateTime("2015-01-01T07:00:00"), DateTime("2015-01-01T07:30:00"),DateTime("2015-01-01T10:00:00")]
        TS = [DateTime("2015-01-01T11:00:00"), DateTime("2015-01-01T11:30:00")]

        network = PSCOPF.Networks.Network()
        # Buses
        PSCOPF.Networks.add_new_bus!(network, "bus_1")
        PSCOPF.Networks.add_new_bus!(network, "bus_2")
        # Branches
        PSCOPF.Networks.add_new_branch!(network, "branch_1_2", 500.);
        # PTDF
        PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_1", 0.5)
        PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_2", -0.5)
        #Generators - Limitables
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "wind_1", PSCOPF.Networks.LIMITABLE,
                                                0., 200.,
                                                0., 10.,
                                                Dates.Second(15*60), Dates.Second(15*60)) #dmo, dp : always 0. ?
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "wind_2", PSCOPF.Networks.LIMITABLE,
                                                0., 200.,
                                                0., 11.,
                                                Dates.Second(15*60), Dates.Second(15*60))
        #Generators - Pilotables
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "ccg_1", PSCOPF.Networks.PILOTABLE,
                                                150., 600., #pmin, pmax
                                                45000., 30., #start_cost, prop_cost
                                                Dates.Second(4*3600), Dates.Second(15*60)) #dmo, dp
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "tac_1", PSCOPF.Networks.PILOTABLE,
                                                10., 300.,
                                                12000., 100.,
                                                Dates.Second(30*60), Dates.Second(15*60))
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "ccg_2", PSCOPF.Networks.PILOTABLE,
                                                100., 600., #pmin, pmax
                                                50000., 20., #start_cost, prop_cost
                                                Dates.Second(4*3600), Dates.Second(15*60)) #dmo, dp

        uncertainties = PSCOPF.Uncertainties()
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_1", TS[1], "S1", 610.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_1", TS[2], "S1", 590.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_2", TS[1], "S1", 110.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_2", TS[2], "S1", 100.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_1", TS[1], "S1", 49.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_1", TS[2], "S1", 50.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_2", TS[1], "S1", 78.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_2", TS[2], "S1", 80.)
        #S2
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_1", TS[1], "S2", 640.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_1", TS[2], "S2", 615.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_2", TS[1], "S2", 110.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "bus_2", TS[2], "S2", 105.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_1", TS[1], "S2", 18.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_1", TS[2], "S2", 23.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_2", TS[1], "S2", 62.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[1], "wind_2", TS[2], "S2", 58.)
        # Load increased, wind prod decreased => start tac_1 (not firm)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_1", TS[1], "S1", 630.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_1", TS[2], "S1", 600.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_2", TS[1], "S1", 100.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_2", TS[2], "S1", 110.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_1", TS[1], "S1", 40.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_1", TS[2], "S1", 41.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_2", TS[1], "S1", 81.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_2", TS[2], "S1", 78.)
        #S2
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_1", TS[1], "S2", 620.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_1", TS[2], "S2", 610.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_2", TS[1], "S2", 120.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "bus_2", TS[2], "S2", 110.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_1", TS[1], "S2", 35.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_1", TS[2], "S2", 31.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_2", TS[1], "S2", 60.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[2], "wind_2", TS[2], "S2", 62.)
        # Finally the increase is not that high => don't start tac_1
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_1", TS[1], "S1", 615.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_1", TS[2], "S1", 613.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_2", TS[1], "S1", 100.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_2", TS[2], "S1", 97.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_1", TS[1], "S1", 42.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_1", TS[2], "S1", 40.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_2", TS[1], "S1", 78.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_2", TS[2], "S1", 75.)
        #S2
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_1", TS[1], "S2", 620.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_1", TS[2], "S2", 610.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_2", TS[1], "S2", 95.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "bus_2", TS[2], "S2", 90.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_1", TS[1], "S2", 53.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_1", TS[2], "S2", 50.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_2", TS[1], "S2", 76.)
        PSCOPF.add_uncertainty!(uncertainties, ECH[3], "wind_2", TS[2], "S2", 75.)

        # initial generators state
        generators_init_state = SortedDict(
                        "ccg_1" => PSCOPF.OFF,
                        "ccg_2" => PSCOPF.OFF,
                        "tac_1" => PSCOPF.OFF,
                    )
        mode = PSCOPF.PSCOPF_MODE_1
        sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)
        exec_context = PSCOPF.PSCOPFContext(network, TS, mode, generators_init_state,
                                            uncertainties, nothing, out_path)
        PSCOPF.run!(exec_context, sequence)
    end

    println("\n\n\n\n\n")

    @testset "example_2bus" begin
        network = PSCOPF.Networks.Network()

        # Buses
        PSCOPF.Networks.add_new_bus!(network, "bus_1")
        PSCOPF.Networks.add_new_bus!(network, "bus_2")

        # Branches
        PSCOPF.Networks.add_new_branch!(network, "branch_1_2", 500.);

        # PTDF
        PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_1", 0.5)
        PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_2", -0.5)
        #Alternatively,
        # network.ptdf = SortedDict{String,SortedDict{String, Float64}}(
        #                         "branch_1_2" => SortedDict{String, Float64}(
        #                             "bus_1" => 0.5,
        #                             "bus_2" => -0.5,
        #                             ),
        #                         )

        # Generators
        #Limitables
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "wind_1_0", PSCOPF.Networks.LIMITABLE,
                                                0., 0., #pmin, pmax : Not concerned ? min is always 0, max is the limitation
                                                0., 10., #start_cost, prop_cost : start cost is always 0 ?
                                                Dates.Second(0), Dates.Second(0)) #dmo, dp : always 0. ?
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "wind_2_0", PSCOPF.Networks.LIMITABLE,
                                                0., 0.,
                                                0., 11.,
                                                Dates.Second(0), Dates.Second(0))
        #Pilotables
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "ccg_1_0", PSCOPF.Networks.PILOTABLE,
                                                10., 200., #pmin, pmax
                                                45000., 30., #start_cost, prop_cost
                                                Dates.Second(4*3600), Dates.Second(15*60)) #dmo, dp
        PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "tac_2_0", PSCOPF.Networks.PILOTABLE,
                                                10., 200.,
                                                12000., 120.,
                                                Dates.Second(30*60), Dates.Second(15*60))

        # initial generators state
        generators_init_state = SortedDict(
                        "ccg_1_0" => PSCOPF.ON,
                        "tac_2_0" => PSCOPF.OFF,
                    )

        # Uncertainties
        # uncertainties = PSCOPF.PSCOPFio.read_uncertainties(data_folder) #filename must be : pscopf_uncertainties.txt
        uncertainties = PSCOPF.Uncertainties(
                                DateTime("2015-01-01T07:00:00") => SortedDict(
                                        "wind_1_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "wind_2_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_1" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_2" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                ),
                                DateTime("2015-01-01T10:00:00") => SortedDict(
                                        "wind_1_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "wind_2_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_1" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_2" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                ),
                                DateTime("2015-01-01T10:30:00") => SortedDict(
                                        "wind_1_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "wind_2_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_1" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_2" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                ),
                                DateTime("2015-01-01T10:45:00") => SortedDict(
                                        "wind_1_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "wind_2_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_1" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_2" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                ),
                                DateTime("2015-01-01T11:00:00") => SortedDict(
                                        "wind_1_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "wind_2_0" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_1" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                        "bus_2" => SortedDict(
                                            DateTime("2015-01-01T11:00:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:15:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:30:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                            DateTime("2015-01-01T11:45:00") => SortedDict(
                                                                                "S1" => 0.,
                                                                                "S2" => 0.,
                                                                                ),
                                        ),
                                ),
        )
        #Alternatively, PSCOPF.add_uncertainty!(uncertainties, ech, nodal_injection_name, ts, scenario_name, value)

        # Timesteps
        TS = [DateTime("2015-01-01T11:00:00"),
                DateTime("2015-01-01T11:15:00"),
                DateTime("2015-01-01T11:30:00"),
                DateTime("2015-01-01T11:45:00")]

        mode = PSCOPF.PSCOPF_MODE_1
        ECH = PSCOPF.generate_ech(network, TS, mode)

        sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)

        exec_context = PSCOPF.PSCOPFContext(network, TS, mode, generators_init_state, uncertainties, nothing)
        PSCOPF.run!(exec_context, sequence)

        market_schedule = PSCOPF.get_market_schedule(exec_context)
        tso_schedule = PSCOPF.get_tso_schedule(exec_context)
    end

end

max_stars_repo_name: klorel/pscopf

max_issues_repo_name: klorel/pscopf

max_forks_repo_name: klorel/pscopf


text:
using FTCTests  # reexport FaulTolerantControl
using Transducers
using Random
using Test

"""
This is used for 2nd-year report.
"""
function run_multiple_sim_2nd_year_report(N=1;
        manoeuvres=[:hovering, :forward],
        methods=[:adaptive, :adaptive2optim],
        t0=0.0, tf=20.0,
        h_threshold=5.0,  # m (nothing: no constraint)
        actual_time_limit=60.0,  # s
        N_thread=Threads.nthreads(),
        collector=Transducers.tcollect, will_plot=false, seed=2021,
    )
    fault_time = 0.0
    _fault_list_single = [[
                          [LoE(fault_time, index, effectiveness)]
                          for index in 1:6
                          for effectiveness in [0.9, 0.5, 0.1, 0.0]
                         ]...]
    _fault_list_double = [[
                          [LoE(fault_time, index, effectiveness),
                           LoE(fault_time, index2, effectiveness)]
                          for index in 1:5
                          for index2 in index+1:6
                          for effectiveness in [0.9, 0.5, 0.1, 0.0]
                         ]...]
    _fault_list_single_failure_single_fault = [[
                                               [LoE(fault_time, index, 0.0),
                                                LoE(fault_time, index2, effectiveness)]
                                               for index in 1:6
                                               for index2 in filter(x -> x != index, 1:6)
                                               for effectiveness in [0.9, 0.5, 0.1]
                                              ]...]
    _fault_list = [_fault_list_single..., _fault_list_double..., _fault_list_single_failure_single_fault...]
    _faults = 1:N |> Map(i -> rand(_fault_list)) |> collect  # randomly sampled N faults
    θs = [[0, 0, -10.0]]  # Control points of Bezier curve; constant position tracking
    θs_array = 1:N |> Map(i -> θs) |> collect
    run_multiple_sim(N,
                     manoeuvres,
                     methods,
                     _faults,
                     θs_array,
                     t0, tf,
                     h_threshold, actual_time_limit, N_thread, collector,
                     will_plot, seed
                    )
end


text:
set_floor!(env.vis, z=0.03)

# ## Animation
z = [minimal_to_maximal(env.mechanism, x) for x in traj]
z = [[z[1] for t = 1:40]..., z..., [z[end] for t = 1:40]...]
T = length(z)
anim = MeshCat.Animation(convert(Int, floor(1.0 / env.mechanism.timestep)))
build_robot(env.vis, env.mechanism, color=magenta)
for t = 1:T
    MeshCat.atframe(anim, t) do
        set_robot(env.vis, env.mechanism, z[t])
    end
end
MeshCat.setanimation!(env.vis, anim)

# ## Ghost
env = get_environment(:halfcheetah, timestep=0.05)
open(env.vis)
setvisible!(env.vis[:robot], false)
timesteps = [1, 50, 60, 70, 80, 90, 100, 108, T]
for t in timesteps
    name = Symbol("robot_$t")
    color = (t == T ? magenta : magenta_light)
    build_robot(env.vis, env.mechanism, color=color, name=name)
    set_robot(env.vis, env.mechanism, z[t], name=name)
end

# ## Save/Load policy
# θ = policy.θ
# @save joinpath(@__DIR__, "halfcheetah_policy.jld2") θ
@load joinpath(@__DIR__, "halfcheetah_policy.jld2") θ

## test random policy
env = get_environment(:halfcheetah, representation=:minimal, g=-9.81, timestep=0.05)
initialize!(env.mechanism, :halfcheetah)
open(env.vis)
# storage = simulate!(env.mechanism, 1.0, record=true, verbose=false)
# visualize(env.mechanism, storage, vis=env.vis, show_contact=true)

reset(env)
render(env)
x0 = get_minimal_state(env.mechanism)

for i = 1:100
    u = rand(Uniform(-1.0, 1.0), env.num_inputs)
    x0, r, _ = step(env, x0, u)
    @show r
    render(env)
end


text:
using UnPack

struct Foo{N}
    xyz::NTuple{N,T} where {T}
end

function Base.getproperty(foo::Foo,s::Symbol)
    if s==:x
        return getproperty(foo, :xyz)[1]
    else
        return getfield(foo, s)
    end
end

foo = Foo((1.0,2.0,3.0))

function bar(foo)
    @unpack x = foo
    return x
end

bar(foo)

# checking type stability of Base.getproperties
N = 3
K1D = 2
rd = RefElemData(Tri(),N)
VX,VY,EToV = uniform_mesh(Tri(),K1D)
md = MeshData(VX,VY,EToV,rd)
function foo(rd::RefElemData,md::MeshData)
    @unpack Dr = rd
    @unpack x,rxJ = md
    return x,Dr,rxJ
end


text:
using Test
module constants
#MathConstants.
global const ϕ = nothing
global const ∞ = Inf
global const -∞ = -Inf

end
#end
#Ε(x) = exp(x)
#error function  ϵ (generic , cherry-picked, smooth)
#global ϵ(x) =  exp(x)^-1

#end
#end

#--- Testing area #craziness allowed
module accurateDigits
#digits accuracy
"""
```inputs:
y: error function
returns number of accurate (at least)
TODO: How to test this test?

Yeilds an error ϵ with
at least the number of Digits required (or better)

note: this is not a linear function
it exponentially goes to z

the logic:using the rule
"if an error has a digit 0, that means the digit
    (associated with this error) is accurate"

a homebrewed way, fabricated out of the spaciousness  #novel
#TODO: How to verify this is correct? #need for a verification method
"""
global ϵ(x) = exp(x)^-1 ##beging with a monotonically decreasomg (survival) funtion- that is smooth  -this enforces a constantly increasing
#=
function valuate(value)
    if value isa Inf value = ∞
    elseif value isa -Inf value = -∞
    elseif value isa nothing || Nothing ||"" value = ϕ

    return value
end
=#

using Test, Plots
#digi(n::Int64, x::Int64, y::Float64
function digi(;n = 6, x = 1) # y = exp(x))
    print(typeof(x))
    return ϵ(log2(n) * x)
end

digi(n=1)
digi(n=0)

digi(n=10^6)
digi(n=10^10)

end
#digi

#=
#diffDigi(a::Int64, b::Int64)
function diffDigi(a = 5, b = 6)
    return digi(b) - digi(a)
end
#=
    digi = 6
    _6 = ϵ(digi * exp(1))
    ϵ(12)

    digi1 = 5
    _5 = ϵ(digi1 * exp(1))

    #for getting rid of non-linearity (hopefully)
    #subtract between 2 consequtive (non-linear) numbers

    diff = _5 - _6
    #=
    as of result we'd lost a bit of accuracy compared to 6
    but 'diff' function is linear
    =#

    _7 = ϵ(7 * exp(1))
=#

function maxEuclidDigi(;a = 5, b = 6)
    _a = digi(a)
    _b = digi(b)
    #maximal #mean euclidean distance
    meanDiff = (max(_a, _b) - min(_a, _b)) / 2
    return meanDiff
end
#= TODO:UNCOMMENT
@test _ϵ = maxEuclidDigi(5, 6)
@test diffDigi(5, 6)
@test diffDigi(5, 6) > maxEuclidDigi(5, 6) #there was an error during testing
end
end

@test maxEuclidDigi(5, 6)
@test diffDigi(5, 6)
@test diffDigi(5, 6) > maxEuclidDigi(5, 6) #there was an error during testing
#=
digi=6
_6 = ϵ(digi*exp(1))
ϵ(12)

digi1 = 5
_5 = ϵ(digi1*exp(1))

#for getting rid of non-linearity
#subtract between 2 consequtive numbers

diff = _5 - _6
=#

#=
as of result we'd lost a bit of accuracy compared to 6
but 'diff' function is linear

_7 = ϵ(7*exp(1))

#maximal euclidean distance
diff = (max(_5,_6) - min(_5,_6) )/2

#@test
end
=#

max_stars_repo_name: adamwillisMastery/DeepLearner

max_issues_repo_name: adamwillisMastery/DeepLearner

max_forks_repo_name: adamwillisXanax/DeepLearner


text:
#=
Cyclical Coordinate Descent for lasso with binary features
=#

function soft_thresh(
        z::Real,
        λ::Real
    )::Float64
    if abs(z) ≤ λ
        return 0
    elseif z > 0
        return z - λ
    else
        return z + λ
    end
end

function coordinate_descent(
        X::Bases,
        Y::Vector{Float64};
        λ::Real = 1,
        β::Dict{BasisIndex, Float64} = Dict{BasisIndex, Float64}(INTERCEPT => 0.0),
        tol::Number = 1e-4,
    )::Tuple{Dict{BasisIndex, Float64}, Vector{Float64}, Vector{Float64}}

    #=
    A fast coordinate descent algorithm for lasso with binary features. Can be warm-started by passing β.
    =#

    n = length(Y)

    residual = Y - X*β # can pass from prev iter
    loss(residual, β) = (1/n) * ( (1/2)sum(residual.^2) + λ*sum(abs.(values(β))) )

    loss_0 = loss(residual, β)
    cycle_loss = [loss_0]

    loop_bases = keys(X)

    @inbounds while true
        if length(cycle_loss) > 10000
            print(cycle_loss[(end-10):end])
            break
        end

        # update cycle
        for b in loop_bases
            βb = get(β, b, 0.0)
            sb = X.sum[b]
            ρ = residual * X[b]

            β̃b = soft_thresh(βb*sb + ρ, λ) / sb
            Δβb = β̃b - βb

            if Δβb ≠ 0 # tried with deleting β[b] if βb=0, faster as-is
                β[b] = β̃b
                for i in X[b].nzind
                    residual[i] -= Δβb
                end
            end
        end

        # check convergence and set loop over active set
        push!(cycle_loss, loss(residual, β))
        if ( abs(cycle_loss[end] - cycle_loss[end-1]) < tol ) # check loss convergence
            if loop_bases == keys(X)
                break
            end
            loop_bases = keys(X)
        else
            loop_bases = keys(β)
        end

    end

    β = Dict(
        b=>βb for (b,βb) in β
        if (βb ≠ 0.0) | ( b == INTERCEPT )
    )
    return β, residual, cycle_loss
end


text:
type Font
    ptr::Ptr{Void}

    function Font(ptr::Ptr{Void})
        f = new(ptr)
        finalizer(f, destroy)
        f
    end
end

function Font(filename::AbstractString)
    Font(ccall((:sfFont_createFromFile, libcsfml_graphics), Ptr{Void}, (Ptr{Cchar},), filename))
end

function copy(font::Font)
    Font(ccall((:sfFont_createFromFile, libcsfml_graphics), Ptr{Void}, (Ptr{Void},), font.ptr))
end

function destroy(font::Font)
    ccall((:sfFont_destroy, libcsfml_graphics), Void, (Ptr{Void},), font.ptr)
end


text:
# define MAF types.

# Each MAFRecord is made up of MAFSpecies entries.
mutable struct MAFSpecies
    name::String
    chr::String
    position::Int
    strand::Bool
    sequence::BioSequence{DNAAlphabet{4}}
end

Base.:(==)( a::MAFSpecies, b::MAFSpecies ) = a.name == b.name && a.position == b.position &&
                                             a.strand == b.strand && a.sequence == b.sequence ? true : false

struct MAFRecord
   species::Vector{MAFSpecies}

   MAFRecord() = new(Vector{MAFSpecies}())
end

Base.endof( maf::MAFRecord ) = endof(maf.species)
Base.getindex( maf::MAFRecord, ind ) = maf.species[ind]

function Base.reverse!( maf::MAFRecord )
   for s in maf.species
      reverse!( s.sequence )
   end
end

function BioSequences.reverse_complement!( mblock::MAFRecord, strandflip::Bool=true )
   for s in mblock.species
      BioSequences.reverse_complement!( s.sequence )
      if strandflip
         s.strand = s.strand ? false : true
      end
   end
end

Base.length( maf::MAFRecord ) = length(maf.species)

function deletegaps!( mblock::MAFRecord )
   str = convert(String, mblock.species[1].sequence)
   ind = search(str, r"-+")
   if length(ind) > 0
      for i in mblock.species
         deleteat!( i.sequence, ind )
      end
      deletegaps!( mblock )
      return
   else
      return
   end
end

function stitch!( a::MAFRecord, b::MAFRecord, index::Dict{String,Int} )
   const aref = a.species[1]
   const bref = b.species[1]
   const aref_len = length(aref.sequence)
   const bref_len = length(bref.sequence)
   sort!( a.species, by=x->index[x.name] )
   sort!( b.species, by=x->index[x.name] )
   aref.name != bref.name && error("Invalid stitching of alignment blocks with two different references $(aref.name) & $(bref.name)!!")
   if aref.position+length(aref.sequence) == bref.position
       i,j = 1,1
       while (i <= length(a.species) || j <= length(b.species))
          if (i <= length(a.species) && j <= length(b.species)) &&
              a.species[i].name == b.species[j].name
             a.species[i].sequence *= b.species[j].sequence
             i += 1
             j += 1
          elseif j > length(b.species) || (i <= length(a.species) && index[a.species[i].name] < index[b.species[j].name])
             a.species[i].sequence *= dna"-" ^ bref_len
             i += 1
          else
             b.species[j].sequence = (dna"-" ^ aref_len) * b.species[j].sequence
             insert!( a.species, i,  b.species[j] )
             i += 1
             j += 1
          end
       end
       return true
   else
      return false
   end
end

# Code modified and expanded from Automa.jl multi-line 'FASTA' example:
# https://github.com/BioJulia/Automa.jl/blob/master/example/fasta.jl

const re = Automa.RegExp

# Create a machine of MAF
const record_machine = (function ()

    lf          = re"\n"
    newline     = re"\r?" * lf
    lf.actions[:enter]          = [:countline]

    anything    = re.rep(re.diff( re".", newline ))
    header      = re.cat(re" ?#", anything, newline)

    float       = re.cat(re"[-+]?[0-9]*", re.opt("\."), re.opt(re"[0-9]*"))
    float.actions[:enter]       = [:mark]
    float.actions[:exit]        = [:score]

    sln         = re.rep1(re.space())
    emptyline   = re.cat(re" *", newline)

    scoreline   = re.cat(re" ?a score=", float, re.alt(newline, re.cat(re" ", anything, newline)))
    #scoreline.actions[:enter]   = [:setstart]

    organism    = re"[0-9A-Za-z\.\_\-]+"
    organism.actions[:enter]    = [:mark]
    organism.actions[:exit]     = [:organism]

    position    = re"[0-9]*"
    position.actions[:enter]    = [:mark]
    position.actions[:exit]     = [:position]

    size        = re"[0-9]*"
    size.actions[:enter]        = [:mark]
    size.actions[:exit]         = [:size]

    strand      = re"[+-]"
    strand.actions[:enter]      = [:mark]
    strand.actions[:exit]       = [:strand]

    sequence    = re"[ATGCNatgcn-]+"
    sequence.actions[:enter]    = [:mark]
    sequence.actions[:exit]     = [:sequence]

    entry       = re.cat(re" ?s", sln, organism, sln, position, sln, size, sln, strand, sln, re"[0-9]*", sln, sequence, newline)
    entry.actions[:enter]       = [:setpos]
    entry.actions[:exit]        = [:entry]

    fullheader  = re.cat(re.rep(header), re.rep(emptyline))
    fullheader.actions[:exit]   = [:anchor, :mark]

    record      = re.cat(re.opt(fullheader), re.opt(scoreline), entry, re.rep(entry), re.rep1(emptyline))
    record.actions[:enter]      = [:anchor]
    record.actions[:exit]       = [:escape]

    return Automa.compile(re.rep(record))
end)()

# It is useful to visualize the state machine for debugging.
# write("maf.dot", Automa.machine2dot(record_machine))
# run(`dot -Tsvg -o maf.svg maf.dot`)

# Bind Julia code to each action name (see the `parse_fasta` function defined below)

maf_actions_stream = Dict(
            :countline   => :(linenum += 1),
            :mark        => :(mark = p),
            :score => quote
                if !bad_record
                   range = (mark:p-1)
                   score = length(range) <= 0 || mark == 0 ? "" : convert(String, data[range])
                end
                mark = 0
            end,
            :organism => quote
                if !bad_record
                   range = (mark:p-1)
                   organism = mark == 0 ? "" : convert(String, data[range])
                end
                mark = 0
            end,
            :position => quote
                if !bad_record
                   range = (mark:p-1)
                   position = length(range) <= 0 || mark == 0 ? "" : convert(String, data[range])
                end
                mark = 0
            end,
            :size => quote
                if !bad_record
                   range = (mark:p-1)
                   size = length(range) <= 0 || mark == 0 ? "" : convert(String, data[range])
                end
                mark = 0
            end,
            :strand => quote
                if !bad_record
                   range = (mark:p-1)
                   strand = mark == 0 ? true : Char(data[last(range)]) == '+'
                end
                mark = 0
            end,
            :sequence => quote
                if !bad_record
                   range = (mark:p-1)
                   sequence = mark == 0 ? "" : chomp(String(data[range]))
                end
                mark = 0
            end,
            :setpos => :(stream.position = p; mark_cs = cs), #println("Set stream position $p at cs $cs")),
            :entry  => quote
                try
                   if first
                      const len    = parse(Int, size)
                      const pscore = parse(Float64, score)
                      const pos    = parse(Int, position)
                      org,chr = parsename( organism, reader.keepname )
                      if (len < reader.minlen || pscore < reader.minscore) ||
                         (regionbool && !hasintersection( regions, Interval(chr, pos, pos+len-1) )) ||
                         (secbool    && !hasintersection( secregions, Interval(chr, pos, pos+len-1) ))
                         bad_record = true
                      end
                   end
                   first = false
                   if !bad_record
                      org,chr = parsename( organism, reader.keepname )
                      push!(record.species, MAFSpecies(org, chr, parse(Int, position), strand, DNASequence(sequence)))
                   end
                catch
                   println(STDERR, "size: $size and score: $score")
                   println(STDERR, "Bad Record at $p, setting p to $(stream.position)")
                   p = stream.position
                   @escape
                end
            end,
            :countline   => :(linenum += 1),
            :anchor      => :(stream.anchor = p), #; println("Anchoring at $p")),
            :escape      => :(found_record=true; @escape))

mutable struct MAFReader{T<:BufferedInputStream}
    stream::T
    cs::Int
    p_eof::Int
    linenum::Int
    minlen::Int
    minscore::Float64
    keepname::Bool
end

function MAFReader{BS <: BufferedInputStream}(input::BS, keepname::Bool=false)
    hits_eof = fillbuffer!(input) == 0
    p_eof = hits_eof ? input.available : -1
    return MAFReader{BS}(input, record_machine.start_state, p_eof, 1, 0, typemin(Float64), keepname)
end

Base.done(reader::MAFReader) = reader.cs <= 0

iscomment(line::String) = (length(line) >= 1 && line[1] == '#') || (length(line) >= 2 && line[2] == '#')
isheader(line::String)  = iscomment(line)

function discard_header!(reader::MAFReader)
    for l in eachline(reader.stream)
       reader.linenum += 1
       if isheader(l)
           continue
       end
       anchor!(reader.stream)
       hits_eof = fillbuffer!(reader.stream) == 0
       if hits_eof
          BufferedStreams.shiftdata!(reader.stream)
          reader.p_eof = reader.stream.available
       end
       reader.linenum -= 1
       return reader
    end
    reader.linenum -= 1
    reader
end

function parsename( name::String, keepname::Bool )
   if keepname
      return name,name
   else
      spl = split(name, '.')
      length(spl) < 2 && error("Must have a . in the name if MAFReader( ..., keepname=true ): $name")
      return convert(String, spl[1]),convert(String, spl[2])
   end
end

const EMPTY_INTCOL = IntervalCollection{Void}()

const context = Automa.CodeGenContext(generator=:goto, checkbounds=false)

@eval function Base.read!(reader::MAFReader, record::MAFRecord;
                          regionbool::Bool=false, regions::IntervalCollection=EMPTY_INTCOL,
                          secbool::Bool=false,    secregions::IntervalCollection=EMPTY_INTCOL)
    cs      = reader.cs
    stream  = reader.stream
    data    = stream.buffer
    p       = stream.position
    p_end   = stream.available
    p_eof   = reader.p_eof
    linenum = reader.linenum
    mark = 0
    mark_cs = 0

    scoreline = organism = sequence = ""
    size = position = score = ""
    bad_record   = false
    found_record = false
    rebuffered   = false
    retry        = false
    first        = true

    if cs < 0
        error("the reader is in error state")
    elseif cs == 0
        error("the reader is finished")
    end

    while true

        p = p < 1 ? stream.anchor : p

        $(Automa.generate_exec_code(context, record_machine, maf_actions_stream))

        reader.cs              = cs
        reader.p_eof           = p_eof
        reader.linenum = linenum

        if found_record && !bad_record
           reader.stream.position = p
           return record
        elseif found_record && bad_record
           bad_record = false
           found_record = false
           first = true
           empty!(record.species)
           continue
        elseif (p == p_eof+1 && data[p_eof] == 10)
           if !bad_record && organism != ""
              org,chr = parsename( organism, reader.keepname )
              spec = MAFSpecies(org, chr, parse(Int, position), strand, DNASequence(sequence))
              length(record.species) > 0 && !(record.species[end] == spec) && push!(record.species, spec)
           end
           reader.cs = 0
           return record
        elseif p > p_eof ≥ 0
           error("incomplete MAF input on line ", linenum)
        elseif p == p_end+1
            hits_eof = BufferedStreams.fillbuffer!(reader.stream) == 0
            if hits_eof
               reader.p_eof = p_eof = p_end
            else
               p     = stream.position
               p_end = stream.available
            end
            cs = record_machine.start_state
        elseif cs < 0
           if !retry && p <= 4
              retry = true
              p = 1
              cs = record_machine.start_state
           else
              println(String(data))
              println("cs:$cs p:$p p_end:$p_end $record anchor:$(stream.anchor) avail:$(stream.available) \"$(map(Char, data[1:p+10]))\"")
              error("MAF file format error on line ", linenum)
           end
        else
           error("Shouldn't happen cs:$cs p:$p p_eof:$p_eof $record")
        end

    end
end

const MAFInterval = Interval{MAFRecord}
const MAFCollection = IntervalCollection{MAFRecord}

function readmaf!( reader::MAFReader, order::Dict{String,Int};
                   minspecies::Int=1, minlength::Int=1, minscore::Float64=-Inf,
                   regionbool=false, regions=EMPTY_INTCOL,
                   secbool=false, secregions=EMPTY_INTCOL)
   maf = MAFCollection()
   readmaf!( maf, reader, order, minspecies=minspecies, minlength=minlength, minscore=minscore,
             regionbool=regionbool, regions=regions, secbool=secbool, secregions=secregions )
end

function readmaf!( maf::MAFCollection, reader::MAFReader, order::Dict{String,Int};
                   minspecies::Int=1, minlength::Int=1, minscore::Float64=-Inf,
                   regionbool=false, regions=EMPTY_INTCOL,
                   secbool=false, secregions=EMPTY_INTCOL)
   prev = MAFRecord()
   while !done( reader )
      rec = MAFRecord()
      read!( reader, rec, regionbool=regionbool, regions=regions, secbool=secbool, secregions=secregions )
      if length(rec.species) >= minspecies
         try
            deletegaps!(rec)
         catch
            println(rec)
            error()
         end
         if length(prev) > 0
            if !(stitch!( prev, rec, order ))
               const ref = prev.species[1]
               if length(ref.sequence) > minlength
                  push!( maf, MAFInterval(ref.chr, ref.position, ref.position+length(ref.sequence)-1, ref.strand ? '+' : '-', prev) )
               end
               prev = rec
            end
         else
            prev = rec
         end
      end
   end
   if length(prev) > 0
      const ref = prev.species[1]
      push!( maf, MAFInterval(ref.chr, ref.position, ref.position+length(ref.sequence)-1, ref.strand ? '+' : '-', prev) )
   end
   maf
end


text:
"""
    EdgeGradient

This type is used to generate an edge gradient landscape, where values change
as a bilinear function of the *x* and *y* coordinates. The direction is
expressed as a floating point value, which will be in *[0,360]*. The inner
constructor takes the mod of the value passed and 360, so that a value that is
out of the correct interval will be corrected.
"""
struct EdgeGradient <: NeutralLandscapeMaker
    direction::Float64
    EdgeGradient(x::T) where {T <: Real} = new(mod(x, 360.0))
end

EdgeGradient() = EdgeGradient(360rand())

function _landscape!(mat, alg::EdgeGradient) where {IT <: Integer}
    _landscape!(mat, PlanarGradient(alg.direction))
    mat .= -2.0abs.(0.5 .- mat) .+ 1.0
end


text:
module TestSPPL

using SPPL
using Test

include("examples.jl")

end # module