text:
"""
Selectors indicate that index values are not indices, but points to
be selected from the dimension values, such as DateTime objects on a Time dimension.
"""
abstract type Selector{T} end

val(m::Selector) = m.val

(::Type{T})(args...) where T <: Selector = T{typeof(args)}(args)

"""
    At(x)

Selector that exactly matches the value on the passed-in dimensions, or throws an error.
For ranges and arrays, every value must match an existing value - not just the end points.
"""
struct At{T} <: Selector{T}
    val::T
end

"""
    Near(x)

Selector that selects the nearest index to its contained value(s)
"""
struct Near{T} <: Selector{T}
    val::T
end

"""
    Between(a, b)

Selector that retreive all indices located between 2 values.
"""
struct Between{T<:Union{Tuple{Any,Any},Nothing}} <: Selector{T}
    val::T
end
Between(x::Tuple) = Between{typeof(x)}(x)

# Get the dims in the same order as the grid
# This would be called after RegularGrid and/or CategoricalGrid
# dimensions are removed
dims2indices(grid::TransformedGrid, dims::Tuple, lookups::Tuple, emptyval) =
    sel2indices(grid, dims, map(val, permutedims(dimz, dims(grid))))

sel2indices(A::AbstractArray, lookup) = sel2indices(dims(A), lookup)
sel2indices(dims::Tuple, lookup) = sel2indices(dims, (lookup,))
sel2indices(dims::Tuple, lookup::Tuple) = sel2indices(map(grid, dims), dims, lookup)
sel2indices(grids, dims::Tuple, lookup::Tuple) =
    (sel2indices(grids[1], dims[1], lookup[1]),
     sel2indices(tail(grids), tail(dims), tail(lookup))...)
    sel2indices(grids::Tuple{}, dims::Tuple{}, lookup::Tuple{}) = ()
sel2indices(grid, dim::AbDim, sel::At) = at(dim, val(sel))
sel2indices(grid, dim::AbDim, sel::At{<:Tuple}) = [at.(Ref(dim), val(sel))...]
sel2indices(grid, dim::AbDim, sel::At{<:AbstractVector}) = at.(Ref(dim), val(sel))
sel2indices(grid, dim::AbDim{<:Any,<:AbstractCategoricalGrid}, sel::Near) =
    throw(ArgumentError("`Near` has no meaning in a `CategoricalGrid`. Use `At`"))
sel2indices(grid, dim::AbDim, sel::Near) = near(dim, val(sel))
sel2indices(grid, dim::AbDim, sel::Near{<:Tuple}) = [near.(Ref(dim), val(sel))...]
sel2indices(grid, dim::AbDim, sel::Near{<:AbstractVector}) = near.(Ref(dim), val(sel))
sel2indices(grid::IndependentGrid{<:Ordered}, dim::AbDim, sel::Between{<:Tuple}) =
    between(dim, val(sel))
sel2indices(grid::IndependentGrid{Unordered}, dim::AbDim, sel::Between{<:Tuple}) =
    throw(ArgumentError("`Between` has no meaning with an `Unordered` grid"))

# We use the transformation from the first TransformedGrid dim.
# In practice the others could be empty.
sel2indices(grids::Tuple{Vararg{<:TransformedGrid}}, dims::AbDimTuple,
            sel::Tuple{Vararg{<:Selector}}) =
    map(to_int, sel, val(dims[1])([map(val, sel)...]))

to_int(::At, x) = convert(Int, x)
to_int(::Near, x) = round(Int, x)

# Do the input values need some kind of scalar conversion?
# what is the scale of these lookup matrices?
sel2indices(grid::LookupGrid, sel::Tuple{Vararg{At}}) =
    lookup(grid)[map(val, sel)...]

at(dim::AbDim, selval) = at(val(dim), selval)
at(dim, selval) = begin
    # This is not particularly efficient.
    # It should be separated out for unordered categorical
    # dims and otherwise treated as an ordered list.
    ind = findfirst(x -> x == selval, val(dim))
    ind == nothing ? throw(ArgumentError("$selval not found in $dim")) : ind
end

near(list, selval) = begin
    list = val(list)
    ind = searchsortedfirst(list, selval)
    if ind <= firstindex(list)
        ind
    elseif abs(list[ind] - selval) < abs(list[ind-1] - selval)
        ind
    else
        ind - 1
    end
end

between(dim::AbDim, sel) = between(indexorder(dim), dim, sel)
between(::Forward, dim::AbDim, sel) =
    rangeorder(dim, searchsortedfirst(val(dim), first(sel)), searchsortedlast(val(dim), last(sel)))
between(::Reverse, dim::AbDim, sel) =
    rangeorder(dim, searchsortedlast(val(dim), last(sel); rev=true),
                    searchsortedfirst(val(dim), first(sel); rev=true))

rangeorder(dim::AbDim, lower, upper) = rangeorder(arrayorder(dim), dim, lower, upper)
rangeorder(::Forward, dim::AbDim, lower, upper) = lower:upper
rangeorder(::Reverse, dim::AbDim, lower, upper) = length(val(dim)) - upper + 1:length(val(dim)) - lower + 1

# Selector indexing without dim wrappers. Must be in the right order!
Base.@propagate_inbounds Base.getindex(a::AbstractArray, I::Vararg{Selector}) =
    getindex(a, sel2indices(a, I)...)
Base.@propagate_inbounds Base.setindex!(a::AbstractArray, x, I::Vararg{Selector}) =
    setindex!(a, x, sel2indices(a, I)...)
Base.view(a::AbstractArray, I::Vararg{Selector}) =
    view(a, sel2indices(a, I)...)


text:
#############
## Actions ##
#############

import Base.collect

#### Count ####

# Count all the keys in an RDD
function count(master::Master, rdd::RDD)
    op = Action("count", Dict())
    results = doop(master, rdd, op)
    return sum(results)
end

function count(worker::Worker, rdd::WorkerRDD, part_id::Int64, args::Dict)
    return length(collect(keys(rdd.partitions[part_id].data)))
end

#### Collect ####

# Collect all keys in the RDD
function collect(master::Master, rdd::RDD)
    op = Action("collect", Dict())
    results = doop(master, rdd, op)
    r = {}
    for i in results
        append!(r, i)
    end
    return r
end

function collect(worker::Worker, rdd::WorkerRDD, part_id::Int64, args::Dict)
    data = rdd.partitions[part_id].data
    return {(k, data[k]) for k in keys(data)}
end

#TODO reduce
function reduce(worker::Worker, rdd::WorkerRDD, args::Dict{Any, Any})
    return None
end

#### Lookup ####

# Look up a particular item in the RDD
function lookup(master::Master, rdd::RDD, key::Any)
    op = Action("lookup", {"key" => key})
    results = doop(master, rdd, op)
    final = {}
    for r in results
        final = cat(1, final, r)
    end
    return final
end

function lookup(worker::Worker, rdd::WorkerRDD, part_id::Int64, args::Dict)
    key = args["key"]
    if key in keys(rdd.partitions[part_id].data)
        return rdd.partitions[part_id].data[args["key"]]
    else
        return {}
    end
end

ext: jl


text:
module LoweredCodeUtils

# We use a code structure where all `using` and `import`
# statements in the package that load anything other than
# a Julia base or stdlib package are located in this file here.
# Nothing else should appear in this file here, apart from
# the `include("packagedef.jl")` statement, which loads what
# we would normally consider the bulk of the package code.
# This somewhat unusual structure is in place to support
# the VS Code extension integration.

using JuliaInterpreter
using JuliaInterpreter: SSAValue, SlotNumber, Frame
using JuliaInterpreter: @lookup, moduleof, pc_expr, step_expr!, is_global_ref, is_quotenode, whichtt,
                        next_until!, finish_and_return!, get_return, nstatements, codelocation, linetable,
                        is_return, lookup_return, is_GotoIfNot, is_ReturnNode

include("packagedef.jl")

end # module


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

mutable struct AutomaticOSUpgradePolicy <: SwaggerModel
    enableAutomaticOSUpgrade::Any # spec type: Union{ Nothing, Bool } # spec name: enableAutomaticOSUpgrade
    disableAutomaticRollback::Any # spec type: Union{ Nothing, Bool } # spec name: disableAutomaticRollback

    function AutomaticOSUpgradePolicy(;enableAutomaticOSUpgrade=nothing, disableAutomaticRollback=nothing)
        o = new()
        validate_property(AutomaticOSUpgradePolicy, Symbol("enableAutomaticOSUpgrade"), enableAutomaticOSUpgrade)
        setfield!(o, Symbol("enableAutomaticOSUpgrade"), enableAutomaticOSUpgrade)
        validate_property(AutomaticOSUpgradePolicy, Symbol("disableAutomaticRollback"), disableAutomaticRollback)
        setfield!(o, Symbol("disableAutomaticRollback"), disableAutomaticRollback)
        o
    end
end # type AutomaticOSUpgradePolicy

const _property_map_AutomaticOSUpgradePolicy = Dict{Symbol,Symbol}(Symbol("enableAutomaticOSUpgrade")=>Symbol("enableAutomaticOSUpgrade"), Symbol("disableAutomaticRollback")=>Symbol("disableAutomaticRollback"))
const _property_types_AutomaticOSUpgradePolicy = Dict{Symbol,String}(Symbol("enableAutomaticOSUpgrade")=>"Bool", Symbol("disableAutomaticRollback")=>"Bool")
Base.propertynames(::Type{ AutomaticOSUpgradePolicy }) = collect(keys(_property_map_AutomaticOSUpgradePolicy))
Swagger.property_type(::Type{ AutomaticOSUpgradePolicy }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_AutomaticOSUpgradePolicy[name]))}
Swagger.field_name(::Type{ AutomaticOSUpgradePolicy }, property_name::Symbol) =  _property_map_AutomaticOSUpgradePolicy[property_name]

function check_required(o::AutomaticOSUpgradePolicy)
    true
end

function validate_property(::Type{ AutomaticOSUpgradePolicy }, name::Symbol, val)
end


text:
using Base.Broadcast: broadcastable
using BenchmarkTools
using ChainRulesCore
using LinearAlgebra: Diagonal, dot
using FiniteDifferences
using Test

@testset "ChainRulesCore" begin
    @testset "differentials" begin
        include("differentials/abstract_zero.jl")
        include("differentials/one.jl")
        include("differentials/thunks.jl")
        include("differentials/composite.jl")
    end

    include("ruleset_loading.jl")
    include("rules.jl")
    include("rule_definition_tools.jl")

    @testset "demos" begin
        include("demos/forwarddiffzero.jl")
        include("demos/reversediffzero.jl")
    end
end


text:
"""
    Tri{T<:Number, S<:Integer} <: CyclicSchedule
    Tri(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period`.
The output conforms to
```
abs(λ0 - λ1) * (2 / π) * abs(asin(sin(π * (t - 1) / period))) + min(λ0, λ1)
```

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
"""
struct Tri{T<:Number, S<:Integer} <: CyclicSchedule
    λ0::T
    λ1::T
    period::S
end
Tri(;λ0, λ1, period) = Tri(λ0, λ1, period)

startvalue(schedule::Tri) = schedule.λ0
endvalue(schedule::Tri) = schedule.λ1
cycle(schedule::Tri, t) = (2 / π) * abs(asin(sin(π * (t - 1) / schedule.period)))

Base.eltype(::Type{<:Tri{T}}) where T = T
Base.IteratorSize(::Type{<:Tri}) = Base.IsInfinite()

"""
    TriDecay2{T<:Tri} <: CyclicSchedule
    TriDecay2(λ0, λ1, period)
    TriDecay2(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and half the amplitude each cycle.
The output conforms to
```
abs(λ0 - λ1) * Tri(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Tri(t)` is the [`cycle`](#) of [`Tri`](#).

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
"""
struct TriDecay2{T<:Tri} <: CyclicSchedule
    tri::T
end
TriDecay2(λ0, λ1, period) = TriDecay2(Tri(λ0, λ1, period))
TriDecay2(;λ0, λ1, period) = TriDecay2(λ0, λ1, period)

startvalue(schedule::TriDecay2) = schedule.tri.λ0
endvalue(schedule::TriDecay2) = schedule.tri.λ1
cycle(schedule::TriDecay2, t) = cycle(schedule.tri, t) / (2^fld(t - 1, schedule.tri.period))

Base.eltype(::Type{<:TriDecay2{Tri{T}}}) where T = T
Base.IteratorSize(::Type{<:TriDecay2}) = Base.IsInfinite()

"""
    TriExp{F<:Number, T<:Tri{F}, S<:Exp{F}} <: CyclicSchedule
    TriExp(λ0, λ1, period, γ)
    TriExp(;λ0, λ1, period, γ)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and an exponentially decaying amplitude.
The output conforms to
```
abs(λ0 - λ1) * Tri(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Tri(t)` is the [`cycle`](#) of [`Tri`](#).

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
- `γ::Number`: the decay rate
"""
struct TriExp{F<:Number, T<:Tri{F}, S<:Exp{F}} <: CyclicSchedule
    tri::T
    exp::S
end
TriExp(λ0, λ1, period, γ) = TriExp(Tri(λ0, λ1, period), Exp(one(λ0), γ))
TriExp(;λ0, λ1, period, γ) = TriExp(λ0, λ1, period, γ)

startvalue(schedule::TriExp) = schedule.tri.λ0
endvalue(schedule::TriExp) = schedule.tri.λ1
cycle(schedule::TriExp, t) = decay(schedule.exp, t) * cycle(schedule.tri, t)

Base.eltype(::Type{<:TriExp{T}}) where T = T
Base.IteratorSize(::Type{<:TriExp}) = Base.IsInfinite()

"""
    Sin{T<:Number, S<:Integer} <: CyclicSchedule
    Sin(;λ0, λ1, period)

A sine wave schedule with `period`.
The output conforms to
```
abs(λ0 - λ1) * abs(sin(π * (t - 1) / period)) + min(λ0, λ1)
```

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
"""
struct Sin{T<:Number, S<:Integer} <: CyclicSchedule
    λ0::T
    λ1::T
    period::S
end
Sin(;λ0, λ1, period) = Sin(λ0, λ1, period)

startvalue(schedule::Sin) = schedule.λ0
endvalue(schedule::Sin) = schedule.λ1
cycle(schedule::Sin, t) = abs(sin(π * (t - 1) / schedule.period))

Base.eltype(::Type{<:Sin{T}}) where T = T
Base.IteratorSize(::Type{<:Sin}) = Base.IsInfinite()

"""
    SinDecay2{T<:Sin} <: CyclicSchedule
    SinDecay2(λ0, λ1, period)
    SinDecay2(;λ0, λ1, period)

A sine wave schedule with `period` and half the amplitude each cycle.
The output conforms to
```
abs(λ0 - λ1) * Sin(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Sin(t)` is the [`cycle`](#) of [`Sin`](#).

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
"""
struct SinDecay2{T<:Sin} <: CyclicSchedule
    sine::T
end
SinDecay2(λ0, λ1, period) = SinDecay2(Sin(λ0, λ1, period))
SinDecay2(;λ0, λ1, period) = SinDecay2(λ0, λ1, period)

startvalue(schedule::SinDecay2) = schedule.sine.λ0
endvalue(schedule::SinDecay2) = schedule.sine.λ1
cycle(schedule::SinDecay2, t) = cycle(schedule.sine, t) / (2^fld(t - 1, schedule.sine.period))

Base.eltype(::Type{<:SinDecay2{Sin{T}}}) where T = T
Base.IteratorSize(::Type{<:SinDecay2}) = Base.IsInfinite()

"""
    SinExp{F<:Number, T<:Sin{F}, S<:Exp{F}} <: CyclicSchedule
    SinDecay2(λ0, λ1, period, γ)
    SinDecay2(;λ0, λ1, period, γ)

A sine wave schedule with `period` and an exponentially decaying amplitude.
The output conforms to
```
abs(λ0 - λ1) * Sin(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Sin(t)` is the [`cycle`](#) of [`Sin`](#).

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
- `γ::Number`: the decay rate
"""
struct SinExp{F<:Number, T<:Sin{F}, S<:Exp{F}} <: CyclicSchedule
    sine::T
    exp::S
end
SinExp(λ0, λ1, period, γ) = SinExp(Sin(λ0, λ1, period), Exp(one(λ0), γ))
SinExp(;λ0, λ1, period, γ) = SinExp(λ0, λ1, period, γ)

startvalue(schedule::SinExp) = schedule.sine.λ0
endvalue(schedule::SinExp) = schedule.sine.λ1
cycle(schedule::SinExp, t) = decay(schedule.exp, t) * cycle(schedule.sine, t)

Base.eltype(::Type{<:SinExp{T}}) where T = T
Base.IteratorSize(::Type{<:SinExp}) = Base.IsInfinite()

"""
    Cos{T<:Number, S<:Integer} <: CyclicSchedule
    Cos(;λ0, λ1, period)

A cosine wave schedule with `period`.
The output conforms to
```
abs(λ0 - λ1) * (1 + cos(2 * π * (t - 1) / period)) / 2 + min(λ0, λ1)
```

# Arguments
- `λ0::Number`: the start value
- `λ1::Number`: the end value
- `period::Integer`: the period
"""
struct Cos{T<:Number, S<:Integer} <: CyclicSchedule
    λ0::T
    λ1::T
    period::S
end
Cos(;λ0, λ1, period) = Cos(λ0, λ1, period)

startvalue(schedule::Cos) = schedule.λ0
endvalue(schedule::Cos) = schedule.λ1
cycle(schedule::Cos, t) = (1 + cos(2 * π * (t - 1) / schedule.period)) / 2

Base.eltype(::Type{<:Cos{T}}) where T = T
Base.IteratorSize(::Type{<:Cos}) = Base.IsInfinite()


text:
function numericRoot(residFnc::Function, measurement, parameters, x0::Vector{Float64})
	return (nlsolve(   (res, X) -> residFnc(res, measurement, parameters, X), x0 )).zero
end

function shuffleXAltD(X::Vector{Float64}, Alt::Vector{Float64}, d::Int, p::Vector{Int})
		# n = length(X)
    Y = deepcopy(Alt)
    for i in 1:d
			Y[p[i]] = X[i]
		end
    return Y
end

# residual function must have the form residFnc!(res, x)
# function numericRootGenericRandomizedFncOld(
#       residFnc!::Function,
#       zDim::Int,
#       xDim::Int,
#       x0::Vector{Float64};
#       perturb::Float64=1e-5  )
#
# 	# xDim = length(x0)
# 	if zDim < xDim
# 		p = collect(1:xDim);
# 		shuffle!(p);
# 		# p1 = p.==1; p2 = p.==2; p3 = p.==3
#     # TODO -- refactor to use functors instead
# 		r = nlsolve(    (res, x) -> residFnc!(shuffleXAltD(x, x0, zDim, p), res ),
#                     x0[p[1:zDim]] + perturb*randn(zDim)
#                )
#
#     return shuffleXAltD(r.zero, x0, zDim, p );
# 	else
#     # return (nlsolve(  (res, x) -> residFnc!(res, x),
#     #                   x0      )).zero
#     # @show x0
#     return ( nlsolve(  residFnc!, x0 + perturb*randn(zDim) ) ).zero
# 	end
# end

#-------------------------------------------------------------------------------
# first in place attempt for generic wrap and root finding below

function (p::GenericWrapParam)(res, x)
  # approximates by not considering cross indices among parameters
  # @show length(p.params), p.varidx, p.particleidx, size(x), size(res), size(p.measurement)
  p.params[p.varidx][:, p.particleidx] = x
  p.usrfnc!(res, p.particleidx, p.measurement, p.params...)
end

# Shuffle incoming X into random permutation in fr.Y
# shuffled fr.Y will be placed back into fr.X[:,fr.gwp.particleidx] upon fr.gwp.usrfnc(x, res)
function shuffleXAltD!(fr::FastRootGenericWrapParam, X::Vector{Float64})
  fr.Y[1:fr.xDim] = fr.X[1:fr.xDim,fr.gwp.particleidx]
  # copy!(fr.Y, fr.X[:,fr.gwp.particleidx])
  for i in 1:fr.zDim
    fr.Y[fr.p[i]] = X[i]
  end
  nothing
end
function (fr::FastRootGenericWrapParam)( res::Vector{Float64}, x::Vector{Float64} )
  shuffleXAltD!(fr, x)
  fr.gwp( res, fr.Y ) #function (p::GenericWrapParam)(res, x)
end

## TODO desperately needs cleaning up and refactoring
# Solve free variable x by root finding residual function fgr.usrfnc(x, res)
# randomly shuffle x dimensions if underconstrained by measurement z dimensions
# small random perturbation used to prevent trivial solver cases, div by 0 etc.
# result stored in fgr.Y
# fr.X must be set to memory ref the param[varidx] being solved, at creation of fr
function numericRootGenericRandomizedFnc!(
      fr::FastRootGenericWrapParam{T};
      perturb::Float64=1e-10,
      testshuffle::Bool=false ) where {T}
  #
  # info("numericRootGenericRandomizedFnc! FastRootGenericWrapParam{T}")
  if fr.zDim < fr.xDim && !fr.gwp.partial || testshuffle
    shuffle!(fr.p)
    for i in 1:fr.xDim
      fr.perturb[1:fr.zDim] = perturb*randn(fr.zDim)
      fr.X[fr.p[1:fr.zDim], fr.gwp.particleidx] += fr.perturb
      r = nlsolve(  fr,
                    fr.X[fr.p[1:fr.zDim], fr.gwp.particleidx] # this is x0
                 )
      if r.f_converged
        shuffleXAltD!( fr, r.zero )
        break;
      else
        # TODO -- report on this bottleneck, useful for optimization of code
        # @show i, fr.p, fr.xDim, fr.zDim
        temp = fr.p[end]
        fr.p[2:end] = fr.p[1:(end-1)]
        fr.p[1] = temp
        if i == fr.xDim
          error("numericRootGenericRandomizedFnc could not converge, i=$(i), fr.gwp.usrfnc!=$(typeof(fr.gwp.usrfnc!))")
        end
      end
    end
    #shuffleXAltD!( fr, r.zero ) # moved up
  elseif fr.zDim >= fr.xDim && !fr.gwp.partial
    fr.perturb[1:fr.xDim] = perturb*randn(fr.xDim)
    fr.X[1:fr.xDim, fr.gwp.particleidx] += fr.perturb[1:fr.xDim] # moved up
    r = nlsolve( fr.gwp, fr.X[1:fr.xDim,fr.gwp.particleidx] )
    if sum(isnan.(( r ).zero)) == 0
      fr.Y[1:fr.xDim] = ( r ).zero
    else
      warn("got NaN, fr.gwp.particleidx = $(fr.gwp.particleidx), r")
      @show fr.gwp.usrfnc!
      for thatlen in 1:length(fr.gwp.params)
        @show thatlen, fr.gwp.params[thatlen][:, fr.gwp.particleidx]
      end
    end
  elseif fr.gwp.partial
    # improve memory management in this function
    fr.p[1:length(fr.gwp.usrfnc!.partial)] = Int[fr.gwp.usrfnc!.partial...] # TODO -- move this line up and out of inner loop
    r = nlsolve(  fr,
                  fr.X[fr.p[1:fr.zDim], fr.gwp.particleidx] # this is x0
               )
    shuffleXAltD!( fr, r.zero )
  else
    error("Unresolved numeric solve case")
	end
  fr.X[:,fr.gwp.particleidx] = fr.Y
  nothing
end

#-------------------------------------------------------------------------------

# this will likely expand with more internal bells and whistles
# to perform in place memory operations for array values in

# see FastRootGenericWrapParam{T}

# TODO -- part of speed and flexibility refactoring exercise
type FastGenericRoot{T} <: Function
  p::Vector{Int}
  perturb::Vector{Float64}
  X::Vector{Float64}
  Y::Vector{Float64}
  xDim::Int
  zDim::Int
  usrfnc::T
  FastGenericRoot{T}(xDim::Int, zDim::Int, residfnc::T) where {T} =
      new(collect(1:xDim), zeros(zDim), zeros(xDim), zeros(xDim), xDim, zDim, residfnc)
end
function shuffleXAltD!(fr::FastGenericRoot, X::Vector{Float64})
  copy!(fr.Y, fr.X)
  for i in 1:fr.zDim
    fr.Y[fr.p[i]] = X[i]
  end
  nothing
end
function (fr::FastGenericRoot)( res::Vector{Float64}, x::Vector{Float64} )
  shuffleXAltD!(fr, x)  #(fr.Y, x, fr.x0, fr.zDim, fr.p)
  fr.usrfnc( res, fr.Y )
end
# Solve free variable x by root finding residual function fgr.usrfnc(x, res)
# randomly shuffle x dimensions if underconstrained by measurement z dimensions
# small random perturbation used to prevent trivial solver cases, div by 0 etc.
# result stored in fgr.Y
function numericRootGenericRandomizedFnc!(
      fgr::FastGenericRoot{T};
      perturb::Float64=1e-5,
      testshuffle::Bool=false ) where {T}
  #
  # info("numericRootGenericRandomizedFnc!(fgr::FastGenericRoot{T}...)  ")
  fgr.perturb[1:fgr.zDim] = perturb*randn(fgr.zDim)
  if fgr.zDim < fgr.xDim || testshuffle
    shuffle!(fgr.p)
    r = nlsolve(  fgr,
                fgr.X[fgr.p[1:fgr.zDim]] + fgr.perturb # this is x0
             )
    # copy!(fgr.X, x0) # should need this line?
    shuffleXAltD!( fgr, r.zero )
    # copy!(fgr.X, r.zero)
  else
    # @show "direct solve"
    fgr.Y = ( nlsolve(  fgr.usrfnc, fgr.X + fgr.perturb ) ).zero
    # copy!(fgr.X, y)
  end
  nothing
end

function numericRootGenericRandomizedFnc(
      residFnc!::Function,
      zDim::Int,
      xDim::Int,
      x0::Vector{Float64};
      perturb::Float64=1e-5,
      testshuffle::Bool=false   )
  #
  # TODO -- this only start of refactoring for inplace, more to come
  # xDim = length(x0)
  fgr = FastGenericRoot{typeof(residFnc!)}(xDim, zDim, residFnc!)
  shuffle!(fgr.p);
  fgr.perturb[1:fgr.zDim] = perturb*randn(fgr.zDim)
  copy!(fgr.X, x0)

  numericRootGenericRandomizedFnc!( fgr, perturb=perturb, testshuffle=testshuffle )
  fgr.Y
end

# function oneparams!(res::Array{Float64}, p::GenericWrapParam)
#   p.usrfnc!(res, p.params[1][:,p.particleidx])
# end
# function twoparams!(res::Array{Float64}, p::GenericWrapParam)
#   p.usrfnc!(res, p.params[1][:,p.particleidx], p.params[2][:,p.particleidx])
# end
# function threeparams!(res::Array{Float64}, p::GenericWrapParam)
#   p.usrfnc!(res, p.params[1][:,p.particleidx], p.params[2][:,p.particleidx], p.params[3][:,p.particleidx])
# end
# function fourparams!(res::Array{Float64}, p::GenericWrapParam)
#   p.usrfnc!(res, p.params[1][:,p.particleidx], p.params[2][:,p.particleidx], p.params[3][:,p.particleidx], p.params[4][:,p.particleidx])
# end

#


text:
function ueberpruefenaechsterPunkt(Parameter1,Punkt,blick1,blick2,b1,M,A3,b3,nx1,nx2,nx3)

a1 = b1-M[:,1]*Parameter1;

if norm(a1-transpose(Punkt)) <= 0.01
    println("Ihr habt den nächsten Punkt korrekt berechnet! Ihr könnt blick1 und blick2 so verändern, dass man sieht, dass der Punkt in der Ebene liegt.")

    U2,S2,V2 = svd(A3)
    u = U2[:,1];
    v = U2[:,2];
    s = -20*u-20*v;
    a = b3[1];
    b = b3[2];
    c = b3[3];
    d = Punkt[1];
    e = Punkt[2];
    f = Punkt[3];
    ## Berechnung Winkel für blick1
    #x2 = -nx1/nx2;
    #Vec1 = [1; x2; 0];
    #Vec2 = [1; 0; 0];
    #alpha = acosd(transpose(Vec1)*Vec2/(norm(Vec1)*norm(Vec2)));
    #x3 = -nx1/nx3;
    #Vec3 = [1; 0; x3];
    #Vec4 = [0; 0; 1];
    #beta = acosd(transpose(Vec3)*Vec4/(norm(Vec3)*norm(Vec4)));

    plt = surface([(s[1],s[2],s[3]),(30*u[1],30*u[2],30*u[3]),(30*v[1],30*v[2],30*v[3])],alpha=.1, fillalpha=.1)
    plot3d!([a],[b],[c],marker=2,xlim=(-14,14),ylim=(-14,14),zlim=(-14,14), camera = (blick1,blick2),cbar= :none, leg = false,xlabel = "x_1",ylabel = "x_2",zlabel = "x_3")
    plot3d!([d],[e],[f],marker=2)
    display(plt);
else
    println("Überprüft noch einmal, ob ihr die Parameter mit den richtigen Vektoren multipliziert habt.")
end

end


text:
# TODO, make sure that names are unique
"""
    dependency_path(paths...)

Path to serve downloaded dependencies
"""
dependency_path(paths...) = joinpath(@__DIR__, "..", "js_dependencies", paths...)

mediatype(asset::Asset) = asset.media_type

const server_proxy_url = Ref{String}()

function url(str::String)
    return server_proxy_url[] * str
end
function url(asset::Asset)
    if !isempty(asset.online_path)
        return asset.online_path
    else
        return url(AssetRegistry.register(asset.local_path))
    end
end

"""
    Asset(path_onload::Pair{String, JSCode})

Convenience constructor to make `Asset.(["path/to/asset" => js"onload"])`` work!
"""
Asset(path_onload::Pair{String, JSCode}) = Asset(path_onload...)

function Asset(online_path::String, onload::Union{Nothing, JSCode} = nothing)
    local_path = ""; real_online_path = ""
    if is_online(online_path)
        local_path = try
            #download(online_path, dependency_path(basename(online_path)))
            ""
        catch e
            @warn "Download for $online_path failed" exception=e
            local_path = ""
        end
        real_online_path = online_path
    else
        local_path = online_path
    end
    return Asset(Symbol(getextension(online_path)), real_online_path, local_path, onload)
end

"""
    getextension(path)
Get the file extension of the path.
The extension is defined to be the bit after the last dot, excluding any query
string.
# Examples
```julia-repl
julia> WebIO.getextension("foo.bar.js")
"js"
julia> WebIO.getextension("https://my-cdn.net/foo.bar.css?version=1")
"css"
```
"""
getextension(path) = lowercase(last(split(first(split(path, "?")), ".")))

"""
    islocal(path)
Determine whether or not the specified path is a local filesystem path (and not
a remote resource that is hosted on, for example, a CDN).
"""
is_online(path) = any(startswith.(path, ("//", "https://", "http://", "ftp://")))

function Dependency(name::Symbol, urls::AbstractVector)
    Dependency(
        name,
        Asset.(urls),
    )
end

function serialize_string(io::IO, assets::Set{Asset})
    for asset in assets
        serialize_string(io, asset)
        println(io)
    end
end

function serialize_string(io::IO, asset::Asset)
    if mediatype(asset) == :js
        println(
            io,
            "<script src='$(url(asset))'></script>"
        )
    elseif mediatype(asset) == :css
        println(
            io,
            "<link href = $(repr(url(asset))) rel = \"stylesheet\",  type=\"text/css\">"
        )
    else
        error("Unrecognized asset media type: $(mediatype(asset))")
    end
end

function serialize_string(io::IO, dependency::Dependency)
    print(io, dependency.name)
end

# With this, one can just put a dependency anywhere in the dom to get loaded
function jsrender(session::Session, x::Dependency)
    push!(session, x)
    # TODO implement returning nothing to just not be in the dom directly
    div(display = "none", visibility = "hidden")
end

const JSCallLib = Asset("https://simondanisch.github.io/JSServe.jl/js_dependencies/core.js")
const JSCallLibLocal = Asset(dependency_path("core.js"))


text:
export exportImage, exportMovie, exportMovies

exportImage(filename, im::ImageMeta; kargs...) = exportImage(filename, arraydata(im); kargs...)

#old syntax
function exportImage(filename, im::AbstractMatrix{T};
                    colormap::Union{Array, String}="grays", kargs...) where {T<:Real}
  exportImage(filename, im, colormap; kargs...)
end

function exportImage(filename, im::AbstractMatrix{T}, colorm::String;kargs...) where {T<:Real}
  return exportImage(filename, im, cmap(colorm); kargs...)
end

function exportImage(filename, im::AbstractMatrix{T}, colorm::Vector{C}; vmin=0.0, vmax=1.0,
                    normalize=true, pixelResizeFactor=1) where {T<:Real, C<:Colorant}
  imC = colorize( im, vmin, vmax, colorm, normalize=normalize )
  exportImage(filename, imC; pixelResizeFactor=pixelResizeFactor)
end

function exportImage(filename, im::AbstractMatrix{T}; pixelResizeFactor=1) where {T<:Colorant}
  file, ext = splitext(filename)

  imR = repeat(im, inner=[pixelResizeFactor,pixelResizeFactor])

  minPxSpacing = minimum(pixelspacing(im))
  newSize = ceil.(Int64, collect(pixelspacing(im)) / minPxSpacing .* collect(size(imR)) )

  dataResized = Images.imresize(imR,(newSize[1],newSize[2]))
  rgbdata = RGB.(dataResized)
  ImageMagick.save(filename, rgbdata)
end

function exportImage(filename, data::Vector{T}; kargs...) where {T<:AbstractMatrix}
    file, ext = splitext(filename)
    exportImage(file*"_xy.png", data[1]; kargs...)
    exportImage(file*"_xz.png", data[2]; kargs...)
    exportImage(file*"_yz.png", data[3]; kargs...)
end

### export movies ###

exportMovie(filename, data::ImageMeta; kargs...) = exportMovie(filename, data.data; kargs...)

#old syntax
function exportMovie(filename, data::AbstractArray{T,3};
                    colormap::Union{Array, String}="grays", kargs...) where {T<:Real}
  exportMovie(filename, data, colormap; kargs...)
end

function exportMovie(filename, data::AbstractArray{T,3}, colorm::String; kargs...) where {T<:Real}
  return exportMovie(filename, data, cmap(colorm); kargs...)
end

function exportMovie(filename, data::AbstractArray{T,3}, colorm::Vector{C}; vmin=0.0, vmax=1.0,
                    normalize=true, pixelResizeFactor=1) where {T<:Real, C<:Colorant}
    imC = colorize( data, vmin, vmax, colorm, normalize=normalize )
    exportMovie(filename, imC; pixelResizeFactor=pixelResizeFactor)
end

function exportMovie(filename, data::AbstractArray{T,3}; pixelResizeFactor=1) where {T<:Colorant}
  file, ext = splitext(filename)

  if pixelResizeFactor > 1
    data = repeat(data, inner=[pixelResizeFactor,pixelResizeFactor,1])
  end

  minPxSpacing = minimum(pixelspacing(data))
  newSize = ceil.(Int64, collect(pixelspacing(data)[1:2]) / minPxSpacing .* collect(size(data)[1:2]) )

  datai = similar(data, (newSize[1],newSize[2],size(data,3)))

  for l=1:size(data,3)
    datai[:,:,l] = Images.imresize(data[:,:,l],(newSize[1],newSize[2]))
  end

  rgbdata = RGB.(datai)
  @debug "saving $filename"
  ImageMagick.save(file*".gif", rgbdata)
end

function exportMovies(filename, data::Vector; kargs...)
  file, ext = splitext(filename)

  exportMovie(file*"_xy.gif", data[1]; kargs...)
  exportMovie(file*"_xz.gif", data[2]; kargs...)
  exportMovie(file*"_yz.gif", data[3]; kargs...)
end


text:
################################################################################
# MIT License
#
# Copyright (c) 2021 Hajime Nakagami<nakagami@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
################################################################################

mutable struct Statement <: DBInterface.Statement
    conn::Connection
    sql::String
    handle::Int32
    stmt_type::Int32
    xsqlda::Vector{XSQLVAR}

    function Statement(conn::Connection, sql::String)
        _op_allocate_statement(conn.wp)

        stmt_handle::Int32 = -1
        if conn.wp.accept_type == ptype_lazy_send
            conn.wp.lazy_response_count += 1
            stmt_handle = -1
        else
            stmt_handle, _, _ = _op_response(wp)
        end

        _op_prepare_statement(conn.wp, conn.transaction.handle, stmt_handle, sql)

        if conn.wp.accept_type == ptype_lazy_send && conn.wp.lazy_response_count > 0
            conn.wp.lazy_response_count -= 1
            stmt_handle, _, _ = _op_response(conn.wp)
        end

        _, _, buf = _op_response(conn.wp)
        stmt_type, xsqlda = parse_xsqlda(conn.wp, buf, stmt_handle)

        new(conn, sql, stmt_handle, stmt_type, xsqlda)
    end

end

"""
    DBInterface.prepare(conn::Firebird.Connection, sql) => Firebird.Statement

Send a `sql` SQL string to the database to be prepared, returning a `Firebird.Statement` object

"""
function DBInterface.prepare(conn::Connection, sql::AbstractString)::Statement
    Statement(conn, sql)
end

"""
    DBInterface.close!(stmt)

Close a prepared statement.
"""
function DBInterface.close!(stmt::Statement)
    if stmt.handle != -1
        wp = stmt.conn.wp
        _op_free_statement(wp, stmt.handle, DSQL_drop)
        if wp.accept_type == ptype_lazy_send
            wp.lazy_response_count += 1
        else
            _op_response(wp)
        end
    end
    stmt.handle = -1
end


text:
## MONOHALFPAGE ============================================================= ##
# Application:
#   - side-by-side figures over fullwidth of document
#   - fill one column of two-column document
monohalfpage = Theme(
    ## FIGURE ------------------------------------------------------------------
    font = joinpath(dirname(pathof(SmolkaFormat)), "..", "fonts/JuliaMono/JuliaMono-Medium.ttf"),
    # font = "../fonts/JuliaMono/JuliaMono-Medium.ttf",
    fontsize = 16,
    resolution = (600,500),

    # AXIS (2D) ----------------------------------------------------------------
    axis = Theme(
        backgroundcolor = :white,
        titlefont = joinpath(dirname(pathof(SmolkaFormat)), "..", "fonts/JuliaMono/JuliaMono-Bold.ttf"),
        # titlefont = "../fonts/JuliaMono/JuliaMono-Bold.ttf",
        titlesize = 18,
    ),

    # LEGEND -------------------------------------------------------------------
    # legend / axislegend
    legend = Theme(
        labelsize = 12,
        titlefont = joinpath(dirname(pathof(SmolkaFormat)), "../fonts/JuliaMono/JuliaMono-Bold.ttf"),
        # titlefont = "../fonts/JuliaMono/JuliaMono-Bold.ttf",
    ),

    ## PLOT OBJECTS (requires capital first letter) ----------------------------
    # lines, lines!
    Lines = Theme(
        linewidth = 3,
    )
)


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule normaliz_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("normaliz")
JLLWrappers.@generate_main_file("normaliz", UUID("6690c6e9-4e12-53b8-b8fd-4bffaef8839f"))
end  # module normaliz_jll


text:
"""
    wrapped_graph(g::MyGraphWrapper)

Method to implement for creating graph wrapper types.
Should return the graph that is wrapped inside `g`.

### See also
[`wrapped_graph_type`](@ref), [`@wrap_graph!`](@ref)
"""
function wrapped_graph(::AbstractValGraph) end

"""
    wrapped_graph_type{::Type{<:MyGraphWrapper}}

Method to implement for creating graph wrapper types.
Should return the type of the graph that is wrapped inside
a graph wrapper.

### See also
[`wrapped_graph`](@ref), [`@wrap_graph!`](@ref)

"""
function wrapped_graph_type(::Type) end

_DEFAULT_INCLUDE = [
    :nv,
    :is_directed,
    :has_edge,
    :get_graphval,
    :get_vertexval,
    :get_edgeval,
    :set_graphval!,
    :set_vertexval!,
    :set_edgeval!,
    :add_vertex!,
    :rem_vertex!,
    :add_edge!,
    :rem_edge!,
   ]

_EXTRA_INCLUDE = [
    :ne,
    :outneighbors,
    :inneighbors,
    :outedgevals,
    :inedgevals,
    :edges
]

"""
    @wrap_graph! MyGraphWrapper include=[] exclude=[]

Generate default method implementations for graph wrappers.
These default implementations simply call the wrapped graph. For that, one
needs to implement `wrapped_graph` and `wrapped_graph_type` for the custom graph wrapper.

Methods given as a `Vector` to the keyword argument `exclude` are not created.
By default, methods for the following functions are generated:
$(join(map(s -> '`' * string(s) * '`', _DEFAULT_INCLUDE), ", ")).

There are additional methods that can be generated but are not generated by default, as
these methods all have a default implementation in terms of other functions. To generate
these additional methods, specify them in a `Vector` to the keyword argument `include`.
The following methods can be additionally generated:
$(join(map(s -> '`' * string(s) * '`', _EXTRA_INCLUDE), ", ")).

Note: To completely implement the `SimpleValueGraphs` interface for some graph one
would also have to implement `zero`. This macro currently cannot do that, as this
would knowledge of how to construct a new graph wrapper. Therefore one has to implement this
method separately.

# Examples
```julia
# generate methods for ne, that are not generated by default and
# do not generate methods for get_graphval and set_graphval! that would
# be generated by default
@wrap_graph! MyGraphWrapper include=[ne] exclude=[get_graphval, set_graphval!]
```
### See also
[`wrapped_graph`](@ref), [`wrapped_graph_type`](@ref)
"""
macro wrap_graph!(GT, args...)

    functions_to_generate = copy(_DEFAULT_INCLUDE)

    # TODO maybe we can somehow assert that GT is something like a type
    # for now we just use an `typeintersect`. If GT is not an `AbstractValGraph`
    # this will simply result in a `Union{}` type
    GT = :(typeintersect(SimpleValueGraphs.AbstractValGraph, $GT))

    for arg in args
        if !(arg isa Expr) ||
                arg.head != :(=) ||
                length(arg.args) != 2 ||
                arg.args[1] ∉ (:include, :exclude) ||
                !(arg.args[2] isa Expr) ||
                arg.args[2].head != :vect

            error("Argument must be of the form `include=[function names...]` or exclude=[function names...]`")
        end
        # TODO we could also fail if there are unknown functions specified in one of the lists
        if arg.args[1] == :include
            include_list = intersect(arg.args[2].args, _EXTRA_INCLUDE)
            union!(functions_to_generate, include_list)
        else
            exclude_list = arg.args[2].args
            setdiff!(functions_to_generate, exclude_list)
        end

    end

    return Expr(:block, map(f -> _generate_wrapped_function!(Val(f), esc(GT)), functions_to_generate)...)
end

function _generate_wrapped_function!(::Val{:nv}, GT)

    return :(SimpleValueGraphs.nv(g::$GT) = nv(wrapped_graph(g)))
end

function _generate_wrapped_function!(::Val{:is_directed}, GT)

    return :(SimpleValueGraphs.is_directed(G::Type{<:$GT}) = is_directed(wrapped_graph_type(G)))
end

function _generate_wrapped_function!(::Val{:has_edge}, GT)

    return :(SimpleValueGraphs.has_edge(g::$GT, s::Integer, d::Integer) = has_edge(wrapped_graph(g), s, d))
end

function _generate_wrapped_function!(::Val{:add_vertex!}, GT)

    return :(SimpleValueGraphs.add_vertex!(g::$GT, vals) = add_vertex!(wrapped_graph(g), vals))
end

function _generate_wrapped_function!(::Val{:rem_vertex!}, GT)

    return :(SimpleValueGraphs.rem_vertex!(g::$GT, v::Integer) = rem_vertex!(wrapped_graph(g), v))
end

function _generate_wrapped_function!(::Val{:add_edge!}, GT)

    return :(SimpleValueGraphs.add_edge!(g::$GT, s::Integer, d::Integer, vals) = add_edge!(wrapped_graph(g), s, d, vals))
end

function _generate_wrapped_function!(::Val{:rem_edge!}, GT)

    return :(SimpleValueGraphs.rem_edge!(g::$GT, s::Integer, d::Integer) = rem_edge!(wrapped_graph(g), s, d))
end

function _generate_wrapped_function!(::Val{:get_vertexval}, GT)

    return quote
        SimpleValueGraphs.get_vertexval(g::$GT, v::Integer, key::Integer) = get_vertexval(wrapped_graph(g), v, key)
        SimpleValueGraphs.get_vertexval(g::$GT, v::Integer, key::Symbol) = get_vertexval(wrapped_graph(g), v, key)
        SimpleValueGraphs.get_vertexval(g::$GT, v::Integer, ::Colon) = get_vertexval(wrapped_graph(g), v, :)
    end
end

function _generate_wrapped_function!(::Val{:set_vertexval!}, GT)

    return quote
        SimpleValueGraphs.set_vertexval!(g::$GT, v::Integer, key::Integer, value) = set_vertexval!(wrapped_graph(g), v, key, value)
        SimpleValueGraphs.set_vertexval!(g::$GT, v::Integer, key::Symbol, value) = set_vertexval!(wrapped_graph(g), v, key, value)
        SimpleValueGraphs.set_vertexval!(g::$GT, v::Integer, ::Colon, values) = set_vertexval!(wrapped_graph(g), v, :, values)
    end
end

function _generate_wrapped_function!(::Val{:get_edgeval}, GT)

    return quote
        SimpleValueGraphs.get_edgeval(g::$GT, s::Integer, d::Integer, key::Integer) = get_edgeval(wrapped_graph(g), s, d, key)
        SimpleValueGraphs.get_edgeval(g::$GT, s::Integer, d::Integer, key::Symbol) = get_edgeval(wrapped_graph(g), s, d, key)
        SimpleValueGraphs.get_edgeval(g::$GT, s::Integer, d::Integer, ::Colon) = get_edgeval(wrapped_graph(g), s, d, :)
    end
end

function _generate_wrapped_function!(::Val{:set_edgeval!}, GT)

    return quote
        SimpleValueGraphs.set_edgeval!(g::$GT, s::Integer, d::Integer, key::Integer, value) = set_edgeval!(wrapped_graph(g), s, d, key, value)
        SimpleValueGraphs.set_edgeval!(g::$GT, s::Integer, d::Integer, key::Symbol, value) = set_edgeval!(wrapped_graph(g), s, d, key, value)
        SimpleValueGraphs.set_edgeval!(g::$GT, s::Integer, d::Integer, ::Colon, values) = set_edgeval!(wrapped_graph(g), s, d, :, values)
    end
end

function _generate_wrapped_function!(::Val{:get_graphval}, GT)

    return quote
        SimpleValueGraphs.get_graphval(g::$GT, key::Integer) = get_graphval(wrapped_graph(g), key)
        SimpleValueGraphs.get_graphval(g::$GT, key::Symbol) = get_graphval(wrapped_graph(g), key)
        SimpleValueGraphs.get_graphval(g::$GT, ::Colon) = get_graphval(wrapped_graph(g), :)
    end
end

function _generate_wrapped_function!(::Val{:set_graphval!}, GT)

    return quote
        SimpleValueGraphs.set_graphval!(g::$GT, key::Integer, value) = set_graphval!(wrapped_graph(g), key, value)
        SimpleValueGraphs.set_graphval!(g::$GT, key::Symbol, value) = set_graphval!(wrapped_graph(g), key, value)
        SimpleValueGraphs.set_graphval!(g::$GT, ::Colon, values) = set_graphval!(wrapped_graph(g), :, values)
    end
end

function _generate_wrapped_function!(::Val{:ne}, GT)

    return :(SimpleValueGraphs.ne(g::$GT) = ne(wrapped_graph(g)))
end

function _generate_wrapped_function!(::Val{:outneighbors}, GT)

    return :(SimpleValueGraphs.outneighbors(g::$GT, u) = outneighbors(wrapped_graph(g), u))
end

function _generate_wrapped_function!(::Val{:inneighbors}, GT)

    return :(SimpleValueGraphs.inneighbors(g::$GT, v) = inneighbors(wrapped_graph(g), v))
end

function _generate_wrapped_function!(::Val{:outedgevals}, GT)

    return quote
        SimpleValueGraphs.outedgevals(g::$GT, u, key::Integer) = outedgevals(wrapped_graph(g), u, key)
        SimpleValueGraphs.outedgevals(g::$GT, u, key::Symbol) = outedgevals(wrapped_graph(g), u, key)
        SimpleValueGraphs.outedgevals(g::$GT, u, ::Colon) = outedgevals(wrapped_graph(g), u, :)
    end
end

function _generate_wrapped_function!(::Val{:inedgevals}, GT)

    return quote
        SimpleValueGraphs.inedgevals(g::$GT, v, key::Integer) = inedgevals(wrapped_graph(g), v, key)
        SimpleValueGraphs.inedgevals(g::$GT, v, key::Symbol) = inedgevals(wrapped_graph(g), v, key)
        SimpleValueGraphs.inedgevals(g::$GT, v, ::Colon) = inedgevals(wrapped_graph(g), v, :)
    end
end

function _generate_wrapped_function!(::Val{:edges}, GT)

    return :(SimpleValueGraphs.edges(g::$GT, key=nothing) = edges(wrapped_graph(g), key))
end


text:
using MendelImpute
using Test
using Random

# write your own tests here
include("intersect_test.jl")
include("searchbreakpoint_test.jl")
include("dynamic_programming_test.jl")


text:
@testset "utilities.jl" begin

f_test(x, a, b) = a .* x .+ b

@test EasyPhys.number_of_arguments(f_test) == 3

@test EasyPhys.argument_names(f_test) == [:x, :a, :b]

EasyPhys.@partially_applicable g_test(x, a, b; kwargs...) = (a * x + b)
for x in 1:5, a in 6:10, b in 1:10
    @test g_test(x, a, b; test=true) == x |> g_test(a, b; test=false) == a * x + b
end

EasyPhys.@partially_applicable g_test(x; kwargs...) = (println(kwargs); 5*x)
@test g_test(2; test=:somearg) == (2::Int) |> g_test(test=:someotherarg) == 10

end


text:
import HTTP

const guideline_registry_consistency_tests_pass =
    Guideline("Registy consistency tests",
              data -> meets_registry_consistency_tests_pass(data.registry_head,
                                                            data.registry_deps))

function meets_registry_consistency_tests_pass(registry_head::String,
                                               registry_deps::Vector{String})
    try
        RegistryCI.test(registry_head; registry_deps = registry_deps)
        return true, ""
    catch ex
        @error "" exception=(ex, catch_backtrace())
    end
    return false, "The registry consistency tests failed"
end

const guideline_compat_for_julia =
    Guideline("Compat with upper bound for julia",
              data -> meets_compat_for_julia(data.registry_head,
                                             data.pkg,
                                             data.version))

function meets_compat_for_julia(working_directory::AbstractString, pkg, version)
    compat = Pkg.TOML.parsefile(joinpath(working_directory, uppercase(pkg[1:1]), pkg, "Compat.toml"))
    # Go through all the compat entries looking for the julia compat
    # of the new version. When found, test
    # 1. that it is a bounded range,
    # 2. that the upper bound is not 2 or higher,
    # 3. that the range includes at least one 1.x version.
    for version_range in keys(compat)
        if version in Pkg.Types.VersionRange(version_range)
            if haskey(compat[version_range], "julia")
                julia_compat = Pkg.Types.VersionSpec(compat[version_range]["julia"])
                if !isempty(intersect(julia_compat,
                                      Pkg.Types.VersionSpec("$(typemax(Base.VInt))-*")))
                    return false, "The compat entry for `julia` is unbounded."
                elseif !isempty(intersect(julia_compat,
                                          Pkg.Types.VersionSpec("2-*")))
                    return false, "The compat entry for `julia` has an upper bound of 2 or higher."
                elseif isempty(intersect(julia_compat,
                                         Pkg.Types.VersionSpec("1")))
                    # For completeness, although this seems rather
                    # unlikely to occur.
                    return false, "The compat entry for `julia` doesn't include any 1.x version."
                else
                    return true, ""
                end
            end
        end
    end

    return false, "There is no compat entry for `julia`."
end

const guideline_compat_for_all_deps =
    Guideline("Compat (with upper bound) for all dependencies",
              data -> meets_compat_for_all_deps(data.registry_head,
                                                data.pkg,
                                                data.version))

function meets_compat_for_all_deps(working_directory::AbstractString, pkg, version)
    deps = Pkg.TOML.parsefile(joinpath(working_directory, uppercase(pkg[1:1]), pkg, "Deps.toml"))
    compat = Pkg.TOML.parsefile(joinpath(working_directory, uppercase(pkg[1:1]), pkg, "Compat.toml"))
    # First, we construct a Dict in which the keys are the package's
    # dependencies, and the value is always false.
    dep_has_compat_with_upper_bound = Dict{String, Bool}()
    for version_range in keys(deps)
        if version in Pkg.Types.VersionRange(version_range)
            for name in keys(deps[version_range])
                if !is_jll_name(name) && !is_julia_stdlib(name)
                    @debug("Found a new (non-stdlib non-JLL) dependency: $(name)")
                    dep_has_compat_with_upper_bound[name] = false
                end
            end
        end
    end
    # Now, we go through all the compat entries. If a dependency has a compat
    # entry with an upper bound, we change the corresponding value in the Dict
    # to true.
    for version_range in keys(compat)
        if version in Pkg.Types.VersionRange(version_range)
            for (name, value) in compat[version_range]
                if value isa Vector
                    if !isempty(value)
                        value_ranges = Pkg.Types.VersionRange.(value)
                        each_range_has_upper_bound = _has_upper_bound.(value_ranges)
                        if all(each_range_has_upper_bound)
                            @debug("Dependency \"$(name)\" has compat entries that all have upper bounds")
                            dep_has_compat_with_upper_bound[name] = true
                        end
                    end
                else
                    value_range = Pkg.Types.VersionRange(value)
                    if _has_upper_bound(value_range)
                        @debug("Dependency \"$(name)\" has a compat entry with an upper bound")
                        dep_has_compat_with_upper_bound[name] = true
                    end
                end
            end
        end
    end
    meets_this_guideline = all(values(dep_has_compat_with_upper_bound))
    if meets_this_guideline
        return true, ""
    else
        bad_dependencies = Vector{String}()
        for name in keys(dep_has_compat_with_upper_bound)
            if !(dep_has_compat_with_upper_bound[name])
                @error("Dependency \"$(name)\" does not have a compat entry that has an upper bound")
                push!(bad_dependencies, name)
            end
        end
        sort!(bad_dependencies)
        message = string(
            "The following dependencies do not have a `[compat]` entry ",
            "that is upper-bounded and only includes a finite number ",
            "of breaking releases: ",
            join(bad_dependencies, ", ")
        )
        return false, message
    end
end

const guideline_patch_release_does_not_narrow_julia_compat =
    Guideline("If it is a patch release, then it does not narrow the Julia compat range",
              data -> meets_patch_release_does_not_narrow_julia_compat(data.pkg,
                                                                       data.version;
                                                                       registry_head = data.registry_head,
                                                                       registry_master = data.registry_master))

function meets_patch_release_does_not_narrow_julia_compat(pkg::String,
                                                          new_version::VersionNumber;
                                                          registry_head::String,
                                                          registry_master::String)
    old_version = latest_version(pkg, registry_master)
    if old_version.major != new_version.major || old_version.minor != new_version.minor
        # Not a patch release.
        return true, ""
    end
    julia_compats_for_old_version = julia_compat(pkg, old_version, registry_master)
    julia_compats_for_new_version = julia_compat(pkg, new_version, registry_head)
    if Set(julia_compats_for_old_version) == Set(julia_compats_for_new_version)
        return true, ""
    end
    meets_this_guideline = range_did_not_narrow(julia_compats_for_old_version, julia_compats_for_new_version)
    if meets_this_guideline
        return true, ""
    else
        if (old_version >= v"1") || (new_version >= v"1")
            msg = string("A patch release is not allowed to narrow the ",
                         "supported ranges of Julia versions. ",
                         "The ranges have changed from ",
                         "$(julia_compats_for_old_version) ",
                         "(in $(old_version)) ",
                         "to $(julia_compats_for_new_version) ",
                         "(in $(new_version)).")
            return false, msg
        else
            @info("Narrows Julia compat, but it's OK since package is pre-1.0")
            return true, ""
        end
    end
end

const guideline_name_length =
    Guideline("Name not too short",
              data -> meets_name_length(data.pkg))

function meets_name_length(pkg)
    meets_this_guideline = length(pkg) >= 5
    if meets_this_guideline
        return true, ""
    else
        return false, "Name is not at least five characters long"
    end
end

const guideline_name_ascii =
    Guideline("Name is composed of ASCII characters only",
              data -> meets_name_ascii(data.pkg))

function meets_name_ascii(pkg)
    if isascii(pkg)
        return true, ""
    else
        return false, "Name is not ASCII"
    end
end

const guideline_julia_name_check =
    Guideline("Name does not include \"julia\" or start with \"Ju\"",
              data -> meets_julia_name_check(data.pkg))

function meets_julia_name_check(pkg)
    if occursin("julia", lowercase(pkg))
        return false, "Lowercase package name $(lowercase(pkg)) contains the string \"julia\"."
    elseif startswith(pkg, "Ju")
        return false, "Package name starts with \"Ju\"."
    else
        return true, ""
    end
end

damerau_levenshtein(name1, name2) = StringDistances.DamerauLevenshtein()(name1, name2)
sqrt_normalized_vd(name1, name2) = VisualStringDistances.visual_distance(name1, name2; normalize=x -> 5 + sqrt(x))

const guideline_distance_check =
    Guideline("Name is not too similar to existing package names",
              data -> meets_distance_check(data.pkg, data.registry_master))

function meets_distance_check(pkg_name::AbstractString,
                              registry_master::AbstractString;
                              kwargs...)
    other_packages = get_all_non_jll_package_names(registry_master)
    return meets_distance_check(pkg_name, other_packages; kwargs...)
end

function meets_distance_check(pkg_name::AbstractString,
                              other_packages::Vector;
                              DL_lowercase_cutoff = 1,
                              DL_cutoff = 2,
                              sqrt_normalized_vd_cutoff = 2.5,
                              comment_collapse_cutoff = 10)
    problem_messages = Tuple{String, Tuple{Float64, Float64, Float64}}[]
    for other_pkg in other_packages
        if pkg_name == other_pkg
            # We short-circuit in this case; more information doesn't help.
            return  (false, "Package name already exists in the registry.")
        elseif lowercase(pkg_name) == lowercase(other_pkg)
            # We'll sort this first
            push!(problem_messages, ("Package name matches existing package name $(other_pkg) up to case.", (0,0,0)))
        else
            msg = ""

            # Distance check 1: DL distance
            dl = damerau_levenshtein(pkg_name, other_pkg)
            if dl <= DL_cutoff
                msg = string(msg, " Damerau-Levenshtein distance $dl is at or below cutoff of $(DL_cutoff).")
            end

            # Distance check 2: lowercase DL distance
            dl_lowercase = damerau_levenshtein(lowercase(pkg_name), lowercase(other_pkg))
            if dl_lowercase <= DL_lowercase_cutoff
                msg = string(msg, " Damerau-Levenshtein distance $(dl_lowercase) between lowercased names is at or below cutoff of $(DL_lowercase_cutoff).")
            end

            # Distance check 3: normalized visual distance,
            # gated by a `dl` check for speed.
            if (sqrt_normalized_vd_cutoff > 0 && dl <= 4)
                nrm_vd = sqrt_normalized_vd(pkg_name, other_pkg)
                if nrm_vd <= sqrt_normalized_vd_cutoff
                    msg = string(msg, " Normalized visual distance ", Printf.@sprintf("%.2f", nrm_vd), " is at or below cutoff of ", Printf.@sprintf("%.2f", sqrt_normalized_vd_cutoff), ".")
                end
            else
                # need to choose something for sorting purposes
                nrm_vd = 10.0
            end

            if msg != ""
                # We must have found a clash.
                push!(problem_messages, (string("Similar to $(other_pkg).", msg), (dl, dl_lowercase, nrm_vd)))
            end
        end
    end

    isempty(problem_messages) && return (true, "")
    sort!(problem_messages, by = Base.tail)
    message = string("Package name similar to $(length(problem_messages)) existing package",
                    length(problem_messages) > 1 ? "s" : "", ".\n")
    use_spoiler = length(problem_messages) > comment_collapse_cutoff
    # we indent each line by two spaces in all the following
    # so that it nests properly in the outer list.
    if use_spoiler
        message *=  """
                      <details>
                      <summary>Similar package names</summary>

                    """
    end
    numbers = string.("  ", 1:length(problem_messages))
    message *= join(join.(zip(numbers, first.(problem_messages)), Ref(". ")), '\n')
    if use_spoiler
        message *= "\n\n  </details>\n"
    end
    return (false, message)
end

const guideline_normal_capitalization =
    Guideline("Normal capitalization",
              data -> meets_normal_capitalization(data.pkg))

function meets_normal_capitalization(pkg)
    meets_this_guideline = occursin(r"^[A-Z]\w*[a-z]\w*[0-9]?$", pkg)
    if meets_this_guideline
        return true, ""
    else
        return false, "Name does not meet all of the following: starts with an uppercase letter, ASCII alphanumerics only, not all letters are uppercase."
    end
end

const guideline_repo_url_requirement =
    Guideline("Repo URL ends with /name.jl.git",
              data -> meets_repo_url_requirement(data.pkg;
                                                 registry_head = data.registry_head))

function meets_repo_url_requirement(pkg::String; registry_head::String)
    package_toml_parsed = Pkg.TOML.parsefile(
        joinpath(
            registry_head,
            uppercase(pkg[1:1]),
            pkg,
            "Package.toml",
        )
    )

    url = package_toml_parsed["repo"]
    subdir = get(package_toml_parsed, "subdir", "")
    is_subdirectory_package = occursin(r"[A-Za-z0-9]", subdir)
    meets_this_guideline = url_has_correct_ending(url, pkg)

    if is_subdirectory_package
        return true, "" # we do not apply this check if the package is a subdirectory package
    end
    if meets_this_guideline
        return true, ""
    end
    return false, "Repo URL does not end with /name.jl.git, where name is the package name"
end

function _invalid_sequential_version(reason::AbstractString)
    return false, "Does not meet sequential version number guideline: $reason", :invalid
end

function _valid_change(old_version::VersionNumber, new_version::VersionNumber)
    diff = difference(old_version, new_version)
    @debug("Difference between versions: ", old_version, new_version, diff)
    if diff == v"0.0.1"
        return true, "", :patch
    elseif diff == v"0.1.0"
        return true, "", :minor
    elseif diff == v"1.0.0"
        return true, "", :major
    else
        return _invalid_sequential_version("increment is not one of: 0.0.1, 0.1.0, 1.0.0")
    end
end

const guideline_sequential_version_number =
    Guideline("Sequential version number",
              data -> meets_sequential_version_number(data.pkg,
                                                      data.version;
                                                      registry_head = data.registry_head,
                                                      registry_master = data.registry_master))

function meets_sequential_version_number(existing::Vector{VersionNumber}, ver::VersionNumber)
    always_assert(!isempty(existing))
    if ver in existing
        return _invalid_sequential_version("version $ver already exists")
    end
    issorted(existing) || (existing = sort(existing))
    idx = searchsortedlast(existing, ver)
    idx > 0 || return _invalid_sequential_version("version $ver less than least existing version $(existing[1])")
    prv = existing[idx]
    always_assert(ver != prv)
    nxt = thismajor(ver) != thismajor(prv) ? nextmajor(prv) :
          thisminor(ver) != thisminor(prv) ? nextminor(prv) : nextpatch(prv)
    ver <= nxt || return _invalid_sequential_version("version $ver skips over $nxt")
    return _valid_change(prv, ver)
end

_has_prerelease_andor_build_data(version) = !isempty(version.prerelease) || !isempty(version.build)

function meets_sequential_version_number(pkg::String,
                                         new_version::VersionNumber;
                                         registry_head::String,
                                         registry_master::String)
    if _has_prerelease_andor_build_data(new_version)
        return false, "Version number is not allowed to contain prerelease or build data", :invalid
    end
    _all_versions = all_versions(pkg, registry_master)
    return meets_sequential_version_number(_all_versions, new_version)
end

const guideline_standard_initial_version_number =
    Guideline("Standard initial version number ",
              data -> meets_standard_initial_version_number(data.version))

function meets_standard_initial_version_number(version)
    if _has_prerelease_andor_build_data(version)
        return false, "Version number is not allowed to contain prerelease or build data"
    end
    meets_this_guideline = version == v"0.0.1" || version == v"0.1.0" || version == v"1.0.0" || _is_x_0_0(version)
    if meets_this_guideline
        return true, ""
    else
        return false, "Version number is not 0.0.1, 0.1.0, 1.0.0, or X.0.0"
    end
end

function _is_x_0_0(version::VersionNumber)
    if _has_prerelease_andor_build_data(version)
        return false
    end
    result = (version.major >= 1) && (version.minor == 0) && (version.patch == 0)
    return result
end

const guideline_code_can_be_downloaded =
    Guideline("Code can be downloaded",
              data -> meets_code_can_be_downloaded(data.registry_head,
                                                   data.pkg, data.version, data.pr;
                                                     pkg_code_path = data.pkg_code_path))

function meets_code_can_be_downloaded(registry_head, pkg, version, pr; pkg_code_path)
    uuid, package_repo, subdir, tree_hash_from_toml = parse_registry_pkg_info(registry_head, pkg, version)

    # We get the `tree_hash` two ways and check they agree, which helps ensures the `subdir` parameter is correct. Two ways:
    # 1. By the commit hash in the PR body and the subdir parameter
    # 2. By the tree hash in the Versions.toml

    commit_hash = commit_from_pull_request_body(pr)

    local tree_hash_from_commit, tree_hash_from_commit_success
    clone_success = load_files_from_url_and_tree_hash(pkg_code_path, package_repo, tree_hash_from_toml) do dir
        tree_hash_from_commit, tree_hash_from_commit_success = try
            readchomp(Cmd(`git rev-parse $(commit_hash):$(subdir)`; dir=dir)), true
        catch e
            @error e
            "", false
        end
    end

    if !clone_success
        return false, "Cloning repository failed."
    end

    if !tree_hash_from_commit_success
        return false, "Could not obtain tree hash from commit hash and subdir parameter. Possibly this indicates that an incorrect `subdir` parameter was passed during registration."
    end

    if tree_hash_from_commit != tree_hash_from_toml
        @error "`tree_hash_from_commit != tree_hash_from_toml`" tree_hash_from_commit tree_hash_from_toml
        return false, "Tree hash obtained from the commit message and subdirectory does not match the tree hash in the Versions.toml file. Possibly this indicates that an incorrect `subdir` parameter was passed during registration."
    else
        return true, ""
    end
end

function _generate_pkg_add_command(pkg::String,
                                   version::VersionNumber)::String
    return "Pkg.add(Pkg.PackageSpec(name=\"$(pkg)\", version=v\"$(string(version))\"));"
end

is_valid_url(str::AbstractString) = !isempty(HTTP.URI(str).scheme) && isvalid(HTTP.URI(str))

const guideline_version_can_be_pkg_added =
    Guideline("Version can be `Pkg.add`ed",
              data -> meets_version_can_be_pkg_added(data.registry_head,
                                                     data.pkg,
                                                     data.version;
                                                     registry_deps = data.registry_deps))

function meets_version_can_be_pkg_added(working_directory::String,
                                        pkg::String,
                                        version::VersionNumber;
                                        registry_deps::Vector{<:AbstractString} = String[])
    pkg_add_command = _generate_pkg_add_command(pkg,
                                                version)
    _registry_deps = convert(Vector{String}, registry_deps)
    _registry_deps_is_valid_url = is_valid_url.(_registry_deps)
    code = """
        import Pkg;
        Pkg.Registry.add(Pkg.RegistrySpec(path=\"$(working_directory)\"));
        _registry_deps = $(_registry_deps);
        _registry_deps_is_valid_url = $(_registry_deps_is_valid_url);
        for i = 1:length(_registry_deps)
            regdep = _registry_deps[i]
            if _registry_deps_is_valid_url[i]
                Pkg.Registry.add(Pkg.RegistrySpec(url = regdep))
            else
                Pkg.Registry.add(regdep)
            end
        end
        @info("Attempting to `Pkg.add` package...");
        $(pkg_add_command)
        @info("Successfully `Pkg.add`ed package");
        """

    cmd_ran_successfully = _run_pkg_commands(working_directory, pkg,
                                version; code = code,
                                before_message = "Attempting to `Pkg.add` the package")
    if cmd_ran_successfully
        @info "Successfully `Pkg.add`ed the package"
        return true, ""
    else
        @error "Was not able to successfully `Pkg.add` the package"
        return false, string("I was not able to install the package ",
                             "(i.e. `Pkg.add(\"$(pkg)\")` failed). ",
                             "See the CI logs for details.")
    end
end

const guideline_version_has_osi_license =
    Guideline("Version has OSI-approved license",
              data -> meets_version_has_osi_license(data.pkg; pkg_code_path = data.pkg_code_path))

function meets_version_has_osi_license(pkg::String; pkg_code_path)
    pkgdir = pkg_code_path
    if !isdir(pkgdir) || isempty(readdir(pkgdir))
        return false, "Could not check license because could not access package code. Perhaps the `can_download_code` check failed earlier."
    end

    license_results = LicenseCheck.find_licenses(pkgdir)

    # Failure mode 1: no licenses
    if isempty(license_results)
        @error "Could not find any licenses"
        return false, string("No licenses detected in the package's top-level folder. An OSI-approved license is required.")
    end

    flat_results = [(filename = lic.license_filename, identifier=identifier, approved=LicenseCheck.is_osi_approved(identifier)) for lic in license_results for identifier in lic.licenses_found ]

    osi_results = [ string(r.identifier, " license in ", r.filename) for r in flat_results if r.approved ]
    non_osi_results = [ string(r.identifier, " license in ", r.filename) for r in flat_results if !r.approved ]

    osi_string = string("Found OSI-approved license(s): ", join(osi_results, ", ", ", and "), ".")
    non_osi_string = string("found non-OSI license(s): ", join(non_osi_results, ", ", ", and "), ".")

    # Failure mode 2: no OSI-approved licenses, but has some kind of license detected
    if isempty(osi_results)
        @error "Found no OSI-approved licenses" non_osi_string
        return false, string("Found no OSI-approved licenses. ",  uppercasefirst(non_osi_string))
    end

    # Pass: at least one OSI-approved license, possibly other licenses.
    @info "License check passed; results" osi_results non_osi_results
    if !isempty(non_osi_results)
        return true, string(osi_string, " Also ", non_osi_string)
    else
        return true, string(osi_string, " Found no other licenses.")
    end
end

const guideline_version_can_be_imported =
    Guideline("Version can be `import`ed",
              data -> meets_version_can_be_imported(data.registry_head,
                                                    data.pkg,
                                                    data.version;
                                                    registry_deps = data.registry_deps))

function meets_version_can_be_imported(working_directory::String,
                                       pkg::String,
                                       version::VersionNumber;
                                       registry_deps::Vector{<:AbstractString} = String[])
    pkg_add_command = _generate_pkg_add_command(pkg,
                                                version)
    _registry_deps = convert(Vector{String}, registry_deps)
    _registry_deps_is_valid_url = is_valid_url.(_registry_deps)
    code = """
        import Pkg;
        Pkg.Registry.add(Pkg.RegistrySpec(path=\"$(working_directory)\"));
        _registry_deps = $(_registry_deps);
        _registry_deps_is_valid_url = $(_registry_deps_is_valid_url);
        for i = 1:length(_registry_deps)
            regdep = _registry_deps[i]
            if _registry_deps_is_valid_url[i]
                Pkg.Registry.add(Pkg.RegistrySpec(url = regdep))
            else
                Pkg.Registry.add(regdep)
            end
        end
        @info("Attempting to `Pkg.add` package...");
        $(pkg_add_command)
        @info("Successfully `Pkg.add`ed package");
        @info("Attempting to `import` package");
        import $(pkg);
        @info("Successfully `import`ed package");
        """

    cmd_ran_successfully = _run_pkg_commands(working_directory, pkg,
                                version; code = code,
                                before_message = "Attempting to `import` the package")

    if cmd_ran_successfully
        @info "Successfully `import`ed the package"
        return true, ""
    else
        @error "Was not able to successfully `import` the package"
        return false, string("I was not able to load the package ",
                             "(i.e. `import $(pkg)` failed). ",
                             "See the CI logs for details.")
    end
end

function _run_pkg_commands(working_directory::String,
                           pkg::String,
                           version::VersionNumber;
                           code,
                           before_message)
    original_directory = pwd()
    tmp_dir_1 = mktempdir()
    atexit(() -> rm(tmp_dir_1; force = true, recursive = true))
    cd(tmp_dir_1)
    # We need to be careful with what environment variables we pass to the child
    # process. For example, we don't want to pass an environment variable containing
    # our GitHub token to the child process. Because if the Julia package that we are
    # testing has malicious code in its __init__() function, it could try to steal
    # our token. So we only pass these environment variables:
    # 1. HTTP_PROXY. If it's set, it is delegated to the child process.
    # 2. HTTPS_PROXY. If it's set, it is delegated to the child process.
    # 3. JULIA_DEPOT_PATH. We set JULIA_DEPOT_PATH to the temporary directory that
    #    we created. This is because we don't want the child process using our
    #    real Julia depot. So we set up a fake depot for the child process to use.
    # 4. JULIA_PKG_SERVER. If it's set, it is delegated to the child process.
    # 5. JULIA_REGISTRYCI_AUTOMERGE. We set JULIA_REGISTRYCI_AUTOMERGE to "true".
    # 6. PATH. If we don't pass PATH, things break. And PATH should not contain any
    #    sensitive information.
    # 7. PYTHON. We set PYTHON to the empty string. This forces any packages that use
    #    PyCall to install their own version of Python instead of using the system
    #    Python.
    # 8. R_HOME. We set R_HOME to "*".
    # 9. HOME. Lots of things need HOME.

    env = Dict(
        "JULIA_DEPOT_PATH" => mktempdir(),
        "JULIA_REGISTRYCI_AUTOMERGE" => "true",
        "PYTHON" => "",
        "R_HOME" => "*",
    )
    for k in ("HOME", "PATH", "HTTP_PROXY", "HTTPS_PROXY", "JULIA_PKG_SERVER")
        if haskey(ENV, k)
            env[k] = ENV[k]
        end
    end

    cmd = Cmd(`$(Base.julia_cmd()) -e $(code)`; env=env)

    # GUI toolkits may need a display just to load the package
    xvfb = Sys.which("xvfb-run")
    @info("xvfb: ", xvfb)
    if xvfb !== nothing
        pushfirst!(cmd.exec, "-a")
        pushfirst!(cmd.exec, xvfb)
    end
    @info(before_message)
    cmd_ran_successfully = success(pipeline(cmd, stdout=stdout, stderr=stderr))
    cd(original_directory)

    rmdir(tmp_dir_1)

    return cmd_ran_successfully
end

function rmdir(dir)
    try
        chmod(dir, 0o700, recursive = true)
    catch
    end
    rm(dir; force = true, recursive = true)
end

url_has_correct_ending(url, pkg) = endswith(url, "/$(pkg).jl.git")


text:
struct NNODE{C,O,P,K} <: NeuralNetDiffEqAlgorithm
    chain::C
    opt::O
    initθ::P
    autodiff::Bool
    kwargs::K
end
function NNODE(chain,opt=Optim.BFGS(),init_params = nothing;autodiff=false,kwargs...)
    if init_params === nothing
        if chain isa FastChain
            initθ = DiffEqFlux.initial_params(chain)
        else
            initθ,re  = Flux.destructure(chain)
        end
    else
        initθ = init_params
    end
    NNODE(chain,opt,initθ,autodiff,kwargs)
end

function DiffEqBase.solve(
    prob::DiffEqBase.AbstractODEProblem,
    alg::NeuralNetDiffEqAlgorithm,
    args...;
    dt,
    timeseries_errors = true,
    save_everystep=true,
    adaptive=false,
    abstol = 1f-6,
    verbose = false,
    maxiters = 100)

    DiffEqBase.isinplace(prob) && error("Only out-of-place methods are allowed!")

    u0 = prob.u0
    tspan = prob.tspan
    f = prob.f
    p = prob.p
    t0 = tspan[1]

    #hidden layer
    chain  = alg.chain
    opt    = alg.opt
    autodiff = alg.autodiff

    #train points generation
    ts = tspan[1]:dt:tspan[2]
    initθ = alg.initθ

    if chain isa FastChain
        #The phi trial solution
        if u0 isa Number
            phi = (t,θ) -> u0 + (t-tspan[1])*first(chain(adapt(typeof(θ),[t]),θ))
        else
            phi = (t,θ) -> u0 + (t-tspan[1])*chain(adapt(typeof(θ),[t]),θ)
        end
    else
        _,re  = Flux.destructure(chain)
        #The phi trial solution
        if u0 isa Number
            phi = (t,θ) -> u0 + (t-tspan[1])*first(re(θ)(adapt(typeof(θ),[t])))
        else
            phi = (t,θ) -> u0 + (t-tspan[1])*re(θ)(adapt(typeof(θ),[t]))
        end
    end

    if autodiff
        dfdx = (t,θ) -> ForwardDiff.derivative(t->phi(t,θ),t)
    else
        dfdx = (t,θ) -> (phi(t+sqrt(eps(t)),θ) - phi(t,θ))/sqrt(eps(t))
    end

    function inner_loss(t,θ)
        sum(abs2,dfdx(t,θ) - f(phi(t,θ),p,t))
    end
    loss(θ) = sum(abs2,inner_loss(t,θ) for t in ts) # sum(abs2,phi(tspan[1],θ) - u0)

    cb = function (p,l)
        verbose && println("Current loss is: $l")
        l < abstol
    end
    res = DiffEqFlux.sciml_train(loss, initθ, opt; cb = cb, maxiters=maxiters, alg.kwargs...)

    #solutions at timepoints
    if u0 isa Number
        u = [first(phi(t,res.minimizer)) for t in ts]
    else
        u = [phi(t,res.minimizer) for t in ts]
    end

    sol = DiffEqBase.build_solution(prob,alg,ts,u,calculate_error = false)
    DiffEqBase.has_analytic(prob.f) && DiffEqBase.calculate_solution_errors!(sol;timeseries_errors=true,dense_errors=false)
    sol
end #solve