text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: codestar_connections
using AWS.Compat
using AWS.UUIDs

"""
    CreateConnection()

Creates a connection that can then be given to other AWS services like CodePipeline so that it can access third-party code repositories. The connection is in pending status until the third-party connection handshake is completed from the console.

# Required Parameters
- `ConnectionName`: The name of the connection to be created. The name must be unique in the calling AWS account.

# Optional Parameters
- `HostArn`: The Amazon Resource Name (ARN) of the host associated with the connection to be created.
- `ProviderType`: The name of the external provider where your third-party code repository is configured.
- `Tags`: The key-value pair to use when tagging the resource.
"""
create_connection(ConnectionName; aws_config::AWSConfig=global_aws_config()) = codestar_connections("CreateConnection", Dict{String, Any}("ConnectionName"=>ConnectionName); aws_config=aws_config)
create_connection(ConnectionName, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("CreateConnection", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ConnectionName"=>ConnectionName), args)); aws_config=aws_config)

"""
    CreateHost()

Creates a resource that represents the infrastructure where a third-party provider is installed. The host is used when you create connections to an installed third-party provider type, such as GitHub Enterprise Server. You create one host for all connections to that provider.  A host created through the CLI or the SDK is in `PENDING` status by default. You can make its status `AVAILABLE` by setting up the host in the console.

# Required Parameters
- `Name`: The name of the host to be created. The name must be unique in the calling AWS account.
- `ProviderEndpoint`: The endpoint of the infrastructure to be represented by the host after it is created.
- `ProviderType`: The name of the installed provider to be associated with your connection. The host resource represents the infrastructure where your provider type is installed. The valid provider type is GitHub Enterprise Server.

# Optional Parameters
- `VpcConfiguration`: The VPC configuration to be provisioned for the host. A VPC must be configured and the infrastructure to be represented by the host must already be connected to the VPC.
"""
create_host(Name, ProviderEndpoint, ProviderType; aws_config::AWSConfig=global_aws_config()) = codestar_connections("CreateHost", Dict{String, Any}("Name"=>Name, "ProviderEndpoint"=>ProviderEndpoint, "ProviderType"=>ProviderType); aws_config=aws_config)
create_host(Name, ProviderEndpoint, ProviderType, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("CreateHost", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Name"=>Name, "ProviderEndpoint"=>ProviderEndpoint, "ProviderType"=>ProviderType), args)); aws_config=aws_config)

"""
    DeleteConnection()

The connection to be deleted.

# Required Parameters
- `ConnectionArn`: The Amazon Resource Name (ARN) of the connection to be deleted.  The ARN is never reused if the connection is deleted.

"""
delete_connection(ConnectionArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("DeleteConnection", Dict{String, Any}("ConnectionArn"=>ConnectionArn); aws_config=aws_config)
delete_connection(ConnectionArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("DeleteConnection", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ConnectionArn"=>ConnectionArn), args)); aws_config=aws_config)

"""
    DeleteHost()

The host to be deleted. Before you delete a host, all connections associated to the host must be deleted.  A host cannot be deleted if it is in the VPC_CONFIG_INITIALIZING or VPC_CONFIG_DELETING state.

# Required Parameters
- `HostArn`: The Amazon Resource Name (ARN) of the host to be deleted.

"""
delete_host(HostArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("DeleteHost", Dict{String, Any}("HostArn"=>HostArn); aws_config=aws_config)
delete_host(HostArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("DeleteHost", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("HostArn"=>HostArn), args)); aws_config=aws_config)

"""
    GetConnection()

Returns the connection ARN and details such as status, owner, and provider type.

# Required Parameters
- `ConnectionArn`: The Amazon Resource Name (ARN) of a connection.

"""
get_connection(ConnectionArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("GetConnection", Dict{String, Any}("ConnectionArn"=>ConnectionArn); aws_config=aws_config)
get_connection(ConnectionArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("GetConnection", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ConnectionArn"=>ConnectionArn), args)); aws_config=aws_config)

"""
    GetHost()

Returns the host ARN and details such as status, provider type, endpoint, and, if applicable, the VPC configuration.

# Required Parameters
- `HostArn`: The Amazon Resource Name (ARN) of the requested host.

"""
get_host(HostArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("GetHost", Dict{String, Any}("HostArn"=>HostArn); aws_config=aws_config)
get_host(HostArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("GetHost", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("HostArn"=>HostArn), args)); aws_config=aws_config)

"""
    ListConnections()

Lists the connections associated with your account.

# Optional Parameters
- `HostArnFilter`: Filters the list of connections to those associated with a specified host.
- `MaxResults`: The maximum number of results to return in a single call. To retrieve the remaining results, make another call with the returned nextToken value.
- `NextToken`: The token that was returned from the previous ListConnections call, which can be used to return the next set of connections in the list.
- `ProviderTypeFilter`: Filters the list of connections to those associated with a specified provider, such as Bitbucket.
"""
list_connections(; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListConnections"; aws_config=aws_config)
list_connections(args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListConnections", args; aws_config=aws_config)

"""
    ListHosts()

Lists the hosts associated with your account.

# Optional Parameters
- `MaxResults`: The maximum number of results to return in a single call. To retrieve the remaining results, make another call with the returned nextToken value.
- `NextToken`: The token that was returned from the previous ListHosts call, which can be used to return the next set of hosts in the list.
"""
list_hosts(; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListHosts"; aws_config=aws_config)
list_hosts(args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListHosts", args; aws_config=aws_config)

"""
    ListTagsForResource()

Gets the set of key-value pairs (metadata) that are used to manage the resource.

# Required Parameters
- `ResourceArn`: The Amazon Resource Name (ARN) of the resource for which you want to get information about tags, if any.

"""
list_tags_for_resource(ResourceArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListTagsForResource", Dict{String, Any}("ResourceArn"=>ResourceArn); aws_config=aws_config)
list_tags_for_resource(ResourceArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("ListTagsForResource", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ResourceArn"=>ResourceArn), args)); aws_config=aws_config)

"""
    TagResource()

Adds to or modifies the tags of the given resource. Tags are metadata that can be used to manage a resource.

# Required Parameters
- `ResourceArn`: The Amazon Resource Name (ARN) of the resource to which you want to add or update tags.
- `Tags`: The tags you want to modify or add to the resource.

"""
tag_resource(ResourceArn, Tags; aws_config::AWSConfig=global_aws_config()) = codestar_connections("TagResource", Dict{String, Any}("ResourceArn"=>ResourceArn, "Tags"=>Tags); aws_config=aws_config)
tag_resource(ResourceArn, Tags, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("TagResource", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ResourceArn"=>ResourceArn, "Tags"=>Tags), args)); aws_config=aws_config)

"""
    UntagResource()

Removes tags from an AWS resource.

# Required Parameters
- `ResourceArn`: The Amazon Resource Name (ARN) of the resource to remove tags from.
- `TagKeys`: The list of keys for the tags to be removed from the resource.

"""
untag_resource(ResourceArn, TagKeys; aws_config::AWSConfig=global_aws_config()) = codestar_connections("UntagResource", Dict{String, Any}("ResourceArn"=>ResourceArn, "TagKeys"=>TagKeys); aws_config=aws_config)
untag_resource(ResourceArn, TagKeys, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("UntagResource", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ResourceArn"=>ResourceArn, "TagKeys"=>TagKeys), args)); aws_config=aws_config)

"""
    UpdateHost()

Updates a specified host with the provided configurations.

# Required Parameters
- `HostArn`: The Amazon Resource Name (ARN) of the host to be updated.

# Optional Parameters
- `ProviderEndpoint`: The URL or endpoint of the host to be updated.
- `VpcConfiguration`: The VPC configuration of the host to be updated. A VPC must be configured and the infrastructure to be represented by the host must already be connected to the VPC.
"""
update_host(HostArn; aws_config::AWSConfig=global_aws_config()) = codestar_connections("UpdateHost", Dict{String, Any}("HostArn"=>HostArn); aws_config=aws_config)
update_host(HostArn, args::AbstractDict{String, <:Any}; aws_config::AWSConfig=global_aws_config()) = codestar_connections("UpdateHost", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("HostArn"=>HostArn), args)); aws_config=aws_config)


text:
"""
    δ_extrema(x)
Compute both the difference between the maximum and the minimum of `x`.
"""
function δ_extrema(x)
    δ_min, δ_max = extrema(x)
    return δ_max - δ_min
end

ext: jl


text:
import Base.LinAlg: axpy!

############
# Addition #
############

_add!(O::AbstractArray, ::Tuple{}) = O
_add!(O::AbstractArray, As::Tuple) = _add!(axpy!(1, As[1], O), Base.tail(As))
_add!{N}(::Type{Inplace{N}}, As...) = _add!(As[N], cat_tuple(As[1:N-1],As[N+1:end]))
_add!(O::AbstractArray, As...) = _add!(copy!(O, As[1]), Base.tail(As))

################
# Substraction #
################

# Turns out to be more expensive than not allocating memory
function _sub!(O::AbstractArray, A::AbstractArray, B::AbstractArray)
    size(O) == size(A) == size(B) || throw(DimensionMismatch("dimensions must match"))
    for i in 1:length(O)
        O[i] = A[i] - B[i]
    end
    O
end

_sub!{T}(::Type{Inplace{1}}, O::AbstractArray, A::AbstractArray{T}) = axpy!(T(-1), A, O)
_sub!{T}(::Type{Inplace{2}}, A::AbstractArray, O::AbstractArray{T}) = axpy!(1, A, scale!(T(-1),O))


text:
@with_kw struct SimulatorOptions{T}
    warmstart::Bool = true
    z_warmstart::T = 0.001
    κ_warmstart::T = 0.001
	failure_abort::Int = 50
end

mutable struct SimulatorStatistics{T}
    dt::AbstractVector{T}
    μ_dt::T
    σ_dt::T
end

function SimulatorStatistics()
    dt = zeros(0)
    μ_dt = 0.0
    σ_dt = 0.0
    return SimulatorStatistics(dt, μ_dt, σ_dt)
end

struct Simulator{T}
    s::Simulation

    traj::ContactTraj
    deriv_traj::ContactDerivTraj

    p::Policy
    uL::Vector{T}
    uU::Vector{T}

    d::Disturbances

    ip::AbstractIPSolver

    opts::SimulatorOptions{T}
    stats::SimulatorStatistics{T}
end

function simulator(s::Simulation, q0::SVector, q1::SVector, h::S, H::Int;
    p = no_policy(s.model),
    uL = -Inf * ones(s.model.dim.u),
    uU = Inf * ones(s.model.dim.u),
    d = no_disturbances(s.model),
    r! = s.res.r!,
    rz! = s.res.rz!,
    rθ! = s.res.rθ!,
    rz = s.rz,
    rθ = s.rθ,
    space = Euclidean(num_var(s.model, s.env)),
    ip_type::Symbol = :interior_point,
    ip_opts = eval(interior_point_options(ip_type))(
		undercut = Inf,
		γ_reg = 0.0,
		r_tol = 1e-8,
		κ_tol = 1e-8,
		),
    sim_opts = SimulatorOptions{S}()) where S

    model = s.model
    env = s.env

    # initialize trajectories
    traj = contact_trajectory(model, env, H, h)
    traj.q[1] = q0
    traj.q[2] = q1

    # initialize interior point solver (for pre-factorization)
    z = zeros(num_var(model, s.env))
    θ = zeros(num_data(model))
    z_initialize!(z, model, env, traj.q[2])
    θ_initialize!(θ, model, traj.q[1], traj.q[2], traj.u[1], traj.w[1], model.μ_world, h)

    ip = eval(ip_type)(
			 z,
			 θ,
			 s = space,
			 oss = OptimizationSpace13(model, env),
			 r! = r!,
			 rz! = rz!,
			 rθ! = rθ!,
			 rz = rz,
			 rθ = rθ,
			 opts = ip_opts)

    # pre-allocate for gradients
    traj_deriv = contact_derivative_trajectory(model, env, ip.δz, H)
    stats = SimulatorStatistics()

    Simulator(
        s,
        traj,
        traj_deriv,
        p, uL, uU,
        d,
        ip,
        sim_opts,
        stats)
end

function step!(sim::Simulator, t)
    # simulation
    model = sim.s.model
    env = sim.s.env

    # unpack
    q = sim.traj.q
    u = sim.traj.u
    w = sim.traj.w
    h = sim.traj.h
    ip = sim.ip
    z = ip.z
    θ = ip.θ

    # t = 1 2 3
    # u1 = traj.u[t]
    # w1 = traj.w[t]
    # γ1 = traj.γ[t]
    # b1 = traj.b[t]
    # q0 = traj.q[t]
    # q1 = traj.q[t+1]
    # q2 = traj.q[t+2]

    # policy
    dt = @elapsed u[t] = control_saturation(policy(sim.p, q[t+1], sim.traj, t), sim.uL, sim.uU)
    push!(sim.stats.dt, dt)

    # disturbances
    w[t] = disturbances(sim.d, q[t+1], t)

    # initialize
    if sim.opts.warmstart
        z_warmstart!(z, model, env, q[t+1], sim.opts.z_warmstart)
        # sim.ip.opts.κ_init = sim.opts.κ_warmstart
    else
        z_initialize!(z, model, env, q[t+1])
    end
    θ_initialize!(θ, model, q[t], q[t+1], u[t], w[t], model.μ_world, h)

    # solve
    status = interior_point_solve!(ip)

    if status
        # parse result
        q2, γ, b, _ = unpack_z(model, env, z)
        sim.traj.z[t] = copy(z) # we need a copy here otherwize we get a constant trajectory = last computed z element # TODO: maybe not use copy
        sim.traj.θ[t] = copy(θ) # we need a copy here otherwize we get a constant trajectory = last computed θ element
        sim.traj.q[t+2] = q2
        sim.traj.γ[t] = γ
        sim.traj.b[t] = b
        sim.traj.κ[1] = ip.κ[1]

        if sim.ip.opts.diff_sol
            sim.deriv_traj.dq2dq0[t] = sim.deriv_traj.vqq
            sim.deriv_traj.dq2dq1[t] = sim.deriv_traj.vqqq
            sim.deriv_traj.dq2du[t] = sim.deriv_traj.vqu
            sim.deriv_traj.dγdq0[t] = sim.deriv_traj.vγq
            sim.deriv_traj.dγdq1[t] = sim.deriv_traj.vγqq
            sim.deriv_traj.dγdu[t] = sim.deriv_traj.vγu
            sim.deriv_traj.dbdq0[t] = sim.deriv_traj.vbq
            sim.deriv_traj.dbdq1[t] = sim.deriv_traj.vbqq
            sim.deriv_traj.dbdu[t] = sim.deriv_traj.vbu
        end
    end

    return status
end

"""
    simulate
    - solves 1-step feasibility problem for H time steps
    - initial configurations: q0, q1
    - time step: h
"""
function simulate!(sim::Simulator; verbose = false)

    verbose && println("\nSimulation")

    # initialize configurations for first step
    z_initialize!(sim.ip.z, sim.s.model, sim.s.env, sim.traj.q[2])

    status = true

    # simulate
    for t = 1:sim.traj.H
        verbose && println("t = $t / $(sim.traj.H)")
        status = step!(sim, t)
        !status && (@error "failed step (t = $t)")
		!status && break
    end

    return status
end

function process!(stats::SimulatorStatistics, N_sample::Int)
    H_sim = length(stats.dt)
    H = Int(H_sim / N_sample)
    dt = sum(reshape(stats.dt, (H, N_sample)), dims=2)
    stats.μ_dt = mean(dt)
    stats.σ_dt = sqrt(mean( (dt .- stats.μ_dt).^2 ))
    return nothing
end

function process!(sim::Simulator)
    process!(sim.stats, sim.p.N_sample)
end


text:
"""
    GFrames

Ability to define frames in a global fashion inside [`Action`](@ref).

# Example
```julia
red_circ = Object(1:90, (args...)->circ("red"))
blue_circ = Object(21:90, (args...)->circ("blue"))
act!([red_circ, blue_circ], Action(GFrames(85:90), disappear(:fade)))
```
is the same as
```julia
red_circ = Object(1:90, (args...)->circ("red"))
blue_circ = Object(21:90, (args...)->circ("blue"))
act!(red_circ, Action(85:90, disappear(:fade)))
act!(blue_circ, Action(65:70, disappear(:fade)))
```

# Fields
- frames::UnitRange defines the frames in a global fashion.
"""
struct GFrames
    frames::UnitRange
end


text:
struct MNIST <: GrayImage{MLImage} end
getModule(::MNIST) = MLDatasets.MNIST
size(::MNIST) = (60000, 0, 10000)
problem(::MNIST) = Classification

max_stars_repo_name: GruAna/PreprocessData

max_issues_repo_name: GruAna/PreprocessData

max_forks_repo_name: GruAna/PreprocessData


text:
using Distributed
@time include("required.jl")

# Load data from CSV files (from file cut with terminal)
@time df = CSV.read(
    joinpath("data", "raw", "ebd_warblers_cut.csv"); header=true, delim="\t"
)

# Prepare data (arrange values & columns)
df = prepare_ebd_data(df)

# Remove duplicates (BUT not ok for counts, so-so for dates, see group-observation.jl in src/test/)
df_nogroups = filter(x -> ismissing(x[:groupIdentifier]), df)
df_groups = dropmissing(df, :groupIdentifier)
unique!(df_groups, [:species, :groupIdentifier])
append!(df_nogroups, df_groups)
newdf = df_nogroups

ycount = by(newdf, :year; n=:year => length, sort=true)
show(ycount; allrows=true)
bar(ycount.year, ycount.n)
filter(x -> x.year > 1980, ycount) |> x -> bar(x.year, x.n)
filter(x -> x.year > 1980, ycount) |> x -> bar(x.year, x.n; yaxis=:log)
filter(x -> x.year < 2000, ycount) |> x -> bar(x.year, x.n)
filter(x -> 1980 <= x.year <= 2000, ycount) |> x -> bar(x.year, x.n)

by(newdf, [:year, :protocolType]; n=:year => length, sort=true)

mcount = by(newdf, :month; n=:month => length, sort=true)
bar(mcount.month, mcount.n)

max_stars_repo_name: gabrieldansereau/betadiversity-hotspots

max_issues_repo_name: gabrieldansereau/betadiversity-hotspots

max_forks_repo_name: gabrieldansereau/betadiversity-hotspots


text:
using DPC, CairoMakie
include("utils.jl")

config = """
neurons:
  - id: neuron
    branches:
      - id: S
        branches:
          - id: S1
            branches:
              - id: S11
                branches:
                  - id: S111
                  - id: S112
                  - id: S113
                  - id: S114
                  - id: S115
                  - id: S116
              - id: S12
                branches:
                  - id: S121
                  - id: S122
                  - id: S123
                  - id: S124
                  - id: S125
                  - id: S126
          - id: S2
            branches:
              - id: S21
                branches:
                  - id: S211
                  - id: S212
                  - id: S213
                  - id: S214
                  - id: S215
                  - id: S216
              - id: S22
                branches:
                  - id: S221
                  - id: S222
                  - id: S223
                  - id: S224
                  - id: S225
                  - id: S226
"""

(net,objects) = load_network(YAML_source=config)

fig = Figure(resolution=(600,600),aspect=DataAspect())
ax = fig[1,1] = Axis(fig, backgroundcolor=:transparent)

diams = get_branch_diameters(objects[:neuron], root_diam=0.2)
lengths = DefaultDict(0.75, :S=>2.0, :S1=>1.5, :S2=>1.5, :S11=>1.0, :S12=>1.0, :S21=>1.0, :S22=>1.0)
α = 0/180*π
angles = DefaultDict(20/180*π, :S=>α, :S1=>α, :S2=>α, :S11=>α, :S12=>α, :S21=>α, :S22=>α)

cols = cgrad(:viridis, 6, categorical=true, rev=true)
dists = get_root_distances(objects[:neuron])
colors = Dict(k=>cols[2+v] for (k,v) in pairs(dists))
plot!(ax, objects[:neuron], angle_between=angles, branch_width=diams, branch_length=lengths, color=colors, padding = (0.0, 0.0))

hidedecorations!(ax)
hidespines!(ax)
xlims!(ax, [-2.5,3.5])
ylims!(ax, [-0.5,5.5])

save(joinpath("figures", "bio_schema.svg"), fig)
save(joinpath("figures", "bio_schema.png"), fig)
save(joinpath("figures", "bio_schema.pdf"), fig)
fig


text:
function Base.vcat(x::MOInput, y::MOInput)
    x.out_dim == y.out_dim || throw(error("out_dim mismatch"))
    return MOInput(vcat(x.x, y.x), x.out_dim)
end

@testset "oilmm" begin
    @testset "single output" begin
        rng = MersenneTwister(123456)

        # Specify inputs and generate observations.
        x = collect(range(-3.37, 3.12; length=11))
        tr_idx = randperm(rng, length(x))[1:6]
        te_idx = setdiff(eachindex(x), tr_idx)
        x_tr_raw = x[tr_idx]
        x_te_raw = x[te_idx]

        # Noise variance.
        σ² = 0.12

        # Specify the equivalent GPPP.
        a = randn(rng)
        f_naive = @gppp let
            f = a * GP(Matern52Kernel())
        end

        # Specify equivalent OILMM.
        fs = [GP(Matern52Kernel())]
        U = reshape([1.0], 1, 1)
        S = Diagonal([abs2(a)])
        D = Diagonal([0.0])
        f = OILMM(fs, U, S, D)
        x_tr = MOInput(x_tr_raw, 1)
        x_te = MOInput(x_te_raw, 1)
        y_tr = rand(rng, f(x_tr, σ²))
        y_te = rand(rng, f(x_te, σ²))

        consistency_tests(
            rng, f, f_naive;
            x_tr=x_tr,
            x_te=x_te,
            x_naive_tr=GPPPInput(:f, x_tr_raw),
            x_naive_te=GPPPInput(:f, x_te_raw),
            y_tr=y_tr,
            y_te=y_te,
            σ²=σ²,
        )
    end
    @testset "P independent processes" begin
        rng = MersenneTwister(123456)
        P = 3
        N = 11

        # Specify inputs and generate observations.
        x = collect(range(-3.37, 3.12; length=N))
        tr_idx = randperm(rng, length(x))[1:2]
        te_idx = setdiff(eachindex(x), tr_idx)
        x_tr_raw = x[tr_idx]
        x_te_raw = x[te_idx]

        # Noise variance.
        σ² = 0.12

        # Specify a collection of GPs
        as = randn(rng, P)
        gpc = Stheno.GPC()
        fs = map(p -> as[p] * wrap(GP(Matern52Kernel()), gpc), 1:P)
        f_naive = Stheno.GPPP(fs, gpc)

        x_naive_tr = BlockData(map(p -> GPPPInput(p, x_tr_raw), 1:P))
        x_naive_te = BlockData(map(p -> GPPPInput(p, x_te_raw), 1:P))

        # Specify equivalent OILMM.
        U = collect(Diagonal(ones(P)))
        S = Diagonal(abs2.(as))
        D = Diagonal(zeros(P))
        f = OILMM([GP(Matern52Kernel()) for p in 1:P], U, S, D)
        x_tr = MOInput(x_tr_raw, P)
        x_te = MOInput(x_te_raw, P)
        y_tr = rand(rng, f(x_tr, σ²))
        y_te = rand(rng, f(x_te, σ²))

        consistency_tests(
            rng, f, f_naive;
            x_tr=x_tr,
            x_te=x_te,
            x_naive_tr=x_naive_tr,
            x_naive_te=x_naive_te,
            y_tr=y_tr,
            y_te=y_te,
            σ²=σ²,
        )
    end
    @testset "Full Rank, Dense H" begin
        rng = MersenneTwister(123456)
        P = 3
        N = 15

        # Specify inputs and generate observations.
        x = collect(range(-3.37, 3.12; length=N))
        tr_idx = randperm(rng, length(x))[1:2]
        te_idx = setdiff(eachindex(x), tr_idx)
        x_tr_raw = x[tr_idx]
        x_te_raw = x[te_idx]

        # Noise variance.
        σ² = 0.12

        # Construct a random orthogonal H.
        U, S_diag, _ = svd(randn(rng, P, P))
        H = U * Diagonal(sqrt.(S_diag))

        # Specify a collection of GPs
        gpc = Stheno.GPC()
        zs = [wrap(GP(Matern52Kernel()), gpc) for p in 1:P]
        fs = [sum(H[p, :] .* zs) for p in 1:P]
        f_naive = Stheno.GPPP(fs, gpc)

        x_naive_tr = BlockData(map(p -> GPPPInput(p, x_tr_raw), 1:P))
        x_naive_te = BlockData(map(p -> GPPPInput(p, x_te_raw), 1:P))

        # Specify equivalent OILMM.
        D = Diagonal(zeros(P))
        f = OILMM(zs, U, Diagonal(S_diag), D)

        # Specify inputs and generate observations.
        x_tr = MOInput(x_tr_raw, P)
        x_te = MOInput(x_te_raw, P)
        y_tr = rand(rng, f(x_tr, σ²))
        y_te = rand(rng, f(x_te, σ²))

        consistency_tests(
            rng, f, f_naive;
            x_tr=x_tr,
            x_te=x_te,
            x_naive_tr=x_naive_tr,
            x_naive_te=x_naive_te,
            y_tr=y_tr,
            y_te=y_te,
            σ²=σ²,
        )
    end
end


text:
function BatchSAC(;π::ActorCritic{T, DoubleNetwork{ContinuousNetwork, ContinuousNetwork}},
                   S,
                   ΔN=50,
                   SAC_α::Float32=1f0,
                   SAC_H_target::Float32 = Float32(-prod(dim(action_space(π)))),
                   𝒟_train,
                   SAC_α_opt::NamedTuple=(;),
                   a_opt::NamedTuple=(;),
                   c_opt::NamedTuple=(;),
                   log::NamedTuple=(;),
                   𝒫::NamedTuple=(;),
                   param_optimizers=Dict(),
                   normalize_training_data = true,
                   kwargs...) where T
    normalize_training_data && (𝒟_train = normalize!(deepcopy(𝒟_train), S, action_space(π)))
    𝒟_train = 𝒟_train |> device(π)

    𝒫 = (SAC_log_α = [Base.log(SAC_α)], SAC_H_target = SAC_H_target, 𝒫...)
    BatchSolver(;agent=PolicyParams(π=π, π⁻=deepcopy(π)),
                 S=S,
                 𝒫=𝒫,
                 𝒟_train=𝒟_train,
                 log=LoggerParams(;dir = "log/batch_sac", log...),
                 param_optimizers=Dict(Flux.params(𝒫[:SAC_log_α]) => TrainingParams(;loss=SAC_temp_loss, name="temp_", SAC_α_opt...), param_optimizers...),
                 a_opt=TrainingParams(;loss=SAC_actor_loss, name="actor_", a_opt...),
                 c_opt=TrainingParams(;loss=double_Q_loss(), name="critic_", epochs=ΔN, c_opt...),
                 target_fn=SAC_target(π),
                 kwargs...)

end


text:
# This is a pure implementation of traversals
using Game2048Core
using Game2048Core: DIRS, Bitboard, count0, simulate_bb, bitboard_to_array
# using Game2048: print_score, print_board
using BenchmarkTools
using StatsBase

const MASK = UInt8(15)

# should be
mutable struct TreeNode
    parent::Union{TreeNode,Nothing}
    board::Bitboard
    children::Vector{TreeNode}
    children_are_moves::Bool
    TreeNode(parent, board, children_are_moves) = new(parent, board, [], children_are_moves)
end

Base.print(tn::TreeNode) = print(tn.board)
Base.show(tn::TreeNode) = show(tn.board)
Base.display(tn::TreeNode) = display(tn.board)

function largest_val_in_corner(bitboard::Bitboard)
    max_tile = maximum(bitboard)
    max_tile_in_corner = (bitboard.board & MASK) == max_tile
    max_tile_in_corner
end

function largest_val_in_corner(tn::TreeNode)
    largest_val_in_corner(tn.board)
end

function add_children!(tn::TreeNode)
    if tn.children_are_moves
        candidates = move.(Ref(tn.board), DIRS)
        diff_to_before = Ref(tn.board) .!= candidates
        # at_least_1_cand = map(largest_val_in_corner, candidates) .& diff_to_before

        # if any(at_least_1_cand)
        #     tn.children = [TreeNode(tn, b, false) for (b, addit) in zip(candidates, at_least_1_cand) if addit]
        # else
        #     tn.children = [TreeNode(tn, b, false) for (b, addit) in zip(candidates, diff_to_before) if addit]
        # end
        tn.children = [TreeNode(tn, b, false) for (b, ok) in zip(candidates, diff_to_before) if ok]
    else
        # if you are here this means that your parent was a move
        # so do not move if you are the same as your parent
        if tn.board == tn.parent.board
            return
        end
        # count the number of 0 spots
        n = count0(tn.board)
        tn.children = reshape([TreeNode(tn, add_tile(tn.board, i, tf), true) for (i, tf) in Iterators.product(1:n, 1:2)], :)
        # tn.children = [TreeNode(tn, add_tile(tn.board, i, 1), true) for i in 1:n]
    end
end

function grow!(tn::TreeNode, levels)
    if levels == 0
        return
    end
    add_children!(tn)
    for c in tn.children
        grow!(c, levels - 1)
    end
end

function grow(board::Bitboard, args...)
    tn = TreeNode(nothing, board, true)
    grow!(tn, args...)
    tn
end

# find number of nodes
function count_nodes(tn::TreeNode)
    if length(tn.children) == 0
        return 1
    else
        return mapreduce(count_nodes, +, tn.children)
    end
end

function max2(f, itrs)
    reduce(@view itrs[3:end]; init = (f(itrs[1]), f(itrs[2]))) do (l1, l2), new_val
        fnv = f(new_val)
        fnv > l1 ? (fnv, l1) : (l1, max(fnv, l2))
    end
end

function sum2(bitboard::Bitboard)
    mapreduce(+, 0:4:60) do s
        1 << ((bitboard.board >> s) & MASK)
    end
end

function make_value(nsims)
    function value(bitboard::Bitboard)
        # # is the maximum tile in the corner
        # nsims = 100
        mapreduce(+, 1:nsims) do _
            # 1 << maximum(simulate_bb(bitboard))
            sb = simulate_bb(bitboard)
            sum2(sb) + 10 * maximum(sb)
        end / nsims

        # # max_tile_in_corner = mapreduce(|, (0, 12, 48, 60)) do s
        # #     val = (bitboard.board >> s) & MASK
        # #     val == max_tile
        # # end

        # max_tile_in_corner = (bitboard.board & MASK) == max_tile
        # next_tile_is_also_good = ((bitboard.board >> 4) & MASK) in (max_tile, max_tile - 1)

        # val_add = next_tile_is_also_good ? 5 : 4

        # # map((0, 12, 48, 60)) do s
        # #     (bitboard.board >> s) & MASK
        # # end .|> Int

        # val2 = max_tile_in_corner ? val_add : 1

        # # val1 = mapreduce(+, 0:4:60) do s
        # #     power = (bitboard.board >> s) & MASK
        # #     1 << power
        # # end

        # c0 = count0(bitboard)

        # return (1 << max_tile) * (1 + c0 / 16) * val2
        # return
    end
end

# simulate each node 10 times
function get_score(tn::TreeNode, value)::Float64
    if length(tn.children) == 0
        return Float64(value(tn.board))
    else
        if tn.children_are_moves
            return maximum(c -> get_score(c, value), tn.children)
        else
            scores = get_score.(tn.children, Ref(value))
            l = length(scores)
            @assert iseven(l)
            return mean(@view scores[1:div(l, 2)]) * 0.9 + mean(@view scores[div(l, 2)+1:l]) * 0.1
        end
    end
end

function grow_nodes!(tn::TreeNode, n)
    if length(tn.children) == 0
        if tn.children_are_moves
            grow!(tn, n)
            return
        else
            return
        end
    else
        grow_nodes!.(tn.children, n)
    end
    return
end

function play(value)
    b = initbboard()
    tn = grow(b, 2)

    # count_nodes(tn)

    while true
        best_child = argmax(c -> get_score(c, value), tn.children)

        if best_child.board == tn.board
            return tn
        end

        # add a tile
        new_board = add_tile(best_child.board)

        for c in best_child.children
            if c.board == new_board
                tn = c
                break
            end
        end

        # cntn = count_nodes(tn)
        # if cntn < 10_000
        grow_nodes!(tn, 2)
        # end
        # cntn = count_nodes(tn)

        # if cntn < 10_000
        #     grow_nodes!(tn, 2)
        # end

        # println(count_nodes(tn))
        # println(cntn)
        # display(tn)

        if length(tn.children) == 0
            return tn
        end
    end
end

value20 = make_value(20)
@time tn=play(value20)

@time res_prefer_corner = [play(value20).board for _ in 1:10];

a = DataFrame(maxtile = maximum.(res_prefer_corner), score = Game2048Core.score.(res_prefer_corner))
# prefering corner in a simplistic way makes the score worse

if false
    svalues = [make_value(i) for i in 10:10:100]

    @time play(svalues[2])
    res = Dict{Tuple{Int,Int},Bitboard}()

    using ProgressMeter
    parts = sum([10 * i for i in 1:10])
    p = Progress(parts)

    @time Threads.@threads for i in 11:20
        Threads.@threads for j in 1:10
            a = play(svalues[j])
            res[(i, j)] = a.board
            for _ in 1:j
                next!(p)
            end
        end
    end

    # @time res1 = [play().board for _ in 1:10]

    @time res2 = [maximum(r) |> Int for r in res]

    @time res2 = [Game2048Core.score(r) |> Int for r in res]

    using DataFrames, DataFrameMacros, Chain
    a = DataFrame(k = keys(res) |> collect, v = [v for (_, v) in res])

    a1 = @chain a begin
        @transform begin
            :try_id = :k[1]
            :nsims = :k[2] * 10
            :score = Game2048Core.score(:v)
            :maxtile = maximum(:v)
        end
    end

    @chain a1 begin
        @transform :win = :maxtile >= 11
        groupby(:nsims)
        @combine(std(:score), mean(:score), mean(:win))
    end

    @chain a1 begin
        @subset :nsims == 90
        sort!(:score, rev = true)
    end

    @chain a1 begin
        @subset :nsims == 100
        sort!(:score, rev = true)
    end

    using Serialization

    serialize("./data/runs_from_sims_10_to_100.csv", a1)

    mean(res2)
    # 2886.4 for 10 trieson
    # 4410.8 for 20 trieson

    countmap(res2)

    # value(bb)

    # @time res = [simulate_bb() for _ in 1:1000]
    # x = res[1]

    # @time tn = grow(b, 6);
    # @time count_nodes(tn)
    # tn.board
    # value(tn.board)
    # @time get_score.(tn.children)

    # @time res = play()
end

max_stars_repo_path: src/2022-02-27 pure-tree-traversal.jl

max_issues_repo_path: src/2022-02-27 pure-tree-traversal.jl

max_forks_repo_path: src/2022-02-27 pure-tree-traversal.jl


text:
using Revise, MDToolbox, Plots, JLD2, Printf, DelimitedFiles

function create_image(pdbFileNum, quatenionNum, saveFileName)
    # read structure and remove center of mass
    pdb = MDToolbox.readpdb("model/0000$(pdbFileNum).pdb")
    MDToolbox.decenter!(pdb)

    # read quaternions
    quater = DelimitedFiles.readdlm("quaternion/QUATERNION_LIST_576_Orient")

    # rotate by to a quaternion value
    pdb_rotated = MDToolbox.rotate(pdb, quater[quatenionNum, :])
    afmData = MDToolbox.afmize(pdb_rotated, (150.0, 150.0), (16, 16))
    @save saveFileName afmData
end

pdbFiles = [1 2 3 4]
quatenionNums = [7, 52, 91, 500]
fileNameList = []
for i in 1:4
    saveFileName = @sprintf "testCase/pdb%02d_quaternion%04d.jld2" pdbFiles[i] quatenionNums[i]
    create_image(pdbFiles[i], quatenionNums[i], saveFileName)
    push!(fileNameList, saveFileName)
end
@save "testCase/fileNameList.jld2" fileNameList

max_stars_repo_name: matsunagalab/BioAFM

max_issues_repo_name: matsunagalab/BioAFM

max_forks_repo_name: matsunagalab/BioAFM


text:
function coreVIdiag(logl::Function, μarray::Array{Array{Float64,1},1}, Σarray::Array{Array{Float64,1},1}; gradlogl = gradlogl, seed = 1, S = 100, optimiser=Optim.LBFGS(), iterations = 1, numerical_verification = false, Stest=0, show_every=-1, inititerations=0)

    D = length(μarray[1])

    @assert(D == length(Σarray[1]))

    @assert(length(μarray) == length(Σarray))

    @printf("Running VI diagonal with S=%d, D=%d for %d iterations\n", S, D, iterations)

    #----------------------------------------------------
    # generate latent variables
    #----------------------------------------------------

    Ztrain = generatelatentZ(S = S, D = D, seed = seed)

    Ztest  = generatelatentZ(S = Stest, D = D, seed = seed+1)

    #----------------------------------------------------
    function unpack(param)
    #----------------------------------------------------

        @assert(length(param) == D+D)

        local μ = param[1:D]

        local Cdiag = reshape(param[D+1:D+D], D)

        return μ, Cdiag

    end

    #----------------------------------------------------
    function minauxiliary(param)
    #----------------------------------------------------

        local μ, Cdiag = unpack(param)

        return -1.0 * elbo(μ, Cdiag, Ztrain)

    end

    #----------------------------------------------------
    function minauxiliary_grad(param)
    #----------------------------------------------------

        local μ, Cdiag = unpack(param)

        return -1.0 * elbo_grad(μ, Cdiag, Ztrain)

    end

    #----------------------------------------------------
    function getcov(Cdiag)
    #----------------------------------------------------

        Diagonal(Cdiag.^2)

    end

    #----------------------------------------------------
    function getcovroot(Cdiag)
    #----------------------------------------------------

        return Cdiag

    end

    #----------------------------------------------------
    function elbo(μ, Cdiag, Z)
    #----------------------------------------------------

        mean(map(z -> logl(μ .+ Cdiag.*z), Z)) + ApproximateVI.entropy(Cdiag)

    end

    #----------------------------------------------------
    function elbo_grad(μ, Cdiag, Z)
    #----------------------------------------------------

        local gradC = 1.0 ./ Cdiag # entropy contribution

        local gradμ = zeros(eltype(μ), D)

        local S     = length(Z)

        for s=1:S
            g      = gradlogl(μ .+ Cdiag .* Z[s])
            gradC += g .* Z[s] / S
            gradμ += g / S
        end

        return [vec(gradμ); gradC]

    end

    gradhelper(storage, param) = copyto!(storage, minauxiliary_grad(param))

    #----------------------------------------------------
    # Numerically verify gradient
    #----------------------------------------------------

    if numerical_verification

        local C = sqrt.(Σarray[1])
        adgrad = ForwardDiff.gradient(minauxiliary, [μarray[1]; C])
        angrad = minauxiliary_grad([μarray[1]; C])
        @printf("gradient from AD vs analytical gradient\n")
        display([vec(adgrad) vec(angrad)])
        @printf("maximum absolute difference is %f\n", maximum(abs.(vec(adgrad) - vec(angrad))))

    end

    #----------------------------------------------------
    # Evaluate initial solutions for few iterations
    #----------------------------------------------------

    initoptimise(μ, Σ) = Optim.optimize(minauxiliary, gradhelper, [μ; vec(sqrt.(Σ))], optimiser, Optim.Options(iterations = inititerations))

    results = if inititerations>0
        @showprogress "Initial search with random start " map(initoptimise, μarray, Σarray)
    else
        map(initoptimise, μarray, Σarray)
    end

    bestindex = argmin(map(r -> r.minimum, results))

    bestinitialsolution = results[bestindex].minimizer

    #----------------------------------------------------
    # Call optimiser
    #----------------------------------------------------

    options = Optim.Options(extended_trace = false, store_trace = false, show_trace = true, show_every=show_every, iterations = iterations, g_tol = 1e-6)

    result  = Optim.optimize(minauxiliary, gradhelper, bestinitialsolution, optimiser, options)

    μopt, Copt = unpack(result.minimizer)

    #----------------------------------------------------
    # Return results
    #----------------------------------------------------

    Σ = getcov(Copt)

    return MvNormal(μopt, Σ), elbo(μopt, Copt, generatelatentZ(S = 10*S, D = D, seed = seed+2))

end


text:
module jluna

    """
    `safe_call(::Function, args...) ::Tuple{Any, Bool, Exception, String)`

    safely call any function, while forwarding any exception that may have occurred
    """
    function safe_call(f::Function, args...)

        res::Any = undef

        backtrace::String = ""
        exception_occurred::Bool = false
        exception::Union{Exception, UndefInitializer} = undef

        try
            res = f(args...)
        catch e
            exception = e
            backtrace = sprint(Base.showerror, exception, catch_backtrace())
            exception_occurred = true
        end

        return (res, exception_occurred, exception, backtrace)
    end

    """
    `dot(::Array, field::Symbol) -> Any`

    wrapped dot operator
    """
    function dot(x::Array, field_name::Symbol) ::Any

        index_maybe = parse(Int, string(field_name));
        @assert index_maybe isa Integer
        return x[index_maybe];
    end
    export dot;

    """
    `dot(::Module, field::Symbol) -> Any`

    wrapped dot operator
    """
    dot(x::Module, field_name::Symbol) = return x.eval(field_name);

    """
    `dot(x::Any, field::Symbol) -> Any`

    wrapped dot operator, x.field
    """
    dot(x::Any, field_name::Symbol) = return eval(:($x.$field_name))

    """
    `unroll_type(::Type) -> Type`

    unroll type declaration
    """
    function unroll_type(type::Type) ::Type

        while hasproperty(type, :body)
            type = type.body
        end

        return type
    end

    """
    `is_name_typename(::Type, ::Type) -> Bool`

    unroll type declaration, then check if name is typename
    """
    function is_name_typename(type_in::Type, type_comparison::Type) ::Bool
        return getproperty(type_in, :name) == Base.typename(type_comparison)
    end

    """
    `get_n_fields(::Type) -> Int64`
    """
    function get_n_fields(type::Type) ::Int64
        return length(fieldnames(type))
    end

    """
    `get_fields(::Type) -> Vector{Pair{Symbol, Type}}`

    get field symbols and types, used by jluna::Type::get_fields
    """
    function get_fields(type::Type) ::Vector{Pair{Symbol, Type}}

        out = Vector{Pair{Symbol, Type}}();
        names = fieldnames(type)
        types = fieldtypes(type)

        for i in 1:(length(names))
            push!(out, names[i] => types[i])
        end

        return out
    end

    """
    `get_parameter(::Type) -> Vector{Pair{Symbol, Type}}`

    get parameter symbols and upper type limits, used by jluna::Type::get_parameters
    """
    function get_parameters(type::Type) ::Vector{Pair{Symbol, Type}}

        while !hasproperty(type, :parameters)
            type = type.body
        end

        out = Vector{Pair{Symbol, Type}}();
        parameters = getproperty(type, :parameters)

        for i in 1:(length(parameters))
            push!(out, parameters[i].name => parameters[i].ub)
        end

        return out
    end

    """
    `get_n_parameters(::Type) -> Int64`
    """
    function get_n_parameters(type::Type) ::Int64

        type = unroll_type(type)

        return length(getproperty(type, :parameters))
    end

    """
    `assign_in_module(::Module, ::Symbol, ::T) -> T`

    assign variable in other module, throws if variable does not exist
    """
    function assign_in_module(m::Module, variable_name::Symbol, value::T) ::T where T

        if (!isdefined(m, variable_name))
            throw(UndefVarError(Symbol(string(m) * "." * string(variable_name))))
        end

        return Base.eval(m, :($variable_name = $value))
    end

    """
    `create_in_module(::Module, ::Symbol, ::T) -> T`

    assign variable in other module, if variable does not exist, create then assign
    """
    function create_or_assign_in_module(m::Module, variable_name::Symbol, value::T) ::T where T
        return Base.eval(m, :($variable_name = $value))
    end

    """
    `get_names(::Module) -> IdDict{Symbol, Any}`

    access all module members as dict
    """
    function get_names(m::Module) ::IdDict{Symbol, Any}

        out = IdDict{Symbol, Any}()

        for n in names(m; all = true)
            if string(n)[1] != '#'
                out[n] = m.eval(n)
            end
        end

        return out
    end

    """
    `get_nth_method(::Function, ::Integer) -> Method`

    wrap method access, used by jlune::Method
    """
    function get_nth_method(f::Function, i::Integer) ::Method

        return methods(f)[i]
    end

    """
    `get_return_type_of_nth_method(::Function, ::Integer) -> Type`

    used by jluna::Function to deduce method signature
    """
    function get_return_type_of_nth_method(f::Function, i::Integer) ::Type

        return Base.return_types(test)[i]
    end

    """
    `get_argument_type_of_nths_methods(::Function, ::Integer) -> Vector{Type}`

    used by jluna::Function to deduce method signature
    """
    function get_argument_types_of_nth_method(f::Function, i::Integer) ::Vector{Type}

        out = Vector{Type}()
        types = methods(f)[i].sig.types

        for i in 2:length(types)
            push!(out, types[i])
        end

        return out
    end

    """
    `get_length_of_generator(::Base.Generator) -> Int64`

    deduce length of Base.Generator object
    """
    function get_length_of_generator(gen::Base.Generator) ::Int64

        if (Base.haslength(gen))
            return length(gen)
        else
            # heuristically deduce length
            for i in Iterators.reverse(gen.iter.itr)
                if gen.iter.flt(i)
                    return i
                end
            end
        end
    end

    """
    `new_array(::Type, dims::Int64...) -> Array{Type, length(dims))`
    """
    function new_array(value_type::Type, lengths::Int64...)

        length = 1;
        for i in lengths
            length *= i
        end

        out = Array{value_type, 1}(undef, length)
        return reshape(out, lengths...)
    end

    """
    `get_value_type_of_array(::Array{T}) -> Type`

    forward value type of array
    """
    function get_value_type_of_array(_::Array{T}) ::Type where T

        return T
    end

    """
    `new_vector(::Integer, ::T) -> Vector{T}`

    create vector by deducing argument type
    """
    function new_vector(size::Integer, _::T) where T
        return Vector{T}(undef, size)
    end

    """
    `new_vector(::Integer, ::T) -> Vector{T}`

    create vector by deducing argument type
    """
    function new_vector(size::Integer, type::Type)
        return Vector{type}(undef, size)
    end

    """
    `new_complex(:T, :T) -> Complex{T}`

    wrap complex ctor
    """
    function new_complex(real::T, imag::T) ::Complex{T} where T
        return Complex{T}(real, imag)
    end

    """
    `invoke(function::Any, arguments::Any...) -> Any`

    wrap function call for non-function objects
    """
    function invoke(x::Any, args...) ::Any
        return x(args...)
    end

    """
    `create_or_assign(::Symbol, ::T) -> T`

    assign variable in main, or if none exist, create it and assign
    """
    function create_or_assign(symbol::Symbol, value::T) ::T where T

        return Main.eval(Expr(:(=), symbol, value))
    end

    """
    `serialize(<:AbstractDict{T, U}) -> Vector{Pair{T, U}}`

    transform dict into array
    """
    function serialize(x::T) ::Vector{Pair{Key_t, Value_t}} where {Key_t, Value_t, T <: AbstractDict{Key_t, Value_t}}

        out = Vector{Pair{Key_t, Value_t}}()
        for e in x
            push!(out, e)
        end
        return out;
    end

    """
    `serialize(::Set{T}) -> Vector{T}`

    transform dict into array
    """
    function serialize(x::T) ::Vector{U} where {U, T <: AbstractSet{U}}

        out = Vector{U}()

        for e in x
            push!(out, e)
        end

        return out;
    end

    """
    `new_dict(key_t::Type, value_t::Type, ::Integer) -> Dict{key_t, value_t}`

    create new dict from type, also provides sizehint
    """
    function new_dict(key_t::Type, value_t::Type, sizehint_maybe::Integer = 0)

        out = Dict{key_t, value_t}();
        sizehint!(out, sizehint_maybe);
        return out;
    end

    """
    `new_set(::Type, ::Integer) -> Set`

    create new set from type, also provides sizehint
    """
    function new_set(value_t::Type, sizehint_maybe::Integer = 0)

        out = Set{value_t}();
        sizehint!(out, sizehint_maybe);
        return out;
    end

    """
    `forward_as_pointer(t::Type, ::Ptr{Cvoid}) -> Ptr{t}`
    """
    function forward_as_pointer(type::Type, ptr::Ptr{Cvoid}) ::Ptr{type}
        return Ptr{type}(ptr)
    end

    """
    `new_lock() -> Base.ReentrantLock`
    """
    function new_lock()
        return Base.ReentrantLock()
    end

    module gc_sentinel

        struct ListNode{T}
            _previous::Union{Base.RefValue{ListNode{T}}, Nothing}
            _value::Union{T, Nothing}
            _is_root::Bool

            function ListNode{T}() where T
                new(nothing, nothing, true)
            end

            function ListNode{T}(previous::Base.RefValue{ListNode{T}}, value) where T
                return new(Ref{ListNode{T}}(previous.x), value, false)
            end
        end

        mutable struct List{T}

            _front::Base.RefValue{ListNode{T}}

            function List{T}() where T
                return new(Ref{ListNode{T}}(ListNode{T}()))
            end
        end

        function append!(list::List{T}, value::T) where T
           list._front = Ref{ListNode{T}}(ListNode{T}(list._front, value))
        end

        function front(list::List{T}) ::T where T
            return list._front[]._value
        end

        function pop!(list::List{T}) ::Nothing where T

            if !list._front[]._is_root
                list._front = list._front[]._previous
            end

            return nothing
        end

        function isempty(list::List{T}) ::Bool where T
            return list._front[]._is_root
        end

        # ---

        const _gc_stack = NTuple{Threads.nthreads(), List{Base.RefValue{Any}}}([List{Base.RefValue{Any}}() for i in 1:Threads.nthreads()])

        function gc_push(ptr::Ptr{Cvoid}) ::Nothing
            append!(_gc_stack[Threads.threadid()], Ref{Any}(unsafe_pointer_to_objref(ptr)))
            return nothing
        end

        function gc_pop() ::Nothing
            pop!(_gc_stack[Threads.threadid()])
        end

        function shutdown() ::Nothing

            for i in 1:Threads.nthreads()
                while !isempty(_gc_stack[i])
                    pop!(_gc_stack[i])
                end
            end
            return nothing
        end
    end

    """
    offers julia-side memory management for C++ jluna
    """
    module memory_handler

        const _current_id = Ref(UInt64(0)) # modified atomically through locks
        const _refs = Ref(Dict{UInt64, Base.RefValue{Any}}())
        const _ref_counter = Ref(Dict{UInt64, UInt64}())

        const _refs_lock = Base.ReentrantLock()
        const _refs_counter_lock = Base.ReentrantLock()

        const _ref_id_marker = '#'
        const _refs_expression = Meta.parse("jluna.memory_handler._refs[]")

        # proxy id that is actually an expression, the ID of topmodule Main is
        ProxyID = Union{Expr, Symbol, Nothing}

        # make as unnamed
        make_unnamed_proxy_id(id::UInt64) = return Expr(:ref, Expr(:ref, _refs_expression, id))

        # make as named with owner and symbol name
        function make_named_proxy_id(id::Symbol, owner_id::ProxyID) ::ProxyID

            if owner_id == :Main
                return id
            else
                return Expr(:(.), owner_id, QuoteNode(id))
            end
        end

        # make as named with main as owner and symbol name
        make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return id

        # make as named with owner and array index name
        make_named_proxy_id(id::Number, owner_id::ProxyID) ::ProxyID = return Expr(:ref, owner_id, convert(Int64, id))

        # assign to proxy id
        function assign(new_value::T, name::ProxyID) where T

            if new_value isa Symbol || new_value isa Expr
                return Main.eval(Expr(:(=), name, QuoteNode(new_value)))
            else
                return Main.eval(Expr(:(=), name, new_value));
            end
        end

        # eval proxy id
        evaluate(name::ProxyID) ::Any = return Main.eval(name)
        evaluate(name::Symbol) ::Any = return Main.eval(:($name))

        """
        `get_name(::ProxyID) -> String`

        parse name from proxy id
        """
        function get_name(id::ProxyID) ::String

            if length(id.args) == 0
                return "Main"
            end

            current = id
            while current.args[1] isa Expr && length(current.args) >= 2

                if current.args[2] isa UInt64
                    current.args[2] = convert(Int64, current.args[2])
                end

                current = current.args[1]
            end

            out = string(id)
            reg = r"\Q((jluna.memory_handler._refs[])[\E(.*)\Q])[]\E"
            captures = match(reg, out)

            if captures != nothing
                out = replace(out, reg => "<unnamed proxy #" * string(tryparse(Int64, captures.captures[1])) * ">")
            end

            return out;
        end

        get_name(::Nothing) ::String = return "Main"
        get_name(s::Symbol) ::String = return string(s)
        get_name(i::Integer) ::String = return "[" * string(i) * "]"

        """
        `print_refs() -> Nothing`

        pretty print _ref state, for debugging
        """
        function print_refs() ::Nothing

            println("jluna.memory_handler._refs: ");
            for e in _refs[]
                println("\t", Int64(e.first), " => ", e.second[], " (", typeof(e.second[]), ") ")
            end
        end

        """
        `create_reference(::Ptr{Cvoid}, ::Any) -> UInt64`

        add reference to _refs
        """
        function create_reference(to_wrap::Ptr{Cvoid}) ::UInt64

            lock(_refs_lock)
            lock(_refs_counter_lock)

            global _current_id[] += 1
            key::UInt64 = _current_id[];

            if (haskey(_refs[], key))
                _ref_counter[][key] += 1
            else
                _refs[][key] = Base.RefValue{Any}(unsafe_pointer_to_objref(to_wrap))
                _ref_counter[][key] = 1
            end

            unlock(_refs_lock)
            unlock(_refs_counter_lock)

            return key;
        end

        create_reference(_::Nothing) ::UInt64 = return 0

        """
        `set_reference(::UInt64, ::T) -> Nothing`

        update the value of a reference in _refs without adding a new entry or changing it's key, ref pointers C++ side stay valid
        """
        function set_reference(key::UInt64, new_value::T) ::Base.RefValue{Any} where T

            lock(_refs_lock)
            result = begin _refs[][key] = Base.RefValue{Any}(new_value) end
            unlock(_refs_lock)
            return result
        end

        """
        `get_reference(::Int64) -> Any`

        access reference in _refs
        """
        function get_reference(key::UInt64) ::Any

            if (key == 0)
                return nothing
            end

            lock(_refs_lock)
            result = _refs[][key]
            unlock(_refs_lock)
            return result
        end

        """
        `free_reference(::UInt64) -> Nothing`

        free reference from _refs
        """
        function free_reference(key::UInt64) ::Nothing

            if (key == 0)
                return nothing;
            end

            lock(_refs_lock)

            if _refs[][key][] isa Module
                unlock(_refs_lock)
                return
            end

            lock(_refs_counter_lock)
            global _ref_counter[][key] -= 1
            count::UInt64 = _ref_counter[][key]

            if (count == 0)
                delete!(_ref_counter[], key)
                delete!(_refs[], key)
            end

            unlock(_refs_lock)
            unlock(_refs_counter_lock)
            return nothing;
        end

        """
        `force_free() -> Nothing`

        immediately deallocate all C++-managed values
        """
        function force_free() ::Nothing

            lock(_refs_lock)
            lock(_refs_counter_lock)

            for k in keys(_refs)
                delete!(_refs[], k)
                delete!(_ref_counter[], k)
            end

            unlock(_refs_lock)
            unlock(_refs_counter_lock)
            return nothing;
        end

        ### GC Sentinel: protects values from being collected

    """
        # declare abstract type
        abstract type AbstractGCSentinel end

        # global storage, one sentinel per thread
        const _sentinels = NTuple{Threads.nthreads(), Vector{Base.Ref{Union{AbstractGCSentinel, Nothing}}}}([[] for _ in 1:(Threads.nthreads())])

        # sentinel, holds N values
        mutable struct GCSentinel{N} <: AbstractGCSentinel
            _values::NTuple{N, Base.RefValue{Any}}
            _n_preserved::Int64
            _index::Int64
        end

        # update sentinel
        function new_sentinel(n::Integer) ::Nothing

            index = length(_sentinels[Threads.threadid()]);
            push!(_sentinels[Threads.threadid()], GCSentinel{n}(NTuple{N, Base.Ref{Any}}([undef for _ in 1:n]), 1, length))

            finalizer(last(_sentinels[Threads.threadid()]), function (x::GCSentinel{n})
                deleteat!(_sentinels[Threads.threadid()], x._index)
            end)
            return nothing
        end

        # add value to sentinel
        function preserve(ptr::Ptr{Cvoid}) where N

            sentinel_ref = _sentinels[Threads.threadid()]
            sentinel_ref[]._values[sentinel_ref[]._n_preserved][] = Ref{Any}(unsafe_pointer_to_objref(ptr))
            sentinel_ref[]._n_preserved += 1
            return nothing
        end

        # release sentinel, all held values can be collected
        function release()
            _sentinels[Threads.threadid()][] = nothing
        end

        """
    end

    module cppcall

        #const _c_adapter_path = "<call jluna::initialize to initialize this field>";

        function verify_library() ::Bool
            return ccall((:verify, _c_adapter_path), Bool, ());
        end

        """
        object that is callable like a function, but executes C++-side code
        """
        mutable struct UnnamedFunction{NArgs}

            _native_handle::Ptr{Cvoid}
            # points to C-side function

            _n_args::Cint
            #  0: (void) -> Any
            #  1: (Any) -> Any
            #  2: (Any, Any) -> Any
            #  3: (Any, Any, Any) -> Any
            # all others invalid

            function UnnamedFunction{N}(ptr::Ptr{Cvoid}) where N

                out = new{N}(ptr, N)
                finalizer(function (t::UnnamedFunction{N})
                    ccall((:free_lambda, cppcall._c_adapter_path), Cvoid, (Csize_t, Cint), t._native_handle, t._n_args)
                end, out);

                return out;
            end
        end

        """
        `make_unnamed_function(::Ptr{Cvoid}, n::Integer) -> UnnamedFunction{n}

        wrapper for UnnamedFunction ctor
        """
        function make_unnamed_function(ptr::Ptr{Cvoid}, n::Integer)
            return UnnamedFunction{n}(ptr)
        end

        """
        `invoke_function(::UnnamedFunction) -> Ptr{Any}`

        invoke function with 0 args
        """
        function invoke_function(f::UnnamedFunction{0}) ::Ptr{Any}
            return ccall((:invoke_lambda_0, cppcall._c_adapter_path), Ptr{Any}, (Ptr{Cvoid},), f._native_handle);
        end

        # overload for 1 arg
        function invoke_function(f::UnnamedFunction{1}, arg1::Ptr{Any}) ::Ptr{Any}
            return ccall((:invoke_lambda_1, cppcall._c_adapter_path), Ptr{Any}, (Ptr{Cvoid}, Ptr{Any}), f._native_handle, arg1);
        end

        # overload for 2 args
        function invoke_function(f::UnnamedFunction{2}, arg1::Ptr{Any}, arg2::Ptr{Any}) ::Ptr{Any}
            return ccall((:invoke_lambda_2, cppcall._c_adapter_path), Ptr{Any}, (Ptr{Cvoid}, Ptr{Any}, Ptr{Any}), f._native_handle, arg1, arg2);
        end

        # overload for 3 args
        function invoke_function(f::UnnamedFunction{3}, arg1::Ptr{Any}, arg2::Ptr{Any}, arg3::Ptr{Any}) ::Ptr{Any}
            return ccall((:invoke_lambda_3, cppcall._c_adapter_path), Ptr{Any}, (Ptr{Cvoid}, Ptr{Any}, Ptr{Any}, Ptr{Any}), f._native_handle, arg1, arg2, arg3);
        end

        """
        `to_pointer(::Any) -> Ptr{Any}`

        get pointer to any object (including immutable ones)
        """
        function to_pointer(x) ::Ptr{Any}
            return ccall((:to_pointer, cppcall._c_adapter_path), Ptr{Cvoid}, (Any,), x)
        end

        """
        `from_pointer(::Ptr{Any}) -> Any`

        wrap unsafe_pointer_to_objref
        """
        function from_pointer(ptr::Ptr{Any})
            return unsafe_pointer_to_objref(ptr)
        end

        """
        `UnnamedFunction(xs...) -> Any`

        invoke UnnamedFunction, checks for correct number of arguments
        """
        function (f::UnnamedFunction{N})(xs...) where N

            n = length(xs)

            if n == N == 0
                return from_pointer(invoke_function(f));
            elseif n == N == 1
                return from_pointer(invoke_function(f, to_pointer(xs[1])));
            elseif n == N == 2
                return from_pointer(invoke_function(f, to_pointer(xs[1]), to_pointer(xs[2])));
            elseif n == N == 3
                return from_pt(invoke_function(f, to_pointer(xs[1]), to_pointer(xs[2]), to_pointer(xs[3])));
            elseif N != 0 && N != 1 && N != 2 & N != 3
                return from_pointer(invoke_function(f, to_pointer([xs...])));
            else
                throw(ErrorException(
                    "MethodError: when trying to invoke <C++ Lambda#" * string(f._native_handle) * ">" *
                    ": wrong number of arguments. expected " * string(N) * ", got " * string(n) * "."
                   *  (n <= 3 ? "" : "\n\nTo create a C++-function that can take n > 3 arguments, simply make a 1-argument function with the only argument being an n-sized tuple or collection.")
                ))
            end
        end

        """
        `make_task(::UInt64) -> Task`
        """
        function make_task(ptr::UInt64)
            return Task() do;
                res_ptr = ccall((:invoke_from_task, _c_adapter_path), Csize_t, (Csize_t,), ptr);
                return unsafe_pointer_to_objref(Ptr{Any}(res_ptr))
            end
        end
    end

    # obfuscate internal state to encourage using operator[] sytanx
    struct ProxyInternal

        _fieldnames_in_order::Vector{Symbol}
        _fields::Dict{Symbol, Union{Any, Missing}}
        _lock::Base.ReentrantLock

        ProxyInternal() = new(Vector{Symbol}(), Dict{Symbol, Union{Any, Missing}}(), Base.ReentrantLock())
    end

    # proxy as deepcopy of cpp-side usertype object
    struct Proxy

        _typename::Symbol
        _value::ProxyInternal

        Proxy(name::Symbol) = new(name, ProxyInternal())
    end

    """
    `new_proxy(::Symbol) -> Proxy`

    wrap proxy ctor
    """
    new_proxy(name::Symbol) = return Proxy(name)

    """
    `implement(::Proxy, ::Module) -> Type`

    translate a usertype proxy into an actual julia type
    """
    function implement(template::Proxy, m::Module = Main) ::Type

        @lock template._value._lock begin
            out::Expr = :(mutable struct $(template._typename) end)
            deleteat!(out.args[3].args, 1)

            for name in template._value._fieldnames_in_order
                push!(out.args[3].args, Expr(:(::), name, :($(typeof(template._value._fields[name])))))
            end

            new_call::Expr = Expr(:(=), Expr(:call, template._typename), Expr(:call, :new))

            for name in template._value._fieldnames_in_order
                push!(new_call.args[2].args, template._value._fields[name])
            end

            push!(out.args[3].args, new_call)
            Base.eval(m, out)
        end
        return m.eval(template._typename)
    end

    """
    `setindex!(::Proxy, <:Any, ::Symbol) -> Nothing`
    extend base.setindex!
    """
    function Base.setindex!(proxy::Proxy, value, key::Symbol) ::Nothing

        @lock proxy._value._lock begin
            if (!haskey(proxy._value._fields, key))
                push!(proxy._value._fieldnames_in_order, key)
            end

            proxy._value._fields[key] = value
        end
        return nothing
    end

    """
    `getindex(::Proxy, ::Symbol) -> Any`
    extend base.getindex
    """
    function Base.getindex(proxy::Proxy, value, key::Symbol) #::Auto

        out::Any = undef
        @lock proxy._value._lock begin
            out = proxy._value._fields[key]
        end
        return out
    end
end

return true # used for testing

max_stars_repo_name: Clemapfel/jluna

max_issues_repo_name: Clemapfel/jluna

max_forks_repo_name: Clemapfel/jluna


text:
using LimitedLDLFactorizations
using MatrixMarket

# Warmup
A = sprand(10, 10, .5); A = A * A' + spdiagm(rand(10))
(L, d, α) = lldl(A, memory=5)

# obtain matrix from
# https://www.cise.ufl.edu/research/sparse/matrices/HB/bcsstk09.html
K = MatrixMarket.mmread("bcsstk09.mtx")
# make K lower triangular and extract diagonal here
# so we don't time tril() and diag().
Kdiag = diag(K)
K1 = tril(K, -1)
nnzK1 = nnz(K1)
n = size(K, 1)

@printf("%2s %6s %6s %8s %7s %8s\n",
        "p", "nnz(K)", "nnz(L)", "‖LDL'-K‖", "α", "time")
for p = 0 : 5 : 50
  res, t, b, g, m = @timed lldl(K1, Kdiag, memory=p)
  (L, d, α) = res
  L = L + speye(n)
  @printf("%2d %6d %6d %8.2e %7.1e %8.2e\n",
          p, nnzK1, nnz(L) - n, norm(L*diagm(d)*L' - K, 1) / norm(K, 1), α, t)
end


text:
using CodeTools, LNR, Media, Requires

import CodeTools: getthing

LNR.cursor(data::Associative) = cursor(data["row"], data["column"])

exit_on_sigint(on) = ccall(:jl_exit_on_sigint, Void, (Cint,), on)

function modulenames(data, pos)
  main = haskey(data, "module") ? data["module"] :
         haskey(data, "path") ? CodeTools.filemodule(data["path"]) :
         "Main"
  main == "" && (main = "Main")
  sub = CodeTools.codemodule(data["code"], pos)
  main, sub
end

function getmodule(data, pos)
  main, sub = modulenames(data, pos)
  getthing("$main.$sub", getthing(main, Main))
end

handle("module") do data
  main, sub = modulenames(data, cursor(data))
  return d(:main => main,
           :sub  => sub,
           :inactive => (getthing(main) == nothing),
           :subInactive => (getthing("$main.$sub") == nothing))
end

handle("allmodules") do
  sort!([string(m) for m in CodeTools.allchildren(Main)])
end

isselection(data) = data["start"] ≠ data["stop"]

macro errs(ex)
  :(try
      $(esc(ex))
    catch e
      EvalError(isa(e, LoadError) ? e.error : e, catch_backtrace())
    end)
end

withpath(f, path) =
  Requires.withpath(f, path == nothing || isuntitled(path) ? nothing : path)

const evallock = ReentrantLock()

handle("eval") do data
  @destruct [text, line, path, mod] = data
  mod = getthing(mod)

  lock(evallock)
  result = @dynamic let Media.input = Editor()
    withpath(path) do
      @errs include_string(mod, text, path, line)
    end
  end
  unlock(evallock)

  display = Media.getdisplay(typeof(result), Media.pool(Editor()), default = Editor())
  display ≠ Editor() && render(display, result)
  render(Editor(), result)
end

handle("evalall") do data
  @destruct [setmod = :module || nothing, path || "untitled", code] = data
  mod = Main
  if setmod ≠ nothing
    mod = getthing(setmod, Main)
  elseif isabspath(path)
    mod = getthing(CodeTools.filemodule(path), Main)
  end
  lock(evallock)
  @dynamic let Media.input = Editor()
    withpath(path) do
      try
        include_string(mod, code, path)
      catch e
        ee = EvalError(e, catch_backtrace())
        render(Console(), ee)
        @msg error(d(:msg => "Error evaluating $(basename(path))",
                     :detail => sprint(showerror, e, ee.bt),
                     :dismissable => true))
      end
    end
  end
  unlock(evallock)
  return
end

handle("evalrepl") do data
  @destruct [mode || nothing, code, mod || "Main"] = data
  mod = getthing(mod)
  if mode == "shell"
    code = "Base.repl_cmd(`$code`, STDOUT)"
  elseif mode == "help"
    code = "@doc $code"
  end
  if Debugger.isdebugging()
    render(Console(), @errs Debugger.interpret(code))
  else
    try
      lock(evallock)
      @dynamic let Media.input = Console()
        withpath(nothing) do
          render(@errs eval(mod, :(ans = include_string($code, "console"))))
        end
      end
      unlock(evallock)
    catch e
      showerror(STDERR, e, catch_backtrace())
    end
  end
  return
end

handle("docs") do data
  @destruct [mod || "Main", word] = data
  mod = getthing(mod)
  docstring = @errs HTML(sprint(show, MIME"text/html"(), include_string(mod, "@doc $word")))
  mtable = try include_string(mod, "methods($word)") catch [] end
  isa(docstring, EvalError) ?
    d(:error    =>  true)   :
    d(:type     => :dom,
      :tag      => :div,
      :contents =>  map(x -> render(Editor(), x), [docstring; mtable]))
end

handle("methods") do data
  @destruct [mod || "Main", word] = data
  mod = include_string(mod)
  wordtype = try
    include_string("typeof($word)")
  catch
    Function
  end
  if wordtype == Function
    result = @errs include_string(mod, "methods($word)")
  elseif wordtype == DataType
    result = @errs include_string(mod, "methodswith($word)")
  end
  d(:result => render(Editor(), result))
end

ismacro(f::Function) = startswith(string(methods(f).mt.name), "@")

wstype(x) = nothing
wstype(::Module) = "module"
wstype(f::Function) = ismacro(f) ? "mixin" : "function"
wstype(::Type) = "type"
wstype(::Expr) = "mixin"
wstype(::Symbol) = "tag"
wstype(::AbstractString) = "property"
wstype(::Number) = "constant"
wstype(::Exception) = "tag"

wsicon(x) = nothing
wsicon(f::Function) = ismacro(f) ? "icon-mention" : nothing
wsicon(::AbstractArray) = "icon-file-binary"
wsicon(::AbstractVector) = "icon-list-ordered"
wsicon(::AbstractString) = "icon-quote"
wsicon(::Expr) = "icon-code"
wsicon(::Symbol) = "icon-code"
wsicon(::Exception) = "icon-bug"
wsicon(::Number) = "n"

wsnamed(name, val) = false
wsnamed(name, f::Function) = name == methods(f).mt.name
wsnamed(name, m::Module) = name == module_name(m)
wsnamed(name, T::DataType) = name == Symbol(T.name)

function wsitem(name::Symbol, val)
  d(:name  => wsnamed(name, val) ? nothing : name,
    :value => render(Inline(), val),
    :type  => wstype(val),
    :icon  => wsicon(val))
end

wsitem(mod::Module, name::Symbol) = wsitem(name, getfield(mod, name))

handle("workspace") do mod
  mod = getthing(mod)
  ns = names(mod)
  filter!(n -> isdefined(mod, n), ns)
  # TODO: only filter out imported modules
  filter!(n -> !isa(getfield(mod, n), Module), ns)
  contexts = [d(:context => string(mod), :items => map(n -> wsitem(mod, n), ns))]
  if Debugger.isdebugging()
    prepend!(contexts, Debugger.contexts())
  end
  return contexts
end


text:
function notebook2html!(notebook; kwargs...)
    session = ServerSession()
    run_notebook!(notebook, session)
    return notebook2html(notebook; kwargs...)
end

@testset "html" begin
    html = "foo"
    block = PlutoStaticHTML.code_block(html)
    @test contains(block, "<b>foo</b>")

    notebook = Notebook([
        Cell("x = 1 + 1"),
        Cell("using Images: load"),
        Cell("PKGDIR = \"$PKGDIR\""),
        Cell("""im_file(ext) = joinpath(PKGDIR, "test", "im", "im.\$ext")"""),
        Cell("""load(im_file("png"))""")
    ])
    html = notebook2html!(notebook)
    lines = split(html, '\n')

    @test contains(lines[1], "1 + 1")
    @test contains(lines[2], "2")

    @test contains(lines[end-1], """<img src=\"data:image/png;base64,""")

    notebook = Notebook([
        Cell("md\"This is **markdown**\"")
    ])
    html = notebook2html!(notebook)
    lines = split(html, '\n')
    @test contains(lines[2], "")

    notebook = Notebook([
        Cell("""("pluto", "tree", "object")"""),
        Cell("""["pluto", "tree", "object"]"""),
        Cell("""[1, (2, (3, 4))]""")
    ])
    html = notebook2html!(notebook)
    lines = split(html, '\n')
    @test contains(lines[2], "(\"pluto\", \"tree\", \"object\")")
    @test contains(lines[2], "<pre")
    @test contains(lines[5], "[\"pluto\", \"tree\", \"object\"]")

    @test contains(lines[8], "[1, (2, (3, 4))]")

    notebook = Notebook([
        Cell("struct A end"),
        Cell("""
            struct B
                x::Int
                a::A
            end
            """
            ),
        Cell("B(1, A())")
    ])
    html = notebook2html!(notebook)
    lines = split(html, '\n')
    @test contains(lines[end-1], "B(1, A())")

    notebook = Notebook([
        Cell("md\"my text\"")
    ])
    html = notebook2html!(notebook; hide_md_code=true)
    lines = split(html, '\n')
    @test lines[1] == ""

    html = notebook2html!(notebook; hide_md_code=false)
    lines = split(html, '\n')
    @test lines[1] != ""
end

@testset "from_file" begin
    mktempdir() do dir
        file = joinpath(dir, "tmp.jl")
        content = pluto_notebook_content("x = 1 + 2")
        write(file, content)
        html = notebook2html(file)
        @test contains(html, "3")
    end
end


text:
using Pkg
Pkg.activate(ENV["JULIA_DQMC"])

using Revise, BenchmarkTools
include(joinpath(ENV["JULIA_DQMC"], "src/dqmc_framework.jl"))

if !(@isdefined infile)
    if isfile("dqmc.in.xml")
        global infile = "dqmc.in.xml"
    else
        error("Variable 'infile' != path to infile.in.xml")
    end
end

p = Params()
p.output_file = "live.out.h5.running"
xml2parameters!(p, infile)

mc = DQMC(p)

init!(mc)

display(mc)
nothing

max_stars_repo_name: crstnbr/dqmc

max_issues_repo_name: crstnbr/dqmc

max_forks_repo_name: crstnbr/dqmc


text:
using EzXML, TypeTransform
import EzXML: Document, Node

# to prevent EzXML error: https://github.com/bicycle1885/EzXML.jl/pull/125
# function Base.show(io::IO, node::Node)
#     # prefix = isdefined(Main, :Node) ? "Node" : "EzXML.Node"
#     prefix = "Node"
#     ntype = EzXML.nodetype(node)
#     if ntype ∈ (EzXML.ELEMENT_NODE, EzXML.ATTRIBUTE_NODE) && EzXML.hasnodename(node)
#         desc = string(repr(ntype), '[', EzXML.nodename(node), ']')
#     else
#         desc = repr(ntype)
#     end
#     print(io, "$(prefix)(<$desc>)")
# end

################################################################
import Tables
# DataTypesSupport
include("xmlutils/TypesSupport/amlTables.jl")
################################################################
include("xmlutils/initializer.jl")
include("xmlutils/addnode.jl")
include("xmlutils/nodeparse.jl")
include("xmlutils/findnode.jl")
include("xmlutils/findcontent.jl")
include("xmlutils/updater.jl")
################################################################

@deprecate addelm!(args...) addnode!(args...)
@deprecate initialize_node(args...) createnode(args...)


text:
module InfiniteVector

include("AbstractInfiniteVector.jl")
include("RangeInfiniteVector.jl")
include("DiscreteInfiniteVector.jl")

end # module


text:
struct Vertex{T, U<:Number}
  position::AbstractVector{T}
  value::U
end
Vertex(x::AbstractVector{T}, f::F) where {T, F} = Vertex(x, f(x))

value(v::Vertex) = v.value
position(v::Vertex) = v.position
newposition(a, ϵ, b) = a + ϵ .* (a - b)

function vertexpositions(ic::AbstractVector{T}, initial_steps::AbstractVector{U}
    ) where {T<:Number, U<:Number}
  if any(iszero.(initial_steps))
    throw(ArgumentError("initial_steps, $initial_steps  must not have any zero
                        values"))
  end
  if length(ic) != length(initial_steps)
    throw(ArgumentError("ic, $ic must be same length as initial_steps
                        $initial_steps"))
  end
  dim = length(ic)
  positions = Vector{Vector{promote_type(T,U)}}()
  for i ∈ 1:dim+1
    x = [ic[j] + ((j == i) ? initial_steps[j] : zero(U)) for j ∈ 1:dim]
    push!(positions, x)
  end
  return positions
end

# must explicitly use <= and >= because == can't overridden and will
# be used in conjunction with < to create a <=
import Base: isless, +, -, <=, >=, isequal, isnan, hash
Base.isless(a::Vertex, b::Vertex) = abs(value(a)) < abs(value(b))
Base.:<=(a::Vertex, b::Vertex) = abs(value(a)) <= abs(value(b))
Base.:>=(a::Vertex, b::Vertex) = abs(value(a)) >= abs(value(b))
Base.:+(a::Vertex, b) = position(a) .+ b
Base.:-(a::Vertex, b::Vertex) = position(a) .- position(b)
#Base.:-(a, b::Vertex) = a .- position(b)
function Base.isequal(a::Vertex, b::Vertex)
  values_equal = value(a) == value(b) || (isnan(a) && isnan(b))
  positions_equal = all(position(a) .== position(b))
  return values_equal && positions_equal
end
#Base.isequal(a::Vertex, b::AbstractVector) = all(position(a) .== b)

Base.isnan(a::Vertex) = isnan(value(a))

function vertexsorter(x::Vertex{T, <:Complex}, y::Vertex{T, <:Complex}) where
  {T}
  xa, ya = angle(value(x)), angle(value(y))
  xa == ya || return isless(xa, ya)

  return vertexsorter_abs_position(x::Vertex, y::Vertex)
end

vertexsorter(x::Vertex, y::Vertex) = vertexsorter_abs_position(x, y)

function vertexsorter_abs_position(x::Vertex, y::Vertex)
  xv, yv = abs(value(x)), abs(value(y))
  xv == yv || return isless(xv, yv)

  xp, yp = position(x), position(y)
  return any(i->isless(xp[i], yp[i]), eachindex(xp))
end


text:
#Main script to initialize:
#-number of workers for parallel computing
#-neural network and lattice
#-hyper-parameters of ground state and dynamic simulations

############################ Parallelizzation ##################################
using Distributed
#How many workers do you want? Set Nworkers to number of CPUs
#you want to add on top of the master CPU. Default Nworkers = 0
Nworkers = 3
addprocs(Nworkers)

################################################################################
###################### Neural network and lattice ##############################
#How large is your lattice? Set the lattice size Lx and Ly
Lx = 4
Ly = 4

#How many hidden units M=Alpha*Lx*Ly do you want? Set Alpha
Alpha = 2

#What is the exchange interaction along x and y? Set Jx and Jy (default = 1)
Jx = 1
Jy = 1

################################################################################
####################### Ground state optimization ##############################
#Do you want to run a ground state optimization? Set gs  (default = true)
gs = true

#Set hyperparameters of ground state optimization
#How many ground state iterations do you want? Set nIter
nIter = 300

#How many Monte Carlo samples do you want to use? Set nSampleGs
nSampleGs = 2000

#How small do you want the learning rate? Set stepSize
stepSize = 0.005

################################################################################
############################## Dynamics ########################################
#Do you want to run dynamics? Set dy (default dy = true)
dy = false

#What is the total integration time you want to simulate? Set totTime
totTime = 10.0

#How many Monte Carlo samples do you want to use? Set nSampleDyn
nSampleDyn = 1000

#How small do you want the step size of the Heun integrator? Set stepSizeHeun
stepSizeHeun = 0.001

#Do you want to invert the covariance matrix with MINRES or Pseudoinverse?
#Set minres_ = true to solve with MINRES, false to solve with Pseudoinverse (default = true)
minres_ = true

#What are the initial parameters for the dynamic W(t=0)? Import independent variational parameters
#Note that real and imaginary parts are conveniently stored in separate files
#First import package DelimitedFiles.jl to read and write files
using DelimitedFiles
#WRBMInit = readdlm("src/pre_optimized_parameters/W_RBM_4_2_real.jl") .+ im*readdlm("src/pre_optimized_parameters/W_RBM_4_2_imag.jl")

#What is the perturbation do you want to use along y? Define DeltaJ(t) (default DeltaJ(t)=0.1)
@everywhere function DeltaJ(t::Float64)
    return 0.1*exp(-(t - 0.987)^2 / (2*0.4^2))*sin(8*t)
end
################################# end ##########################################
################################################################################
include("../main/init.jl")

max_stars_repo_name: GiamFbn/MyULTRAFAST

max_issues_repo_name: GiamFbn/MyULTRAFAST

max_forks_repo_name: GiamFbn/MyULTRAFAST


text:
"""
    main_generate_instance
A main file allowing to generate a pscopf instance and write its describing files.

For that,
1- set the parameter `instance_out_path` to the directory where instance files will be written.
2- define `create_network` to create the desired network.
This main provides an example for creating the network using PSCOPF's api
3- define `create_init_state` to create an initial state for the units with pmin>0.
This main provides an example for that.
"""

using Dates
using DataStructures

root_path = dirname(@__DIR__)
push!(LOAD_PATH, root_path);
cd(root_path)
include(joinpath(root_path, "src", "PSCOPF.jl"));

#############################
# PARAMS
#############################

# instance_out_path is the path that will contain the partial instance data (missing pscopf_uncertainties):
# pscopf_units : the description of the units
# pscopf_init : the description of the initial state of units (just before the first interest timepoit T)
# pscopf_limits : the flow capacity of each branch
# pscopf_ptdf : the ptdf coefficients per (branch, bus_id)
instance_out_path = ( length(ARGS) > 0 ? ARGS[1] :
                            joinpath(@__DIR__, "..", "data", "2buses_small_usecase", "generate_instance") )

#############################
# CREATE INSTANCE
#############################
function create_network()
    println("Creating the network")
    network = PSCOPF.Networks.Network()
    # Buses
    println("adding buses")
    PSCOPF.Networks.add_new_bus!(network, "bus_1")
    PSCOPF.Networks.add_new_bus!(network, "bus_2")
    # Branches
    println("adding branches")
    PSCOPF.Networks.add_new_branch!(network, "branch_1_2", 500.);
    # PTDF
    println("adding the PTDF")
    PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_1", 0.5)
    PSCOPF.Networks.add_ptdf_elt!(network, "branch_1_2", "bus_2", -0.5)
    # Alternatively, PSCOPF.PSCOPFio.read_ptdf!(network, dirpath, filename)
    #Generators - Limitables
    println("adding Limitables")
    PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "wind_1", PSCOPF.Networks.LIMITABLE,
                                            0., 200.,
                                            0., 10.,
                                            Dates.Second(15*60), Dates.Second(15*60)) #dmo, dp : always 0. ?
    PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "wind_2", PSCOPF.Networks.LIMITABLE,
                                            0., 200.,
                                            0., 11.,
                                            Dates.Second(15*60), Dates.Second(15*60))
    #Generators - Pilotables
    println("adding Pilotables")
    PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "ccg_1", PSCOPF.Networks.PILOTABLE,
                                            150., 600., #pmin, pmax
                                            45000., 30., #start_cost, prop_cost
                                            Dates.Second(4*3600), Dates.Second(15*60)) #dmo, dp
    PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_1", "tac_1", PSCOPF.Networks.PILOTABLE,
                                            10., 300.,
                                            12000., 100.,
                                            Dates.Second(30*60), Dates.Second(15*60))
    PSCOPF.Networks.add_new_generator_to_bus!(network, "bus_2", "ccg_2", PSCOPF.Networks.PILOTABLE,
                                            100., 600., #pmin, pmax
                                            50000., 20., #start_cost, prop_cost
                                            Dates.Second(4*3600), Dates.Second(15*60)) #dmo, dp

    return network
end

function create_init_state(network::Union{Nothing,PSCOPF.Network}=nothing;
                        all_off=false, all_on=false)
    println("Setting the initial state of generators (ON/OFF)")
    generators_init_state = SortedDict{String, PSCOPF.GeneratorState}()
    state = nothing
    if all_off && all_on
        throw(error("cannot set to ON and OFF at the same time"))
    elseif all_off && !isnothing(network)
        state = PSCOPF.OFF
    elseif all_on && !isnothing(network)
        state = PSCOPF.ON
    end

    if !isnothing(state)
        for generator in PSCOPF.get_generators(network)
            if PSCOPF.get_p_min(generator) > 0
                generators_init_state[PSCOPF.get_id(generator)] = state
            end
        end
    else
        throw(error("You need to define generators initial states"))
        #otherwise, we can define a dictionary
        # generators_init_state = SortedDict(
        #                     "ccg_1" => PSCOPF.ON,
        #                     "ccg_2" => PSCOPF.OFF,
        #                     "tac_1" => PSCOPF.OFF,
        #                 )
    end

    return generators_init_state
end

#############################
# GENERATE INSTANCE FILES
#############################
function generate_instance(dir_path)
    network = create_network() # User-defined
    PSCOPF.check(network)
    println("Network created!")

    generators_init_state = create_init_state(network, all_off=true) # User-defined
    PSCOPF.check_initial_state(generators_init_state, network)
    println("Generators' initial state created!")

    PSCOPF.PSCOPFio.write(dir_path, network)
    PSCOPF.PSCOPFio.write(dir_path, generators_init_state)
    println("instance files written to ", dir_path)
end

generate_instance(instance_out_path)

max_stars_repo_name: klorel/pscopf

max_issues_repo_name: klorel/pscopf

max_forks_repo_name: klorel/pscopf


text:
using ModelingToolkit
using Test

@Param t σ ρ β
@Unknown x(t) y(t) z(t)
@Deriv D'~t # Default of first derivative, Derivative(t,1)
@Const c=0

# Default values
p = Parameter(:p)
u = Unknown(:u, dependents = [t])

σ*(y-x)
D(x)
D(x) ~ -σ*(y-x)
D(y) ~ x*(ρ-z)-sin(y)

@test D(t) == Constant(1)


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

module Meshes

using Tables
using StaticArrays
using CircularArrays
using SimpleTraits
using RecipesBase
using LinearAlgebra
using Random

using IterTools: ivec
using StatsBase: Weights
using SpecialFunctions: gamma
using Distances: PreMetric, Euclidean, Mahalanobis, evaluate
using ReferenceFrameRotations: angle_to_dcm
using NearestNeighbors: KDTree, BallTree, knn, inrange

# import categorical arrays as a temporary solution for plot recipes
using CategoricalArrays: CategoricalValue, levelcode

import Tables
import Random
import Base: values, ==, +, -, *
import StatsBase: sample
import Distances: evaluate
import NearestNeighbors: MinkowskiMetric

# numerical tolerances
include("tolerances.jl")

# basic types
include("vectors.jl")
include("points.jl")
include("angles.jl")

# type traits
include("traits/variable.jl")
include("traits/domain.jl")
include("traits/data.jl")
include("traits/optional.jl")

# point sets
include("pointsets.jl")

# basic geometries
include("geometries.jl")

# geometric distances
include("distances.jl")

# geometry sets
include("geometrysets.jl")

# connectivities and meshes
include("connectivities.jl")
include("conventions.jl")
include("mesh.jl")

# domain and data views
include("views.jl")

# paths for traversal
include("paths.jl")

# neighborhoods and searches
include("neighborhoods.jl")
include("neighborsearch.jl")

# partitions
include("partitions.jl")

# algorithms
include("viewing.jl")
include("sampling.jl")
include("partitioning.jl")
include("intersections.jl")
include("discretization.jl")
include("simplification.jl")
include("boundingboxes.jl")

# utilities
include("utils.jl")

# plot recipes
include("plotrecipes/domain.jl")
include("plotrecipes/data.jl")
include("plotrecipes/geometries.jl")
include("plotrecipes/pointsets.jl")
include("plotrecipes/cartesiangrids.jl")
include("plotrecipes/mesh.jl")
include("plotrecipes/partitions.jl")

export
  # points
  Point, Point1, Point2, Point3, Point1f, Point2f, Point3f,
  embeddim, coordtype, coordinates,
  ⪯, ≺, ⪰, ≻,

  # vectors
  Vec, Vec1, Vec2, Vec3, Vec1f, Vec2f, Vec3f,

  # angles
  ∠,

  # domain/data traits
  Domain, Data, Variable,
  domain, constructor, asarray,
  variables, name, mactype,

  # optional traits
  IsGrid, isgrid,

  # geometries
  Geometry,
  embeddim, paramdim, coordtype,
  measure, area, volume, boundary,
  centroid,

  # primitives
  Primitive,
  Line, Ray, Plane, BezierCurve, Box, Ball, Sphere, Cylinder,
  ncontrols, degree, Horner, DeCasteljau,
  center, radius, height, sides,
  points, measure, diagonal,

  # polytopes
  Polytope, Polygon, Polyhedron,
  Segment, Triangle, Quadrangle,
  Pyramid, Tetrahedron, Hexahedron,
  Chain, PolyArea,
  vertices, nvertices,
  faces, facets,
  windingnumber, chains, segments,
  isclosed, issimple, hasholes,
  angles, innerangles, close!, open!,
  orientation, bridge,

  # orientation algorithms
  WindingOrientation,
  TriangleOrientation,

  # geometry sets
  GeometrySet,

  # connectivities
  Connectivity,
  polytopetype, connect, materialize,

  # ordering conventions
  OrderingConvention,
  GMSH,
  connectivities,

  # rotation conventions
  RotationConvention,
  GSLIB, Leapfrog, Datamine,
  TaitBryanExtr, TaitBryanIntr,
  EulerExtr, EulerIntr,
  axesseq, orientation, angleunits,
  mainaxis, isextrinsic, rotmat,

  # point sets
  PointSet,
  nelements,

  # meshes
  Mesh,
  CartesianGrid, UnstructuredMesh,
  faces, elements, spacing,
  nelements,

  # views
  DomainView, DataView,

  # paths
  Path,
  LinearPath, RandomPath,
  ShiftedPath, SourcePath,
  traverse,

  # neighborhoods
  Neighborhood, MetricBall,
  NormBall, Ellipsoid,
  metric, radius,

  # neighbordhood search
  NeighborSearchMethod,
  BoundedNeighborSearchMethod,
  NeighborhoodSearch,
  KNearestSearch,
  KBallSearch,
  BoundedSearch,
  search!, search,

  # partitions
  Partition,
  subsets, metadata,

  # viewing
  viewindices,
  slice,

  # sampling
  SamplingMethod,
  RegularSampling,
  UniformSampling,
  WeightedSampling,
  BallSampling,
  sample,

  # partitioning
  PartitionMethod,
  PredicatePartitionMethod,
  SPredicatePartitionMethod,
  RandomPartition,
  FractionPartition,
  BlockPartition,
  BisectPointPartition,
  BisectFractionPartition,
  BallPartition,
  PlanePartition,
  DirectionPartition,
  PredicatePartition,
  SpatialPredicatePartition,
  ProductPartition,
  HierarchicalPartition,
  partition, split, →,

  # intersections
  Intersection,
  CrossingSegments,
  MidTouchingSegments,
  CornerTouchingBoxes,
  OverlappingSegments,
  NonIntersectingSegments,
  OverlappingBoxes,
  FaceTouchingBoxes,
  CornerTouchingBoxes,
  NonIntersectingBoxes,
  intersecttype,

  # discretization
  DiscretizationMethod,
  FIST,
  discretize,

  # simplification
  SimplificationMethod,
  DouglasPeucker,
  simplify,

  # bounding boxes
  boundingbox,

  # utililities
  signarea,
  sideof

end # module


text:
# Autogenerated wrapper script for hidapi_jll for x86_64-apple-darwin
export hidapi

using libusb_jll
JLLWrappers.@generate_wrapper_header("hidapi")
JLLWrappers.@declare_library_product(hidapi, "@rpath/libhidapi.0.dylib")
function __init__()
    JLLWrappers.@generate_init_header(libusb_jll)
    JLLWrappers.@init_library_product(
        hidapi,
        "lib/libhidapi.0.dylib",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
# Файл с некоторыми скриптами,
# Которые были полезны во время работы, но не могут быть внесены на
# постоянной основе в скрипты, которые представляют реальный интерес.

# В одном из экспериментов потребовалось запустить вычисления с сохраненных
# ранее сериализованных результатов, здесь их чтение.
Js_old = deserialize("../Sep11/Js.jld")
Qs_old = deserialize("../Sep11/QQs.jld")

Js_old = Js;
Qs_old = Qs;

Js_old_1 = Js;
Qs_old_1 = Qs;

q₀ = Qs[:,end];


text:
module QuAlgorithmZoo

using LuxurySparse, LinearAlgebra
using MacroTools: @forward
using Yao, Yao.ConstGate, BitBasis
using YaoArrayRegister: u1rows!
import Yao: mat, dispatch!, niparams, getiparams, setiparams!, cache_key, print_block, apply!, PrimitiveBlock, ishermitian, isunitary, isreflexive
import YaoBlocks: render_params
import Base: ==, copy, hash

export openbox

"""
    openbox(block::AbstractBlock) -> AbstractBlock

For a black box, like QFTBlock, you can get its white box (loyal simulation) using this function.
"""
function openbox end

include("Miscellaneous.jl")
include("sequence.jl")
include("Diff.jl")
include("Adam.jl")
include("QFT.jl")
include("CircuitBuild.jl")
include("QCOptProblem.jl")
include("RotBasis.jl")
include("Grover.jl")
include("PhaseEstimation.jl")
include("HHL.jl")
include("hamiltonian_solvers.jl")
include("HadamardTest.jl")
include("lin_diffEq_HHL.jl")

end # module