text:
"""
    ShearX <: Augmentor.AffineOperation

Description
--------------

Shear the image horizontally for the given `degree`. This
operation can only be performed as an affine transformation and
will in general cause other operations of the pipeline to use
their affine formulation as well (if they have one).

It will always perform the transformation around the center of
the image. This can be particularly useful when combining the
operation with [`CropNative`](@ref).

Usage
--------------

    ShearX(degree)

Arguments
--------------

- **`degree`** : `Real` or `AbstractVector` of `Real` that denote
    the shearing angle(s) in degree. If a vector is provided,
    then a random element will be sampled each time the operation
    is applied.

See also
--------------

[`ShearY`](@ref), [`CropNative`](@ref), [`augment`](@ref)

Examples
--------------

```julia
using Augmentor
img = testpattern()

# shear horizontally exactly 5 degree
augment(img, ShearX(5))

# shear horizontally between 10 and 20 degree to the right
augment(img, ShearX(10:20))

# shear horizontally one of the five specified degrees
augment(img, ShearX([-10, -5, 0, 5, 10]))
```
"""
struct ShearX{T<:AbstractVector} <: AffineOperation
    degree::T

    function ShearX{T}(degree::T) where {T<:AbstractVector{S} where S<:Real}
        length(degree) > 0 || throw(ArgumentError("The number of different angles passed to \"ShearX(...)\" must be non-zero"))
        (minimum(degree) >= -70 && maximum(degree) <= 70) || throw(ArgumentError("The specified shearing angle(s) must be in the interval [-70, 70]"))
        new{T}(degree)
    end
end
ShearX(degree::T) where {T<:AbstractVector} = ShearX{T}(degree)
ShearX(degree::Real) = ShearX(degree:degree)

@inline supports_eager(::Type{<:ShearX}) = false

randparam(op::ShearX, img) = Float64(safe_rand(op.degree))

function toaffinemap(op::ShearX, img::AbstractMatrix, angle)
    recenter(@SMatrix([1. 0.; tan(-deg2rad(Float64(angle))) 1.]), center(img))
end

function Base.show(io::IO, op::ShearX)
    if get(io, :compact, false)
        if length(op.degree) == 1
            print(io, "ShearX ", first(op.degree), " degree")
        else
            print(io, "ShearX by ϕ ∈ ", op.degree, " degree")
        end
    else
        print(io, "Augmentor.")
        showconstruction(io, op)
    end
end

# --------------------------------------------------------------------

"""
    ShearY <: Augmentor.AffineOperation

Description
--------------

Shear the image vertically for the given `degree`. This operation
can only be performed as an affine transformation and will in
general cause other operations of the pipeline to use their
affine formulation as well (if they have one).

It will always perform the transformation around the center of
the image. This can be particularly useful when combining the
operation with [`CropNative`](@ref).

Usage
--------------

    ShearY(degree)

Arguments
--------------

- **`degree`** : `Real` or `AbstractVector` of `Real` that denote
    the shearing angle(s) in degree. If a vector is provided,
    then a random element will be sampled each time the operation
    is applied.

See also
--------------

[`ShearX`](@ref), [`CropNative`](@ref), [`augment`](@ref)

Examples
--------------

```julia
using Augmentor
img = testpattern()

# shear vertically exactly 5 degree
augment(img, ShearY(5))

# shear vertically between 10 and 20 degree upwards
augment(img, ShearY(10:20))

# shear vertically one of the five specified degrees
augment(img, ShearY([-10, -5, 0, 5, 10]))
```
"""
struct ShearY{T<:AbstractVector} <: AffineOperation
    degree::T

    function ShearY{T}(degree::T) where {T<:AbstractVector{S} where S<:Real}
        length(degree) > 0 || throw(ArgumentError("The number of different angles passed to \"ShearY(...)\" must be non-zero"))
        (minimum(degree) >= -70 && maximum(degree) <= 70) || throw(ArgumentError("The specified shearing angle(s) must be in the interval [-70, 70]"))
        new{T}(degree)
    end
end
ShearY(degree::T) where {T<:AbstractVector} = ShearY{T}(degree)
ShearY(degree::Real) = ShearY(degree:degree)

@inline supports_eager(::Type{<:ShearY}) = false

randparam(op::ShearY, img) = Float64(safe_rand(op.degree))

function toaffinemap(op::ShearY, img::AbstractMatrix, angle)
    recenter(@SMatrix([1. tan(-deg2rad(Float64(angle))); 0. 1.]), center(img))
end

function Base.show(io::IO, op::ShearY)
    if get(io, :compact, false)
        if length(op.degree) == 1
            print(io, "ShearY ", first(op.degree), " degree")
        else
            print(io, "ShearY by ψ ∈ ", op.degree, " degree")
        end
    else
        print(io, "Augmentor.")
        showconstruction(io, op)
    end
end

# --------------------------------------------------------------------

function showconstruction(io::IO, op::Union{Rotate,ShearX,ShearY})
    if length(op.degree) == 1
        print(io, typeof(op).name.name, '(', first(op.degree), ')')
    else
        print(io, typeof(op).name.name, '(', op.degree, ')')
    end
end


text:
struct Gernert <: AbstractHelmholtzSolver end

function core_pt_flash(method::Gernert,
    model::M,
    P0::Real,
    T0::Real,
    x0,
    options=nothing) where M <: AbstractHelmholtzModel
    return gernert_pt(model,P0,T0,x0)
end

function _rachford_rice(z,K) #find phase fraction, with z:feed phase and K:vaporization constant
    function f0(ψ)
        res = zero(eltype(K))
        for i = 1:length(K)
            if (K[i] == Inf) || (K[i] == -Inf)
                res += z[i]/ψ
            else
                res +=z[i]*(K[i]-1)/(1-ψ*(1-K[i]))
            end
        end
        return res
    end
    df0(ψ) = ForwardDiff.derivative(f0,ψ)
    try
        return Roots.find_zero((f0,df0),0.5,Roots.Newton())
    catch
        return Inf
    end
end

function _rachford_rice(y,K,ψ)
    if  ψ == one(ψ)
        return sum(y .* (1 .- 1 ./ K))
    elseif ψ == zero(ψ)
        return sum(y .* (K .-1))
    else
        res = zero(eltype(K))
        for i = 1:length(K)
            if (K[i] == Inf) || (K[i] == -Inf)
                res += y[i]/ψ
            else
                res +=y[i]*(K[i]-1)/(1-ψ*(1-K[i]))
            end
        end
        return res
    end
end

_K0(model,P0,T0,type::Symbol) = _K0(model,P0,T0,Val(type))
function _K0(model,P0,T0,type::Val{:mollerup})
    return (critical_pressure(model) ./ P0).*exp.(5.42.*(1.0 .- (critical_temperature(model)./T0)))
end
function _K0(model,P0,T0,type::Val{:wilson})
    return exp.( log.(critical_pressure(model) ./ P0)  .+ 5.373 .* (1.0 .+ acentric_factor(model)) .* (1.0 .- critical_temperature(model) ./ T0))
end

function gernert_pt(model::AbstractHelmholtzModel,P0::Real,T0::Real,x0)

    min_v = dot(x0,covolumes(model))
    max_v = 40*T0/P0
    v0 = Threads.@spawn volume_solver(model,P0,T0,x0)
    lnϕ0 = Threads.@spawn core_logfugacity_coefficient(model,fetch(v0),T0,x0)
    #=Step 1: Generation of trial phase compositions
    Starting with the assumption that two coexisting phases are present,
    initial estimates of trial phase compositions are generated
    using the generalized Wilson correlation [25] to calculate K-values for all N components
    =#
    K0 = _K0(model,P0,T0,:wilson) #this can be replaced if you have an starter that doesnt require critical data

    g_0 = _rachford_rice(x0,K0,0.0)
    g_1 = _rachford_rice(x0,K0,1.0)
    if g_0 <= 0  #bubble point assumption
        xi1 = copy(x0)
        xi2 = x0 .* K0 ./ (g_0 + 1)
    elseif g_1 >= 0 #dew point assumption
        xi1 = x0 ./ (-g_1 + 1) ./ K0
        xi2 = copy(x0)
    else #two phase assumption
        ϕ = _rachford_rice(x0,K0)
        xi1 = x0 ./(1.0 .- ϕ .* (1.0 .- K0))
        xi2 = K0 .* x0 ./(1.0 .- ϕ .* (1.0 .- K0))
    end
    normalizefrac!(xi1)
    normalizefrac!(xi2)

    #= Step 2: Successive substitution method
    Based on the initial estimates of the phase compositions,
    three (3) steps of successive substitution are performed
    in order to increase the accuracy of the estimates.
    The successive substitution has been introduced
    for the solution of phase equilibria conditions by Prausnitz and Chueh =#

    #initial step
    vi1 = Threads.@spawn volume_solver(model,P0,T0,xi1)
    vi2 = Threads.@spawn volume_solver(model,P0,T0,xi2)

    lnϕi1 =similar(xi1)
    lnϕi2=similar(xi2)
    vv1 = fetch(vi1)
    vv2 = fetch(vi2)

    lnϕi1 = core_logfugacity_coefficient(model,P0,fetch(vv1),T0,xi1)
    lnϕi2 = core_logfugacity_coefficient(model,P0,fetch(vv2),T0,xi2)
    stable_phase = false

    for i = 1:3

        K0 .= exp.(lnϕi1 .- lnϕi2)

        ϕ = _rachford_rice(x0,K0)
        if !(0 < ϕ < 1) #proceed to stability analysis
            stable_phase = true
            break
        end
        if i == 3 #partial loop
            break
        end

        xi1 .= x0 ./(1.0 .- ϕ .* (1.0 .- K0))
        xi2 .= K0 .* x0 ./(1.0 .- ϕ .* (1.0 .- K0))
        normalizefrac!(xi1)
        normalizefrac!(xi2)
        lnϕi1 .= core_logfugacity_coefficient(model,P0,fetch(vi1),T0,xi1)
        lnϕi2 .= core_logfugacity_coefficient(model,P0,fetch(vi2),T0,xi2)
    end
    v0 = fetch(v0)
    lnϕ0 = fetch(lnϕ0)
    @show lnϕ0
    function tpd(xi, lnϕ)
        return  sum(xi.*(log.(xi) .+lnϕ - log.(x0) .- lnϕ0))
    end
    if stable_phase == false
        tpd1 =   tpd(xi1,lnϕi1)
        tpd2 =   tpd(xi2,lnϕi2)
        ΔG = (1-ϕ)*tpd1 + ϕ*tpd2
        if (ΔG < 0) || (tpd1 < 0) || (tpd2 < 0)
            stable_phase = true
        end
    end
    #= the algorithm continues with a more detailed stability analysis.
    In principle, the whole composition range needs to be checked
    for negative tangent plane distances.
    Since such a multi-dimensional search is impractical for multi-component mixtures
    another more practical approach was suggested by Michelsen and Mollerup [23].
    From the initial Wilson K-values and the feed composition,
     heavy and light trial phase compositions are calculated =#

     vt_h = 0.0
     vt_l = 0.0
     vt_hmin = 0.0
     vt_lmin = 0.0
    if stable_phase == true
        K0=_K0(model,P0,T0,:wilson)
        xt_h = normalizefrac(x0./K0)
        xt_l = normalizefrac(x0 .*K0)
        xt_hmin = copy(xt_h)
        xt_lmin = copy(xt_l)
        vt_h = Threads.@spawn volume_solver(model,P0,T0,xt_h,vt_h)
        vt_l = Threads.@spawn volume_solver(model,P0,T0,xt_l,vt_l)

        vt_h_old = fetch(vt_h)
        vt_l_old = fetch(vt_l)
        vt_hmin =vt_h_old
        vt_lmin = vt_l_old
        lnϕt_l = core_logfugacity_coefficient(model,fetch(vt_l_old),T0,xt_l)
        lnϕt_h = core_logfugacity_coefficient(model,fetch(vt_h_old),T0,xt_h)
        tpd_hmin = tpd(xt_h,lnϕt_h)
        tpd_lmin = tpd(xt_l,lnϕt_l)
        xt_h .= x0.* exp.(lnϕ0) ./ exp.(lnϕt_h)
        xt_l .= x0.* exp.(lnϕ0) ./ exp.(lnϕt_l)
        normalizefrac!(xt_h)
        normalizefrac!(xt_l)
        i = 0
        iters = 5
        tpd_h = Inf
        tpd_l = Inf
        while i < iters
            i +=1
            vt_h_old = fetch(vt_h)
            vt_l_old = fetch(vt_l)
            vt_h = Threads.@spawn volume_solver(model,P0,T0,xt_h,vt_h_old)
            vt_l = Threads.@spawn volume_solver(model,P0,T0,xt_l,vt_l_old)
            vt_h_old = fetch(vt_h)
            vt_l_old = fetch(vt_l)
            lnϕt_h = core_logfugacity_coefficient(model,fetch(vt_h_old),T0,xt_h)
            lnϕt_l = core_logfugacity_coefficient(model,fetch(vt_l_old),T0,xt_l)
            tpd_h = tpd(xt_h,lnϕt_h)
            tpd_l = tpd(xt_l,lnϕt_l)
            #println(tpd_h)
            #println(tpd_l)
            if tpd_h < tpd_hmin
                tpd_hmin = tpd_h
                xt_hmin .= xt_h
                vt_hmin = fetch(vt_h)
            end

            if tpd_l < tpd_lmin
                tpd_lmin = tpd_l
                xt_lmin .= xt_l
                vt_lmin = fetch(vt_l)
            end
            if (tpd_lmin < 0) && (stable_phase == true)
                stable_phase = false
                iters +=3
            end
            if (tpd_hmin < 0) && (stable_phase == true)
                stable_phase = false
                iters +=3
            end

            if xt_h ≈ x0
                stable_phase = true
                break
            end
            if xt_l ≈ x0
                stable_phase = true
                break
            end

            xt_h .= x0.* exp.(lnϕ0) ./ exp.(lnϕt_h)
            xt_l .= x0.* exp.(lnϕ0) ./ exp.(lnϕt_l)

            normalizefrac!(xt_h)
            normalizefrac!(xt_l)
            @show xt_h
            @show xt_l
            @show xt_lmin
            @show xt_hmin
        end

        vi1 = 0.0
        vi2 = 0.0
        if stable_phase == false
            xi1 .= xt_hmin
            xi2 .= xt_lmin
            vi1 = vt_hmin
            vi2 = vt_lmin
        end
    end

    if stable_phase == true
        return [helmholtz_phase(model,v0,T0,x0)]
    end

    if stable_phase == false
       return _f0_isothermal6!(model,P0,T0,x0,xi1,xi2,vi1,vi2,fetch(v0))
    end
end

function _f0_isothermal6!(model,P0,T0,x0,x010,x020,v010,v020,v000)
    len = length(x0)
    x01 = zeros(eltype(x0),len)
    x02 = zeros(eltype(x0),len)
    ψ00 = Ref(zero(eltype(x0)))
    vv = zeros(eltype(x0),2)
    function f_0(z,v1,v2)
        x1 = vcat(z[1:end-1],1-sum(z[1:end-1]))
        β = z[end]
        @show x1
        return  β*core_gibbs(model,P0,v1,T0,x1) + (1-β)*core_gibbs(model,P0,v2,T0,(x0 .- β .* x1)/(1-β))
    end
    function fj!(F, J, z)
        β1 = z[end]
        sumx1 = one(eltype(x0))
        sumx2 = one(eltype(x0))
        for i = 1:len-1
            x01[i]= z[i]
            sumx1 -=x01[i]
            x02[i] =(x0[i] - β1*z[i])/(1- β1)
            sumx2 -=x02[i]
        end
        x01[end] = sumx1
        x02[end] = sumx2
        ψ00[]=β1

        vi1 = Threads.@spawn volume_solver(model,P0,T0,x01,vv[1])
        vi2 = Threads.@spawn volume_solver(model,P0,T0,x02,vv[2])
        v01 = fetch(vi1)
        v02 = fetch(vi2)
        vv[1] = v01
        vv[2] = v02

        #@show v01
        #@show v02

        f_01(z) = f_0(z,v01,v02)
        #@show f_01(z)
        if !(J == nothing)
            ForwardDiff.hessian!(J,f_01,z)
        end
        if !(F == nothing)
            ForwardDiff.gradient!(F,f_01,z)
        end
    end
F0 = similar(x0)
J0 = zeros(eltype(x0),len,len)
z0 = copy(x010)
ψ0 = abs((x0[1]-x020[1])/(x020[1]-x010[1]))
z0[end] = ψ0

dz = similar(z0)
cachedz = similar(z0)
fj!(F0,J0,z0)
#sol = nlsolve(only_fj(fj!),z0)
#
#sol)
τ = eps(eltype(x0))
ε2 = sqrt(τ)
ε3 = (τ)^(1/3)
ix = 0
isconverged = false
for i = 1:100
    fj!(F0,J0,z0)
    dz .= J0\-F0
    cholesky!(Positive, J0)
    α =1.0
    minz,maxz = extrema(z0 .+ α .* dz)
    while ! (0 <=minz < maxz <=1.0)
        α *=0.6321205588285577
        minz,maxz = extrema(z0 .+ α .* dz)
    end
    @show z0
    @show vv
    f_j(z) = f_0(z,vv[1],vv[2])
    fj = f_j(z0)
    Θ = τ*(1+abs(fj))
    #@show Θ
    #@show  f_j(z0 .+ α .*dz)- fj
    #@show  fj
    #@show  f_j(z0 .+ α .*dz)
    #@show norm(dz)
    #@show sqrt(τ)*(1+norm(z0))

    if abs(f_j(z0 .+ α .*dz)- fj) < Θ
        isconverged =true
        break
    elseif norm(dz) < sqrt(τ)*(1+norm(z0))
        isconverged =true
        break
    elseif norm(x01-x02) < ε3
        #println("fractions are the same")
        break
    elseif abs(vv[1]-vv[2]) < ε2
        #println("volumes are the same")
        break
    elseif i == 100
        break
    end
    ix +=1

    z0 .= z0 .+ α .*dz
end
    if isconverged == true
        n1 = mol_number(x01,molecular_weight(model),ψ00[])
        n2 = mol_number(x02,molecular_weight(model),1-ψ00[])
        return [helmholtz_phase(model,vv[1],T0,n1),helmholtz_phase(model,vv[2],T0,n2)]
    else
        return [helmholtz_phase(model,v000,T0,x0)]
    end
end

#mono_t_psiflash(model::AbstractHelmholtzModel,T0::Real,ψ0::Real,x0)
#Tr = T0/dot(x0,critical_temperature(model))
#if Tr>one(Tr)
#
#else
#end

ext: jl


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder, Pkg

name = "FGlT"
version = v"1.0.0"

# Collection of sources required to complete build
sources = [
    GitSource("https://github.com/fcdimitr/fglt.git", "c3c0c683a76fef56473314527760b74ffd271455")
]

# Bash recipe for building across all platforms
script = raw"""
cd $WORKSPACE/srcdir/fglt/
meson --cross-file=${MESON_TARGET_TOOLCHAIN} build
cd build/
ninja -j${nproc}
ninja install
"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line
platforms = supported_platforms(; experimental=true)

# [FIXED!] FGlT contains std::string values!  This causes incompatibilities across the GCC 4/5 version boundary.
# platforms = expand_cxxstring_abis(platforms)

# The products that we will ensure are always built
products = [
    LibraryProduct("libfglt", :libfglt)
]

# Dependencies that must be installed before this package can be built
dependencies = [
    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae")),
    Dependency(PackageSpec(name="cilkrts_jll", uuid="71772805-00bc-5a29-9044-a26d819b7806")),
]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version = v"7.1.0")  # CILK support

max_stars_repo_name: sharanry/Yggdrasil

max_issues_repo_name: sharanry/Yggdrasil

max_forks_repo_name: sharanry/Yggdrasil


text:
# QuEST_jl/src/base/operators.jl
#

function applyDiagonalOp(qureg          :: QuEST_Types.Qureg,
                         op             :: QuEST_Types.DiagonalOp)  ::Nothing

    ccall(:applyDiagonalOp, Cvoid, (QuEST_Types.Qureg, QuEST_Types.DiagonalOp), qureg, op)
    nothing;
end

function applyMatrix2(qureg             :: QuEST_Types.Qureg,
                      targetQubit       :: Integer,
                      u                 :: QuEST_Types.ComplexMatrix2)  ::Nothing

    ccall(:applyMatrix2, Cvoid,
          (QuEST_Types.Qureg, Cint,        QuEST_Types.ComplexMatrix2),
          qureg,              targetQubit, u)
end

function applyMatrix4(qureg             :: QuEST_Types.Qureg,
                      targetQubit1      :: Integer,
                      targetQubit2      :: Integer,
                      u                 :: QuEST_Types.ComplexMatrix4)   ::Nothing

    ccall(:applyMatrix4,  Cvoid,
          (QuEST_Types.Qureg, Cint,          Cint,         QuEST_Types.ComplexMatrix4),
          qureg,              targetQubit1,  targetQubit2, u)
end

function applyMatrixN(qureg             :: QuEST_Types.Qureg,
                      targetQubits      :: Vector{QubitIdx},
                      u                 :: QuEST_Types.ComplexMatrixN)   ::Nothing

    local numTargs  ::Cint = length(targetQubits)
    @assert numTargs == u.numQubits

    ccall(:applyMatrixN, Cvoid,
          (QuEST_Types.Qureg, Ptr{QubitIdx}, Cint,      QuEST_Types.ComplexMatrixN),
          qureg,              targetQubits,  numTargs,  u)
end

function applyMultiControlledMatrixN(qureg          :: QuEST_Types.Qureg,
                                     ctrlQubits     :: Vector{QubitIdx},
                                     targQubits     :: Vector{QubitIdx},
                                     u              :: QuEST_Types.ComplexMatrixN)  ::Nothing

    local numCtrls ::Cint = length(ctrlQubits)
    local numTargs ::Cint = length(targQubits)

    @assert numTargs == u.numQubits
    @assert isdisjoint(ctrlQubits,targQubits)

    ccall(:applyMultiControlledMatrixN, Cvoid,
          (QuEST_Types.Qureg, Ptr{Cint},  Cint,      Ptr{Cint},  Cint,     QuEST_Types.ComplexMatrixN),
          qureg,              ctrlQubits, numCtrls,  targQubits, numTargs, u)
end

function applyPauliHamil(inQureg        :: QuEST_Types.Qureg,
                         hamil          :: QuEST_Types.PauliHamil,
                         outQureg       :: QuEST_Types.Qureg)        ::Nothing

    ccall(:applyPauliHamil, Cvoid,
          (QuEST_Types.Qureg, QuEST_Types.PauliHamil, QuEST_Types.Qureg),
          inQureg,            hamil,                  outQureg)
end

function applyPauliSum(inQureg        ::QuEST_Types.Qureg,
                       allPauliCodes  ::Vector{QuEST_Types.pauliOpType},
                       termCoeffs     ::Vector{Qreal},
                       numSumTerms    ::Integer,
                       outQureg       ::QuEST_Types.Qureg)          ::Nothing

    @assert length(allPauliCodes) == numSumTerms * inQureg.numQubitsRepresented

    ccall(:applyPauliSum, Cvoid,
          (QuEST_Types.Qureg, Ptr{QuEST_Types.pauliOpType}, Ptr{Qreal}, Cint,        QuEST_Types.Qureg),
          inQureg,            allPauliCodes,                termCoeffs, numSumTerms, outQureg)
end

function applyTrotterCircuit(qureg          :: QuEST_Types.Qureg,
                             hamil          :: QuEST_Types.PauliHamil,
                             time           :: Qreal,
                             order          :: Integer,
                             reps           :: Integer)          ::Nothing

    ccall(:applyTrotterCircuit, Cvoid,
          (QuEST_Types.Qureg, QuEST_Types.PauliHamil, Qreal, Cint,  Cint),
          qureg,              hamil,                  time,  order, reps)
end

#EOF


text:
### A Pluto.jl notebook ###
# v0.14.2

using Markdown
using InteractiveUtils

# ╔═╡ 794a5356-6e05-44b4-ad9d-375a1fa3613b
begin
	using PlutoUI
	using DataFrames
	using Random
	using Test
end

# ╔═╡ b3dcf3f4-9d68-11eb-01a6-317fe0e0cd7e
md"""
## Introduction DataFrames / 6 - Manipulating rows

ref. [Introduction to DataFrames, Bogumił Kamiński](https://github.com/bkamins/Julia-DataFrames-Tutorial/blob/master/06_rows.ipynb)

$(html"
© Pascal, April 2021
")
"""

# ╔═╡ d6993a5b-8158-4ce3-accb-8e8c514229b3
PlutoUI.TableOfContents(indent=true, depth=4, aside=true)

# ╔═╡ de97ded0-3952-4188-9a43-a8f383e1ae65
md"""
### Selecting rows
"""

# ╔═╡ 9ccb6d8f-a8e0-42c2-8691-2eac247e591a
Random.seed!(42)

# ╔═╡ 385aee9f-5c17-4bd2-b3b2-58171fcc5990
df = DataFrame(rand(4, 5), :auto)

# ╔═╡ 472fe012-dd24-46d3-b1b1-f217054c40f0
md"""
using `:` as row selector will **copy** columns
"""

# ╔═╡ 147bcb72-1d40-458e-abb4-02c8d8ddfa76
df[:, :]

# ╔═╡ 7699775c-8dec-4467-af21-89bd8354d409
@assert df[:, :] == copy(df)   # meaning copy(df) ≡ df[:, :]

# ╔═╡ 09617e8f-1d09-4145-831f-38fd98313dd1
md"""
We can get a subset of rows of a DataFrame without copying using view to get a `SubDataFrame`
"""

# ╔═╡ 80277a09-24ea-4dae-9a41-39853a46fbaa
vdf = view(df, 1:3, 1:3)

# ╔═╡ 9b61d582-6d26-483d-bb66-174b44b38ff4
md"""
we still have a detailed reference to the parent:
"""

# ╔═╡ 5e7c5b0f-be5d-43aa-b13a-4202b1cb34bf
parent(vdf), parentindices(vdf)

# ╔═╡ 26a2e983-0a3c-41b3-ae2a-79395644e2d4
md"""
Selecting a single row returns a `DataFrameRow` object which is also a view
"""

# ╔═╡ f7dbfd3d-2450-4db3-9d25-d08011c62bf2
dfr = df[3, :]

# ╔═╡ cdee6bee-59d8-4fe4-80d9-183b1fa3e18b
parent(dfr), parentindices(dfr), rownumber(dfr)

# ╔═╡ 96b5f96f-57de-4fef-bc73-3f092b00be74
md"""
Let us add a column to a DataFrame by assigning a scalar broadcasting.
"""

# ╔═╡ 912c5b66-38d9-49f4-881c-4f2a988104c2
df[!, :Z] .= 1

# ╔═╡ 560497da-2699-411f-96f9-334919bfffdf
df

# ╔═╡ d114f107-4e31-4585-915f-a17369023a74
md"""
Earlier we used `:` for column selection in a view (`SubDataFrame` and `DataFrameRow`). In this case a view will have all columns of the parent after the parent is mutated.
"""

# ╔═╡ 5b92be5d-a394-4f8e-b081-bfaef94f3165
dfr

# ╔═╡ 3ac61fe5-3e33-44f5-86e9-c5945a2fc03c
parent(dfr), parentindices(dfr), rownumber(dfr)

# ╔═╡ e7da86b9-595c-450c-8a42-60cb63633982
md"""
**Note** that `parent` and `parentindices` refer to the true source of data for a `DataFrameRow` and `rownumber` refers to row number in the direct object that was used to create `DataFrameRow`
"""

# ╔═╡ 6a211df8-ec06-4c24-a362-5e7f604b944a
df₁ = DataFrame(a=1:4)

# ╔═╡ d552bdef-2106-4465-8244-0c8c2144d1f7
dfv₁ = view(df₁, [3, 2], :)

# ╔═╡ f4c515e2-a550-42ec-836c-6f6968c383bc
dfr₁ = dfv₁[1, :]

# ╔═╡ 31a0adc2-c50d-4949-a234-f71e36f46c3c
parent(dfr₁), parentindices(dfr₁), rownumber(dfr₁)

# ╔═╡ ca9927e5-3ff2-4c73-8206-4d7495668312
md"""
### Reordering rows

We create some random data frame (and hope that x.x is not sorted :), which is quite likely with 12 rows)

"""

# ╔═╡ 5d950253-8184-492f-a649-9123161ba9cc
x₀ = DataFrame(id=1:12, x = rand(12), y = [zeros(6); ones(6)])

# ╔═╡ 479bbe19-5faf-4f94-8fc2-45f541fffba2
md"""
Check if a DataFrame or a subset of its columns is sorted
"""

# ╔═╡ e68e64a2-4bd7-446d-b550-0cfa157831e4
issorted(x₀), issorted(x₀, :x)

# ╔═╡ 96733dd9-e40a-4ee9-85f9-082feab93144
sort!(x₀, :x)   ## in-place sort

# ╔═╡ 2d6c26cc-f52f-4d77-a17e-def865adc34c
y₀ = sort(x₀, :id)  ## Create new dataframe

# ╔═╡ e7b9b478-e2a7-4633-b963-2408a203ef33
md"""
We sort by two columns, first is decreasing, second is increasing.
"""

# ╔═╡ 9e30faf0-2617-43db-9177-64fad45f7b03
sort(x₀, [:y, :x], rev=[true, false])

# ╔═╡ fe56f87b-50df-4a8a-b531-232a0a51293c
@assert sort(x₀, [:y, :x], rev=[true, false]) == sort(x₀, [order(:y, rev=true), :x])
# 2 equivalent ways of sorting

# ╔═╡ 1c003f86-9321-4809-a4b9-a98add2ad39d
md"""
Now let us try some more fancy sorting stuff
"""

# ╔═╡ bace7faf-5eee-4f3d-af48-f9c7ff31ce27
sort(x₀, [order(:y, rev=true), order(:x, by=v -> -v)])
# :y descendant then :x  descendant (as well)

# ╔═╡ 09530879-8714-4332-a925-6f9298d6dc8b
md"""
And here is how we can reorder rows (here randomly)
"""

# ╔═╡ 00818c32-887c-4e3d-8a5f-18c1dcaedba1
x₀[shuffle(1:10), :]

# ╔═╡ 800fcbb6-e1ca-4c63-96af-30bf4680a97b
md"""
It is also easy to swap rows using broadcasted assignment
"""

# ╔═╡ 86916666-4b3e-49dd-bb97-1321f34097f4
begin
	sort!(x₀, :id)
	x₀[[1, 10], :] .= x₀[[10, 1], :]
	x₀
end

# ╔═╡ a4615e9b-d3d9-443a-9146-e411f9bbd749
md"""
### Merging/Adding rows
"""

# ╔═╡ b82a4d73-57f9-48fa-9c76-7d2cf853edd3
x₁ = DataFrame(rand(3, 5), :auto)

# ╔═╡ 6c8bd34e-2f6f-44bd-a6cd-e7b79597f7ae
md"""
merge by rows - dataframes must have the same column names; the same is `vcat`
"""

# ╔═╡ 69d2f06d-866a-4e95-92e9-9b4f06eaff04
[x₁; x₁]

# ╔═╡ 05bdca3f-033a-41c6-9dac-888b08b55cce
md"""
we can efficiently `vcat` a vector of DataFrames using `reduce`
"""

# ╔═╡ 6e4115db-22b2-487a-bda1-255364597014
reduce(vcat, [x₁, x₁, x₁])

# ╔═╡ b03679d0-2ecc-4b64-9aa6-735c6d18a407
md"""
get y with other order of names
"""

# ╔═╡ 1b40287c-b7b6-4ed8-bbe1-1141ef6fb04a
y₁ = x₁[:, reverse(names(x₁))]

# ╔═╡ bd4429da-3378-4392-9cd9-bba21ff9ec37
md"""
`vcat` is still possible as it does column name matching
"""

# ╔═╡ 4becce78-c6d2-43ec-a8ed-58edbf3290ff
vcat(x₁, y₁)

# ╔═╡ e741cbed-3da6-4713-ac85-93b8a683fd87
md"""
However column names must still match or...
"""

# ╔═╡ 39f529a7-8ffa-4149-9780-0d98feee2975
@test_throws ArgumentError vcat(x₁, y₁[:, 1:3])

# ╔═╡ 9df70270-20db-4a83-9c40-55edc37c99b4
md"""
...Unless we pass `:intersect`, `:union` or specific column names as keyword argument `cols`
"""

# ╔═╡ 037843b8-810b-4e78-8262-f9eb631986c4
vcat(x₁, y₁[:, 1:3], cols=:intersect)

# ╔═╡ 5a2aa52a-8739-4c49-95af-e2cd82a96523
vcat(x₁, y₁[:, 1:3], cols=:union)

# ╔═╡ af43af30-7221-4fbb-aef8-63f08956b010
vcat(x₁, y₁[:, 1:3], cols=[:x1, :x5])

# ╔═╡ 198ba262-a6b4-4a2f-8d87-23249c78cb99
md"""
`append!` modifies x in place
"""

# ╔═╡ dd22d95d-840d-41fa-8e1e-1cafb160676e
append!(x₁, y₁)

# ╔═╡ 7fea33ff-04e0-4d67-9bf5-d450269ed8a5
md"""
Standard `repeat` function works on rows; also `inner` and `outer` keyword arguments are accepted
"""

# ╔═╡ a8d073a5-e8ac-4cc0-a668-4bc478533c10
repeat(x₁, 2)

# ╔═╡ 6eb15245-caf0-4598-8741-00bb545ab7dd
md"""
`push!` adds one row to x at the end; one must pass a correct number of values unless `cols` keyword argument is passed.
"""

# ╔═╡ d43682a9-6399-404b-9d88-7a15a198eb08
push!(x₁, 1:5); x₁

# ╔═╡ 70fbd712-7a30-4776-ae1b-109a0a0ca1fe
md"""
It also works with dictionaries
"""

# ╔═╡ 8f227a0c-0083-4abf-8857-7518add1ace8
push!(x₁, Dict(:x1=> 11, :x2=> 12, :x3=> 13, :x4=> 14, :x5=> 15)); x₁

# ╔═╡ ed3ed957-056d-46c9-9877-10490d054122
md"""
and `NamedTuples` via name matching
"""

# ╔═╡ 83a04fdc-93c6-4377-ab35-c18151ca668c
push!(x₁, (x2=2, x1=1, x4=4, x3=3, x5=5))

# ╔═╡ 770b0df0-6095-4e39-b08e-ca49d64cd778
md"""
and DataFrameRow also via name matching
"""

# ╔═╡ 014c6489-a973-48fe-ad5d-a3f487b510b2
push!(x₁, x₁[1, :]); x₁

# ╔═╡ 84aed6e2-afbd-4657-8453-5e1afb91a061
md"""
For more consult the documentation of `push!`, `append!` and `vcat` for allowed values of `cols` keyword argument. This keyword argument governs the way these functions perform column matching of passed arguments. Also `append!` and `push!` support a `promote` keyword argument that decides if column type promotion is allowed.

Let us here just give a quick example of how heterogeneous data can be stored in the dataframe using these functionalities:
"""

# ╔═╡ 3f12d46a-786b-4a20-83ec-7eca121f2770
src = [(a=1, b=2), (a=missing, b=10, c=20), (b="s", c=1, d=1)]

# ╔═╡ b981bce1-dd57-4451-86a5-e9ee14eeb185
dfₛ = DataFrame();

# ╔═╡ 98973c9c-dcd3-44dc-835e-b60d2e7ac4b0
for r in src
    push!(dfₛ, r, cols=:union) ## if cols is :union then promote is true by default
end

# ╔═╡ fa7a9bc6-6609-4e84-93c6-7924bd653fde
dfₛ

# ╔═╡ cf16bd85-9da2-44ac-8e1a-051f04546b60
md"""
We see that `push!` dynamically added columns as needed and updated their element types
"""

# ╔═╡ 7ff49b11-9b98-4e8c-af33-f8ecbc2565c8
md"""
### Subsetting/Removing rows
"""

# ╔═╡ c1f5641e-adad-4ab5-8118-c7a9763cff6f
x₂ = DataFrame(id=1:10, val='a':'j')

# ╔═╡ 3db600e4-3fbb-4fb8-8df6-9438753f18b9
md"""
by using indexing
"""

# ╔═╡ 1474158e-c2aa-457b-abbb-f10089054f60
x₂[1:2, :]

# ╔═╡ d3050da0-db13-40e0-a5f5-bb290178f2a0
md"""
a single row selection creates a DataFrameRow
"""

# ╔═╡ b6972fc7-2d40-47b3-b8a1-825cfa59ef67
x₂[1, :]

# ╔═╡ e51b2218-e998-415f-aa9c-ae7582a04619
md"""
but this is a DataFrame.
"""

# ╔═╡ aaea674f-88c7-43b7-8327-31abdc145787
x₂[1:1, :]

# ╔═╡ df89ef70-ab45-4fdf-830a-154260225e2f
md"""
And here is a view.
"""

# ╔═╡ 02650cd1-e572-4d70-a39e-da92ce674659
view(x₂, 1:2, :)

# ╔═╡ 94ac6ebb-86b6-4e31-a64d-e4139b850316
md"""
selects columns 1 and 2
"""

# ╔═╡ d9a8ac15-2774-4e51-ab1b-8c0e95cf158c
view(x₂, :, 1:2)

# ╔═╡ d8d13d16-10ad-4e1e-8265-36516448e702
md"""
indexing by Bool, exact length math is required
"""

# ╔═╡ b85c34fc-9d01-4f7e-9104-452ec672101c
x₂[repeat([true, false], 5), :]

# ╔═╡ 7cd99843-0ead-4119-9221-7dd8c03d3b27
md"""
Alternatively we can also create a view
"""

# ╔═╡ d7e6404c-d0a9-4ee1-8372-edd4ee4fa520
view(x₂, repeat([true, false], 5), :)

# ╔═╡ bad9c21d-c35b-4e4e-8e18-51aeb41c57a9
md"""
we can delete one row in place
"""

# ╔═╡ 2d042183-e3f6-4e00-b86c-bdc1f11bc703
delete!(x₂, 7)

# ╔═╡ b80e344e-8550-4f47-bef6-69833a04277c
md"""
or a collection of rows, also in place
"""

# ╔═╡ 61d6ee03-b042-4f79-a017-dcd59e4eca80
delete!(x₂, 6:7)

# ╔═╡ 26e4a50f-eeb7-4bee-ba4e-e87c479c760c
md"""
We can also create a new DataFrame when deleting rows using `Not` indexing
"""

# ╔═╡ 0b758728-d696-4043-aa02-f3fc27189de8
x₂[Not(1:2), :]

# ╔═╡ 2a2a8d79-f872-4139-bd8e-9119ffed0bcd
x₂

# ╔═╡ 223176a3-aba4-4e6e-a5b1-c51d3ac0bf7e
md"""
Now let us move to row filtering
"""

# ╔═╡ d0d96bda-a7ca-4417-9992-7e3bd748d30b
x₃ = DataFrame([1:4, 2:5, 3:6], :auto)

# ╔═╡ 36088a45-a24c-4d43-9ca1-de665cccb25c
md"""
Let us create a new DataFrame where filtering function operates on DataFrameRow
"""

# ╔═╡ fa126b33-51b7-4aa7-9cc6-c7998ec93fcf
filter(r -> r.x1 > 2.5, x₃)

# ╔═╡ 60ba6b50-48ee-460d-bbd9-61e681d72c74
filter(r -> r.x1 > 2.5, x₃, view=true)  ## the same but as a view

# ╔═╡ 3edc2f7f-0b92-421b-ab10-0a5ee432eed0
filter(:x1 => >(2.5), x₃)               ## Another way

# ╔═╡ a7dcb351-fff6-4bff-96b9-287ca467dfc5
md"""
In place modification of x₃, an example with do-block syntax
"""

# ╔═╡ 04d68d08-4c9e-4e59-aeb4-8c1502aa64be
filter!(x₃) do r
    r.x1 > 2.5 && (return r.x2 < 4.5)
    r.x3 < 3.5
end

# ╔═╡ 616f8df8-0bc1-49eb-b161-6be6fe3dc932
md"""
A common operation is selection of rows for which a value in a column is contained in a given set. Here are a few ways in which you can achieve this.
"""

# ╔═╡ eb5af9d1-ab74-448f-a38b-8f6b1521a510
sdf₁ = DataFrame(x=1:12, y=mod1.(1:12, 4))

# ╔═╡ 9dbef9da-3d2a-4a47-98f3-48947176452a
md"""
We select rows for which column `y` has value 1 or 4.
"""

# ╔═╡ 1d1cccb2-d0ce-4cbb-b81c-5a887d382bf9
filter(r -> r.y ∈ [1, 4], sdf₁)

# ╔═╡ bcc5685b-13f8-40d8-a9de-953ae04f08dc
filter(:y => ∈([1,4]), sdf₁)

# ╔═╡ db9f9a23-2301-423e-b4ee-4d3de02a55f4
sdf₁[in.(sdf₁.y, Ref([1, 4])), :]

# ╔═╡ 31249b78-474f-4ccc-9ca4-540c9ef258aa
md"""
### Deduplicating
"""

# ╔═╡ 3d6aab60-11d2-4c78-bd84-4ceb3b5bfc20
begin
	xdf = DataFrame(A=[1,2], B=["x","y"])
	append!(xdf, xdf)
	xdf.C = 1:4
	xdf
end

# ╔═╡ 5b5da74e-c5ea-4154-8515-b625dc2b300d
md"""
Get first unique rows for given index
"""

# ╔═╡ cfaa2f0f-1228-4f61-81ad-a5cc08a901ed
unique(xdf, [1, 2])

# ╔═╡ a412e28a-e2d4-45af-a193-75bdefbecc75
md"""
Now we look at whole rows
"""

# ╔═╡ bcd063bb-3629-4060-a0e5-d1f56458cc36
unique(xdf)

# ╔═╡ 7be5c6d8-a7f2-4243-b50f-109f8090a636
md"""
Get indicators of non-unique rows
"""

# ╔═╡ 02a45544-c83b-4917-84ca-2b7f6474a771
nonunique(xdf, :A)

# ╔═╡ 488788a4-071e-4d29-96ba-319f235076a9
md"""
modify `xdf` in place
"""

# ╔═╡ e16de281-1676-430d-9049-9fd81d402dcb
unique!(xdf, :B)

# ╔═╡ 701b11d4-017f-48ea-998f-2203a58b98fb
md"""
### Extracting one row from a DataFrame into standard collections
"""

# ╔═╡ ee246276-451a-4fb1-ab79-81acef89a416
x = DataFrame(x=[1,missing,2], y=["a", "b", missing], z=[true,false,true])

# ╔═╡ dd5860ed-3bc2-44d4-ba25-c155a68f8e05
cols = [:y, :z]

# ╔═╡ c66d68b3-5410-4612-841b-ae53acc5ed69
md"""
We can use a conversion to a Vector or an Array
"""

# ╔═╡ d22c3b36-bfe0-4c70-bdd4-045c52098e34
Vector(x[1, cols])

# ╔═╡ 21f9eb7e-9b62-4e6a-b19e-0ff3648dd2bd
@assert Vector(x[1, cols]) == Array(x[1, cols]) # the same

# ╔═╡ 8a68f92e-265c-4099-b5db-42adfa4aa94e
md"""
Now we will get a vector of vectors
"""

# ╔═╡ 42960153-7f10-4e9e-952f-5fb084268191
[Vector(x[i, cols]) for i ∈ axes(x, 1)]

# ╔═╡ 69329cda-551a-4876-be22-a4b3cdda3c1d
md"""
It is easy to convert a DataFrameRow into a NamedTuple
"""

# ╔═╡ aa3c7475-c680-4c99-837a-5dae1e0ae789
copy(x[1, cols])

# ╔═╡ f4c268e8-a5a1-4bf7-861a-afb557b9bd43
md"""
or a Tuple
"""

# ╔═╡ 1e312c3c-8cde-4de3-8781-90cbe992a023
Tuple(x[1, cols])

# ╔═╡ 0b445f9c-a038-436f-b6e6-a7cacfdb43a6
md"""
### Working with a collection of rows of a DataFrame

We can use eachrow to get a vector-like collection of DataFrameRows
"""

# ╔═╡ 17f9c040-20b8-427f-b3d0-906480bea76c
ndf = DataFrame(reshape(1:12, 3, 4), :auto)

# ╔═╡ 629b54fb-7c55-4a11-b868-c7ff4a1c6e7d
er_df = eachrow(ndf)

# ╔═╡ 053ee3b7-ed9e-4111-86b0-491d0693b8c3
er_df[1]

# ╔═╡ 94f5aa8c-a4d4-42e9-ab67-60e90defa945
last(er_df)

# ╔═╡ bfd74ab1-6cfd-4f82-b72f-f5d32e198676
@assert last(er_df) == er_df[end]

# ╔═╡ e6b663c6-3fab-4b93-9fc4-151a3511236f
md"""
As DataFrameRows objects keeps connection to the parent dataframe you can get the columns of the parent using `getproperty`
"""

# ╔═╡ 06e1bd78-289c-47af-80fa-d83ae43e87f1
er_df.x1

# ╔═╡ 4134d57c-4472-48b9-a12b-cb6799d0cc75
md"""
### Flattening a DataFrame

Occasionally we have a dataframe whose one column is a vector of collections. We can expand (flatten) such a column using the `flatten` function
"""

# ╔═╡ 8e1c24ba-85c9-4d64-a038-3854d2f733c4
fdf = DataFrame(a = 'a':'c', b = [[1, 2, 3], [4, 5], 6])

# ╔═╡ 9aa32873-4528-4a6b-b0a4-031b0e105a55
flatten(fdf, :b)

# ╔═╡ 58a333e7-1d2c-4911-87d6-cbbfab505044

# ╔═╡ 1d8d14d1-cc81-486d-89e6-0d7266e44d7a
md"""
### Only one row

`only` from Julia Base is also supported in `DataFrames.jl` and succeeds if the dataframe has only one row, in which case it is returned.
"""

# ╔═╡ f2030d95-3dcf-450f-a1cb-acf284d4870a
udf = DataFrame(a=1)

# ╔═╡ d2e6ba37-05bd-4760-9ebc-409fc40adbde
only(udf)

# ╔═╡ b8d0b8cb-021d-42bd-90a6-858496b8a9c0
udf₂ = repeat(udf, 2)

# ╔═╡ fe183565-5068-49d3-b6a9-3bc5de81cc84
@test_throws ArgumentError only(udf₂)

# ╔═╡ c54cdbf1-7787-4f0f-b9ff-d4389f4a7c5e
html"""

"""

# ╔═╡ Cell order:
# ╟─b3dcf3f4-9d68-11eb-01a6-317fe0e0cd7e
# ╠═794a5356-6e05-44b4-ad9d-375a1fa3613b
# ╟─d6993a5b-8158-4ce3-accb-8e8c514229b3
# ╟─de97ded0-3952-4188-9a43-a8f383e1ae65
# ╠═9ccb6d8f-a8e0-42c2-8691-2eac247e591a
# ╠═385aee9f-5c17-4bd2-b3b2-58171fcc5990
# ╟─472fe012-dd24-46d3-b1b1-f217054c40f0
# ╠═147bcb72-1d40-458e-abb4-02c8d8ddfa76
# ╠═7699775c-8dec-4467-af21-89bd8354d409
# ╟─09617e8f-1d09-4145-831f-38fd98313dd1
# ╠═80277a09-24ea-4dae-9a41-39853a46fbaa
# ╟─9b61d582-6d26-483d-bb66-174b44b38ff4
# ╠═5e7c5b0f-be5d-43aa-b13a-4202b1cb34bf
# ╟─26a2e983-0a3c-41b3-ae2a-79395644e2d4
# ╠═f7dbfd3d-2450-4db3-9d25-d08011c62bf2
# ╠═cdee6bee-59d8-4fe4-80d9-183b1fa3e18b
# ╟─96b5f96f-57de-4fef-bc73-3f092b00be74
# ╠═912c5b66-38d9-49f4-881c-4f2a988104c2
# ╠═560497da-2699-411f-96f9-334919bfffdf
# ╟─d114f107-4e31-4585-915f-a17369023a74
# ╠═5b92be5d-a394-4f8e-b081-bfaef94f3165
# ╠═3ac61fe5-3e33-44f5-86e9-c5945a2fc03c
# ╟─e7da86b9-595c-450c-8a42-60cb63633982
# ╠═6a211df8-ec06-4c24-a362-5e7f604b944a
# ╠═d552bdef-2106-4465-8244-0c8c2144d1f7
# ╠═f4c515e2-a550-42ec-836c-6f6968c383bc
# ╠═31a0adc2-c50d-4949-a234-f71e36f46c3c
# ╟─ca9927e5-3ff2-4c73-8206-4d7495668312
# ╠═5d950253-8184-492f-a649-9123161ba9cc
# ╟─479bbe19-5faf-4f94-8fc2-45f541fffba2
# ╠═e68e64a2-4bd7-446d-b550-0cfa157831e4
# ╠═96733dd9-e40a-4ee9-85f9-082feab93144
# ╠═2d6c26cc-f52f-4d77-a17e-def865adc34c
# ╟─e7b9b478-e2a7-4633-b963-2408a203ef33
# ╠═9e30faf0-2617-43db-9177-64fad45f7b03
# ╠═fe56f87b-50df-4a8a-b531-232a0a51293c
# ╟─1c003f86-9321-4809-a4b9-a98add2ad39d
# ╠═bace7faf-5eee-4f3d-af48-f9c7ff31ce27
# ╟─09530879-8714-4332-a925-6f9298d6dc8b
# ╠═00818c32-887c-4e3d-8a5f-18c1dcaedba1
# ╟─800fcbb6-e1ca-4c63-96af-30bf4680a97b
# ╠═86916666-4b3e-49dd-bb97-1321f34097f4
# ╟─a4615e9b-d3d9-443a-9146-e411f9bbd749
# ╠═b82a4d73-57f9-48fa-9c76-7d2cf853edd3
# ╟─6c8bd34e-2f6f-44bd-a6cd-e7b79597f7ae
# ╠═69d2f06d-866a-4e95-92e9-9b4f06eaff04
# ╟─05bdca3f-033a-41c6-9dac-888b08b55cce
# ╠═6e4115db-22b2-487a-bda1-255364597014
# ╟─b03679d0-2ecc-4b64-9aa6-735c6d18a407
# ╠═1b40287c-b7b6-4ed8-bbe1-1141ef6fb04a
# ╟─bd4429da-3378-4392-9cd9-bba21ff9ec37
# ╠═4becce78-c6d2-43ec-a8ed-58edbf3290ff
# ╟─e741cbed-3da6-4713-ac85-93b8a683fd87
# ╠═39f529a7-8ffa-4149-9780-0d98feee2975
# ╟─9df70270-20db-4a83-9c40-55edc37c99b4
# ╠═037843b8-810b-4e78-8262-f9eb631986c4
# ╠═5a2aa52a-8739-4c49-95af-e2cd82a96523
# ╠═af43af30-7221-4fbb-aef8-63f08956b010
# ╟─198ba262-a6b4-4a2f-8d87-23249c78cb99
# ╠═dd22d95d-840d-41fa-8e1e-1cafb160676e
# ╟─7fea33ff-04e0-4d67-9bf5-d450269ed8a5
# ╠═a8d073a5-e8ac-4cc0-a668-4bc478533c10
# ╟─6eb15245-caf0-4598-8741-00bb545ab7dd
# ╠═d43682a9-6399-404b-9d88-7a15a198eb08
# ╟─70fbd712-7a30-4776-ae1b-109a0a0ca1fe
# ╠═8f227a0c-0083-4abf-8857-7518add1ace8
# ╟─ed3ed957-056d-46c9-9877-10490d054122
# ╠═83a04fdc-93c6-4377-ab35-c18151ca668c
# ╟─770b0df0-6095-4e39-b08e-ca49d64cd778
# ╠═014c6489-a973-48fe-ad5d-a3f487b510b2
# ╟─84aed6e2-afbd-4657-8453-5e1afb91a061
# ╠═3f12d46a-786b-4a20-83ec-7eca121f2770
# ╠═b981bce1-dd57-4451-86a5-e9ee14eeb185
# ╠═98973c9c-dcd3-44dc-835e-b60d2e7ac4b0
# ╠═fa7a9bc6-6609-4e84-93c6-7924bd653fde
# ╟─cf16bd85-9da2-44ac-8e1a-051f04546b60
# ╟─7ff49b11-9b98-4e8c-af33-f8ecbc2565c8
# ╠═c1f5641e-adad-4ab5-8118-c7a9763cff6f
# ╟─3db600e4-3fbb-4fb8-8df6-9438753f18b9
# ╠═1474158e-c2aa-457b-abbb-f10089054f60
# ╟─d3050da0-db13-40e0-a5f5-bb290178f2a0
# ╠═b6972fc7-2d40-47b3-b8a1-825cfa59ef67
# ╟─e51b2218-e998-415f-aa9c-ae7582a04619
# ╠═aaea674f-88c7-43b7-8327-31abdc145787
# ╟─df89ef70-ab45-4fdf-830a-154260225e2f
# ╠═02650cd1-e572-4d70-a39e-da92ce674659
# ╟─94ac6ebb-86b6-4e31-a64d-e4139b850316
# ╠═d9a8ac15-2774-4e51-ab1b-8c0e95cf158c
# ╟─d8d13d16-10ad-4e1e-8265-36516448e702
# ╠═b85c34fc-9d01-4f7e-9104-452ec672101c
# ╟─7cd99843-0ead-4119-9221-7dd8c03d3b27
# ╠═d7e6404c-d0a9-4ee1-8372-edd4ee4fa520
# ╟─bad9c21d-c35b-4e4e-8e18-51aeb41c57a9
# ╠═2d042183-e3f6-4e00-b86c-bdc1f11bc703
# ╟─b80e344e-8550-4f47-bef6-69833a04277c
# ╠═61d6ee03-b042-4f79-a017-dcd59e4eca80
# ╟─26e4a50f-eeb7-4bee-ba4e-e87c479c760c
# ╠═0b758728-d696-4043-aa02-f3fc27189de8
# ╠═2a2a8d79-f872-4139-bd8e-9119ffed0bcd
# ╟─223176a3-aba4-4e6e-a5b1-c51d3ac0bf7e
# ╠═d0d96bda-a7ca-4417-9992-7e3bd748d30b
# ╟─36088a45-a24c-4d43-9ca1-de665cccb25c
# ╠═fa126b33-51b7-4aa7-9cc6-c7998ec93fcf
# ╠═60ba6b50-48ee-460d-bbd9-61e681d72c74
# ╠═3edc2f7f-0b92-421b-ab10-0a5ee432eed0
# ╟─a7dcb351-fff6-4bff-96b9-287ca467dfc5
# ╠═04d68d08-4c9e-4e59-aeb4-8c1502aa64be
# ╟─616f8df8-0bc1-49eb-b161-6be6fe3dc932
# ╠═eb5af9d1-ab74-448f-a38b-8f6b1521a510
# ╟─9dbef9da-3d2a-4a47-98f3-48947176452a
# ╠═1d1cccb2-d0ce-4cbb-b81c-5a887d382bf9
# ╠═bcc5685b-13f8-40d8-a9de-953ae04f08dc
# ╠═db9f9a23-2301-423e-b4ee-4d3de02a55f4
# ╟─31249b78-474f-4ccc-9ca4-540c9ef258aa
# ╠═3d6aab60-11d2-4c78-bd84-4ceb3b5bfc20
# ╟─5b5da74e-c5ea-4154-8515-b625dc2b300d
# ╠═cfaa2f0f-1228-4f61-81ad-a5cc08a901ed
# ╟─a412e28a-e2d4-45af-a193-75bdefbecc75
# ╠═bcd063bb-3629-4060-a0e5-d1f56458cc36
# ╟─7be5c6d8-a7f2-4243-b50f-109f8090a636
# ╠═02a45544-c83b-4917-84ca-2b7f6474a771
# ╟─488788a4-071e-4d29-96ba-319f235076a9
# ╠═e16de281-1676-430d-9049-9fd81d402dcb
# ╟─701b11d4-017f-48ea-998f-2203a58b98fb
# ╠═ee246276-451a-4fb1-ab79-81acef89a416
# ╠═dd5860ed-3bc2-44d4-ba25-c155a68f8e05
# ╟─c66d68b3-5410-4612-841b-ae53acc5ed69
# ╠═d22c3b36-bfe0-4c70-bdd4-045c52098e34
# ╠═21f9eb7e-9b62-4e6a-b19e-0ff3648dd2bd
# ╟─8a68f92e-265c-4099-b5db-42adfa4aa94e
# ╠═42960153-7f10-4e9e-952f-5fb084268191
# ╟─69329cda-551a-4876-be22-a4b3cdda3c1d
# ╠═aa3c7475-c680-4c99-837a-5dae1e0ae789
# ╟─f4c268e8-a5a1-4bf7-861a-afb557b9bd43
# ╠═1e312c3c-8cde-4de3-8781-90cbe992a023
# ╟─0b445f9c-a038-436f-b6e6-a7cacfdb43a6
# ╠═17f9c040-20b8-427f-b3d0-906480bea76c
# ╠═629b54fb-7c55-4a11-b868-c7ff4a1c6e7d
# ╠═053ee3b7-ed9e-4111-86b0-491d0693b8c3
# ╠═94f5aa8c-a4d4-42e9-ab67-60e90defa945
# ╠═bfd74ab1-6cfd-4f82-b72f-f5d32e198676
# ╟─e6b663c6-3fab-4b93-9fc4-151a3511236f
# ╠═06e1bd78-289c-47af-80fa-d83ae43e87f1
# ╟─4134d57c-4472-48b9-a12b-cb6799d0cc75
# ╠═8e1c24ba-85c9-4d64-a038-3854d2f733c4
# ╠═9aa32873-4528-4a6b-b0a4-031b0e105a55
# ╠═58a333e7-1d2c-4911-87d6-cbbfab505044
# ╟─1d8d14d1-cc81-486d-89e6-0d7266e44d7a
# ╠═f2030d95-3dcf-450f-a1cb-acf284d4870a
# ╠═d2e6ba37-05bd-4760-9ebc-409fc40adbde
# ╠═b8d0b8cb-021d-42bd-90a6-858496b8a9c0
# ╠═fe183565-5068-49d3-b6a9-3bc5de81cc84
# ╟─c54cdbf1-7787-4f0f-b9ff-d4389f4a7c5e

max_stars_repo_name: pascal-p/julia-notebooks

max_issues_repo_name: pascal-p/julia-notebooks

max_forks_repo_name: pascal-p/julia-notebooks


text:
import Base.show
show(io::IO,eq::Equation)=print(io,ps(eq))
show(io::IO,ex::Factor)=print(io,ps(ex))

ps(eq::Equation)="$(ps(eq.lhs)) ≖ $(ps(eq.rhs))"
function ps(ex::Expression)
	s=""
	for t in ex
		s*=ps(t)*"+"
	end
	s[1:end-1]
end
function ps(t::Term)
	s=""
	for f in t
		if isa(f,Expression) && length(f)>1
			s*="("*ps(f)*")"*"*"
		else
			s*=ps(f)*"*"
		end
	end
	s[1:end-1]
end
function ps(ar::Array{T,1}) where {T}
	if isempty(ar)
		return "$T[]"
	end
	s="$T["
	for a in ar
		s*=ps(a)*","
	end
	try
		s=s[1:end-1]*"]"
	catch err
		s=s[1:end-2]*"]"
	end
	return s
end
#function ps{T}(a::T)
#	"$T"
#end
function ps(a)
	io=IOBuffer()
	show(io,a)
	str=String(take!(io))
end
function ps(c::Component)
	ar=getargs(c)
	s="$(typeof(c))("
	for a in ar
		s*="$(ps(a)),"
	end
	try
		s=s[1:end-1]*")"
	catch err
		s=s[1:end-2]*")"
	end
	return s
end
ps(s::Symbol)=":$s"
ps(n::Number)="$n"

function print(io::IO,u::U)
	print(io,u.x)
	print(io," [")
	print(io,u.units)
	print(io,"]")
end
ps(a::Alt)="Alt($(a.indices))"
function show(io::IO,a::Alt)
	print(io,ps(a))
end
function print(io::IO,a::Alt)
	print(io,"ϵ$(a.indices)")
end


text:
n = @inferred NamedArray(rand(2,4))
Letters = [string(Char(64+i)) for i in 1:26]
letters = [string(Char(64+32+i)) for i in 1:26]

@inferred setnames!(n, ["one", "two"], 1)
@inferred setnames!(n, letters[1:4], 2)

v = @inferred NamedArray(rand(1:100, 6), (letters[1:6],), (:index,))

m = @inferred NamedArray(rand(2,3,4,3,2,3))

bv = @inferred NamedArray(rand(Bool, 25))
bm = @inferred NamedArray(rand(Bool, 10, 10))

if ! @isdefined namesanddim
	namesanddim(n::NamedArray) = (names(n), dimnames(n))
	namesanddim(n::NamedArray, d::Int) = (names(n, d), dimnames(n, d))
end

if ! @isdefined hasdefaultnames
	hasdefaultnames(n::NamedArray, d) = names(n, d) == map(string, 1:size(n, d))
	hasdefaultnames(n::NamedArray) = all(Bool[hasdefaultnames(n, d) for d in 1:ndims(n)])
end

if ! @isdefined hasdefaultdimnames
	hasdefaultdimnames(n::NamedArray, d) = dimnames(n, d) == Symbol(Letters[d])
	hasdefaultdimnames(n::NamedArray) = all([hasdefaultdimnames(n, d) for d in 1:ndims(n)])
end


text:
"""
    BS(segment_cost, n[, β = log(n)])

Runs the Binary segmentation algorithm for the cost function `segment_cost` for a time series of length `n` and a penalty `β` and returns the position of found changepoints, and
the cost of this segmentation.

See also: [`@BS`](@ref), [`@segment_cost`](@ref)

# Returns
* `CP::Vector{Int}`: Vector of indices of detected changepoints
* `cost::Float64`: Cost of optimal segmentation

# Example
```julia-repl
# Sample Normal time series with changing mean
n = 1000
λ = 100
μ, σ = Normal(0.0, 10.0), 1.0
sample, cps = @changepoint_sampler n λ Normal(μ, σ)
# Run binary segmentation
seg_cost = NormalMeanChange(sample, σ)
BS_cps, BS_cost = BS(seg_cost, n)
```

# References
Scott, A.J. and Knott, M. (1974) A Cluster Analysis Method for Grouping Means in the Analysis of Variance, Biometrics 30(3), 507 - 512
"""
function BS( segment_cost::Function , n::Int64; pen::Float64 = log(n) )
    tau = Tuple{Int, Int}[] # Segmentations to test
    CP = Array{Int64}(undef,0)

    push!(tau, (0, n))
    push!(CP, 0)

    # keep adding segments until none contain a changepoint

    while length(tau) > 0
        a, b = pop!(tau)
        x = Array{Float64}(undef, 0)
        for j in (a+1):(b - 1)
            push!(x , segment_cost(a,j) + segment_cost(j,b) + pen)
        end
        minval , ind = findmin(x)
        if minval - segment_cost(a,b) < 0
            # significant so add seg to tau
            chpt = ind + a
            push!(CP,chpt)
            if chpt != a + 1;
                push!(tau, (a, chpt))
            end
            if chpt != b - 1
                push!(tau, (chpt,b))
            end
        end
    end

    cost = 0.0

    CP = sort(CP)

    for j in 1:(length(CP)-1)
        cost = cost + segment_cost(CP[j]+1,CP[j+1]) + pen
    end

    cost = cost + segment_cost(CP[end]+1,n)

    return CP, cost
end


text:
using LinearAlgebra, Statistics, Roots, Optim, Plots, Random
Random.seed!(1234)
n = 1000 # number of observations to draw

# the true regression function
r(x) = 2 + 1/50*x*(30-x)
# the true density function
σy = 3/2
f(x,y) = 3/4000 * 1/√(2π*σy^2) * x*(20-x)*exp(-1/(2σy^2)*(y-r(x))^2)

# x values and y values for a grid
xs = 0:1/2^3:20
ys = 0:1/2^3:10

# F(t) is the CDF of X
F(t) = -t^3/4000 + 3t^2/400

"Sample from the distribution of X (inverse CDF trick)"
function sampleX()
    U = rand()
    find_zero(t->F(t)-U,(0,20),Bisection())
end

"Sample from joint distribution of X and Y"
function sampleXY(r,σ)
    X = sampleX()
    Y = r(X) + σ*randn()
    (X,Y)
end

# Sample n observations
observations = [sampleXY(r,σy) for i=1:n]

max_stars_repo_name: data-gymnasia/julia-binder

max_issues_repo_name: data-gymnasia/julia-binder

max_forks_repo_name: data-gymnasia/julia-binder


text:
module Project

import JSON

export new,add

function new(project_name::AbstractString,version::AbstractString,project_directory::AbstractString="pwd")
  original_pwd = pwd()
  if (project_directory == "pwd")
    project_directory = pwd()
  elseif project_directory[1] == '.'
    project_directory = string(pwd(),"/",project_directory)
  end
  cd(project_directory)
  mkdir(project_name)
  cd(project_name)
  project_settings = Dict(
    "name" => project_name,
    "version" => version,
    "dependencies" => []
  )
  json_string = JSON.json(project_settings)
  project_file = open("./project.json", "w")
  write(project_file,json_string)
  close(project_file)
  cd(original_pwd)
  print("Your project was created successfully.")
end

function add(package_name::AbstractString)
  project_directory = pwd()
  cd(project_directory)
  if (isfile("./project.json"))
    project_file = open("./project.json")
    project_settings = JSON.parse(readall(project_file))
    push!(project_settings["dependencies"],("pkg",package_name))
    try
      if (get(Pkg.installed(), package_name, 0) == 0)
        Pkg.add(package_name)
      end
      project_file = open("./project.json","w")
      json_string = JSON.json(project_settings)
      write(project_file,json_string)
    catch e
      println(string("\x1b[31m","\x1b[1m","An error occured while installing the package $package_name via Pkg.add"))
    end
    close(project_file)
  end
end

function set_version(new_version_string)
  project_directory = pwd()
  cd(project_directory)
  if (isfile("./project.json"))
    project_file = open("./project.json")
    project_settings = JSON.parse(readall(project_file))
    project_settings["version"] = new_version_string
    project_file = open("./project.json","w")
    json_string = JSON.json(project_settings)
    write(project_file,json_string)
  end
end

function install_dependencies(force_reinstall=false)
  project_directory = pwd()
  cd(project_directory)
  if (isfile("./project.json"))
    project_file = open("./project.json")
    project_settings = JSON.parse(readall(project_file))
    for dep in project_settings["dependencies"]
      if dep[1] == "pkg"
        if (get(Pkg.installed(), dep[2], 0) == 0)
          Pkg.add(dep[2])
        elseif force_reinstall
          Pkg.rm(dep[2])
          Pkg.add(dep[2])
        end
      end
    end
  else
    print_with_color(:red,"ERROR: There is no project.json file in your current directory.\n")
  end
end

end # module


text:
using LinearAlgebra
using JLD2
import StatisticalNonlocality: ou_transition_matrix
import StatisticalNonlocality: fourier

filename = "three_state.jld2"

n = 2
M = ou_transition_matrix(n)

Λ, V = eigen(M)
Λ, W = eigen(M')

# Right eigenvectors
V³ = round.(V[:, 1] ./ V[1, 1])
V² = round.(V[:, 2] ./ V[1, 2])
V¹ = round.(V[:, 3] ./ V[1, 3])

# Left eigenvectors
W³ = round.(W[:, 1] ./ W[1, 1])
W² = round.(W[:, 2] ./ W[1, 2])
W¹ = round.(W[:, 3] ./ W[1, 3])

W[:, 1] .= W¹
W[:, 2] .= -W²
W[:, 3] .= W³

U = [-1 0 0; 0 0 0; 0 0 1] # advection operator structure
W * U * inv(W)

N = 128

D, x = fourier(N, a = 0, b = 2π)

γ = 1.0 # 1/ γ is the eddy timescale
U = 1.0
κ = 1.0

fluxkernel =
    -U * inv(κ .* D^2 - γ * I - U * D * inv(κ .* D^2 - 2 * γ * I) * U * D) * U
λ = eigvals(fluxkernel)
σ = svdvals(fluxkernel)

data_directory = "data"
mkpath(data_directory)
filepath = data_directory * "/" * filename
if isfile(filepath)
    rm(filepath)
end

file = jldopen(filepath, "a+")

file["kernel"] = fluxkernel
file["eigenvalues"] = λ
file["singularvalues"] = σ

close(file)

max_stars_repo_name: sandreza/StatisticalNonlocality

max_issues_repo_name: sandreza/StatisticalNonlocality

max_forks_repo_name: sandreza/StatisticalNonlocality


text:
"""This module implements interface for robust optimization with a solver."""
# Author: Henri Gérard <hgerard.proy@gmail.com>
# License: MIT

function KL_opt(Z, ϵ, solver, regression::LinearRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m,θ[1:lt])
    @variable(m, λ >= 0)
    @variable(m, μ)
    @variable(m, s[1:N])

    @NLconstraint(m, cons[i = 1:N], ((sum(θ[j]*X[i,j] for j in 1:lt) - Y[i])^2-μ) <= λ*(log(s[i]+λ) - log(λ)))

    @objective(m, Min, λ*ϵ+μ + sum(1/N*s[i] for i in 1:N))

    status = solve(m)

    return getvalue(θ)
end

function DRO_opt(Z, ϵ, solver, regression::LinearRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m, θ[1:lt])
    @variable(m, τ >= 0)
    @variable(m, s[1:N])

    @NLconstraint(m, myconstr[i=1:N,j=1:N], (sum(θ[k]*Z[j,k] for k in 1:lt)- Z[j,end])^2 - τ*sum((Z[i,k]-Z[j,k])*(Z[i,k]-Z[j,k]) for k in 1:lt+1) <= s[i])

    @objective(m, Min, τ*ϵ + sum(1/N*s[i] for i in 1:N))

    # print(m)

    status = solve(m)
    # println("theta = ", getvalue(θ))
    # println("tau = ", getvalue(τ))
    # println("s = ", getvalue(s))
    return getvalue(θ)
end

function KL_opt(Z, ϵ, solver, regression::LogisticRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m,θ[1:lt])
    @variable(m, λ >= 0)
    @variable(m, μ)
    @variable(m, s[1:N])

    @NLconstraint(m, cons[i = 1:N], (log(1+exp(-Y[i]*sum(θ[j]*X[i,j] for j in 1:lt)))-μ) <= λ*(log(s[i]+λ) - log(λ)))

    @objective(m, Min, λ*ϵ+μ + sum(1/N*s[i] for i in 1:N))

    status = solve(m)

    return getvalue(θ)
end

function DRO_opt(Z, ϵ, solver, regression::LogisticRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m, θ[1:lt])
    @variable(m, τ >= 0)
    @variable(m, s[1:N])

    @NLconstraint(m, myconstr[i=1:N,j=1:N], log(1+exp(-Y[i]*sum(θ[k]*X[i,k] for k in 1:lt))) - τ*sum((Z[i,k]-Z[j,k])*(Z[i,k]-Z[j,k]) for k in 1:lt+1) <= s[j])

    @objective(m, Min, τ*ϵ + sum(1/N*s[i] for i in 1:N))

    # print(m)

    status = solve(m)
    # println("theta = ", getvalue(θ))
    # println("tau = ", getvalue(τ))
    # println("s = ", getvalue(s))
    return getvalue(θ)
end

function normal_opt(Z, solver, regression::LinearRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m,θ[1:lt])

    @NLobjective(m, Min, sum(1/N*(sum(θ[j]*X[i,j] for j in 1:lt) - Y[i])^2 for i in 1:N))

    status = solve(m)

    return getvalue(θ)
end

function normal_opt(Z, solver, regression::LogisticRegression)
    N = size(Z)[1]
    lt = size(Z)[2]-1
    X = Z[:,1:end-1]
    Y = Z[:,end]

    m = Model(solver=solver)

    @variable(m,θ[1:lt])

    @NLobjective(m, Min, sum( log(1+exp(-Y[i]*sum(θ[j]*X[i,j] for j in 1:lt))) for i in 1:N ))

    status = solve(m)

    return getvalue(θ)
end

max_stars_repo_name: Henri-Gerard/robox

max_issues_repo_name: Henri-Gerard/robox

max_forks_repo_name: Henri-Gerard/robox


text:
using OrderedCollections

"""
    LazyBridgeOptimizer{OT<:MOI.ModelLike} <: AbstractBridgeOptimizer

The `LazyBridgeOptimizer` combines several bridges, which are added using the [`add_bridge`](@ref) function.
Whenever a constraint is added, it only attempts to bridge it if it is not supported by the internal model (hence its name `Lazy`).
When bridging a constraint, it selects the minimal number of bridges needed.
For instance, a constraint `F`-in-`S` can be bridged into a constraint `F1`-in-`S1` (supported by the internal model) using bridge 1 or
bridged into a constraint `F2`-in-`S2` (unsupported by the internal model) using bridge 2 which can then be
bridged into a constraint `F3`-in-`S3` (supported by the internal model) using bridge 3,
it will choose bridge 1 as it allows to bridge `F`-in-`S` using only one bridge instead of two if it uses bridge 2 and 3.
"""
mutable struct LazyBridgeOptimizer{OT<:MOI.ModelLike} <: AbstractBridgeOptimizer
    # Internal model
    model::OT
    # Bridged variables
    variable_map::Variable.Map
    var_to_name::Dict{MOI.VariableIndex, String}
    name_to_var::Union{Dict{String, MOI.VariableIndex}, Nothing}
    # Bridged constraints
    constraint_map::Constraint.Map
    con_to_name::Dict{MOI.ConstraintIndex, String}
    name_to_con::Union{Dict{String, MOI.ConstraintIndex}, Nothing}
    # Bridged objective
    objective_map::Objective.Map
    # Bellman-Ford
    # List of types of available bridges
    variable_bridge_types::Vector{Any}
    # (S,) -> Number of bridges that need to be used for constrained variables in `S`
    variable_dist::Dict{Tuple{DataType}, Int}
    # (S,) -> Bridge to be used for constrained variables in `S`
    variable_best::Dict{Tuple{DataType}, DataType}
    # List of types of available bridges
    constraint_bridge_types::Vector{Any}
    # (F, S) -> Number of bridges that need to be used for an `F`-in-`S` constraint
    constraint_dist::Dict{Tuple{DataType, DataType}, Int}
    # (F, S) -> Bridge to be used for an `F`-in-`S` constraint
    constraint_best::Dict{Tuple{DataType, DataType}, DataType}
    # List of types of available bridges
    objective_bridge_types::Vector{Any}
    # (F, S) -> Number of bridges that need to be used for an `F`-in-`S` constraint
    objective_dist::Dict{Tuple{DataType}, Int}
    # (F, S) -> Bridge to be used for an `F`-in-`S` constraint
    objective_best::Dict{Tuple{DataType}, DataType}
end
function LazyBridgeOptimizer(model::MOI.ModelLike)
    return LazyBridgeOptimizer{typeof(model)}(
        model,
        Variable.Map(), Dict{MOI.VariableIndex, String}(), nothing,
        Constraint.Map(), Dict{MOI.ConstraintIndex, String}(), nothing,
        Objective.Map(),
        Any[], Dict{Tuple{DataType}, Int}(),
        Dict{Tuple{DataType}, DataType}(),
        Any[], Dict{Tuple{DataType, DataType}, Int}(),
        Dict{Tuple{DataType, DataType}, DataType}(),
        Any[], Dict{Tuple{DataType}, Int}(),
        Dict{Tuple{DataType}, DataType}())
end

function Variable.bridges(bridge::LazyBridgeOptimizer)
    return bridge.variable_map
end
function Constraint.bridges(bridge::LazyBridgeOptimizer)
    return bridge.constraint_map
end
function Objective.bridges(b::LazyBridgeOptimizer)
    return b.objective_map
end

function _dist(b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet})
    F = MOIU.variable_function_type(S)
    if MOI.supports_constraint(b.model, F, S)
        return 0
    else
        return min(get(b.variable_dist, (S,), typemax(Int)),
                   get(b.constraint_dist, (F, S), typemax(Int) - 1) + 1)
    end
end

function _dist(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction}, S::Type{<:MOI.AbstractSet})
    if MOI.supports_constraint(b.model, F, S)
        return 0
    else
        return get(b.constraint_dist, (F, S), typemax(Int))
    end
end

function _dist(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractScalarFunction})
    if MOI.supports(b.model, MOI.ObjectiveFunction{F}())
        return 0
    else
        return get(b.objective_dist, (F,), typemax(Int))
    end
end

function _added_dist(b::LazyBridgeOptimizer, args...)
    dist = mapreduce(C -> _dist(b, C[1]), +,
                     added_constrained_variable_types(args...), init = 0)
    dist += mapreduce(C -> _dist(b, C[1], C[2]), +,
                      added_constraint_types(args...), init = 0)
    return dist
end
function added_dist(b::LazyBridgeOptimizer,
                    BT::Type{<:Union{Variable.AbstractBridge,
                                     Constraint.AbstractBridge}},
                    args...)
    return _added_dist(b, BT, args...)
end
function added_dist(b::LazyBridgeOptimizer,
                    BT::Type{<:Objective.AbstractBridge},
                    args...)
    F = set_objective_function_type(BT, args...)
    return _added_dist(b, BT, args...) + _dist(b, F)
end

function _supports_added_no_update(b::LazyBridgeOptimizer, args...)
    return all(C -> supports_no_update(b, C[1]), added_constrained_variable_types(args...)) &&
        all(C -> supports_no_update(b, C[1], C[2]), added_constraint_types(args...))
end
function supports_added_no_update(
    b::LazyBridgeOptimizer,
    BT::Type{<:Union{Variable.AbstractBridge,
                     Constraint.AbstractBridge}},
    args...
)
    return _supports_added_no_update(b, BT, args...)
end
function supports_added_no_update(
    b::LazyBridgeOptimizer,
    BT::Type{<:Objective.AbstractBridge},
    args...
)
    F = set_objective_function_type(BT, args...)
    return _supports_added_no_update(b, BT, args...) &&
        supports_no_update(b, F)
end

# Update `b.variable_dist`, `b.constraint_dist` `b.variable_best` and
# `b.constraint_best` for constrained variable types in `variables` and
# constraint types in `constraints`.
function update_dist!(b::LazyBridgeOptimizer, variables, constraints, objectives)
    # Bellman-Ford algorithm
    changed = true # Has b.constraint_dist changed in the last iteration ?
    while changed
        changed = false
        for BT in b.variable_bridge_types
            for (S,) in variables
                if Variable.supports_constrained_variable(BT, S) &&
                    supports_added_no_update(b, BT, S)
                    # Number of bridges needed using BT
                    dist = 1 + added_dist(b, BT, S)
                    # Is it better that what can currently be done ?
                    if dist < _dist(b, S)
                        b.variable_dist[(S,)] = dist
                        b.variable_best[(S,)] = Variable.concrete_bridge_type(BT, S)
                        changed = true
                    end
                end
            end
        end
        for BT in b.constraint_bridge_types
            for (F, S) in constraints
                if MOI.supports_constraint(BT, F, S) &&
                    supports_added_no_update(b, BT, F, S)
                    # Number of bridges needed using BT
                    dist = 1 + added_dist(b, BT, F, S)
                    # Is it better that what can currently be done ?
                    if dist < _dist(b, F, S)
                        b.constraint_dist[(F, S)] = dist
                        b.constraint_best[(F, S)] = Constraint.concrete_bridge_type(BT, F, S)
                        changed = true
                    end
                end
            end
        end
        for BT in b.objective_bridge_types
            for (F,) in objectives
                if Objective.supports_objective_function(BT, F) &&
                    supports_added_no_update(b, BT, F)
                    # Number of bridges needed using BT
                    dist = 1 + added_dist(b, BT, F)
                    # Is it better that what can currently be done ?
                    if dist < _dist(b, F)
                        b.objective_dist[(F,)] = dist
                        b.objective_best[(F,)] = Objective.concrete_bridge_type(BT, F)
                        changed = true
                    end
                end
            end
        end
    end
end

function fill_required!(required_variables::OrderedSet{Tuple{DataType}},
                        required_constraints::OrderedSet{Tuple{DataType, DataType}},
                        required_objectives::OrderedSet{Tuple{DataType}},
                        b::LazyBridgeOptimizer,
                        BT::Type{<:AbstractBridge})
    for C in added_constrained_variable_types(BT)
        # The constrained variables in `C[1]` might be supported while the
        # constraints `F`-in-`C[1]` are not supported. In this case, the second
        # `fill_requires!` would add `F`-in-`C[1]` while we know that:
        # * either `C[1]` is natively supported in which case we don't need to
        #   worry about `F`-in-`C[1]`;
        # * either, `C[1]` was found to be supported via variable bridges in
        #   but in this run of Bellman-Ford, `F`-in-`C[1]` was also in required
        #   so we know that it is not supported and we don't need to add it
        #   again in required as it would arrive to the same conclusion.
        if !supports_no_update(b, C[1])
            fill_required!(required_variables, required_constraints,
                           required_objectives, b, C[1])
            F = MOIU.variable_function_type(C[1])
            fill_required!(required_variables, required_constraints,
                           required_objectives, b, F, C[1])
        end
    end
    for C in added_constraint_types(BT)
        fill_required!(required_variables, required_constraints,
                       required_objectives, b, C[1], C[2])
    end
    if BT <: Objective.AbstractBridge
        fill_required!(required_variables, required_constraints, required_objectives, b,
                       set_objective_function_type(BT))
    end
end

function fill_required!(required_variables::OrderedSet{Tuple{DataType}},
                        required_constraints::OrderedSet{Tuple{DataType, DataType}},
                        required_objectives::OrderedSet{Tuple{DataType}},
                        b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet})
    if supports_no_update(b, S)
        return # The constraint is supported
    end
    if (S,) in required_variables
        return # The requirements for these constrained variables have already been added or are being added
    end
    # The constrained variables are not supported yet, add in
    # `required_variables` the required variables types to bridge it.
    push!(required_variables, (S,))
    for BT in b.variable_bridge_types
        if Variable.supports_constrained_variable(BT, S)
            fill_required!(required_variables, required_constraints,
                           required_objectives, b,
                           Variable.concrete_bridge_type(BT, S))
        end
    end
end

function fill_required!(required_variables::OrderedSet{Tuple{DataType}},
                        required_constraints::OrderedSet{Tuple{DataType, DataType}},
                        required_objectives::OrderedSet{Tuple{DataType}},
                        b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction},
                        S::Type{<:MOI.AbstractSet})
    if supports_no_update(b, F, S)
        return # The constraint is supported
    end
    if (F, S) in required_constraints
        return # The requirements for this constraint have already been added or are being added
    end
    # The constraint is not supported yet, add
    # * in `required_variables` the required constrained variables types and
    # * in `required_constraints` the required constraint types
    # to bridge it.
    push!(required_constraints, (F, S))
    for BT in b.constraint_bridge_types
        if MOI.supports_constraint(BT, F, S)
            fill_required!(required_variables, required_constraints,
                           required_objectives, b,
                           Constraint.concrete_bridge_type(BT, F, S))
        end
    end
end

function fill_required!(required_variables::OrderedSet{Tuple{DataType}},
                        required_constraints::OrderedSet{Tuple{DataType, DataType}},
                        required_objectives::OrderedSet{Tuple{DataType}},
                        b::LazyBridgeOptimizer,
                        F::Type{<:MOI.AbstractScalarFunction})
    if supports_no_update(b, F)
        return # The objective is supported
    end
    if (F,) in required_objectives
        return # The requirements for this objective have already been added or are being added
    end
    # The objective is not supported yet, add
    # * in `required_variables` the required constrained variables types,
    # * in `required_constraints` the required constraint types and
    # * in `required_objectives` the required objective types
    # to bridge it.
    push!(required_objectives, (F,))
    for BT in b.objective_bridge_types
        if Objective.supports_objective_function(BT, F)
            fill_required!(required_variables, required_constraints,
                           required_objectives, b,
                           Objective.concrete_bridge_type(BT, F))
        end
    end
end

function required(b::LazyBridgeOptimizer, types::Tuple)
    required_variables = OrderedSet{Tuple{DataType}}()
    required_constraints = OrderedSet{Tuple{DataType, DataType}}()
    required_objectives = OrderedSet{Tuple{DataType}}()
    fill_required!(required_variables, required_constraints,
                   required_objectives, b, types...)
    return required_variables, required_constraints, required_objectives
end

# Compute dist[(F, S)], dist[(S,)] and best[(F, S)], best[(S,)]
function update!(b::LazyBridgeOptimizer, types::Tuple)
    update_dist!(b, required(b, types)...)
end

# After `add_bridge(b, BT)`, some constrained variables `(S,)` in
# `keys(b.variable_best)` or constraints `(F, S)` in `keys(b.constraint_best)`
# or `(F,)` in `keys(b.objective_best)` may be bridged
# with less bridges than `b.variable_dist[(S,)]`,
# `b.constraint_dist[(F, S)]` or `b.objective_dist[(F,)]` using `BT`.
function _update_key_dists!(b)
    # TODO we should call `fill_required!`.
    update_dist!(b, keys(b.variable_best), keys(b.constraint_best), keys(b.objective_best))
end

"""
    add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Variable.AbstractBridge})

Enable the use of the variable bridges of type `BT` by `b`.
"""
function add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Variable.AbstractBridge})
    push!(b.variable_bridge_types, BT)
    update_dist!(b, keys(b.variable_best), keys(b.constraint_best), keys(b.objective_best))
end

"""
    add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Constraint.AbstractBridge})

Enable the use of the constraint bridges of type `BT` by `b`.
"""
function add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Constraint.AbstractBridge})
    push!(b.constraint_bridge_types, BT)
    update_dist!(b, keys(b.variable_best), keys(b.constraint_best), keys(b.objective_best))
end

"""
    add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Objective.AbstractBridge})

Enable the use of the objective bridges of type `BT` by `b`.
"""
function add_bridge(b::LazyBridgeOptimizer, BT::Type{<:Objective.AbstractBridge})
    push!(b.objective_bridge_types, BT)
    update_dist!(b, keys(b.variable_best), keys(b.constraint_best), keys(b.objective_best))
end

function _remove_bridge(bridge_types::Vector, BT::Type)
    i = findfirst(isequal(BT), bridge_types)
    if i === nothing
        error("Cannot remove bridge `BT` as it was never added or was already",
              " removed.")
    else
        deleteat!(bridge_types, i)
    end
    return
end
function _reset_dist(b::LazyBridgeOptimizer)
    empty!(b.variable_dist)
    empty!(b.variable_best)
    empty!(b.constraint_dist)
    empty!(b.constraint_best)
end

"""
    remove_bridge(b::LazyBridgeOptimizer, BT::Type{<:Variable.AbstractBridge})

Disable the use of the variable bridges of type `BT` by `b`.
"""
function remove_bridge(b::LazyBridgeOptimizer, BT::Type{<:Variable.AbstractBridge})
    _remove_bridge(b.variable_bridge_types, BT)
    _reset_dist(b)
end

"""
    remove_bridge(b::LazyBridgeOptimizer, BT::Type{<:Constraint.AbstractBridge})

Disable the use of the constraint bridges of type `BT` by `b`.
"""
function remove_bridge(b::LazyBridgeOptimizer, BT::Type{<:Constraint.AbstractBridge})
    _remove_bridge(b.constraint_bridge_types, BT)
    _reset_dist(b)
end

# It only bridges when the constraint is not supporting, hence the name "Lazy"
function is_bridged(b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet})
    return !MOI.supports_constraint(b.model, MOIU.variable_function_type(S), S)
end
function is_bridged(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction}, S::Type{<:MOI.AbstractSet})
    return !MOI.supports_constraint(b.model, F, S)
end
function is_bridged(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractScalarFunction})
    return !MOI.supports(b.model, MOI.ObjectiveFunction{F}())
end
# Same as supports_constraint but do not trigger `update!`. This is
# used inside `update!`.
function supports_no_update(b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet})
    F = MOIU.variable_function_type(S)
    return MOI.supports_constraint(b.model, MOIU.variable_function_type(S), S) ||
        (S,) in keys(b.variable_best) || (F, S) in keys(b.constraint_best)
end
function supports_no_update(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction}, S::Type{<:MOI.AbstractSet})
    return MOI.supports_constraint(b.model, F, S) || (F, S) in keys(b.constraint_best)
end
function supports_no_update(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractScalarFunction})
    return MOI.supports(b.model, MOI.ObjectiveFunction{F}()) || (F,) in keys(b.objective_best)
end

supports_constraint_bridges(::LazyBridgeOptimizer) = true
function supports_bridging_constrained_variable(
    b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet}
)
    update!(b, (S,))
    return (S,) in keys(b.variable_best)
end
function supports_bridging_constraint(
    b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction},
    S::Type{<:MOI.AbstractSet}
)
    update!(b, (F, S))
    return (F, S) in keys(b.constraint_best)
end
function supports_bridging_objective_function(
    b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractScalarFunction}
)
    update!(b, (F,))
    return (F,) in keys(b.objective_best)
end
function bridge_type(b::LazyBridgeOptimizer, S::Type{<:MOI.AbstractSet})
    update!(b, (S,))
    result = get(b.variable_best, (S,), nothing)
    if result === nothing
        throw(MOI.UnsupportedConstraint{MOIU.variable_function_type(S), S}())
    end
    return result
end
function bridge_type(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractFunction}, S::Type{<:MOI.AbstractSet})
    update!(b, (F, S))
    result = get(b.constraint_best, (F, S), nothing)
    if result === nothing
        throw(MOI.UnsupportedConstraint{F, S}())
    end
    return result
end
function bridge_type(b::LazyBridgeOptimizer, F::Type{<:MOI.AbstractScalarFunction})
    update!(b, (F,))
    result = get(b.objective_best, (F,), nothing)
    if result === nothing
        throw(MOI.UnsupportedAttribute(MOI.ObjectiveFunction{F}()))
    end
    return result
end

_func_name(::Type{Constraint.ScalarFunctionizeBridge}) = "SingleVariable", "constraint"
_func_name(::Type{Constraint.VectorFunctionizeBridge}) = "VectorOfVariables", "constraint"
_func_name(::Type{Objective.FunctionizeBridge}) = "SingleVariable", "objective"
function _functionize_bridge(bridge_types, target_type)
    index = findfirst(bridge_type -> bridge_type <: target_type,
                      bridge_types)
    if index === nothing
        func, name = _func_name(target_type)
        error("Need to apply a `$target_type` to a",
              " `$func` $name because the variable is",
              " bridged but no such $name bridge type was added. Add one",
              " with `add_bridge`.")
    end
    return bridge_types[index]
end
function constraint_vector_functionize_bridge(b::LazyBridgeOptimizer)
    return _functionize_bridge(b.constraint_bridge_types, Constraint.VectorFunctionizeBridge)
end
function constraint_scalar_functionize_bridge(b::LazyBridgeOptimizer)
    return _functionize_bridge(b.constraint_bridge_types, Constraint.ScalarFunctionizeBridge)
end
function objective_functionize_bridge(b::LazyBridgeOptimizer)
    return _functionize_bridge(b.objective_bridge_types, Objective.FunctionizeBridge)
end


text:
#=
union_types:
- Julia version: 1.5.0
- Author: shisa
- Date: 2020-08-09
=#

module UnionTypes
    # type
    IntOrString = Union{Int, String}

    # methods
    function get_union_type(x::IntOrString)
        return typeof(x)
    end

    function main()
        println("Type(x=10) = ", get_union_type(10))
        println("Type(x='Hello') = ", get_union_type("Hello"))
    end
end

if abspath(PROGRAM_FILE) == @__FILE__
    using .UnionTypes
    UnionTypes.main()
end

max_stars_repo_name: ShisatoYano/JuliaPractice

max_issues_repo_name: ShisatoYano/JuliaPractice

max_forks_repo_name: ShisatoYano/JuliaPractice


text:
"""
File Appender.
"""
type File <: Appender
    name::AbstractString
    layout::LAYOUT
    filter::FILTER
    state::LifeCycle.State

    filename::AbstractString
    append::Bool
    flush::Bool
    locking::Bool

    target::IO

    File(nm::AbstractString, lyt::LAYOUT, flt::FILTER) = new(nm, lyt, flt, LifeCycle.INITIALIZED)
end

function File(config::Dict{AbstractString,Any})
    nm = get(config, "name", "FileAppender")
    fn = get(config, "filename", nothing)
    lyt= get(config, "layout", nothing)
    flt= get(config, "filter", nothing)
    doappend = get(config, "append",  true)
    doflush  = get(config, "flush",   true)
    dolock   = get(config, "locking", false)

    file = File(nm, LAYOUT(lyt), FILTER(flt))
    if fn !== nothing
        file.filename = fn
    else
        file.state = LifeCycle.INVALID
        error(LOGGER, "No `filename` provided for the appender `File`.")
    end
    file.append = doappend
    file.flush = doflush
    file.locking = dolock

    return file
end
File(;kwargs...) = map(e->(string(e[1]),e[2]), kwargs) |> Dict{AbstractString,Any} |> File
show(io::IO, apnd::File) = print(io, "FileAppender($(isdefined(apnd, :filename) ? apnd.filename : ""), $(apnd.state))")

name(apnd::File) = isempty(apnd.name) ? string(typeof(apnd)) : apnd.name
layout(apnd::File) = apnd.layout

function start(apnd::File)
    invoke(start, Tuple{Appender}, apnd)
    apnd.target = open(apnd.filename, apnd.append ? "a" : "w")
end

function stop(apnd::File)
    invoke(stop, Tuple{Appender}, apnd)
    close(apnd.target)
end

function append!(apnd::File, evnt::Event)
    if !isnull(apnd.layout)
        if apnd.locking
            lock(apnd.target)
            try
                write!(apnd, evnt)
            finally
                unlock(apnd.target)
            end
        else
            write!(apnd, evnt)
        end
    end
end

function write!(apnd::File, evnt::Event)
    write(apnd.target, serialize(apnd.layout, evnt))
    apnd.flush && flush(apnd.target)
end


text:
"""
Memory-efficient vector of tokens.

Vector elements are tokens, and optionally Nothing and Missing.
Some Missing/Nothing support is already built into all AbstractToken subtypes:
they have an internal encoding for missing and nothing, and Functions ismissing and isnothing
have methods for AbstractToken which test on that encoding. You will need
Missing and/or Nothing in the element type only if some code does not use ismissing/isnothing
but explicitly tests instances on identity (or equality) to nothing and missing.

To have explicit Missing or Nothing support for vector elements,
specify the appropriate Union for type parameter T.
This will *not* increase memory consumption per element, but cause some overhead on
array read accesses, to check for nothing or missing and replace a token instance by nothing or missing.
"""
struct TokenVector{T<:Union{Nothing,Missing,Token}}  <: AbstractVector{T}
    vec :: Vector{HybridFly}
    heap :: IOShared # sharing and appending from outside is ok, other operations not.
    function TokenVector{T}(heap::IOShared) where T
        vec = Vector{HybridFly}()
        push!(heap.registered,vec)
        return new(vec,heap)
    end
end

const HTokenVector = TokenVector{Token{HybridFly}}
const BTokenVector = TokenVector{Token{BufferFly}}
const EXPECTED_TOKENSIZE = 10

TokenVector{T}(undef::UndefInitializer, n) where T = TokenVector{T}(undef,n,EXPECTED_TOKENSIZE)

#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# TODO elegante Methode den externen token Type aus einer Tokenvector{T} instanz zu ermitteln.

function TokenVector{T}(undef::UndefInitializer, vectorsize, tokensize) where T
    ret = TokenVector{T}(IOShared(vectorsize*tokensize))
    resize!(ret.vec,vectorsize)
    fill!(ret.vec,T(nothing))
end

function TokenVector{T}(v::AbstractVector{S}, expectedTokensize=EXPECTED_TOKENSIZE) where {T,S}
    size = length(v)
    ret = TokenVector{T}(undef,size, expectedTokensize)
    for i in 1:size
        ret[i] = T(v[i])
    end
end

Base.IndexStyle(::Type{<:TokenVector}) = IndexLinear()

Base.eltype(::Type{TokenVector{T}}) where T = T

Base.sizehint!(tv:: TokenVector{T}, n) where T = sizehint!(tv.vec,n)

function Base.empty!(tv::TokenVector)
    empty!(tv.vec)
    # empty!(tv.heap) # not allowed - heap is designed to be nonexclusive.
    return tv
end

Base.size(tv::TokenVector) = size(tv.vec)
Base.length(tv) = length(tv.vec)
Base.firstindex(tv::TokenVector) = 1
Base.lastindex(tv::TokenVector) = length(tv)

function Base.getindex(tv::TokenVector{T}, i::Int) where T
    v = tv.vec[i]
    if Missing <:T && ismissing(v)
        return missing
    end
    if Nothing <:T && isnothing(v)
        return nothing
    end
    sharedbuf = tv.heap.buffer
    #= commented out because 1-2 checks on any read access
    # should not be necessary, but could prevend unnecessary references to heap.buffer
    sharedbuf = EMPTYSTRING
    if !isdirect(v) && usize(v)>0
        sharedbuf = tv.heap.buffer
    end
    =#
    return HToken <:T ? HToken(v,sharedbuf) : BToken(bf(v),sharedbuf)
end

"""

Element assignment with conversion.

t maybe one of the following:

 * AbstractString: stores a token with given content and category T_TEXT

 * AbstractToken: stores a token with given content and category

 * Integer: stores a token with decimal coded integer value and category T_INT

 * Real:  stores a token with decimal coded floating point number and ategory T_REAL

 * Missing: stores a token t with Category T_SYMBOL and empty string, giving ismissing(t) == true

 * Nothing: stores a token with Category T_SPECIAL and empty string, giving isnothing(t) == true

 """
function Base.setindex!(tv::TokenVector{T}, s, i::Int) where T
    tv.vec[i] = hf(put(tv.heap, s, T).fly)
end

"""
    compress(tv::TokenVector, reserve::UInt32)

Build a reorganized copy of a TokenVector with (usually) reduced memory consumption.
Important use case: reorganization before serialization to external storage.

A new IOShared is allocated, a new TokenVector is built using it as buffer,
compression is turned on. reserve specifies the maximum of free space in
the new buffer - small values will probably force a reallocation.

Warning

    * worstcase runtime O((sum of token sizes)**2)
    * heap is set to search mode, subsequent assignments are O(heapsize)

New vector is returned. Old vector is cleared (all content is deleted)
"""
function compress(tv::TokenVector{T},reserve::UInt32) where T
    totalsize = 0
    for i in 1:length(tv.vec)
        f = tv.vec[i]
        if !isdirect(f)
            s = usize(f)
            if (! HToken <: T) || s > MAX_DIRECT_SIZE
                totalsize += s
            end
        end
    end
    heap = IOShared(totalsize)
    ret = TokenVector{T}(heap)
    compressOnPut(heap,true)
    ii = sortperm(tv.vec,by = t -> -(t.len%Int) )
    for i in ii
        t = tv[i]
        if !isdirect(t)
            s = usize(f)
            if HToken <: T && s <= MAX_DIRECT_SIZE
                t = T(DirectFly(t))
            else
                t = put(heap,t)
            end
        end
        ret.vec[i] = hf(t.fly)
    end
    shrink!(heap,reserve,false)
    empty!(tv)
    return ret
end

"Resize tv to contain n elements. If n is smaller than the current collection length, the first n elements will be retained. If n is larger, the new elements are set to nothing."
function Base.resize!(tv::TokenVector{T}, n::Integer) where T<:Union{Nothing,Missing,Token{BufferFly},Token{HybridFly}}
    oldsize = length(tv.vec)
    resize!(tv.vec,n)
    t = HybridFly(nothing) # initialization value
    while oldsize < n
        oldsize += 1
        tv.vec[oldsize] = t
    end
end

"store content of item in heap of tv and return a FlyToken referencing that content"
function _put(tv::TokenVector{T}, item) where T
    if HToken <: T
        return hf(put(tv.heap,item,HToken).fly)
    else
        return hf(put(tv.heap,item,BToken).fly)
    end
end

"inserts a copy of item with content in tv.heap (or direct) as an element in tv at *index*. item can be anything accepted by token constructor"
Base.insert!(tv::TokenVector{T}, index::Integer, item) where T = insert!(tv.vek,index,_put(tv,item))

function Base.append!(tv::TokenVector{T},items::AbstractVector{S}) where {T, S }
    n = length(items)
    last = lastindex(tv)
    resize!(tv.vec, last+n)
    for i in 1:n
        setindex!(tv,items[i],last+i)
    end
    return tv
end

# push should be defined for AbstractVector using append
#push!(collection, items...) -> collection
# iterate defined for AbstractVector

"copy shares content readonly, source keeps its write rights on its buffer"
Base.copy(tv::TokenVector{T}) where T = TokenVector{T}(copy(tv.vec),copy(tv.heap))

"fill with anything that can be converted (via constructor) to a Token"
function Base.fill!(tv::TokenVector{T}, s) where T
    t = Token{F}(s)
    t = put(tv.heap,t)
    fill!(tv.vec,t.fly)
    return tv
end

function Base.sizehint!(tv, n)
    sizehint!(tv.vec,n)
    return tv
end

function Base.convert(::Type{TokenVector{F}}, arr::AbstractVector{S}) where {F<:FlyToken, S}
    tv = TokenVector{F}(undef, length(arr))
    @inbounds for i in eachindex(arr)
        tv[i] = Token{F}(arr[i])
    end
    return tv
end

#= analog implementieren??

Base.convert(::Type{StringArray}, arr::AbstractArray{T}) where {T<:STR} = StringArray{T}(arr)
Base.convert(::Type{StringArray{T, N} where T}, arr::AbstractArray{S}) where {S<:STR, N} = StringVector{S}(arr)
StringVector{T}() where {T} = StringVector{T}(UInt8[], UInt64[], UInt32[])
StringVector() = StringVector{String}()
StringVector{T}(::UndefInitializer, len::Int) where {T} = StringArray{T}(undef, len)
StringVector(::UndefInitializer, len::Int) = StringArray{String}(undef, len)
# special constructor where a full byte buffer is provided and offsets/lens will be filled in later
StringVector{T}(buffer::Vector{UInt8}, len::Int) where {T} = StringVector{T}(buffer, fill(UNDEF_OFFSET, len), fill(zero(UInt32), len))

(T::Type{<:StringArray})(arr::AbstractArray{<:STR}) = convert(T, arr)

function Base.similar(a::StringArray, T::Type{<:STR}, dims::Tuple{Vararg{Int64, N}}) where N
    StringArray{T, N}(undef, dims)
end

function Base.resize!(arr::StringVector, len)
    l = length(arr)
    resize!(arr.offsets, len)
    resize!(arr.lengths, len)
    if l < len
        arr.offsets[l+1:len] .= UNDEF_OFFSET # undef
        arr.lengths[l+1:len] .= 0
    end
    arr
end

function Base.push!(arr::StringVector, val::AbstractString)
    l = length(arr.buffer)
    resize!(arr.buffer, l + sizeof(val))
    unsafe_copyto!(pointer(arr.buffer, l + 1), pointer(val,1), sizeof(val))
    push!(arr.offsets, l)
    push!(arr.lengths, sizeof(val))
    arr
end

function Base.push!(arr::StringVector{Union{T, Missing}}, val::Missing) where {T}
    push!(arr.offsets, MISSING_OFFSET)
    push!(arr.lengths, 0)
    arr
end

function Base.deleteat!(arr::StringVector, idx)
    deleteat!(arr.lengths, idx)
    deleteat!(arr.offsets, idx)
    arr
end

function Base.insert!(arr::StringVector, idx::Integer, item::AbstractString)
    l = length(arr.buffer)
    resize!(arr.buffer, l + sizeof(item))
    unsafe_copyto!(pointer(arr.buffer, l + 1), pointer(item), sizeof(item))
    insert!(arr.offsets, idx, l)
    insert!(arr.lengths, idx, sizeof(item))
    arr
end

function Base.insert!(arr::StringVector{Union{T, Missing}}, idx::Integer, item::Missing) where {T}
    insert!(arr.offsets, idx, MISSING_OFFSET)
    insert!(arr.lengths, idx, 0)
    arr
end

function Base.permute!(arr::StringArray{String}, p::AbstractVector)
    permute!(convert(StringArray{WeakRefString{UInt8}}, arr), p)
    arr
end

function Base.sortperm(arr::StringArray{String})
    sortperm(convert(StringArray{WeakRefString{UInt8}}, arr))
end

function Base.sort!(arr::StringArray{String})
    permute!(arr, sortperm(arr))
    arr
end

function Base.vcat(a::StringVector{T}, b::StringVector{T}) where T
    StringVector{T}(vcat(a.buffer, b.buffer), vcat(a.offsets, b.offsets .+ length(a.buffer)), vcat(a.lengths, b.lengths))
end

function Base.append!(a::StringVector{T}, b::StringVector) where T
    append!(a.offsets, b.offsets .+ length(a.buffer))
    append!(a.buffer, b.buffer)
    append!(a.lengths, b.lengths)
    a
end

function Base.append!(a::StringVector{T}, b::AbstractVector) where T
    for x in b
        push!(a, x)
    end
    a
end

const _default_splice = []

function Base.splice!(a::StringVector, i::Integer, ins=_default_splice)
    v = a[i]
    m = length(ins)
    if m == 0
        deleteat!(a, i)
    elseif m == 1
        a[i] = ins[1]
    else
        _growat!(a, i, m-1)
        k = 1
        for x in ins
            a[i+k-1] = x
            k += 1
        end
    end
    return v
end

function Base.splice!(a::StringVector, r::UnitRange{<:Integer}, ins=_default_splice)
    v = a[r]
    m = length(ins)
    if m == 0
        deleteat!(a, r)
        return v
    end

    n = length(a)
    f = first(r)
    l = last(r)
    d = length(r)

    if m < d
        delta = d - m
        _deleteat!(a, (f - 1 < n - l) ? f : (l - delta + 1), delta)
    elseif m > d
        _growat!(a, (f - 1 < n - l) ? f : (l + 1), m - d)
    end

    k = 1
    for x in ins
        a[f+k-1] = x
        k += 1
    end
    return v
end

function _growbeg!(a::StringVector, n)
    Base._growbeg!(a.offsets, n)
    Base._growbeg!(a.lengths, n)
    return
end

function Base.prepend!(a::StringVector, items::AbstractVector)
    itemindices = eachindex(items)
    n = length(itemindices)
    _growbeg!(a, n)
    if a === items
        copyto!(a, 1, items, n+1, n)
    else
        copyto!(a, 1, items, first(itemindices), n)
    end
    return a
end

Base.prepend!(a::StringVector, iter) = _prepend!(a, Base.IteratorSize(iter), iter)
Base.pushfirst!(a::StringVector, iter...) = prepend!(a, iter)

function _prepend!(a, ::Union{Base.HasLength,Base.HasShape}, iter)
    n = length(iter)
    _growbeg!(a, n)
    i = 0
    for item in iter
        @inbounds a[i += 1] = item
    end
    a
end

function _prepend!(a, ::Base.IteratorSize, iter)
    n = 0
    for item in iter
        n += 1
        pushfirst!(a, item)
    end
    reverse!(a, 1, n)
    a
end

function Base.pop!(a::StringVector)
    if isempty(a)
        throw(ArgumentError("array must be non-empty"))
    end
    item = a[end]
    deleteat!(a, length(a))
    return item
end

function Base.pushfirst!(a::StringVector, item)
    _growbeg!(a, 1)
    a[1] = item
    return a
end

function Base.popfirst!(a::StringVector)
    if isempty(a)
        throw(ArgumentError("array must be non-empty"))
    end
    item = a[1]
    deleteat!(a, 1)
    return item
end

=#