text:
using Test, Tables, TableTraits, DataValues, QueryOperators, IteratorInterfaceExtensions, SparseArrays

@testset "utils.jl" begin

    @test getproperty((1, 2), 1) == 1

    NT = NamedTuple{(), Tuple{}}
    @test Tables.names(NT) === ()
    @test Tables.types(NT) === Tuple{}
    @test isempty(Tables.runlength(Tables.types(NT)))
    @test Tables.columnindex(Tables.names(NT), :i) == 0
    @test Tables.columntype(Tables.names(NT), Tables.types(NT), :i) == Union{}

    NT = NamedTuple{(:a, :b, :c), NTuple{3, Int64}}
    @test Tables.names(NT) === (:a, :b, :c)
    @test Tables.types(NT) === Tuple{Int64, Int64, Int64}
    @test Tables.runlength(Tables.types(NT)) == [(Int64, 3)]
    @test Tables.columnindex(Tables.names(NT), :a) == 1
    @test Tables.columnindex(Tables.names(NT), :i) == 0
    @test Tables.columntype(Tables.names(NT), Tables.types(NT), :a) == Int64
    @test Tables.columntype(Tables.names(NT), Tables.types(NT), :i) == Union{}

    NT = NamedTuple{Tuple(Symbol("a$i") for i = 1:20), Tuple{vcat(fill(Int, 10), fill(String, 10))...}}
    @test Tables.names(NT) === Tuple(Symbol("a$i") for i = 1:20)
    @test Tables.types(NT) === Tuple{vcat(fill(Int, 10), fill(String, 10))...}
    @test Tables.runlength(Tables.types(NT)) == [(Int, 10), (String, 10)]
    @test Tables.columnindex(Tables.names(NT), :a20) == 20
    @test Tables.columnindex(Tables.names(NT), :i) == 0
    @test Tables.columntype(Tables.names(NT), Tables.types(NT), :a20) == String
    @test Tables.columntype(Tables.names(NT), Tables.types(NT), :i) == Union{}

    nt = (a=1, b=2, c=3)
    NT = typeof(nt)
    output = [0, 0, 0]
    Tables.eachcolumn(Tables.Schema(Tables.names(NT), Tables.types(NT)), nt) do val, col, nm
        output[col] = val
    end
    @test output == [1, 2, 3]

    nt = NamedTuple{Tuple(Symbol("a$i") for i = 1:101)}(Tuple(i for i = 1:101))
    NT = typeof(nt)
    @test Tables.runlength(Tables.types(NT)) == [(Int, 101)]
    output = zeros(Int, 101)
    Tables.eachcolumn(Tables.Schema(Tables.names(NT), Tables.types(NT)), nt) do val, col, nm
        output[col] = val
    end
    @test output == [i for i = 1:101]

    nt = NamedTuple{Tuple(Symbol("a$i") for i = 1:101)}(Tuple(i % 2 == 0 ? i : "$i" for i = 1:101))
    NT = typeof(nt)
    @test Tables.runlength(Tables.types(NT)) == [i % 2 == 0 ? (Int, 1) : (String, 1) for i = 1:101]
    output = Vector{Any}(undef, 101)
    Tables.eachcolumn(Tables.Schema(Tables.names(NT), Tables.types(NT)), nt) do val, col, nm
        output[col] = val
    end
    @test output == [i % 2 == 0 ? i : "$i" for i = 1:101]

    nt = (a=Ref(0), b=Ref(0))
    Tables.eachcolumn(Tables.Schema((:a, :b), nothing), (a=1, b=2)) do val, col, nm
        nt[nm][] = val
    end
    @test nt.a[] == 1
    @test nt.b[] == 2

    nt = (a=[1,2,3], b=[4,5,6])
    @test Tables.columnindex(nt, :i) == 0
    @test Tables.columnindex(nt, :a) == 1
    @test Tables.columntype(nt, :a) == Int
    @test Tables.columntype(nt, :i) == Union{}

    rows = Tables.rows(nt)
    @test eltype(rows) == Tables.ColumnsRow{typeof(nt)}
    @test Tables.schema(rows) == Tables.Schema((:a, :b), (Int, Int))
    @test rows.a == [1,2,3]
    @test propertynames(rows) == Tables.columnnames(rows) == (:a, :b)
    row = first(rows)
    @test row.a == 1
    @test Tables.getcolumn(row, :a) == 1
    @test Tables.getcolumn(row, 1) == 1
    @test Tables.istable(rows)
    @test Tables.rowaccess(rows)
    @test Tables.rows(rows) === rows
    @test Tables.columnaccess(rows)
    @test Tables.columns(rows) === nt
    @test Tables.materializer(rows) === Tables.materializer(nt)

    @test Tables.rowmerge(row; b=200, hey="hello") == (a=1, b=200, hey="hello")
    @test Tables.rowmerge(row, (hey="hello", a=200)) == (a=200, b=4, hey="hello")
    @test Tables.rowmerge(row, (hey="hello", a=200), row, (x=:x, y=:y, hey="bye")) == (a=1, b=4, hey="bye", x=:x, y=:y)

    @test Tables.sym(1) === 1
    @test Tables.sym("hey") == :hey

    @test propertynames(Tables.Schema((:a, :b), nothing)) == (:names, :types)

    v = Tables.EmptyVector(1)
    @test_throws UndefRefError v[1]
    @test Base.IndexStyle(typeof(v)) == Base.IndexLinear()

    @test Tables.istable(Tables.CopiedColumns)
    @test Tables.columnaccess(Tables.CopiedColumns)
    c = Tables.CopiedColumns(nt)
    @test Tables.columns(c) === c
    @test Tables.materializer(c) == Tables.materializer(nt)
    @test Tables.getcolumn(c, :a) == [1,2,3]
    @test Tables.getcolumn(c, 1) == [1,2,3]

    @test_throws ArgumentError Tables.columntable([1,2,3])

    tt = [(1,2,3), (4,5,6)]
    r = Tables.nondatavaluerows(tt)
    row = first(r)
    @test getproperty(row, 1) == 1
    @test Tables.columntable(tt) == NamedTuple{(Symbol("1"), Symbol("2"), Symbol("3"))}(([1, 4], [2, 5], [3, 6]))

    @test Tables.getarray([1,2,3]) == [1,2,3]
    @test Tables.getarray((1,2,3)) == [1,2,3]

    # empty schema
    sch = Tables.Schema((), ())
    @test sch.names == ()
    @test sch.types == ()
end

@testset "namedtuples.jl" begin

    nt = (a=1, b=2, c=3)
    rt = [nt, nt, nt]
    @test Tables.rows(rt) === rt
    @test Tables.schema(rt).names == Tables.names(typeof(nt))
    @test Tables.namedtupleiterator(eltype(rt), rt) === rt

    rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")]
    nt = (a=[1,2,3], b=[4.0, 5.0, 6.0], c=["7", "8", "9"])
    @test Tables.rowcount(nt) == 3
    @test Tables.schema(nt) == Tables.Schema((:a, :b, :c), Tuple{Int, Float64, String})
    @test Tables.istable(typeof(nt))
    @test Tables.columnaccess(typeof(nt))
    @test Tables.columns(nt) === nt
    @test rowtable(nt) == rt
    @test columntable(rt) == nt
    @test rt == (rt |> columntable |> rowtable)
    @test nt == (nt |> rowtable |> columntable)

    @test Tables.buildcolumns(nothing, rt) == nt
    @test Tables.columntable(nothing, nt) == nt

    # test push!
    rtf = Iterators.Filter(x->x.a >= 1, rt)
    @test Tables.columntable(rtf) == nt
    @test Tables.buildcolumns(nothing, rtf) == nt

    rt = [(a=1, b=4.0, c="7"), (a=2.0, b=missing, c="8"), (a=3, b=6.0, c="9")]
    @test Tables.istable(typeof(rt))
    @test Tables.rowaccess(typeof(rt))
    tt = Tables.buildcolumns(nothing, rt)
    @test isequal(tt, (a = [1.0, 2.0, 3.0], b = Union{Missing, Float64}[4.0, missing, 6.0], c = ["7", "8", "9"]))
    @test tt.a[1] === 1.0
    @test tt.a[2] === 2.0
    @test tt.a[3] === 3.0

    nti = Tables.NamedTupleIterator{Nothing, typeof(rt)}(rt)
    @test Base.IteratorEltype(typeof(nti)) == Base.EltypeUnknown()
    @test Base.IteratorSize(typeof(nti)) == Base.HasShape{1}()
    @test length(nti) == 3
    nti2 = collect(nti)
    @test isequal(rt, nti2)
    nti = Tables.NamedTupleIterator{typeof(Tables.Schema((:a, :b, :c), (Union{Int, Float64}, Union{Float64, Missing}, String))), typeof(rt)}(rt)
    @test eltype(typeof(nti)) == NamedTuple{(:a, :b, :c),Tuple{Union{Float64, Int},Union{Missing, Float64},String}}

    # test really wide tables
    nms = Tuple(Symbol("i", i) for i = 1:101)
    vals = Tuple(rand(Int, 3) for i = 1:101)
    nt = NamedTuple{nms}(vals)
    rt = Tables.rowtable(nt)
    @test length(rt) == 3
    @test length(rt[1]) == 101
    @test eltype(rt).parameters[1] == nms
    @test Tables.columntable(rt) == nt
    @test Tables.buildcolumns(nothing, rt) == nt

    # 228
    @test Tables.columntable(NamedTuple[]) === NamedTuple()
end

@testset "Materializer" begin
    rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")]
    nt = (a=[1,2,3], b=[4.0, 5.0, 6.0], c=["7", "8", "9"])

    @test nt == Tables.materializer(nt)(Tables.columns(nt))
    @test nt == Tables.materializer(nt)(Tables.columns(rt))
    @test nt == Tables.materializer(nt)(rt)
    @test rt == Tables.materializer(rt)(nt)

    function select(table, cols::Symbol...)
        Tables.istable(table) || throw(ArgumentError("select requires a table input"))
        nt = Tables.columntable(table)  # columntable(t) creates a NamedTuple of AbstractVectors
        newcols = NamedTuple{cols}(nt)
        Tables.materializer(table)(newcols)
    end

    @test select(nt, :a, :b, :c) == nt
    @test select(nt, :c, :a) == NamedTuple{(:c, :a)}(nt)
    @test select(rt, :a) == [(a=1,), (a=2,), (a=3,)]

    @test Tables.materializer(1) === Tables.columntable
end

@testset "Matrix integration" begin
    rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")]
    nt = (a=[1,2,3], b=[4.0, 5.0, 6.0])

    mat = Tables.matrix(rt)
    @test nt.a == mat[:, 1]
    @test size(mat) == (3, 3)
    @test eltype(mat) == Any
    @test_throws ArgumentError Tables.rows(mat)
    @test_throws ArgumentError Tables.columns(mat)
    mat2 = Tables.matrix(nt)
    @test eltype(mat2) == Float64
    @test mat2[:, 1] == nt.a
    @test !Tables.istable(mat2)
    @test !Tables.istable(typeof(mat2))
    mat3 = Tables.matrix(nt; transpose=true)
    @test size(mat3) == (2, 3)
    @test mat3[1, :] == nt.a
    @test mat3[2, :] == nt.b
    sp = Tables.table(sparse(mat[:, 1:2]))
    @test Tables.columnnames(sp) == [:Column1, :Column2]
    @test Tables.getcolumn(sp, 1) == [1, 2, 3]

    tbl = Tables.table(mat) |> columntable
    @test keys(tbl) == (:Column1, :Column2, :Column3)
    @test tbl.Column1 == [1, 2, 3]
    tbl2 = Tables.table(mat2) |> rowtable
    @test length(tbl2) == 3
    @test map(x->x.Column1, tbl2) == [1.0, 2.0, 3.0]

    mattbl = Tables.table(mat)
    @test Tables.istable(typeof(mattbl))
    @test Tables.rowaccess(typeof(mattbl))
    @test Tables.rows(mattbl) === mattbl
    @test Tables.columnaccess(typeof(mattbl))
    @test Tables.columns(mattbl) === mattbl
    @test mattbl.Column1 == [1,2,3]
    @test Tables.getcolumn(mattbl, :Column1) == [1,2,3]
    @test Tables.getcolumn(mattbl, 1) == [1,2,3]
    matrow = first(mattbl)
    @test eltype(mattbl) == typeof(matrow)
    @test matrow.Column1 == 1
    @test Tables.getcolumn(matrow, :Column1) == 1
    @test Tables.getcolumn(matrow, 1) == 1
    @test propertynames(mattbl) == propertynames(matrow) == [:Column1, :Column2, :Column3]

    mattbl = Tables.table(mat, header=[:A, :B, :C])
    @test Tables.columnnames(mattbl) == [:A, :B, :C]
    mattbl = Tables.table(mat, header=view(["DUMMY", "A", "B", "C"], 2:4))
    @test Tables.columnnames(mattbl) == [:A, :B, :C]
    mattbl = Tables.table(mat, header=(["DUMMY", "A", "B", "C"][i] for i in 2:4))
    @test Tables.columnnames(mattbl) == [:A, :B, :C]
    mattbl = Tables.table(mat, header=1:3)
    @test Tables.columnnames(mattbl) == Symbol.(1:3)
    @test_throws ArgumentError Tables.table(mat, header=[:A, :B, :C, :D])

    X = [1 2; 3 4; 5 6]
    tbl = Tables.table(X)
    tbl[1][1] = 100
    @test X[1, 1] == 100
    tbl[2][3] = 200
    @test X[3, 2] == 200

    # #155
    m = hcat([1,2,3],[1,2,3])
    T = Tables.table(m)
    M = Tables.matrix(T)
    Mt = Tables.matrix(T, transpose=true)
    @test M[:, 1] == [1, 2, 3]
    # 182
    @test M === m #checks that both are the same object in memory
    @test Mt == permutedims(m)
    # 167
    @test !Tables.istable(Matrix{Union{}}(undef, 2, 3))

    # Tables.Columns roundtrip
    X = [1 0 0; 0 1 0; 0 0 1]
    X′ = Tables.matrix(Tables.Columns(Tables.table(X)))
    @test X′ == X
    # test for presence of superfluous wrapping
    @test typeof(X′) == typeof(X)

    # Treating vectors as 1-column tables:
    x = [1,2,3,4,5]
    tbl = Tables.Columns(Tables.table(x))
    @test keys(tbl) == [:Column1]
    @test keys(Tables.Columns(Tables.table(x, header=["col1"]))) == [:col1]
    x′ = Tables.matrix(tbl)
    @test x′ == reshape(x, :, 1) == Tables.matrix(Tables.table(x))
end

import Base: ==
struct GenericRow
    a::Int
    b::Float64
    c::String
end
==(a::GenericRow, b::GenericRow) = a.a == b.a && a.b == b.b && a.c == b.c

struct GenericRowTable
    data::Vector{GenericRow}
end
==(a::GenericRowTable, b::GenericRowTable) = all(a.data .== b.data)

Base.eltype(g::GenericRowTable) = GenericRow
Base.length(g::GenericRowTable) = length(g.data)
Base.size(g::GenericRowTable) = (length(g.data),)
Tables.istable(::Type{GenericRowTable}) = true
Tables.rowaccess(::Type{GenericRowTable}) = true
Tables.rows(x::GenericRowTable) = x
Tables.schema(x::GenericRowTable) = Tables.Schema((:a, :b, :c), Tuple{Int, Float64, String})

function Base.iterate(g::GenericRowTable, st=1)
    st > length(g.data) && return nothing
    return g.data[st], st + 1
end

genericrowtable(x) = GenericRowTable(collect(map(x->GenericRow(x.a, x.b, x.c), Tables.rows(x))))

struct GenericColumn{T} <: AbstractVector{T}
    data::Vector{T}
end
Base.eltype(g::GenericColumn{T}) where {T} = T
Base.length(g::GenericColumn) = length(g.data)
==(a::GenericColumn, b::GenericColumn) = a.data == b.data
Base.getindex(g::GenericColumn, i::Int) = g.data[i]

struct GenericColumnTable
    names::Dict{Symbol, Int}
    data::Vector{GenericColumn}
end

Tables.istable(::Type{GenericColumnTable}) = true
Tables.columnaccess(::Type{GenericColumnTable}) = true
Tables.columns(x::GenericColumnTable) = x
Tables.schema(g::GenericColumnTable) = Tables.Schema(Tuple(keys(getfield(g, 1))), Tuple{(eltype(x) for x in getfield(g, 2))...})
Base.getproperty(g::GenericColumnTable, nm::Symbol) = getfield(g, 2)[getfield(g, 1)[nm]]
Base.propertynames(g::GenericColumnTable) = Tuple(keys(getfield(g, 1)))

function genericcolumntable(x)
    cols = Tables.columns(x)
    sch = Tables.schema(x)
    data = [GenericColumn(getproperty(cols, nm)) for nm in sch.names]
    return GenericColumnTable(Dict(nm=>i for (i, nm) in enumerate(sch.names)), data)
end
==(a::GenericColumnTable, b::GenericColumnTable) = getfield(a, 1) == getfield(b, 1) && getfield(a, 2) == getfield(b, 2)

@testset "Tables.jl interface" begin

    @test !Tables.istable(1)
    @test !Tables.istable(Int)
    @test !Tables.rowaccess(1)
    @test !Tables.rowaccess(Int)
    @test !Tables.columnaccess(1)
    @test !Tables.columnaccess(Int)
    @test Tables.schema(1) === nothing

    sch = Tables.Schema{(:a, :b), Tuple{Int64, Float64}}()
    @test Tables.Schema((:a, :b), Tuple{Int64, Float64}) === sch
    @test Tables.Schema(NamedTuple{(:a, :b), Tuple{Int64, Float64}}) === sch
    @test Tables.Schema((:a, :b), nothing) === Tables.Schema{(:a, :b), nothing}()
    @test Tables.Schema([:a, :b], [Int64, Float64]) === sch
    show(sch)
    @test sch.names == (:a, :b)
    @test sch.types == (Int64, Float64)
    @test_throws ArgumentError sch.foobar

    gr = GenericRowTable([GenericRow(1, 4.0, "7"), GenericRow(2, 5.0, "8"), GenericRow(3, 6.0, "9")])
    gc = GenericColumnTable(Dict(:a=>1, :b=>2, :c=>3), [GenericColumn([1,2,3]), GenericColumn([4.0, 5.0, 6.0]), GenericColumn(["7", "8", "9"])])
    @test gc == (gr |> genericcolumntable)
    @test gr == (gc |> genericrowtable)
    @test gr == (gr |> genericrowtable)

    @test_throws ArgumentError Tables.columns(Int64)
    @test_throws ArgumentError Tables.rows(Int64)

    # 204
    rows = NamedTuple{(:a, :b), Tuple{Int64, Float64}}[]
    t = Tables.columns(rows)
    @test Tables.schema(t) == Tables.Schema((:a, :b), (Int64, Float64))
    rows = GenericRow[]
    t = Tables.columns(rows)
    @test Tables.schema(t) == Tables.Schema((:a, :b, :c), (Int, Float64, String))
    rows = Tuple{Int64, Float64, String}[]
    t = Tables.columns(rows)
    @test Tables.schema(t) == Tables.Schema((:Column1, :Column2, :Column3), (Int64, Float64, String))
end

@testset "isless" begin
    t = (x = [1, 1, 0, 2], y = [-1, 1, 3, 2])
    a,b,c,d = Tables.rows(t)
    @test isless(a, b)
    @test isless(c, d)
    @test !isless(d, a)
    @test !isequal(a, b)
    @test isequal(a, a)
    @test sortperm([a, b, c, d]) == [3, 1, 2, 4]
end

struct ColumnSource
end

TableTraits.supports_get_columns_copy_using_missing(::ColumnSource) = true

function TableTraits.get_columns_copy_using_missing(x::ColumnSource)
    return (a=[1,2,3], b=[4.,5.,6.], c=["A", "B", "C"])
end

let x=ColumnSource()
    @test Tables.source(Tables.columns(x)) == Tables.source(Tables.CopiedColumns(TableTraits.get_columns_copy_using_missing(x)))
end

struct ColumnSource2
end

IteratorInterfaceExtensions.isiterable(x::ColumnSource2) = true
TableTraits.isiterabletable(::ColumnSource2) = true

IteratorInterfaceExtensions.getiterator(::ColumnSource2) =
    Tables.rows((a=[1,2,3], b=[4.,5.,6.], c=["A", "B", "C"]))

let x=ColumnSource2()
    @test Tables.source(Tables.columns(x)) == (a=[1,2,3], b=[4.,5.,6.], c=["A", "B", "C"])
end

@testset "TableTraits integration" begin
    rt = (a = Real[1, 2.0, 3], b = Union{Missing, Float64}[4.0, missing, 6.0], c = ["7", "8", "9"])

    dv = Tables.datavaluerows(rt)
    @test Base.IteratorSize(typeof(dv)) == Base.HasLength()
    @test eltype(dv) == NamedTuple{(:a, :b, :c),Tuple{Real,DataValue{Float64},String}}
    @test_throws MethodError size(dv)
    rt2 = collect(dv)
    @test rt2[1] == (a = 1, b = DataValue{Float64}(4.0), c = "7")

    ei = Tables.nondatavaluerows(QueryOperators.EnumerableIterable{eltype(dv), typeof(dv)}(dv))
    @test Tables.istable(typeof(ei))
    @test Tables.rowaccess(typeof(ei))
    @test Tables.rows(ei) === ei
    @test Base.IteratorEltype(typeof(ei)) == Base.HasEltype()
    @test Base.IteratorSize(typeof(ei)) == Base.HasLength()
    @test eltype(ei) == Tables.IteratorRow{NamedTuple{(:a, :b, :c),Tuple{Real,DataValue{Float64},String}}}
    @test eltype(typeof(ei)) == Tables.IteratorRow{NamedTuple{(:a, :b, :c),Tuple{Real,DataValue{Float64},String}}}
    @test_throws MethodError size(ei)
    nt = ei |> columntable
    @test isequal(rt, nt)
    rt3 = ei |> rowtable
    @test isequal(rt |> rowtable, rt3)

    # rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")]
    mt = Tables.nondatavaluerows(ei.x |> y->QueryOperators.map(y, x->(a=x.b, c=x.c), Expr(:block)))
    @inferred (mt |> columntable)
    @inferred (mt |> rowtable)

    # uninferrable case
    mt = Tables.nondatavaluerows(ei.x |> y->QueryOperators.map(y, x->(a=x.a, c=x.c), Expr(:block)))
    @test (mt |> columntable) == (a = Real[1, 2.0, 3], c = ["7", "8", "9"])
    @test length(mt |> rowtable) == 3

    rt = (a = Missing[missing, missing], b=[1,2])
    dv = Tables.datavaluerows(rt)
    @test eltype(dv) == NamedTuple{(:a, :b), Tuple{DataValue{Union{}}, Int}}

    # DataValue{Any}
    @test isequal(Tables.columntable(Tables.nondatavaluerows([(a=DataValue{Any}(), b=DataValue{Int}())])), (a = Any[missing], b = Union{Missing, Int64}[missing]))
end

@testset "AbstractDict" begin

    d = Dict(:a => 1, :b => missing, :c => "7")
    n = (a=1, b=missing, c="7")
    drt = [d, d, d]
    rt = [n, n, n]
    dct = Dict(:a => [1, 1, 1], :b => [missing, missing, missing], :c => ["7", "7", "7"])
    ct = (a = [1, 1, 1], b = [missing, missing, missing], c = ["7", "7", "7"])
    @test Tables.istable(drt)
    @test Tables.rowaccess(drt)
    @test Tables.rows(drt) === drt
    @test Tables.schema(drt) === nothing
    @test isequal(Tables.rowtable(drt), rt)
    @test isequal(Tables.columntable(drt), ct)

    @test Tables.istable(dct)
    @test Tables.columnaccess(dct)
    @test Tables.columns(dct) === dct
    @test Tables.schema(dct) == Tables.Schema((:a, :b, :c), Tuple{Int, Missing, String})
    @test isequal(Tables.rowtable(dct), rt)
    @test isequal(Tables.columntable(dct), ct)

    # a Dict w/ scalar values isn't a table
    @test_throws Exception Tables.columns(d)
end

struct Row <: Tables.AbstractRow
    a::Int
    b::Union{Float64, Missing}
    c::String
end

Tables.getcolumn(r::Row, i::Int) = getfield(r, i)
Tables.getcolumn(r::Row, nm::Symbol) = getfield(r, nm)
Tables.getcolumn(r::Row, ::Type{T}, i::Int, nm::Symbol) where {T} = getfield(r, i)
Tables.columnnames(r::Row) = fieldnames(Row)

struct DummyRow <: Tables.AbstractRow end

@testset "AbstractRow" begin

    row = Row(1, missing, "hey")
    row2 = Row(2, 3.14, "ho")

    @test Base.IteratorSize(typeof(row)) == Base.HasLength()
    @test length(row) == 3
    @test firstindex(row) == 1
    @test lastindex(row) == 3
    @test isequal((row[1], row[2], row[3]), (1, missing, "hey"))
    @test isequal((row[:a], row[:b], row[:c]), (1, missing, "hey"))
    @test isequal((row.a, row.b, row.c), (1, missing, "hey"))
    @test isequal((getproperty(row, 1), getproperty(row, 2), getproperty(row, 3)), (1, missing, "hey"))
    @test propertynames(row) == (:a, :b, :c)
    @test keys(row) == (:a, :b, :c)
    @test isequal(values(row), [1, missing, "hey"])
    @test haskey(row, :a)
    @test haskey(row, 1)
    @test get(row, 1, 0) == get(row, :a, 0) == 1
    @test get(() -> 0, row, 1) == get(() -> 0, row, :a) == 1
    @test isequal(collect(row), [1, missing, "hey"])
    @test !isempty(row)
    @test isequal(NamedTuple(row), (a=1, b=missing, c="hey"))
    show(row)

    art = [row, row2]
    ct = (a=[1, 2], b=[missing, 3.14], c=["hey", "ho"])
    @test Tables.istable(art)
    @test Tables.rowaccess(art)
    @test Tables.rows(art) === art
    @test Tables.schema(art) === nothing
    @test isequal(Tables.columntable(art), ct)

    r = DummyRow()
    @test_throws ErrorException("`Tables.getcolumn` must be specifically overloaded for DummyRow <: Union{AbstractRow, AbstractColumns}`") Tables.getcolumn(r, 1)
    @test_throws ErrorException("`Tables.getcolumn` must be specifically overloaded for DummyRow <: Union{AbstractRow, AbstractColumns}`") Tables.getcolumn(r, :a)
    @test_throws ErrorException("`Tables.columnnames` must be specifically overloaded for DummyRow <: Union{AbstractRow, AbstractColumns}`") Tables.columnnames(r)
end

struct TestColumns <: Tables.AbstractColumns
    a::Vector{Int}
    b::Vector{Union{Float64, Missing}}
    c::Vector{String}
end

Tables.getcolumn(r::TestColumns, i::Int) = getfield(r, i)
Tables.getcolumn(r::TestColumns, nm::Symbol) = getfield(r, nm)
Tables.getcolumn(r::TestColumns, ::Type{T}, i::Int, nm::Symbol) where {T} = getfield(r, i)
Tables.columnnames(r::TestColumns) = fieldnames(TestColumns)

struct DummyCols <: Tables.AbstractColumns end

@testset "AbstractColumns" begin

    col = TestColumns([1, 2], [missing, 3.14], ["hey", "ho"])

    @test Base.IteratorSize(typeof(col)) == Base.HasLength()
    @test length(col) == 3
    @test firstindex(col) == 1
    @test lastindex(col) == 3
    @test isequal((col[1], col[2], col[3]), ([1,2], [missing,3.14], ["hey","ho"]))
    @test isequal((col[:a], col[:b], col[:c]), ([1,2], [missing,3.14], ["hey","ho"]))
    @test isequal((col.a, col.b, col.c), ([1,2], [missing,3.14], ["hey","ho"]))
    @test isequal((getproperty(col, 1), getproperty(col, 2), getproperty(col, 3)), ([1,2], [missing,3.14], ["hey","ho"]))
    @test propertynames(col) == (:a, :b, :c)
    @test keys(col) == (:a, :b, :c)
    @test isequal(values(col), [[1,2], [missing,3.14], ["hey","ho"]])
    @test haskey(col, :a)
    @test haskey(col, 1)
    @test get(col, 1, 0) == get(col, :a, 0) == [1,2]
    @test get(() -> 0, col, 1) == get(() -> 0, col, :a) == [1,2]
    @test isequal(collect(col), [[1,2], [missing,3.14], ["hey","ho"]])
    show(col)

    tbl = TestColumns(rand(Int, 100), rand(100), [ "a" for _ = 1:100])
    @test length(sprint(show, tbl)) < 500

    ct = (a=[1, 2], b=[missing, 3.14], c=["hey", "ho"])
    @test Tables.istable(col)
    @test Tables.columnaccess(col)
    @test Tables.columns(col) === col
    @test Tables.schema(col) === nothing
    @test isequal(Tables.columntable(col), ct)

    c = DummyCols()
    @test_throws ErrorException("`Tables.getcolumn` must be specifically overloaded for DummyCols <: Union{AbstractRow, AbstractColumns}`") Tables.getcolumn(c, 1)
    @test_throws ErrorException("`Tables.getcolumn` must be specifically overloaded for DummyCols <: Union{AbstractRow, AbstractColumns}`") Tables.getcolumn(c, :a)
    @test_throws ErrorException("`Tables.columnnames` must be specifically overloaded for DummyCols <: Union{AbstractRow, AbstractColumns}`") Tables.columnnames(c)
end

@testset "Tables.Columns" begin
    X = (A=[1,2], B=[im, -im], C=["kirk", "spock"])

    Xc = Tables.Columns(X)
    @test Tables.schema(X) == Tables.schema(Xc)
    for i ∈ 1:3
        @test Tables.getcolumn(X, i) == Tables.getcolumn(Xc, i)
    end
    for i ∈ (:A, :B, :C)
        @test Tables.getcolumn(X, i) == Tables.getcolumn(Xc, i)
    end
    @test Tables.columnnames(X) == Tables.columnnames(Xc)

    tbl = Tables.Columns((; a=rand(Int, 100), b=rand(100), c=[ "a" for _ = 1:100]))
    @test length(sprint(show, tbl)) < 500
end

struct IsRowTable
    rows::Vector{NamedTuple}
end

Base.iterate(x::IsRowTable) = iterate(x.rows)
Base.iterate(x::IsRowTable, st) = iterate(x.rows, st)
Base.length(x::IsRowTable) = length(x.rows)

Tables.isrowtable(::Type{IsRowTable}) = true

@testset "Tables.isrowtable" begin

    nt = (a=1, b=3.14, c="hey")
    rt = IsRowTable([nt, nt, nt])
    @test Tables.istable(rt)
    @test Tables.rowaccess(rt)
    @test Tables.rows(rt) === rt
    @test Tables.columntable(rt) == Tables.columntable([nt, nt, nt])

end

@testset "Tables.partitions" begin

    ct = (a=[1, 2], b=[missing, 3.14], c=["hey", "ho"])

    for tbl in Tables.partitions(ct)
        @test tbl === ct
    end

    parts = [ct, ct]
    for tbl in Tables.partitions(Tables.partitioner(parts))
        @test tbl === ct
    end

    for tbl in Tables.partitions(Tables.partitioner(x -> x, parts))
        @test Tables.columns(tbl) === ct
    end

end

@testset "Dict tables" begin

    rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")]
    ct = (a=[1, 2, 3], b=[4.0, 5.0, 6.0], c=["7", "8", "9"])

    dct = Tables.dictcolumntable(rt)
    @test Tables.istable(dct)
    @test Tables.columnaccess(dct)
    @test Tables.columns(dct) === dct
    @test Tables.schema(dct) == Tables.Schema((:a, :b, :c), (Int, Float64, String))
    @test dct.a == [1, 2, 3]
    @test dct.b == [4.0, 5.0, 6.0]
    @test dct.c == ["7", "8", "9"]

    dct = Tables.dictcolumntable(ct)
    @test dct.a == [1, 2, 3]
    @test dct.b == [4.0, 5.0, 6.0]
    @test dct.c == ["7", "8", "9"]
    cct = Tables.columntable(dct)
    @test cct.a == [1, 2, 3]
    @test cct.b == [4.0, 5.0, 6.0]
    @test cct.c == ["7", "8", "9"]

    drt = Tables.dictrowtable(rt)
    @test Tables.isrowtable(drt)
    @test Tables.schema(drt) == Tables.Schema((:a, :b, :c), (Int, Float64, String))
    row = first(drt)
    @test (row.a, row.b, row.c) == (1, 4.0, "7")
    cct = Tables.columntable(drt)
    @test cct.a == [1, 2, 3]
    @test cct.b == [4.0, 5.0, 6.0]
    @test cct.c == ["7", "8", "9"]

    rt = [
        (a=1, b=2, c=3),
        (b=4.0, c=missing, d=5),
        (a=6, d=7),
        (a=8, b=9, c=10, d=missing),
        (d=11, c=10, b=9, a=8)
    ]

    drt = Tables.dictrowtable(rt)
    @test Tables.schema(drt) == Tables.Schema((:a, :b, :c, :d), (Union{Missing, Int}, Union{Missing, Float64, Int}, Union{Missing, Int}, Union{Missing, Int}))
    @test length(drt) == 5
    ct = Tables.columntable(drt)
    @test isequal(ct.a, [1, missing, 6, 8, 8])
    @test isequal(ct.b, Union{Int, Float64, Missing}[2, 4.0, missing, 9, 9])
    @test isequal(ct.c, [3, missing, missing, 10, 10])
    @test isequal(ct.d, [missing, 5, 7, missing, 11])

    dct = Tables.dictcolumntable(rt)
    @test isequal(dct.a, [1, missing, 6, 8, 8])
    @test isequal(ct.b, Union{Int, Float64, Missing}[2, 4.0, missing, 9, 9])
    @test isequal(ct.c, [3, missing, missing, 10, 10])
    @test isequal(dct.d, [missing, 5, 7, missing, 11])

end

# extremely wide tables
struct WideTable <: Tables.AbstractColumns
end

Tables.istable(::Type{WideTable}) = true
Tables.columnaccess(::Type{WideTable}) = true
Tables.columns(x::WideTable) = x
Tables.schema(::WideTable) = Tables.Schema([Symbol("x", i) for i = 1:(Tables.SCHEMA_SPECIALIZATION_THRESHOLD + 1)], [Float64 for _ = 1:(Tables.SCHEMA_SPECIALIZATION_THRESHOLD + 1)])
Tables.getcolumn(g::WideTable, nm::Symbol) = rand(100)
Tables.getcolumn(g::WideTable, i::Int) = rand(100)
Base.getindex(::WideTable, i::Int) = rand(100)
Tables.columnnames(::WideTable) = [Symbol("x", i) for i = 1:(Tables.SCHEMA_SPECIALIZATION_THRESHOLD + 1)]

struct WideTable2 <: Tables.AbstractColumns
end

Tables.istable(::Type{WideTable2}) = true
Tables.columnaccess(::Type{WideTable2}) = true
Tables.columns(x::WideTable2) = x
Tables.schema(::WideTable2) = Tables.Schema([Symbol("x", i) for i = 1:1000], [Float64 for _ = 1:1000]; stored=true)
Tables.getcolumn(g::WideTable2, nm::Symbol) = rand(100)
Base.getindex(::WideTable2, i::Int) = rand(100)
Tables.columnnames(::WideTable2) = [Symbol("x", i) for i = 1:1000]

@testset "wide tables" begin
    x = WideTable();
    sch = Tables.schema(x)
    @test sch.names == [Symbol("x", i) for i = 1:(Tables.SCHEMA_SPECIALIZATION_THRESHOLD + 1)]
    @test sch.types == [Float64 for _ = 1:(Tables.SCHEMA_SPECIALIZATION_THRESHOLD + 1)]
    @test typeof(sch) == Tables.Schema{nothing, nothing}
    r = Tables.rows(x)
    Tables.eachcolumn(sch, first(r)) do y, i, nm
        @test y isa Float64
        @test i isa Integer
        @test nm isa Symbol
    end
    Tables.eachcolumns(sch, first(r), x) do y, i, nm, col
        @test y isa Float64
        @test i isa Integer
        @test nm isa Symbol
        @test col isa Vector{Float64}
    end
    @test_throws ArgumentError Tables.columntable(x)
    @test_throws ArgumentError Tables.rowtable(x)
    y = Tables.dictrowtable(x);
    @test length(y) == 100
    y = Tables.dictcolumntable(x);
    @test Tables.schema(y) == Tables.schema(x)
    # y = Tables.matrix(x); # works, just takes a really long time and a lot of memory

    x = WideTable2();
    sch = Tables.schema(x)
    @test sch.names == [Symbol("x", i) for i = 1:1000]
    @test sch.types == [Float64 for _ = 1:1000]
    @test typeof(sch) == Tables.Schema{nothing, nothing}
    r = Tables.rows(x)
    Tables.eachcolumn(sch, first(r)) do y, i, nm
        @test y isa Float64
        @test i isa Integer
        @test nm isa Symbol
    end
    Tables.eachcolumns(sch, first(r), x) do y, i, nm, col
        @test y isa Float64
        @test i isa Integer
        @test nm isa Symbol
        @test col isa Vector{Float64}
    end
end


text:
const MASKCOL = ARGB32(0.5)
const ZEROCOL = ARGB32(0.3)

"""
    ImageGenerator

Abstract supertype for objects that convert a frame of the simulation into an `ARGB32`
image for display. Frames may be a single grid or a `NamedTuple` of multiple grids.
"""
abstract type ImageGenerator end

imagesize(p::ImageGenerator, init::NamedTuple) = imagesize(p, first(init))
imagesize(::ImageGenerator, init::AbstractArray) = size(init)

_allocimage(p::ImageGenerator, init) = fill(ARGB32(0), imagesize(p, init)...)

function grid_to_image!(o::ImageOutput, data::SimData)
    grid_to_image!(imagebuffer(o), imagegen(o), o, data, grids(data))
end

"""
    SingleGridImageGenerator <: ImageGenerator

Abstract supertype for [`ImageGenerator`](@ref)s that convert a single grid
into an image array.

The first grid will be displayed if a `SingleGridImageGenerator` is
used with a `NamedTuple` of grids.
"""
abstract type SingleGridImageGenerator <: ImageGenerator end

function grid_to_image!(
    imagebuffer, p::SingleGridImageGenerator, o::ImageOutput, data::SimData,
    grids::NamedTuple;
    name=string(first(keys(grids))), time=currenttime(data)
)
    grid_to_image!(imagebuffer, p, o, data, first(grids); name=name, time=time)
end
function grid_to_image!(
    imagebuffer, p::SingleGridImageGenerator, o::ImageOutput, data::SimData,
    grids::NamedTuple{(:_default_,)};
    name=nothing, time=currenttime(data)
)
    grid_to_image!(imagebuffer, p, o, data, first(grids); name=name, time=time)
end
function grid_to_image!(
    imagebuffer, p::SingleGridImageGenerator, o::ImageOutput,
    data::SimData{Y,X}, grid::AbstractArray;
    name=nothing, time=currenttime(data), minval=minval(o), maxval=maxval(o),
) where {Y,X}
    for j in 1:X, i in 1:Y
        @inbounds val = grid[i, j]
        pixel = to_rgb(cell_to_pixel(p, mask(o), minval, maxval, data, val, (i, j)))
        @inbounds imagebuffer[i, j] = pixel
    end
    _rendertext!(imagebuffer, textconfig(o), name, time)
    return imagebuffer
end

"""
    Image <: SingleGridImageGenerator

    Image(; scheme=ObjectScheme(), zerocolor=nothing, maskcolor=nothing)

Converts output grids to a colorsheme.

# Keywords

- `scheme`: a ColorSchemes.jl colorscheme, [`ObjectScheme`](@ref) or object that defines
    `Base.get(obj, val)` and returns a `Color` or a value that can be converted to `Color`
    using `ARGB32(val)`.
- `zerocolor`: a `Col` to use when values are zero, or `nothing` to ignore.
- `maskcolor`: a `Color` to use when cells are masked, or `nothing` to ignore.
"""
Base.@kwdef struct Image{S,Z,M} <: SingleGridImageGenerator
    scheme::S      = ObjectScheme()
    zerocolor::Z   = ZEROCOL
    maskcolor::M   = MASKCOL
end
Image(scheme, zerocolor=ZEROCOL) = Image(scheme, zerocolor, MASKCOL)

scheme(p::Image) = p.scheme
zerocolor(p::Image) = p.zerocolor
maskcolor(p::Image) = p.maskcolor

# Show colorscheme in Atom etc
Base.show(io::IO, m::MIME"image/svg+xml", p::Image) = show(io, m, scheme(p))

@inline function cell_to_pixel(p::Image, mask, minval, maxval, data::SimData, val, I)
    if !(maskcolor(p) isa Nothing) && ismasked(mask, I...)
        to_rgb(maskcolor(p))
    else
        normval = normalise(val, minval, maxval)
        if !(zerocolor(p) isa Nothing) && normval == zero(typeof(normval))
            to_rgb(zerocolor(p))
        elseif normval isa Number && isnan(normval)
            zerocolor(p) isa Nothing ? to_rgb(scheme(p), 0) : to_rgb(zerocolor(p))
        else
            to_rgb(scheme(p), normval)
        end
    end
end

"""
    SparseOptInspector()

A [`ImageGenerator`](@ref) that checks [`SparseOpt`](@ref) visually.
Cells that do not run show in gray. Errors show in red, but if they do there's a bug.
"""
struct SparseOptInspector <: SingleGridImageGenerator end

function cell_to_pixel(p::SparseOptInspector, mask, minval, maxval, data::SimData, val, I::Tuple)
    opt(data) isa SparseOpt || error("Can only use SparseOptInspector with SparseOpt grids")
    r = radius(first(grids(data)))
    blocksize = 2r
    blockindex = _indtoblock.((I[1] + r,  I[2] + r), blocksize)
    normedval = normalise(val, minval, maxval)
    status = sourcestatus(first(data))
    # This is done at the start of the next frame, so wont show up in
    # the image properly. So do it preemtively?
    _wrapstatus!(status)
    if status[blockindex...]
        if normedval > 0
            to_rgb(normedval)
        else
            to_rgb((0.0, 0.0, 0.0))
        end
    elseif normedval > 0
        to_rgb((1.0, 0.0, 0.0)) # This (a red cell) would mean there is a bug in SparseOpt
    else
        to_rgb((0.5, 0.5, 0.5))
    end
end

"""
    MultiGridImageGenerator <: ImageGenerator

Abstract type for `ImageGenerator`s that convert a frame containing multiple
grids into a single image.
"""
abstract type MultiGridImageGenerator <: ImageGenerator end

"""
    Layout <: MultiGridImageGenerator

    Layout(layout::Array, imagegens::Matrix)

Layout allows displaying multiple grids in a block layout, by specifying a
layout matrix and a list of [`Image`](@ref)s to be run for each.

# Arguments

- `layout`: A Vector or Matrix containing the keys or numbers of grids in the locations to
    display them. `nothing`, `missing` or `0` values will be skipped.
- `imagegens`: tuple of [`Image`](@ref), one for each grid in the simulation.
    Can be `nothing` or any other value for grids not in layout.
"""
Base.@kwdef struct Layout{L<:AbstractMatrix,P} <: MultiGridImageGenerator
    layout::L
    imagegens::P
    Layout(layouts::L, imagegens::P) where {L,P} = begin
        imgens = map(_asimagegen, imagegens)
        new{L,typeof(imgens)}(layouts, imgens)
    end
end
# Convenience constructor to convert Vector input to a column Matrix
function Layout(layout::AbstractVector, imagegens)
    Layout(reshape(layout, length(layout), 1), imagegens)
end

_asimagegen(p::ImageGenerator) = p
_asimagegen(x) = Image(x)

layout(p::Layout) = p.layout
imagegens(p::Layout) = p.imagegens
imagesize(p::Layout, init::NamedTuple) = size(first(init)) .* size(p.layout)

function grid_to_image!(
    imagebuffer, p::Layout, o::ImageOutput, data::SimData, grids::NamedTuple
)
    ngrids = length(grids)
    if !(minval(o) isa Nothing)
        length(minval(o)) == ngrids || _wronglengtherror(minval, ngrids, length(minval(o)))
    end
    if !(maxval(o) isa Nothing)
        length(maxval(o)) == ngrids || _wronglengtherror(maxval, ngrids, length(maxval(o)))
    end

    grid_ids = layout(p)
    # Loop over the layout matrix
    for i in 1:size(grid_ids, 1), j in 1:size(grid_ids, 2)
        grid_id = grid_ids[i, j]
        # Accept symbol keys and numbers, skip missing/nothing/0
        (ismissing(grid_id) || grid_id === nothing || grid_id == 0)  && continue
        n = if grid_id isa Symbol
            found = findfirst(k -> k === grid_id, keys(grids))
            found === nothing && _grididnotinkeyserror(grid_id, grids)
            found
        else
            grid_id
        end
        I, J = map((i, j), gridsize(data)) do k, s
            (k - 1) * s + 1:k * s
        end
        # Run image generators for section
        grid_to_image!(
            view(imagebuffer, I, J), imagegens(p)[n], o, data, grids[n];
            name=string(keys(grids)[n]), time=nothing,
            minval=_valn(minval(o), n), maxval=_valn(maxval(o), n),
        )
    end
    _rendertime!(imagebuffer, textconfig(o), currenttime(data))
    return imagebuffer
end

_valn(::Nothing, n) = nothing
_valn(vals, n) = vals[n]

@noinline _grididnotinkeyserror(grid_id, grids) =
    throw(ArgumentError("$grid_id is not in $(keys(grids))"))
@noinline _wronglengtherror(f, ngrids, n) =
    throw(ArgumentError("Number of grids ($ngrids) and length of $f ($n) must be the same"))

# Automatically choose an image generator

function autoimagegen(init, scheme)
    Image(first(_iterableschemes(scheme)), ZEROCOL, MASKCOL)
end
function autoimagegen(init::NamedTuple, scheme)
    rows = length(init) ÷ 4 + 1
    cols = (length(init) - 1) ÷ rows + 1
    layout = reshape([keys(init)...], (rows, cols))
    imagegen = autoimagegen.(values(init), _iterableschemes(scheme))
    Layout(layout, imagegen)
end

_iterableschemes(::Nothing) = (ObjectScheme(),)
_iterableschemes(schemes::Union{Tuple,NamedTuple,AbstractArray}) = schemes
_iterableschemes(scheme) = (scheme,)

# Coll conversion tools

# Set a value to be between zero and one, before converting to Color.
# min and max of `nothing` are assumed to be 0 and 1.
normalise(x::X, minv, maxv) where X = max(min((x - minv) / (maxv - minv), oneunit(X)), zero(X))
normalise(x::X, minv, maxv::Nothing) where X = max((x - minv) / (oneunit(X) - minv), zero(X))
normalise(x::X, minv::Nothing, maxv) where X = min(x / maxv, oneunit(X), oneunit(X))
normalise(x, minv::Nothing, maxv::Nothing) = x

# Rescale a value between 0 and 1 to be between `min` and `max`.
# This can be used to shrink the range of a colorsheme that is displayed.
# min and max of `nothing` are assumed to be 0 and 1.
scale(x, min, max) = x * (max - min) + min
scale(x, ::Nothing, max) = x * max
scale(x, min, ::Nothing) = x * (oneunit(typeof(min)) - min) + min
scale(x, ::Nothing, ::Nothing) = x

to_rgb(vals::Tuple) = ARGB32(vals...)
to_rgb(val::Real) = ARGB32(RGB(val))
to_rgb(val::Color) = ARGB32(val)
to_rgb(val::ARGB32) = val
# Handle external colorshemes, such as from ColorSchemes.jl
to_rgb(scheme, val::Real) = to_rgb(get(scheme, val))

ext: jl


text:
using HDF5

@testset "read" begin
    file_name = "test_hdf5.h5"
    data_name = "test_dm"
    n = 35

    arr_real = rand(n, n)
    arr_imag = rand(n, n)
    h5write(file_name, "$data_name/real", arr_real)
    h5write(file_name, "$data_name/imag", arr_imag)

    # the transpose is due to the col-majer and row-majer difference between python and julia
    @test read_ρ(file_name, data_name) == complex.(arr_real', arr_imag')

    rm(file_name)
end


text:
# The master process listens on a well-known port
# Launched workers connect to the master and redirect their STDOUTs to the same
# Workers can join and leave the cluster on demand.

export ElasticManager, elastic_worker

const HDR_COOKIE_LEN = Distributed.HDR_COOKIE_LEN

struct ElasticManager <: ClusterManager
    active::Dict{Int, WorkerConfig}        # active workers
    pending::Channel{TCPSocket}          # to be added workers
    terminated::Set{Int}             # terminated worker ids
    topology::Symbol
    sockname
    printing_kwargs

    function ElasticManager(;addr=IPv4("127.0.0.1"), port=9009, cookie=nothing, topology=:all_to_all, printing_kwargs=())
        Distributed.init_multi()
        cookie !== nothing && cluster_cookie(cookie)

        if addr == :auto
            addr = get_private_ip()
        end

        l_sock = listen(addr, port)

        lman = new(Dict{Int, WorkerConfig}(), Channel{TCPSocket}(typemax(Int)), Set{Int}(), topology, getsockname(l_sock), printing_kwargs)

        @async begin
            while true
                let s = accept(l_sock)
                    @async process_worker_conn(lman, s)
                end
            end
        end

        @async process_pending_connections(lman)

        lman
    end
end

ElasticManager(port) = ElasticManager(;port=port)
ElasticManager(addr, port) = ElasticManager(;addr=addr, port=port)
ElasticManager(addr, port, cookie) = ElasticManager(;addr=addr, port=port, cookie=cookie)

function process_worker_conn(mgr::ElasticManager, s::TCPSocket)
    # Socket is the worker's STDOUT
    wc = WorkerConfig()
    wc.io = s

    # Validate cookie
    cookie = read(s, HDR_COOKIE_LEN)
    if length(cookie) < HDR_COOKIE_LEN
        error("Cookie read failed. Connection closed by peer.")
    end
    self_cookie = cluster_cookie()
    for i in 1:HDR_COOKIE_LEN
        if UInt8(self_cookie[i]) != cookie[i]
            println(i, " ", self_cookie[i], " ", cookie[i])
            error("Invalid cookie sent by remote worker.")
        end
    end

    put!(mgr.pending, s)
end

function process_pending_connections(mgr::ElasticManager)
    while true
        wait(mgr.pending)
        try
            addprocs(mgr; topology=mgr.topology)
        catch e
            showerror(stderr, e)
            Base.show_backtrace(stderr, Base.catch_backtrace())
        end
    end
end

function launch(mgr::ElasticManager, params::Dict, launched::Array, c::Condition)
    # The workers have already been started.
    while isready(mgr.pending)
        wc=WorkerConfig()
        wc.io = take!(mgr.pending)
        push!(launched, wc)
    end

    notify(c)
end

function manage(mgr::ElasticManager, id::Integer, config::WorkerConfig, op::Symbol)
    if op == :register
        mgr.active[id] = config
    elseif  op == :deregister
        delete!(mgr.active, id)
        push!(mgr.terminated, id)
    end
end

function Base.show(io::IO, mgr::ElasticManager)
    iob = IOBuffer()

    println(iob, "ElasticManager:")
    print(iob, "  Active workers : [ ")
    for id in sort(collect(keys(mgr.active)))
        print(iob, id, ",")
    end
    seek(iob, position(iob)-1)
    println(iob, "]")

    println(iob, "  Number of workers to be added  : ", Base.n_avail(mgr.pending))

    print(iob, "  Terminated workers : [ ")
    for id in sort(collect(mgr.terminated))
        print(iob, id, ",")
    end
    seek(iob, position(iob)-1)
    println(iob, "]")

    println(iob, "  Worker connect command : ")
    print(iob, "    ", get_connect_cmd(mgr; mgr.printing_kwargs...))

    print(io, String(take!(iob)))
end

# Does not return. If executing from a REPL try
# @async connect_to_cluster(.....)
# addr, port that a ElasticManager on the master processes is listening on.
function elastic_worker(cookie, addr="127.0.0.1", port=9009; stdout_to_master=true)
    c = connect(addr, port)
    write(c, rpad(cookie, HDR_COOKIE_LEN)[1:HDR_COOKIE_LEN])
    stdout_to_master && redirect_stdout(c)
    start_worker(c, cookie)
end

function get_private_ip()
    if Sys.islinux()
        cmd = `hostname --ip-address`
    elseif Sys.isapple()
        cmd = `ipconfig getifaddr en0`
    else
        error("`addr=:auto` is only supported on Linux and Mac")
    end
    try
        return IPv4(first(split(strip(read(cmd, String)))))
    catch err
        error("""Failed to automatically get host's IP address (output below). Please specify `addr=` explicitly.
        \$ $(repr(cmd))
        $err
        """)
    end
end

function get_connect_cmd(em::ElasticManager; absolute_exename=true, same_project=true)

    ip = string(em.sockname[1])
    port = convert(Int,em.sockname[2])
    cookie = cluster_cookie()
    exename = absolute_exename ? joinpath(Sys.BINDIR, Base.julia_exename()) : "julia"
    project = same_project ? ("--project=$(Pkg.API.Context().env.project_file)",) : ()

    join([
        exename,
        project...,
        "-e 'using ClusterManagers; ClusterManagers.elastic_worker(\"$cookie\",\"$ip\",$port)'"
    ]," ")

end


text:
@testset "547.friend-circles.jl" begin
    @test reverse_words("  hello world!  ") == "world! hello"
    @test reverse_words("a good   example") == "example good a"
    @test reverse_words("  Bob    Loves  Alice   ") == "Alice Loves Bob"
    @test reverse_words("Alice does not even like bob") == "bob like even not does Alice"
end


text:
using PtFEM

ProjDir = dirname(@__FILE__)

data = Dict(
  # Beam(ndim, nst, nxe, nip, direction, finite_element(nod, nodof), axisymmet
  :struc_el => Beam(2, 1, 20, 1, :x, Line(2, 2), false),
  :properties => [1.0; 1.0],
  #:etype => [1, 1, 2, 2],
  :x_coords => 0.0:1.0:20.0,
  :support => [
      (1, [0 0]),
      (21, [0 0])
    ],
  :loaded_nodes => [(i, [-1.0 0.0]) for i in 1:20],
  :penalty => 1e19
)

data |> display
println()

@time m, dis_df, fm_df = p43(data)
println()

if VERSION.minor > 5
  println("Displacements:")
  m.displacements' |> display
  println()

  println("Actions:")
  m.actions' |> display
  println()
else
  using DataFrames
  dis_df = DataFrame(
    translations = m.displacements'[:, 1],
    rotations = m.displacements'[:, 2]
  )
  fm_df = DataFrame(
    xl_Force = m.actions[1, :],
    xl_Moment = m.actions[2, :],
    xr_Force = m.actions[3, :],
    xr_Moment = m.actions[4, :]
  )
  # Correct element forces and moments for equivalent nodal
  # forces and moments introduced for loading between nodes
  if :eq_nodal_forces_and_moments in keys(data)
    eqfm = data[:eq_nodal_forces_and_moments]
    k = data[:struc_el].fin_el.nod * data[:struc_el].fin_el.nodof

    for t in eqfm
      for i in 1:k
        fm_df[t[1], i] -= t[2][i]
      end
    end
  end

  display(dis_df)
  println()
  display(fm_df)
end

if VERSION.minor == 5
  using Plots
  gr(size=(400,600))

  p = Vector{Plots.Plot{Plots.GRBackend}}(undef, 3)
  p[1] = plot(m.x_coords, m.displacements[1,:], ylim=(-500.0, 10.0), lab="Displacement",
   xlabel="x [m]", ylabel="deflection [m]", color=:red)
  p[2] = plot(m.actions'[:,1], lab="Shear force", ylim=(-15, 15), xlabel="element",
    ylabel="shear force [N]", palette=:greens,fill=(0,:auto),α=0.6)
  p[3] = plot(m.actions'[:,4], lab="Moment", ylim=(-30, 35), xlabel="element",
    ylabel="moment [Nm]", palette=:grays,fill=(0,:auto),α=0.6)

  plot(p..., layout=(3, 1))
  savefig(ProjDir*"/figure-23b.png")
end

max_stars_repo_path: examples/Nodal Formulas/figure-23b.jl

max_issues_repo_path: examples/Nodal Formulas/figure-23b.jl

max_forks_repo_path: examples/Nodal Formulas/figure-23b.jl


text:
abstract type AbstractPolynomial{N, T} end

Base.broadcastable(p::AbstractPolynomial) = Ref(p)

"""
```math
\\int_{0}^{t} p\\left(\\tau\\right) e^{c \\tau} d\\tau
```

Can be found using integration by parts.
"""
@inline function exponential_integral(p::AbstractPolynomial{N}, c, t; inv_c = inv(c), exp_c_t=exp(c * t)) where N
    if N === 1
        return inv_c * constant(p) * (exp_c_t - 1)
    else
        return inv_c * (p(t) * exp_c_t - constant(p) - exponential_integral(derivative(p), c, t; inv_c=inv_c, exp_c_t=exp_c_t))
    end
end


text:
function split_adjacency(A)
    graphs = SimpleGraph{Int64}[]

    subinds = connected_components(Graph(A))
    for subset in subinds
        subA = zeros(Int64, size(A)...)
        subA[subset, subset] = A[subset, subset]
        push!(graphs, Graph(subA))
    end

    roots = [subinds[i][1] for i=1:length(subinds)]

    return graphs, roots
end


text:
using Printf

using Keyboard

setmargin(6)

function test_readkey()
    println()
    println("Press any key combination (^C to exit).")
    println("METAKEY, KEYCHAR, SPECIALKEY")
    while true
        r = readkey(prompt = "")
        r.metakey === Keyboard.METAKEYNAMES_TABLE.ctrl && r.keychar == 'C' && break
        show(r.metakey); print(", "); show(r.keychar); print(", "); show(r.specialkey)
        println()
    end
end

function test_readsinglelinekey()
    p = "Do you wish to go riding?"
    k = "YN"
    shouldnt = "SHOULDN'T BE HERE"
    println()
    report(err) = println("ERROR TRAPPED: ", sprint(showerror, err))

    try
        readsingleline(p, "       ")
        println(shouldnt)
    catch e
        report(e)
    end
    try
        readsingleline(p, "YN\u7f")
        println(shouldnt)
    catch e
        report(e)
    end
    try
        readsingleline("       ", k)
        println(shouldnt)
    catch e
        report(e)
    end
    try
        readsingleline(p, k, defaultKey='Z')
        println(shouldnt)
    catch e
    report(e)
    end

    println()

    function assimresult(c::Union{Char, Nothing})::Bool
        println("result: '$c'\n")
        res = queryrepeat()
        println()
        return res
    end

    ctrl_c = Char(3)
    while true
        s = readsingleline(p, k, escape=true, defaultKey='Y', quitCtrlC=true)
        s == ctrl_c && return
    !assimresult(s) && break
    end
    while true
        s = readsingleline(p, k, escape=false, defaultKey='Y', quitCtrlC=true)
        s == ctrl_c && return
    !assimresult(s) && break
    end
    while true
        s = readsingleline(p, k, escape=true, quitCtrlC=true)
        s == ctrl_c && return
    !assimresult(s) && break
    end
    while true
        s = readsingleline(p, k, escape=false, quitCtrlC=true)
        s == ctrl_c && return
    !assimresult(s) && break
    end
end

function test_readmultiline_errors()
    shouldnt = "SHOULDN'T BE HERE "
    report(err) = println("ERROR TRAPPED: ", sprint(showerror, err))
    # report2(err) = println("FALSE ERROR - SHOULDN'T BE HERE: ", sprint(showerror, err))

    L = (
        "\nMiscellaneous:\n",

        [[' ', "aaa"]],
        [['\x13', "bbb"]],
        [[1, :a],4,[:b, 4]],

        "\nIncorrect Lengths:\n",
        [[]],
        [["xxx"],[]],
        [['c',"one",report],['d',"two"]],
        [["four",report],['g',"five",test_readkey]],
        [["six"],['h',"seven" ]],
        [['c',"message", report, 3]],

        "\nErroneous Elements:\n",
        [['e'],['f',"three"]],
        [["eight"],["nine"],[3]],
        [['i', "ten"],[4, "eleven"]],
        [['j', "twelve"],['k', 5]],
        [['l', "thirteen", report],['m', "fourteen", 6]],
        [['n', "fifteen", 7],['o', "sixteen", report]],
        [["seventeen", 7],["eighteen", report]],
        [["twenty", 'r'],['q', "nineteen"]],
        [[8, :abc],['s', "twentyone"]],

        "\nDuplicate key characters:\n",
        [['a', "one"],['A', "two"]],
        )
    for e in L
        if e isa String; println(e)
        else
            try
                readmultimenu("", e)
                println(shouldnt * _vector_sans_type_tostring(e))
            catch e
                report(e)
            end
        end
    end
    # try
    #     L = [['b', "twentytwo"], ['B', "twentythree"]] # ?duplicate keys
    #     readmultilinekey(data=L, case=true)
    # catch e
    #     report2(e) # this shouldn't occur - false error
    # end
end

function _test_readmultiline(;ismultisel::Bool)
    println()
    function doit(prompt, data; case::Bool=false)::Bool
        pre = nothing
        if ismultisel; pre = "Select one or more options. ";
        else; pre = "Select an option. "
        end
        prompt = pre * prompt
        while true
            r = nothing
            !ismultisel && (r = readradiomenu(prompt, data, case=case, quitCtrlC=true))
            ismultisel && (r = readmultimenu(prompt, data, case=case, quitCtrlC=true))
            r === Char(3) && return false
            s = r === nothing ? "ESC key pressed" :
                (ismultisel ? "keys pressed=$r" : "keypress='$(r.keychar)', index=$(r.index)")
            println(s)
            println()
            !queryrepeat() && (println(); break)
            println()
        end
        return true
    end
    L = [
        "aaa",
        "bbb",
        "ccc",
        "ddd",
        "eee",
        ]
    p = "Just Strings supplied, all in a single Vector (keys auto-generated, no functions)"
    !doit(p, L) && return

    L = [
        ["aaa"],
        ["bbb"],
        ["ccc"],
        ["ddd"],
        ["eee"],
        ]
    p = "Just Strings supplied, each in a Vector of it's own (keys auto-generated, no functions)"
    !doit(p, L) && return

    L = [
        ['z',"aaa"],
        ['y',"bbb"],
        ['x',"ccc"],
        ['v',"ddd"],
        ['u',"eee"],
        ]
    p = "Strings and keys supplied (no functions)"
    !doit(p, L) && return

    sp = "Function '%s' was evaluated\n"
    fun_1() = @eval @printf($sp, "fun_1")
    fun_2() = @eval @printf($sp, "fun_2")
    fun_3() = @eval @printf($sp, "fun_3")
    fun_4() = @eval @printf($sp, "fun_4")
    fun_5() = @eval @printf($sp, "fun_5")

    L = [
        ['z',"aaa", fun_1],
        ['y',"bbb", fun_2],
        ['x',"ccc", fun_3],
        ['v',"ddd", fun_4],
        ['u',"eee", fun_5],
        ]
    p = "Strings, keys and functions supplied"
    !doit(p, L) && return

    L = [
        ["aaa", fun_1],
        ["bbb", fun_2],
        ["ccc", fun_3],
        ["ddd", fun_4],
        ["eee", fun_5],
        ]
    p = "Strings and functions supplied (keys are auto-generated)"
    !doit(p, L) && return

    L = [
        ['a', "aaa", fun_1],
        ['A', "bbb", fun_2],
        ['b', "ccc", fun_3],
        ['B', "ddd", fun_4],
        ['c', "eee", fun_5],
        ]
    p = "Strings, keys and functions supplied (keys are case-sensitive)"
    !doit(p, L, case=true) && return

    return
end
test_readradiomenu() = _test_readmultiline(ismultisel=false)
test_readmultimenu() = _test_readmultiline(ismultisel=true)

function testall()
    L = [
        ["test_readmultiline_errors()", test_readmultiline_errors],
        ["test_readmultimenu()", test_readmultimenu],
        ["test_readradiomenu()", test_readradiomenu],
        ["test_readsinglelinekey()", test_readsinglelinekey],
        ["test_readkey()", test_readkey],
        ]
    p = "Select one of the following tests (which will be evaluated):"
    while true
        println()
        res = readradiomenu(p, L)
        res === nothing && return
        readkey()
    end
end


text:
function SnapPicedata2File(folderB,
    folderA,
    StartTime,
    TimePeriod,
    StartFiberLength,
    FLRange,
    ChannelID,
    TgrFolder  )

    t= ccall( (:snapPartdata,"partSnap.dll") ,Int32,
                (Cstring ,Cstring , Cstring,Clong,Clong,Clong ,Clong ,Cstring  ),
                folderB,
                folderA,
                StartTime,
                TimePeriod,
                StartFiberLength,
                FLRange,
                ChannelID,
                TgrFolder
                )
    return t
end

function DtFormat( DtStr)
#orgformat:"20181107045146"
#TgtFormat: #"2019-03-06-18:42:59"
    return  DtStr[1:4] *"-" *DtStr[5:6]* "-" * DtStr[7:8] *"-"* DtStr[9:10] *":"* DtStr[11:12] *":"* DtStr[13:14]
end

function TestSnap()
    folderB="G:/Bd003/data/userdata1"
    folderA="G:/Bd003/data/userdata1"
    t= "Data^Ver00^BD-003^20181107045146^ID00"
    StartT="20181107045146"
    StartTime= DtFormat(StartT)  #"2019-03-06-18:42:59"
    TimePeriod=300 #s
    StartFiberLength=1
    FLRange=20744
    ChannelID=2
    TgrFolder="G:\\dataroot\\log"

    ret = SnapPicedata2File(folderB,
                    folderA,
                    StartTime,
                    TimePeriod,
                    StartFiberLength,
                    FLRange,
                    ChannelID,
                    TgrFolder  )
    println(ret)
end

max_stars_repo_name: wangzhongtian/fiberAlarmAnalyseTool

max_issues_repo_name: wangzhongtian/fiberAlarmAnalyseTool

max_forks_repo_name: wangzhongtian/fiberAlarmAnalyseTool


text:
function seconds_since_the_epoch_utc(zoned_date_time::TimeZones.ZonedDateTime)
    zoned_date_time_utc = TimeZones.astimezone(
        zoned_date_time,
        TimeZones.tz"UTC",
    )
    float_number_of_seconds_since_the_unix_epoch = TimeZones.datetime2unix(
        Dates.DateTime(
            zoned_date_time_utc
        )
    )
    return float_number_of_seconds_since_the_unix_epoch
end

function integer_seconds_since_the_epoch_utc(zoned_date_time::TimeZones.ZonedDateTime)
    float_number_of_seconds_since_the_unix_epoch = seconds_since_the_epoch_utc(
        zoned_date_time
    )
    integer_number_of_seconds_since_the_unix_epoch = round(
        Int,
        float_number_of_seconds_since_the_unix_epoch,
    )
    return integer_number_of_seconds_since_the_unix_epoch
end


text:
# !INPUT/OUTPUT PARAMETERS:
#   sol  : speed of light in atomic units (in,real)
#   kpa  : quantum number kappa (in,integer)
#   e    : energy (in,real)
#   nr   : number of radial mesh points (in,integer)
#   r    : radial mesh (in,real(nr))
#   vr   : potential on radial mesh (in,real(nr))
#   nn   : number of nodes (out,integer)
#   g0   : m th energy derivative of the major component multiplied by r
#          (out,real(nr))
#   g1   : radial derivative of g0 (out,real(nr))
#   f0   : m th energy derivative of the minor component multiplied by r
#          (out,real(nr))
#   f1   : radial derivative of f0 (out,real(nr))
function rdiracint!(sol, kpa, e, nr, r, vr, g0, g1, f0, f1)

    # rescaling limit
    rsc = 1.0e100

    @assert nr >= 4

    # inverse speed of light
    ci = 1.0/sol

    # electron rest energy
    e0 = sol^2
    t1 = 2.0*e0 + e

    # determine the r -> 0 boundary values of F and G
    t2 = kpa/r[1]
    t3 = ci*(t1 - vr[1])
    t4 = ci*(vr[1] - e)
    f0[1] = 1.0
    f1[1] = 0.0
    g0[1] = (f1[1] - t2*f0[1])/t4
    g1[1] = t3*f0[1] - t2*g0[1]

    # extrapolate to the first four points
    g1[2:4] .= g1[1]
    f1[2:4] .= f1[1]

    nn = 0 # number of nodes
    for ir in 2:nr
        t2 = kpa/r[ir]
        t3 = ci*(t1 - vr[ir])
        t4 = ci*(vr[ir] - e)
        ir0 = ir - 3
        if ir0 < 1
            ir0 = 1
        end
        @views g1[ir] = poly3(r[ir0:ir0+2], g1[ir0:ir0+2], r[ir])
        @views f1[ir] = poly3(r[ir0:ir0+2], f1[ir0:ir0+2], r[ir])
        # integrate to find wavefunction
        @views g0[ir] = poly4i( r[ir0:ir0+3], g1[ir0:ir0+3], r[ir]) + g0[ir0]
        @views f0[ir] = poly4i( r[ir0:ir0+3], f1[ir0:ir0+3], r[ir]) + f0[ir0]
        # compute the derivatives
        g1[ir] = t3*f0[ir] - t2*g0[ir]
        f1[ir] = t4*g0[ir] + t2*f0[ir]
        # integrate for correction
        @views g0[ir] = poly4i(r[ir0:ir0+3], g1[ir0:ir0+3], r[ir]) + g0[ir0]
        @views f0[ir] = poly4i(r[ir0:ir0+3], f1[ir0:ir0+3], r[ir]) + f0[ir0]
        # compute the derivatives again
        g1[ir] = t3*f0[ir] - t2*g0[ir]
        f1[ir] = t4*g0[ir] + t2*f0[ir]
        # check for overflow
        if ( (abs(g0[ir]) > rsc) || (abs(g1[ir]) > rsc) ||
             (abs(f0[ir]) > rsc) || (abs(f1[ir]) > rsc) )
            # set the remaining points and return
            g0[ir:nr] .= g0[ir]
            g1[ir:nr] .= g1[ir]
            f0[ir:nr] .= f0[ir]
            f1[ir:nr] .= f1[ir]
            return nn, e
        end
        # check for node
        if g0[ir-1]*g0[ir] < 0.0
            nn = nn + 1
        end
    end
    return nn, e
end

function poly3(xa, ya, x)
    # arguments
    # real(8) xa(3),ya(3),x
    # local variables

    # evaluate the polynomial coefficients
    x0 = xa[1]
    x1 = xa[2] - x0
    x2 = xa[3] - x0
    y0 = ya[1]
    y1 = ya[2] - y0
    y2 = ya[3] - y0
    t0 = 1.0/(x1*x2*(x2-x1))
    t1 = x1*y2
    t2 = x2*y1
    c1 = x2*t2 - x1*t1
    c2 = t1 - t2
    t1 = x - x0

    # evaluate the polynomial
    return y0 + t0*t1*(c1 + c2*t1)
end

function poly4i(xa, ya, x)
    # arguments
    #real(8), intent(in) :: xa(4),ya(4),x

    # evaluate the polynomial coefficients
    x0 = xa[1]
    x1 = xa[2] - x0
    x2 = xa[3] - x0
    x3 = xa[4] - x0
    y0 = ya[1]
    y1 = ya[2] - y0
    y2 = ya[3] - y0
    y3 = ya[4] - y0
    t4 = x1 - x2
    t5 = x1 - x3
    t6 = x2 - x3
    t1 = x1*x2*y3
    t2 = x2*x3*y1
    t3 = x1*x3
    t0 = 1.0/(x2*t3*t4*t5*t6)
    t3 = t3*y2
    c3 = t1*t4 + t2*t6 - t3*t5
    t4 = x1^2
    t5 = x2^2
    t6 = x3^2
    c2 = t1*(t5 - t4) + t2*(t6 - t5) + t3*(t4 - t6)
    c1 = t1*(x2*t4 - x1*t5) + t2*(x3*t5 - x2*t6) + t3*(x1*t6 - x3*t4)
    t1 = x - x0
    # integrate the polynomial
    return t1*(y0 + t0*t1*(0.5*c1 + t1*(c2/3 + 0.25*c3*t1) ) )
end


text:
using Typeclass
using Base.Test

@class Eq T begin
    eq(x::T,y::T)=!noteq(x,y)
    noteq(x::T,y::T)=!eq(x,y)
    ==(x::T,y::T)=eq(x,y)        # ignored unless use @instance! form
end

@instance Eq Int begin
    eq(x::Int,y::Int)=x==y
end

@test noteq(3,4)

@instance Eq Float64 begin
    noteq(x::Float64,y::Float64)=x!=y
end

@test eq(3.0,3.0)

# Create a simple type to check what happens when we don't
# resolve circular definitions
type Bad
    b
end

@instance Eq Bad

# stack overflow because we didn't adequately resolve circulars
@test_throws StackOverflowError eq(Bad(0),Bad(0))

@class Semiring A begin
       plus(x::A,y::A)=x+y
       times(x::A,y::A)=x*y
end

type Foo
    f
end

# default Julia definition of == doesn't hold for Foo
@test (Foo(3)==Foo(3) )==false

@instance Eq Foo begin
   eq(a::Foo, b::Foo) = a.f==b.f
end
# since we used @instance, does not overwrite default Julia ==

# now our eq is defined
@test eq(Foo(3),Foo(3) )==true
# but == behavior is unchanged
@test (Foo(3)==Foo(3) )==false

# Now use the overwriting version which adds a new method for ==,
# overwriting the deafult Julia definition
@instance! Eq Foo begin
   eq(a::Foo, b::Foo) = a.f==b.f
end
# now == agrees with eq
@test (Foo(3)==Foo(3) )==true

# A typeclass of things that are equal if all their fields are equal
@class NameEq T begin
    function eq(x::T,y::T)
        for name in names(x)
            if getfield(x,name)!=getfield(y,name)
                return false
            end
        end
        return true
    end
    ==(x::T,y::T)=eq(x::T,y::T)
    !=(x::T,y::T)=!eq(x::T,y::T)
end

# a type that we'll declare as an instance
type TestNameEq
    field1
    field2
end

# default behavior of ==
@test (TestNameEq(3,4)==TestNameEq(3,4))==false
@instance NameEq TestNameEq
# same issue with needing to override default deliberately
@test (TestNameEq(3,4)==TestNameEq(3,4))==false
@test eq(TestNameEq(3,4),TestNameEq(3,4))==true
@instance! NameEq TestNameEq
@test (TestNameEq(3,4)==TestNameEq(3,4))==true

# Some math
@class Monoid T begin
       munit(::T)::T
       mappend(x::T,y::T)::T
end

@instance Monoid Array{Int} begin
       munit(::Array{Int})=Int[]
       mappend(x::Array{Int},y::Array{Int})=[x;y]
end

@test munit(Int[3])==Int[]
@test mappend([3,4],[4,5])==[3,4,4,5]

# test multiple type parameters
@class Deq S T begin
    deq(x::S,y::T)=x==y
    dneq(x::S,y::T)= !deq(x,y)
end

@instance Deq Int Float64 begin
    deq(a::Int,b::Float64)=a==b
end

@test dneq(3,4.0)
@test deq(3,3.0)

# test function blocks in @instance
# test things in Base
import Base.show #o/w clobbers
@class Showable T begin
    function show(x::T,y::T)
        3==4
        print(x,y)
    end
end

# another circular definition with heavily overloaded operator
@class Addable T begin
    add(x::T,y::T)=x+y
    +(x::T,y::T)=add(x,y)
end

# using @instance not @instance! avoids clobbering +, which would break Julia
@instance Addable Int

# test function blocks in @instance
@instance Addable Foo begin
    function add(x::Foo,y::Foo)
        5==3
        try
            garble
        catch x
            print(x)
        end
        return Foo(x.f + y.f)
    end
end

@test eq(Foo(4)+Foo(3),Foo(7))

type Addable1
    data::Int
end

type Addable2
    data::Int
end

@instance Addable Addable1 begin
    add(x::Addable1,y::Addable1)=x.data+y.data
end

@instance Addable Addable2 begin
    +(x::Addable2,y::Addable2)=x.data+y.data
end

# finally, the kind of stuff this was written to do

# typeclasses in a multiple dispatch language are properties of tuples of types
@class MonoidalCategory Ob Mor begin
    dom(f::Mor)::Ob
    cod(f::Mor)::Ob
    id(A::Ob)::Mor
    compose(f::Mor,g::Mor)::Mor #f*g
    otimes(f::Mor,g::Mor)::Mor
    otimes(A::Ob,B::Ob)::Ob
    munit(::Ob)::Ob
    munit(f::Mor)=munit(dom(f))
    # syntax, using unicode
    ∘(f::Mor,g::Mor)=compose(f,g)
    |(f::Mor,g::Mor)=compose(g,f)
    ⊗(f::Mor,g::Mor)=otimes(f,g)
    ⊗(A::Ob,B::Ob)=otimes(A,B)
end

typealias Mat Matrix{Float64}
@instance MonoidalCategory Int Mat begin
    dom(f::Mat)=size(f)[2]
    cod(f::Mat)=size(f)[1]
    id(A::Int)=eye(A)
    compose(f::Mat,g::Mat)=f*g
    otimes(f::Mat,g::Mat)=kron(f,g)
    otimes(A::Int,B::Int)=A*B
    munit(::Int)=1
end

@test id(2)⊗id(2) == id(4)
@test dom(rand(2,3))==3


text:
function kernel_kmeans(K::Matrix, k::Int, ϵ::Float64 = 1e-5)
    n = size(K, 1)
    C = rand(1:k, n)
    C_next = copy(C)

    s_total = Inf
    s_total_next = 0

    while true
        for i in 1:n
            k_min = 1
            s_min = Inf

            for kk in 1:k
                s1 = K[i, i]

                N_kk = 0
                s2 = 0
                for l in 1:n
                    if C[l] == kk
                        N_kk += 1
                        s2 += K[i, l]
                    end
                end
                s2 = (-2/N_kk) * s2

                s3 = 0
                for l in 1:n, m in 1:n
                    if C[l] == kk && C[m] == kk
                        s3 += K[l, m]
                    end
                end
                s3 = (1/N_kk^2) * s3

                s = s1 + s2 + s3
                if s < s_min
                    s_min = s
                    k_min = kk
                end
            end

            C_next[i] = k_min
            s_total_next += s_min
        end

        println(s_total_next)

        if abs(s_total_next - s_total) < ϵ
            break
        end

        s_total, s_total_next = s_total_next, 0
        C, C_next = C_next, C
    end

    return [findin(C_next, i) for i in 1:k]
end

function max_k_cut{T}(D::Matrix{T}, k::Int)
    n = size(D, 1)
    Y = Semidefinite(n, n)

    constraints = Constraint[]

    for i in 1:n, j in 1:n
        if i == j
            push!(constraints, Y[i, j] == T(1))
        elseif k >= 3
            push!(constraints, Y[i, j] >= -inv(k - 1))
        end
    end

    problem = minimize(trace(D*Y), constraints)
    solve!(problem, MosekSolver())

    return Convex.evaluate(Y)
end

function factor_k_cut(Y::Matrix, k::Int)
    #Y = max_k_cut(D, k)
    U, S, V = svd(Y)
    S = diagm(S[1:k])
    W = U[:, 1:k] * sqrt(S)
end


text:
using OptimalTransport
import OptimalTransport.Dual: Dual

using ForwardDiff

using Distances
using PythonOT: PythonOT

using LinearAlgebra
using Random
using Test

const POT = PythonOT

Random.seed!(100)

@testset "dual.jl" begin
    M = 250
    N = 200

    # create two random histograms
    μ = normalize!(rand(M), 1)
    ν = normalize!(rand(N), 1)

    # create random cost matrix
    C = pairwise(SqEuclidean(), rand(1, M), rand(1, N); dims=2)
    ε = 0.01
    K = exp.(-C / ε)

    @testset "semidual_grad" begin
        ∇_ad = ForwardDiff.gradient(zeros(size(ν))) do xs
            Dual.ot_entropic_semidual(μ, xs, ε, K)
        end

        ∇ = Dual.ot_entropic_semidual_grad(μ, zeros(size(ν)), ε, K)

        @test ∇ ≈ ∇_ad
    end

    @testset "dual_grad" begin
        ∇_ad = ForwardDiff.gradient(zeros(size(ν, 1) + size(μ, 1))) do xs
            Dual.ot_entropic_dual(xs[1:M], xs[(M + 1):end], ε, K)
        end

        ∇ = vcat(Dual.ot_entropic_dual_grad(zeros(size(μ)), zeros(size(ν)), ε, K)...)

        @test ∇ ≈ ∇_ad
    end

    @testset "batch" begin
        L = 10
        μ = rand(M, L)
        μ = μ ./ sum(μ; dims=1)
        u = rand(size(μ)...)
        ν = rand(N, L)
        ν = ν ./ sum(ν; dims=1)
        v = rand(size(ν)...)
        @testset "dual" begin
            @test Dual.ot_entropic_dual(u, v, ε, K) == [
                Dual.ot_entropic_dual(x, y, ε, K) for (x, y) in zip(eachcol(u), eachcol(v))
            ]

            ∇_u, ∇_v = Dual.ot_entropic_dual_grad(u, v, ε, K)
            grad_pairwise = [
                Dual.ot_entropic_dual_grad(x, y, ε, K) for
                (x, y) in zip(eachcol(u), eachcol(v))
            ]
            @test (∇_u ≈ hcat([g[1] for g in grad_pairwise]...)) && (∇_v ≈ hcat([g[2] for g in grad_pairwise]...))

            @test Dual.getprimal_ot_entropic_dual(u, v, ε, K) ≈ cat(
                [
                    Dual.getprimal_ot_entropic_dual(x, y, ε, K) for
                    (x, y) in zip(eachcol(u), eachcol(v))
                ]...;
                dims=3,
            )
        end

        @testset "semidual" begin
            @test Dual.ot_entropic_semidual(μ, v, ε, K) == [
                Dual.ot_entropic_semidual(x, y, ε, K) for
                (x, y) in zip(eachcol(μ), eachcol(v))
            ]
            @test Dual.ot_entropic_semidual_grad(μ, v, ε, K) ≈ hcat(
                [
                    Dual.ot_entropic_semidual_grad(x, y, ε, K) for
                    (x, y) in zip(eachcol(μ), eachcol(v))
                ]...,
            )

            @test Dual.getprimal_ot_entropic_semidual(μ, v, ε, K) ≈ hcat(
                [
                    Dual.getprimal_ot_entropic_semidual(x, y, ε, K) for
                    (x, y) in zip(eachcol(μ), eachcol(v))
                ]...,
            )
        end
    end
end


text:
using Test, DiffEqOperators
using OrdinaryDiffEq

@testset "Parabolic Heat Equation with Dirichlet BCs" begin
    x = collect(-pi : 2pi/511 : pi)
    u0 = @. -(x - 0.5).^2 + 1/12
    A = CenteredDifference(2,2,2π/511,512,:Dirichlet,:Dirichlet;BC=(u0[1],u0[end]))
    heat_eqn = ODEProblem(A, u0, (0.,10.))
    soln = solve(heat_eqn,Tsit5(),dense=false,tstops=0:0.01:10)

    # Broken in different amounts on each CI computer
    for t in 0:0.1:10
        @test_skip soln(t)[1] ≈ u0[1]
        @test_skip soln(t)[end] ≈ u0[end]
    end
end

@testset "Parabolic Heat Equation with Neumann BCs" begin
    N = 512
    dx = 2π/(N-1)
    x = collect(-pi : dx : pi)
    u0 = @. -(x - 0.5)^2 + 1/12
    B = CenteredDifference(1,2,dx,N,:None,:None)
    deriv_start, deriv_end = (B*u0)[1], (B*u0)[end]

    A = CenteredDifference(2,2,dx,N,:Neumann,:Neumann;BC=(deriv_start,deriv_end))

    heat_eqn = ODEProblem(A, u0, (0.,10.))
    soln = solve(heat_eqn,Tsit5(),dense=false,tstops=0:0.01:10)

    first_order_coeffs_start = [-11/6, 3.0, -3/2, 1/3] * (1/dx)
    first_order_coeffs_end = -reverse([-11/6, 3.0, -3/2, 1/3] * (1/dx))

    for t in 0:0.1:10
        @test sum(first_order_coeffs_start .* soln(t)[1:4]) ≈ deriv_start atol=1e-1
        @test sum(first_order_coeffs_end .* soln(t)[end-3:end]) ≈ deriv_end atol=1e-1
    end
end

@testset "Parabolic Heat Equation with Robin BCs" begin
    N = 512
    dx = 2π/(N-1)
    x = collect(-pi : dx : pi)
    u0 = @. -(x - 0.5).^2 + 1/12
    B = CenteredDifference(1,2,dx,N,:None,:None)
    deriv_start, deriv_end = (B*u0)[1], (B*u0)[end]
    params = [1.0,0.5]

    left_RBC = params[1]*u0[1] - params[2]*deriv_start
    right_RBC = params[1]*u0[end] + params[2]*deriv_end

    A = CenteredDifference(2,2,dx,N,:Robin,:Dirichlet;BC=((params[1],params[2],left_RBC),u0[end]));

    heat_eqn = ODEProblem(A, u0, (0.,10.));
    soln = solve(heat_eqn,Tsit5(),dense=false,tstops=0:0.01:10);

    first_order_coeffs_start = [-11/6, 3.0, -3/2, 1/3] * (1/dx)
    first_order_coeffs_end = -reverse([-11/6, 3.0, -3/2, 1/3] * (1/dx))
    val = []
    # Broken in different amounts on each CI computer
    for t in 0.2:0.1:9.8
        @test_skip params[1]*soln(t)[1] + -params[2]*sum(first_order_coeffs_start .* soln(t)[1:4]) ≈ left_RBC atol=1e-1
        # append!(val,params[1]*soln(t)[1] + -params[2]*sum(first_order_coeffs_start .* soln(t)[1:4]) - left_RBC)
        @test_skip params[1]*soln(t)[end] + params[2]*sum(first_order_coeffs_end .* soln(t)[end-3:end]) ≈ right_RBC atol=1e-1
    end
end


text:
module LightGraphsMatching

using LightGraphs

using SparseArrays: spzeros

using JuMP
using MathProgBase: AbstractMathProgSolver
import BlossomV # 'using BlossomV'  leads to naming conflicts with JuMP
using Hungarian

export MatchingResult, maximum_weight_matching, maximum_weight_maximal_matching, minimum_weight_perfect_matching, HungarianAlgorithm, LPAlgorithm

"""
    struct MatchingResult{U}
        weight::U
        mate::Vector{Int}
    end

A type representing the result of a matching algorithm.

    weight: total weight of the matching

    mate:    `mate[i] = j` if vertex `i` is matched to vertex `j`.
             `mate[i] = -1` for unmatched vertices.
"""
struct MatchingResult{U<:Real}
    weight::U
    mate::Vector{Int}
end

include("lp.jl")
include("maximum_weight_matching.jl")
include("blossomv.jl")
include("hungarian.jl")
include("maximum_weight_maximal_matching.jl")

end # module


text:
function node_loadness(g::GenericGraph, index::Int64, dist_key::AbstractString)

  S, P, D = GraphAnalysis.assemble_shortest_paths(g, index, dist_key, distance=true)

  sorted_nodes = Int64[]
  sorted_temp = sort(collect(zip(values(D), keys(D))))

  for temp in sorted_temp
    if temp[1] > 0.0 && temp[1] != Inf
      push!(sorted_nodes, temp[2])
    end
  end

  load = [ k => 1.0 for k=keys(D)]

  while length(sorted_nodes) > 0

    v = pop!(sorted_nodes)

    if haskey(P, v)
      paths_count = length(P[v])

      for x in P[v]

        if x == index
          break
        end

        load[x] += load[v] / float(paths_count)
      end

    end

  end

  for v in keys(load)
    load[v] -= 1
  end

  return load

end

max_stars_repo_name: mpuk/GraphAnalysis

max_issues_repo_name: mpuk/GraphAnalysis

max_forks_repo_name: mpuk/GraphAnalysis


text:
module ChainedCore

using Base: Fix1, Fix2, tail

export
    # Types
    ArgPosition,
    ArgsTrailing,
    ChainedFix,
    NFix,
    # Constants
    Approx,
    In,
    Less,
    Greater,
    Equal,
    NotEqual,
    LessThanOrEqual,
    GreaterThanOrEqual,
    Not,
    NotIn,
    NotApprox,
    PipeChain,
    EndsWith,
    StartsWith,
    @nfix,
    # Constants
    And,
    Or,
    PipeChain,
    # methods
    and,
    getargs,
    getkwargs,
    print_fixed,
    getfxn,
    is_fixed_function,
    ⩓,
    or,
    pipe_chain,
    ⩔

if length(methods(isapprox, Tuple{Any})) == 0
    Base.isapprox(y; kwargs...) = x -> isapprox(x, y; kwargs...)
end
const Approx{Kwargs,T} = typeof(isapprox(Any)).name.wrapper{Kwargs,T}

if length(methods(startswith, Tuple{Any})) == 0
    Base.startswith(s) = Base.Fix2(startswith, s)
end
const StartsWith{T} = Fix2{typeof(startswith),T}
if length(methods(endswith, Tuple{Any})) == 0
    Base.endswith(s) = Base.Fix2(endswith, s)
end

const EndsWith{T} = Fix2{typeof(endswith),T}
const Not{T} = (typeof(!(sum)).name.wrapper){T}
const In{T} = Fix2{typeof(in),T}
const NotIn{T} = (typeof(!in(Any)).name.wrapper){Fix2{typeof(in),T}}
const NotApprox{T,Kwargs} = (typeof(!in(Any)).name.wrapper){Approx{T,Kwargs}}

const Less{T} = Union{Fix2{typeof(<),T},Fix2{typeof(isless),T}}
const Equal{T} = Union{Fix2{typeof(==),T},Fix2{typeof(isequal),T}}
const NotEqual{T} = Fix2{typeof(!=),T}
const Greater{T} = Fix2{typeof(>),T}
const GreaterThanOrEqual{T} = Fix2{typeof(>=),T}
const LessThanOrEqual{T} = Fix2{typeof(<=),T}

"""
    ChainedFix(link, f1, f2)

Internal type for composing functions from [`and`](@ref) and [`or`](@ref). For
example, `and(x::Function, y::Function)` becomes `ChainedFix(and, x, y)`. Calling
`ChainedFix` uses the `link` function to propagate arguments to the linked functions `f1`
and `f2`.

See also: [`and`](@ref), [`or`](@ref), [`pipe_chain`](@ref)
"""
struct ChainedFix{L,F1,F2} <: Function
    "the method the \"links\" `f1` and `f2` (i.e. link(f1, f2))"
    link::L
    "the first position in the `link(x, f1, f2)` call"
    f1::F1
    "the second position in the `link(x, f1, f2)` call"
    f2::F2
end

(cf::ChainedFix)(x) = cf.link(x, cf.f1, cf.f2)

"""
    pipe_chain(x, y, z...)

Chain together a
"""
pipe_chain(x, y, z...) = pipe_chain(x, pipe_chain(y, z...))
pipe_chain(x, y) = ChainedFix(pipe_chain, x, y)

(cf::ChainedFix{typeof(pipe_chain)})(x) = x |> cf.f1 |> cf.f2

const PipeChain{F1,F2} = ChainedFix{typeof(pipe_chain),F1,F2}

"""
    and(x, y)
    x ⩓ y

Synonymous with bitwise `&` operator but may be used to chain multiple `Fix1` or
`Fix2` operations. The `⩓` (`\\And<TAB>`) operator may be used in its place (e.g., `x ⩓ y`).

## Examples
```jldoctest
julia> using ChainedFixes

julia> and(true, <(5))(1)
true

julia> and(<(5), false)(1)
false

julia> and(and(<(5), >(1)), >(2))(3)
true

julia> and(<(5) ⩓ >(1), >(2))(3)  # ⩓ == \\And
true

```
"""
function and end

# 3 - args
function and(x, y::Function, z::Function)
    if y(x)
        return z(x)
    else
        return false
    end
end
function and(x, y, z::Function)
    if y
        return z(x)
    else
        return false
    end
end
function and(x, y::Function, z)
    if y(x)
        return z
    else
        return false
    end
end
and(x, y, z) = throw(MethodError(and, (x, y, z)))

# 2 -args
and(x, y) = x & y
and(x::Function, y) = ChainedFix(and, x, y)
and(x, y::Function) = ChainedFix(and, x, y)
and(x::Function, y::Function) = ChainedFix(and, x, y)
and(f1::Less{T}, f2::Less{T}) where {T} = (f1.x < f2.x) ? f1 : f2
and(f1::LessThanOrEqual{T}, f2::LessThanOrEqual{T}) where {T} = (f1.x < f2.x) ? f1 : f2
and(f1::Greater{T}, f2::Greater{T}) where {T} = (f1.x > f2.x) ? f1 : f2
and(f1::GreaterThanOrEqual{T}, f2::GreaterThanOrEqual{T}) where {T} = (f1.x > f2.x) ? f1 : f2

# \And
⩓(x, y) = and(x, y)

"""
    or(x, y)
    x ⩔ y

Synonymous with bitwise `|` operator but may be used to chain multiple `Fix1` or
`Fix2` operations. The `⩔` (`\\Or<TAB>`) operator may be used in its place (e.g., `x ⩔ y`).

## Examples
```jldoctest
julia> using ChainedFixes

julia> or(true, <(5))(1)
true

julia> or(<(5), false)(1)
true

julia> or(<(5) ⩔ >(1), >(2))(3)  # ⩔ == \\Or
true
```
"""
function or end

# 3 - args
function or(x, y::Function, z::Function)
    if y(x)
        return true
    else
        return z(x)
    end
end
function or(x, y, z::Function)
    if y
        return true
    else
        return z(x)
    end
end
function or(x, y::Function, z)
    if y(x)
        return true
    else
        return z
    end
end
or(x, y, z) = throw(MethodError(or, (x, y, z)))

# 2 -args
or(x, y) = x | y
or(x::Function, y) = ChainedFix(or, x, y)
or(x, y::Function) = ChainedFix(or, x, y)
or(x::Function, y::Function) = ChainedFix(or, x, y)
or(f1::Less{T}, f2::Less{T}) where {T} = (f1.x > f2.x) ? f1 : f2
or(f1::LessThanOrEqual{T}, f2::LessThanOrEqual{T}) where {T} = (f1.x > f2.x) ? f1 : f2
or(f1::Greater{T}, f2::Greater{T}) where {T} = (f1.x < f2.x) ? f1 : f2
or(f1::GreaterThanOrEqual{T}, f2::GreaterThanOrEqual{T}) where {T} = (f1.x < f2.x) ? f1 : f2

#\Or
⩔(x, y) = or(x, y)

const And{F1,F2} = ChainedFix{typeof(and),F1,F2}

const Or{F1,F2} = ChainedFix{typeof(or),F1,F2}

struct ArgPosition{N}
    ArgPosition{N}() where {N} = new{N::Int}()
    ArgPosition(n::Int) = new{n}()
end

struct ArgsTrailing end

struct NFix{F,Args<:Tuple,Kwargs<:NamedTuple} <: Function
    f::F
    args::Args
    kwargs::Kwargs
end

# (max_underscore, has_trailing_underscore)
Base.@pure function underscore_info(::Type{A})::Tuple{Int,Bool} where {A}
    has_trailing_underscore = false
    max_underscore = 0
    for p in A.parameters
        if p <: ArgPosition
            n = p.parameters[1]
            if n > max_underscore
                max_underscore = n
            end
        elseif p <: ArgsTrailing
            has_trailing_underscore = true
        else
            nothing
        end
    end
    return (max_underscore, has_trailing_underscore)
end

@inline (f::NFix)(args...; kwargs...) = _execute(f.f, f.args, f.kwargs, args, kwargs.data)

@generated function _execute(
    f,
    fixed_args::FA,
    fixed_kwargs::NamedTuple{FKN},
    args::A,
    kwargs::NamedTuple{KN}
)  where {Nf,Na,FA<:Tuple{Vararg{Any,Nf}},A<:Tuple{Vararg{Any,Na}},FKN,KN}

    out = Expr(:call, :f)
    if !isempty(FKN)
        kwargsexpr = Expr(:parameters)
        for name in FKN
            push!(kwargsexpr.args, Expr(:kw, name, :(getfield(fixed_kwargs, $(QuoteNode(name))))))
        end
        if !isempty(KN)
            for name in KN
                push!(kwargsexpr.args, Expr(:kw, name, :(getfield(kwargs, $(QuoteNode(name))))))
            end
        end
        push!(out.args, kwargsexpr)
    else
        if !isempty(KN)
            kwargsexpr = Expr(:parameters)
            for name in KN
                push!(kwargsexpr.args, Expr(:kw, name, :(getfield(kwargs, $(QuoteNode(name))))))
            end
            push!(out.args, kwargsexpr)
        end
    end

    max_underscore, has_trailing_underscore = underscore_info(FA)
    if max_underscore === Na
        has_trailing_underscore = false  # don't need to account for this now
    elseif !has_trailing_underscore || max_underscore > Na
        str = "Expected $max_underscore positional arguments but received $Na"
        return :($str)
    end

    argexpr = Any[]
    cnt = 1
    for p in FA.parameters
        if p <: ArgPosition
            push!(argexpr, :(getfield(args, $(p.parameters[1]))))
        elseif !(p <: ArgsTrailing)
            push!(argexpr, :(getfield(fixed_args, $cnt)))
        end
        cnt += 1
    end
    if has_trailing_underscore
        for i in (max_underscore + 1):Na
            push!(argexpr, :(getfield(args, $i)))
        end
    end
    append!(out.args, argexpr)
    return out
end

"""
    @nfix fxn(args...; kwargs...)

Integers following an underscore (`_1`, `_2`) describe the corresponding position of arguments passed to the fixed method.
A trailing underscore (`_...`) indicates that all arguments passed that don't correspond to a fixed underscore position can be used as varargs.
"""
macro nfix(f)
    function_name = f.args[1]
    narg = length(f.args)
    argpos = 2
    if f.args[2] isa Expr && f.args[2].head === :kw
        # we only have kwargs
        pair_names = Expr(:tuple)
        pair_args = Expr(:tuple)
        for kw in f.args[argpos:end]
            push!(pair_names.args, QuoteNode(kw.args[1]))
            push!(pair_args.args, kw.args[2])
        end
        kwargs = :(NamedTuple{$pair_names}($pair_args))
        args = Expr(:tuple)
        pos = Expr(:tuple)
    else
        if f.args[2] isa Expr && f.args[2].head === :parameters
            # we also have kwargs
            pair_names = Expr(:tuple)
            pair_args = Expr(:tuple)
            for kw in f.args[2].args
                push!(pair_names.args, QuoteNode(kw.args[1]))
                push!(pair_args.args, kw.args[2])
            end
            kwargs = :(NamedTuple{$pair_names}($pair_args))
            argpos = 3
        else  # no kwargs
            kwargs = :(NamedTuple{(),Tuple{}}(()))
        end
        args = Expr(:tuple)
        pos = Expr(:tuple)
        n = 1
        underscore_positions = Int[]
        if narg > argpos
            for i in argpos:narg
                arg_i = f.args[i]
                if arg_i == :(_...)
                    if i == narg
                        push!(args.args, :(ChainedFixes.ArgsTrailing()))
                    else
                        error("trailing arguments must be the last positional arguments")
                    end
                elseif arg_i isa Symbol && first(string(arg_i)) == '_'
                    xsplit = split(string(arg_i), "_")
                    if length(xsplit) === 2
                        if xsplit[2] == ""
                            position_i = n
                        else
                            position_i = parse(Int, xsplit[2])
                        end
                        push!(underscore_positions, position_i)
                        push!(args.args, :(ChainedFixes.ArgPosition{$position_i}()))
                        n += 1
                    else
                        # there were multiple underscores in `x` so it isn't underscore followed by Int
                        push!(args.args, f.args[i])
                    end
                else
                    push!(args.args, f.args[i])
                end
            end
        end
        cnt = 1
        for p_i in sort(unique(underscore_positions))
            cnt === p_i || error("expected underscore suffixed with $cnt")
            cnt += 1
        end
    end
    esc(:(ChainedFixes.NFix($function_name, $args, $kwargs)))
end

"""
    print_fixed(io, x)

Used to print each argument and function that is a part of a `ChainedFix` or `NFix` type
"""
print_fixed(io::IO, f::Function) = print(io, "$(nameof(f))")
print_fixed(io::IO, x) = print(io, repr(x))
function print_fixed(io::IO, x::Approx)
    print(io, "≈(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end
function print_fixed(io::IO, x::StartsWith)
    print(io, "startswith(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::EndsWith)
    print(io, "endswith(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::Not)
    print(io, "!")
    print_fixed(io, getargs(x)[1])
end
function print_fixed(io::IO, x::In)
    print(io, "in(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::Less)
    print(io, "<(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::Equal)
    print(io, "==(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::NotEqual)
    print(io, "!=(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::Greater)
    print(io, ">(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::GreaterThanOrEqual)
    print(io, ">=(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end

function print_fixed(io::IO, x::LessThanOrEqual)
    print(io, "<=(")
    print_fixed(io, first(getargs(x)))
    print(io, ")")
end
function print_fixed(io::IO, f::ChainedFix)
    print_fixed(io, f.link)
    print(io, "(")
    print_fixed(io, f.f1)
    print(io, ", ")
    print_fixed(io, f.f2)
    print(io, ")")
end
function print_fixed(io::IO, f::PipeChain)
    print_fixed(io, f.f1)
    print(io, " |> ")
    print_fixed(io, f.f2)
end
print_fixed(io::IO, x::ArgPosition{N}) where {N} = print(io, "_$N")
print_fixed(io::IO, x::ArgsTrailing) = print(io, "_...")

function print_fixed(io::IO, f::NFix) where {P}
    print_fixed(io, getfxn(f))
    print(io, "(")
    cnt = 1
    args = getargs(f)
    nargs = length(args)
    for arg_i in args
        print_fixed(io, arg_i)
        if cnt !== nargs
            cnt += 1
            print(io, ", ")
        end
    end
    if !isempty(f.kwargs)
        print(io, "; ")
        kwargs = getkwargs(f)
        nkwargs = length(kwargs)
        i = 1
        for (k, v) in pairs(kwargs)
            print(io, "$k = ")
            print_fixed(io, v)
            if i !== nkwargs
                print(io, ", ")
            end
            i += 1
        end
    end
    print(io, ")")
end
Base.show(io::IO, ::MIME"text/plain", f::ChainedFix) = print_fixed(io, f)

function Base.show(io::IO, ::MIME"text/plain", f::PipeChain)
    print(io, "|> ")
    print_fixed(io, f)
end
Base.show(io::IO, ::MIME"text/plain", f::NFix) = print_fixed(io, f)

"""
    is_fixed_function(f) -> Bool

Returns `true` if `f` is a callable function that already has arguments fixed to it.
A "fixed" function can only be called on one argument (e.g., `f(arg)`) and all other
arguments are already assigned. Functions that return true should also have `getargs`
defined.
"""
is_fixed_function(::T) where {T} = is_fixed_function(T)
is_fixed_function(::Type{T}) where {T} = false
is_fixed_function(::Type{<:Fix2}) = true
is_fixed_function(::Type{<:Fix1}) = true
is_fixed_function(::Type{<:Approx}) = true
is_fixed_function(::Type{<:ChainedFix}) = true
is_fixed_function(::Type{<:NFix}) = true
is_fixed_function(::Type{<:Not}) = true

"""
    getargs(f) -> Tuple

Return a tuple of fixed positional arguments of the fixed function `f`.

## Examples

```jldoctest
julia> using ChainedFixes

julia> ChainedFixes.getargs(==(1))
(1,)

```
"""
getargs(x::Fix2) = (getfield(x, :x),)
getargs(x::Fix1) = (getfield(x, :x),)
getargs(x::Approx) = (getfield(x, :y),)
getargs(x::NFix) = getfield(x, :args)
getargs(x::Not) = (getfield(x, :f),)
getargs(x::ChainedFix) = (getfield(x, :f1), getfield(x, :f2))

"""
    getkwargs(f) -> NamedTuple

Return the fixed keyword arguments of the fixed function `f`.

## Examples

```jldoctest
julia> using ChainedFixes

julia> ChainedFixes.getkwargs(isapprox(1, atol=2))
(atol = 2,)

```
"""
getkwargs(x) = NamedTuple{(),Tuple{}}(())
getkwargs(x::Approx) = getfield(x, :kwargs).data
getkwargs(x::NFix) = getfield(x, :kwargs)
getkwargs(x::Not) = getkwargs(getfield(x, :f))

"""
    getfxn(f) -> Function

Given a fixed function `f`, returns raw method without any fixed arguments.
"""
getfxn(x) = identity
getfxn(x::ChainedFix) = getfield(x, :link)
getfxn(x::Function) = x
getfxn(x::NFix) = getfield(x, :f)
getfxn(x::Fix1) = getfield(x, :f)
getfxn(x::Fix2) = getfield(x, :f)
getfxn(x::Approx) = isapprox
getfxn(x::Not) = !

end


text:
using StatsBase, StatsFuns, JLD,  Hwloc

counts = Hwloc.histmap(Hwloc.topology_load())
ncores = counts[:Core]
if nworkers()<ncores
    addprocs(ncores   - nprocs())
end

using DynNets
@everywhere using DynNets
@everywhere using StatsFuns
N_est = 900
Tvals = [50 500 5000]#500#
N_ind = length(Tvals)[1]

N = 10 ;
GBAest =1; GWest = N; useStartVal = false

groupsIndsEst = DynNets.distributeAinVecN(ones(Int64,1),N)

est_times = Array{Float64}(zeros(N_est))
est_conv_flag = falses(N_est)
est_flag = falses(N_est)

tmpdegsT = ones(10,N)

#ridefinizioni che servono per allocare
tmpModelEst =  DynNets.SdErgm1(tmpdegsT,[zeros(GWest),zeros(GBAest),zeros(GBAest)],
                                    groupsIndsEst)
# tmpModelDgp =  DynNets.SdErgm1_UmBAgr(tmpdegsT,[zeros(GWdgp),zeros(GBAdgp),zeros(GBAdgp)],
#                                     groupsIndsDgp)
# tmpModelDgp.Par[1] = ones(GWdgp);tmpModelDgp.Par[2]  = ones(GBAdgp);tmpModelDgp.Par[3]  = ones(GBAdgp)
size(tmpModelEst.Par[1])[1]
# Set the value of the parameters common to all replicate of DGP+Estimation
sampleDegsT = 0; n=0;fit_T = 0;
#Initialize the variable to store
for i =1: size(tmpModelEst.Par)[1]
    ex1 = "estPar_$(i)  = SharedArray{Float64}(zeros(N_ind,N_est, size(tmpModelEst.Par[$(i)])[1] ))"
    eval(parse(ex1))
 end

ObsAllT = Array{Array{Int16,3},1}(N_ind)
ParTvDgpAllT = Array{Array{Float64,3},1}(N_ind)

#define save path and list variables to save
save_fold = "./data/estimatesTest/"
    file_name = "EstTest_$(N)_$(Tvals)_$(N_est)_$(useStartVal)_Sin_Miss_Targeting_GBA$(GBAest).jld"
    save_path = save_fold*file_name#

est_times = SharedArray{Float64}(zeros(N_ind,N_est))
est_conv_flag = SharedArray{Bool,2}(N_ind,N_est)
est_flag = SharedArray{Bool,2}(N_ind,N_est)

@progress "T"  for ind = 1:N_ind
#   ind=1

    T = Tvals[ind]
    ObsTmp = SharedArray{Int16,3}(Tvals[ind],N,N_est)
    ParTvDgpTmp = SharedArray{Float64,3}(Tvals[ind],N,N_est)

    @sync @parallel   for n = 1:N_est
#     n=1

        Y_T,fit_T  = DynNets.sampleDynNetsBin1(T,"FIT-MULTI-SIN";N=N)
        sampleDegsT = squeeze(sum(Y_T,2),2)

        ParTvDgpTmp[:,:,n] = fit_T
        ObsTmp[:,:,n] = sampleDegsT

        Model2Est =  DynNets.SdErgm1(sampleDegsT,[N,zeros(GBAest),zeros(GBAest)],
                                                    groupsIndsEst)
        #estimate
        useStartVal ?   start_val = [sim_W, sim_B, sim_A] : start_val = zeros(3,3)
        est_times[ind,n] = @elapsed  hat_pars , conv_flag =
                            DynNets.estimateTargeting(Model2Est )

        estPar_1[ind,n,:] = hat_pars[1]
        estPar_2[ind,n,:] = hat_pars[2]
        estPar_3[ind,n,:] = hat_pars[3]
        est_conv_flag[ind,n] = conv_flag
        est_flag[ind,n] = true
   end

   ObsAllT[ind] = ObsTmp
   ParTvDgpAllT[ind] = ParTvDgpTmp

    # save

    @save(save_path,groupsIndsEst,ObsAllT,ParTvDgpAllT ,estPar_1,estPar_2,estPar_3,est_conv_flag,est_times)
    println([N,GWest,T,sum(est_times[ind,:]),sum(est_conv_flag[ind,:])/sum(est_flag[ind,:])])
end

##

 ##


text:
global max_spectra_to_use = 250
if max_spectra_to_use < 250
   @warn "param.in setting max_spectra_to_use to " * string(max_spectra_to_use)
end

global idx_day_to_use
#global tophap_ccf_mask_scale_factor=1.6

global fits_target_str
@assert fits_target_str == "Solar" || fits_target_str == "Sun"

global linelist_for_ccf_filename = "G2.espresso.mas"

hostname = gethostname()
   if occursin("aci.ics.psu.edu",hostname)
      global ancilary_solar_data_path = "/gpfs/group/ebf11/default/ebf11/neid_solar"
   elseif occursin("nuc8",hostname)  # Eric's home machine :)
      global ancilary_solar_data_path = "/home/eford/Data/SolarSpectra/NEID_solar"
   end
   global ccf_mid_velocity = 0

#global bjd_first_good = 2458745.1296134139
   #global bjd_last_good = 2458745.283
   # Good days identified by Andrea
   good_days = ["20201215", "20201216", "20201218", "20201219", "20201220", "20201226", "20201230", "20210104", "20210109", "20210110", "20210111", "20210112", "20210115", "20210118" ]
   very_good_days = ["20201216", "20201220", "20210109", "20210110", "20210111", "20210112" ]
   #good_days = ["20210115" ]
   function is_good_day(fn::AbstractString)
      m = match(r"neidL1_(\d+)T\d+\.fits", fn)
      if m != Nothing
         substr = m.captures[1]
         return any(map(d->d==substr,good_days))
      else
         return false
      end
   end

global df_files
   global df_files_use = df_files |>
      @filter( _.target == fits_target_str ) |>
      #@filter(bjd_first_good <= _.bjd < bjd_last_good) |>
      #@filter( is_good_day(_.Filename) ) |>
      #@take(max_spectra_to_use) |>
      DataFrame
   global df_files_solar_by_day = df_files_use |> @groupby(floor(Int64,_.bjd)) |> @map({obsjd_int=key(_), data=_ } ) |> DataFrame
   if 1 <= idx_day_to_use <= size(df_files_solar_by_day,1)
      df_files_use = df_files_solar_by_day[idx_day_to_use,:data] |> @orderby(_.bjd) |> DataFrame
   else
      exit()
   end
   if size(df_files_use,1) > max_spectra_to_use
         df_files_use = df_files_use |> @orderby(-_.bjd) |> @take(max_spectra_to_use) |> @orderby(_.bjd) |> DataFrame
   end


text:
#=
These implementations were ported from the wonderful DiffLinearAlgebra
package (https://github.com/invenia/DiffLinearAlgebra.jl).
=#

using LinearAlgebra: BlasFloat

_zeros(x) = fill!(similar(x), zero(eltype(x)))

_rule_via(∂) = Rule(ΔΩ -> isa(ΔΩ, Zero) ? ΔΩ : ∂(extern(ΔΩ)))

#####
##### `BLAS.dot`
#####

frule(::typeof(BLAS.dot), x, y) = frule(dot, x, y)

rrule(::typeof(BLAS.dot), x, y) = rrule(dot, x, y)

function rrule(::typeof(BLAS.dot), n, X, incx, Y, incy)
    Ω = BLAS.dot(n, X, incx, Y, incy)
    ∂X = ΔΩ -> scal!(n, ΔΩ, blascopy!(n, Y, incy, _zeros(X), incx), incx)
    ∂Y = ΔΩ -> scal!(n, ΔΩ, blascopy!(n, X, incx, _zeros(Y), incy), incy)
    return Ω, (DNERule(), _rule_via(∂X), DNERule(), _rule_via(∂Y), DNERule())
end

#####
##### `BLAS.nrm2`
#####

function frule(::typeof(BLAS.nrm2), x)
    Ω = BLAS.nrm2(x)
    return Ω, Rule(Δx -> sum(Δx * cast(@thunk(x * inv(Ω)))))
end

function rrule(::typeof(BLAS.nrm2), x)
    Ω = BLAS.nrm2(x)
    return Ω, Rule(ΔΩ -> ΔΩ * @thunk(x * inv(Ω)))
end

function rrule(::typeof(BLAS.nrm2), n, X, incx)
    Ω = BLAS.nrm2(n, X, incx)
    ∂X = ΔΩ -> scal!(n, ΔΩ / Ω, blascopy!(n, X, incx, _zeros(X), incx), incx)
    return Ω, (DNERule(), _rule_via(∂X), DNERule())
end

#####
##### `BLAS.asum`
#####

frule(::typeof(BLAS.asum), x) = (BLAS.asum(x), Rule(Δx -> sum(cast(sign, x) * Δx)))

rrule(::typeof(BLAS.asum), x) = (BLAS.asum(x), Rule(ΔΩ -> ΔΩ * cast(sign, x)))

function rrule(::typeof(BLAS.asum), n, X, incx)
    Ω = BLAS.asum(n, X, incx)
    ∂X = ΔΩ -> scal!(n, ΔΩ, blascopy!(n, sign.(X), incx, _zeros(X), incx), incx)
    return Ω, (DNERule(), _rule_via(∂X), DNERule())
end

#####
##### `BLAS.gemv`
#####

function rrule(::typeof(gemv), tA::Char, α::T, A::AbstractMatrix{T},
               x::AbstractVector{T}) where T<:BlasFloat
    y = gemv(tA, α, A, x)
    if uppercase(tA) === 'N'
        ∂A = Rule(ȳ -> α * ȳ * x', (Ā, ȳ) -> ger!(α, ȳ, x, Ā))
        ∂x = Rule(ȳ -> gemv('T', α, A, ȳ), (x̄, ȳ) -> gemv!('T', α, A, ȳ, one(T), x̄))
    else
        ∂A = Rule(ȳ -> α * x * ȳ', (Ā, ȳ) -> ger!(α, x, ȳ, Ā))
        ∂x = Rule(ȳ -> gemv('N', α, A, ȳ), (x̄, ȳ) -> gemv!('N', α, A, ȳ, one(T), x̄))
    end
    return y, (DNERule(), Rule(ȳ -> dot(ȳ, y) / α), ∂A, ∂x)
end

function rrule(::typeof(gemv), tA::Char, A::AbstractMatrix{T},
               x::AbstractVector{T}) where T<:BlasFloat
    y, (dtA, _, dA, dx) = rrule(gemv, tA, one(T), A, x)
    return y, (dtA, dA, dx)
end

#####
##### `BLAS.gemm`
#####

function rrule(::typeof(gemm), tA::Char, tB::Char, α::T,
               A::AbstractMatrix{T}, B::AbstractMatrix{T}) where T<:BlasFloat
    C = gemm(tA, tB, α, A, B)
    β = one(T)
    if uppercase(tA) === 'N'
        if uppercase(tB) === 'N'
            ∂A = Rule(C̄ -> gemm('N', 'T', α, C̄, B),
                      (Ā, C̄) -> gemm!('N', 'T', α, C̄, B, β, Ā))
            ∂B = Rule(C̄ -> gemm('T', 'N', α, A, C̄),
                      (B̄, C̄) -> gemm!('T', 'N', α, A, C̄, β, B̄))
        else
            ∂A = Rule(C̄ -> gemm('N', 'N', α, C̄, B),
                      (Ā, C̄) -> gemm!('N', 'N', α, C̄, B, β, Ā))
            ∂B = Rule(C̄ -> gemm('T', 'N', α, C̄, A),
                      (B̄, C̄) -> gemm!('T', 'N', α, C̄, A, β, B̄))
        end
    else
        if uppercase(tB) === 'N'
            ∂A = Rule(C̄ -> gemm('N', 'T', α, B, C̄),
                      (Ā, C̄) -> gemm!('N', 'T', α, B, C̄, β, Ā))
            ∂B = Rule(C̄ -> gemm('N', 'N', α, A, C̄),
                      (B̄, C̄) -> gemm!('N', 'N', α, A, C̄, β, B̄))
        else
            ∂A = Rule(C̄ -> gemm('T', 'T', α, B, C̄),
                      (Ā, C̄) -> gemm!('T', 'T', α, B, C̄, β, Ā))
            ∂B = Rule(C̄ -> gemm('T', 'T', α, C̄, A),
                      (B̄, C̄) -> gemm!('T', 'T', α, C̄, A, β, B̄))
        end
    end
    return C, (DNERule(), DNERule(), Rule(C̄ -> dot(C̄, C) / α), ∂A, ∂B)
end

function rrule(::typeof(gemm), tA::Char, tB::Char,
               A::AbstractMatrix{T}, B::AbstractMatrix{T}) where T<:BlasFloat
    C, (dtA, dtB, _, dA, dB) = rrule(gemm, tA, tB, one(T), A, B)
    return C, (dtA, dtB, dA, dB)
end