text:
module TestDataFrame

using Dates, DataFrames, LinearAlgebra, Statistics, Random, Test
using DataFrames: _columns
using DataFrames: columns
const ≅ = isequal
const ≇ = !isequal

@testset "equality" begin
    @test DataFrame(a=[1, 2, 3], b=[4, 5, 6]) == DataFrame(a=[1, 2, 3], b=[4, 5, 6])
    @test DataFrame(a=[1, 2], b=[4, 5]) != DataFrame(a=[1, 2, 3], b=[4, 5, 6])
    @test DataFrame(a=[1, 2, 3], b=[4, 5, 6]) != DataFrame(a=[1, 2, 3])
    @test DataFrame(a=[1, 2, 3], b=[4, 5, 6]) != DataFrame(a=[1, 2, 3], c=[4, 5, 6])
    @test DataFrame(a=[1, 2, 3], b=[4, 5, 6]) != DataFrame(b=[4, 5, 6], a=[1, 2, 3])
    @test DataFrame(a=[1, 2, 2], b=[4, 5, 6]) != DataFrame(a=[1, 2, 3], b=[4, 5, 6])
    @test DataFrame(a=[1, 3, missing], b=[4, 5, 6]) != DataFrame(a=[1, 2, missing], b=[4, 5, 6])
    @test DataFrame(a=[1, 2, missing], b=[4, 5, 6]) ≅ DataFrame(a=[1, 2, missing], b=[4, 5, 6])
    @test DataFrame(a=[1, 2, missing], b=[4, 5, 6]) ≇ DataFrame(a=[1, 2, 3], b=[4, 5, 6])
end

@testset "copying" begin
    df = DataFrame(a = Union{Int, Missing}[2, 3],
                b = Union{DataFrame, Missing}[DataFrame(c = 1), DataFrame(d = 2)])
    dfc = copy(df)
    dfdc = deepcopy(df)

    df[1, :a] = 4
    df[1, :b][:e] = 5
    names!(df, [:f, :g])

    @test names(dfc) == [:a, :b]
    @test names(dfdc) == [:a, :b]

    @test dfc[1, :a] === 4
    @test dfdc[1, :a] === 2

    @test names(dfc[1, :b]) == [:c, :e]
    @test names(dfdc[1, :b]) == [:c]

    x = DataFrame(a = [1, 2, 3], b = [4, 5, 6])

    #test_group("DataFrame assignment")
    # Insert single column
    x0 = x[Int[], :]
    @test_throws ArgumentError x0[:d] = [1]
    @test_throws ArgumentError x0[:d] = 1:3

    # Insert single value
    x[:d] = 3
    @test x[:d] == [3, 3, 3]

    x0[:d] = 3
    @test x0[:d] == Int[]
end

@testset "similar / missings" begin
    df = DataFrame(a = Union{Int, Missing}[1],
                b = Union{String, Missing}["b"],
                c = CategoricalArray{Union{Float64, Missing}}([3.3]))
    missingdf = DataFrame(a = missings(Int, 2),
                        b = missings(String, 2),
                        c = CategoricalArray{Union{Float64, Missing}}(undef, 2))
    # https://github.com/JuliaData/Missings.jl/issues/66
    # @test missingdf ≅ similar(df, 2)
    @test typeof.(columns(similar(df, 2))) == typeof.(columns(missingdf))
    @test size(similar(df, 2)) == size(missingdf)
end

@testset "Associative methods" begin
    df = DataFrame(a=[1, 2], b=[3.0, 4.0])
    @test haskey(df, :a)
    @test !haskey(df, :c)
    @test haskey(df, 1)
    @test_throws MethodError haskey(df, 1.5)
    @test_throws ArgumentError haskey(df, true)
    @test get(df, :a, -1) === columns(df)[1]
    @test get(df, :c, -1) == -1
    @test !isempty(df)

    dfv = view(df, 1:2, 1:2)
    @test get(df, :a, -1) === columns(df)[1]

    @test empty!(df) === df
    @test isempty(columns(df))
    @test isempty(df)
    @test isempty(DataFrame(a=[], b=[]))

    df = DataFrame(a=Union{Int, Missing}[1, 2], b=Union{Float64, Missing}[3.0, 4.0])
    @test_throws BoundsError insertcols!(df, 5, :newcol => ["a", "b"], )
    @test_throws ErrorException insertcols!(df, 1, :newcol => ["a"])
    @test insertcols!(df, 1, :newcol => ["a", "b"]) == df
    @test names(df) == [:newcol, :a, :b]
    @test df[:a] == [1, 2]
    @test df[:b] == [3.0, 4.0]
    @test df[:newcol] == ["a", "b"]

    @test insertcols!(df, 1, :newcol => ["a1", "b1"], makeunique=true) == df
    @test names(df) == [:newcol_1, :newcol, :a, :b]
    @test df[:a] == [1, 2]
    @test df[:b] == [3.0, 4.0]
    @test df[:newcol] == ["a", "b"]
    @test df[:newcol_1] == ["a1", "b1"]

    df = DataFrame(a=[1,2], a_1=[3,4])
    @test_throws ArgumentError insertcols!(df, 1, :a => [11,12])
    df = DataFrame(a=[1,2], a_1=[3,4])
    insertcols!(df, 1, :a => [11,12], makeunique=true)
    @test names(df) == [:a_2, :a, :a_1]
    insertcols!(df, 4, :a => [11,12], makeunique=true)
    @test names(df) == [:a_2, :a, :a_1, :a_3]
    @test_throws BoundsError insertcols!(df, 10, :a => [11,12], makeunique=true)
    df = DataFrame(a=[1,2], a_1=[3,4])
    insertcols!(df, 1, :a => 11, makeunique=true)
    @test names(df) == [:a_2, :a, :a_1]
    insertcols!(df, 4, :a => 11, makeunique=true)
    @test names(df) == [:a_2, :a, :a_1, :a_3]
    @test_throws BoundsError insertcols!(df, 10, :a => 11, makeunique=true)

    df = DataFrame(x = 1:2)
    @test insertcols!(df, 2, y=2:3) == DataFrame(x=1:2, y=2:3)
    @test_throws ArgumentError insertcols!(df, 2)
    @test_throws ArgumentError insertcols!(df, 2, a=1, b=2)

    df = DataFrame()
    @test insertcols!(df, 1, x=[1]) == DataFrame(x = [1])
end

@testset "DataFrame constructors" begin
    df = DataFrame(Union{Int, Missing}, 10, 3)
    @test size(df, 1) == 10
    @test size(df, 2) == 3
    @test typeof(df[1]) == Vector{Union{Int, Missing}}
    @test typeof(df[2]) == Vector{Union{Int, Missing}}
    @test typeof(df[3]) == Vector{Union{Int, Missing}}
    @test all(ismissing, df[1])
    @test all(ismissing, df[2])
    @test all(ismissing, df[3])
    @test typeof(df[:, 1]) == Vector{Union{Int, Missing}}
    @test typeof(df[:, 2]) == Vector{Union{Int, Missing}}
    @test typeof(df[:, 3]) == Vector{Union{Int, Missing}}
    @test all(ismissing, df[:, 1])
    @test all(ismissing, df[:, 2])
    @test all(ismissing, df[:, 3])

    df = DataFrame([Union{Int, Missing}, Union{Float64, Missing}, Union{String, Missing}], 100)
    @test size(df, 1) == 100
    @test size(df, 2) == 3
    @test typeof(df[1]) == Vector{Union{Int, Missing}}
    @test typeof(df[2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[3]) == Vector{Union{String, Missing}}
    @test all(ismissing, df[1])
    @test all(ismissing, df[2])
    @test all(ismissing, df[3])
    @test typeof(df[:, 1]) == Vector{Union{Int, Missing}}
    @test typeof(df[:, 2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[:, 3]) == Vector{Union{String, Missing}}
    @test all(ismissing, df[:, 1])
    @test all(ismissing, df[:, 2])
    @test all(ismissing, df[:, 3])

    df = DataFrame([Union{Int, Missing}, Union{Float64, Missing}, Union{String, Missing}],
                [:A, :B, :C], 100)
    @test size(df, 1) == 100
    @test size(df, 2) == 3
    @test typeof(df[1]) == Vector{Union{Int, Missing}}
    @test typeof(df[2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[3]) == Vector{Union{String, Missing}}
    @test all(ismissing, df[1])
    @test all(ismissing, df[2])
    @test all(ismissing, df[3])
    @test typeof(df[:, 1]) == Vector{Union{Int, Missing}}
    @test typeof(df[:, 2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[:, 3]) == Vector{Union{String, Missing}}
    @test all(ismissing, df[:, 1])
    @test all(ismissing, df[:, 2])
    @test all(ismissing, df[:, 3])

    df = DataFrame([Union{Int, Missing}, Union{Float64, Missing}, Union{String, Missing}],
                [:A, :B, :C], [false, false, true], 100)
    @test size(df, 1) == 100
    @test size(df, 2) == 3
    @test typeof(df[1]) == Vector{Union{Int, Missing}}
    @test typeof(df[2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[3]) <: CategoricalVector{Union{String, Missing}}
    @test all(ismissing, df[1])
    @test all(ismissing, df[2])
    @test all(ismissing, df[3])
    @test typeof(df[:, 1]) == Vector{Union{Int, Missing}}
    @test typeof(df[:, 2]) == Vector{Union{Float64, Missing}}
    @test typeof(df[:, 3]) <: CategoricalVector{Union{String, Missing}}
    @test all(ismissing, df[:, 1])
    @test all(ismissing, df[:, 2])
    @test all(ismissing, df[:, 3])

    df = convert(DataFrame, zeros(10, 5))
    @test size(df, 1) == 10
    @test size(df, 2) == 5
    @test typeof(df[1]) == Vector{Float64}
    @test typeof(df[:, 1]) == Vector{Float64}

    df = convert(DataFrame, ones(10, 5))
    @test size(df, 1) == 10
    @test size(df, 2) == 5
    @test typeof(df[1]) == Vector{Float64}
    @test typeof(df[:, 1]) == Vector{Float64}

    df = convert(DataFrame, Matrix{Float64}(undef, 10, 5))
    @test size(df, 1) == 10
    @test size(df, 2) == 5
    @test typeof(df[1]) == Vector{Float64}
    @test typeof(df[:, 1]) == Vector{Float64}

    @test DataFrame([Union{Int, Missing}[1, 2, 3], Union{Float64, Missing}[2.5, 4.5, 6.5]],
                    [:A, :B]) ==
        DataFrame(A = Union{Int, Missing}[1, 2, 3], B = Union{Float64, Missing}[2.5, 4.5, 6.5])

    # This assignment was missing before
    df = DataFrame(Column = [:A])
    df[1, :Column] = :Testing

    # zero-row DataFrame and subDataFrame test
    df = DataFrame(x=[], y=[])
    @test nrow(df) == 0
    df = DataFrame(x=[1:3;], y=[3:5;])
    sdf = view(df, df[:x] .== 4, :)
    @test size(sdf, 1) == 0

    # Test that vector type is correctly determined from scalar type
    df = DataFrame(x=categorical(["a"])[1])
    @test df.x isa CategoricalVector{String}

    @test hash(convert(DataFrame, [1 2; 3 4])) == hash(convert(DataFrame, [1 2; 3 4]))
    @test hash(convert(DataFrame, [1 2; 3 4])) != hash(convert(DataFrame, [1 3; 2 4]))
    @test hash(convert(DataFrame, [1 2; 3 4])) == hash(convert(DataFrame, [1 2; 3 4]), zero(UInt))
end

@testset "push!(df, row)" begin
    df=DataFrame( first=[1,2,3], second=["apple","orange","pear"] )

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    dfc= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, Any[3,"pear"])
    @test df == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, (3,"pear"))
    @test df == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, (33.33,"pear"))
    @test dfc == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, (1,"2",3))
    @test dfc == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, ("coconut",22))
    @test dfc == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, (11,22))
    @test dfc == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, Dict(:first=>3, :second=>"pear"))
    @test df == dfb

    df=DataFrame( first=[1,2,3], second=["apple","orange","banana"] )
    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, Dict(:first=>3, :second=>"banana"))
    @test df == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, (first=3, second="banana"))
    @test df == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, (second="banana", first=3))
    @test df == dfb

    df0= DataFrame( first=[1,2], second=["apple","orange"] )
    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, (second=3, first=3))
    @test df0 == dfb

    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    push!(dfb, (second="banana", first=3))
    @test df == dfb

    df0= DataFrame( first=[1,2], second=["apple","orange"] )
    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, Dict(:first=>true, :second=>false))
    @test df0 == dfb

    df0= DataFrame( first=[1,2], second=["apple","orange"] )
    dfb= DataFrame( first=[1,2], second=["apple","orange"] )
    @test_throws ArgumentError push!(dfb, Dict(:first=>"chicken", :second=>"stuff"))
    @test df0 == dfb

    df0=DataFrame( first=[1,2,3], second=["apple","orange","pear"] )
    dfb=DataFrame( first=[1,2,3], second=["apple","orange","pear"] )
    @test_throws ArgumentError push!(dfb, Dict(:first=>"chicken", :second=>1))
    @test df0 == dfb

    df0=DataFrame( first=["1","2","3"], second=["apple","orange","pear"] )
    dfb=DataFrame( first=["1","2","3"], second=["apple","orange","pear"] )
    @test_throws ArgumentError push!(dfb, Dict(:first=>"chicken", :second=>1))
    @test df0 == dfb

    df = DataFrame(x=1)
    push!(df, Dict(:x=>2), Dict(:x=>3))
    @test df[:x] == [1,2,3]

    df = DataFrame(x=1, y=2)
    push!(df, [3, 4], [5, 6])
    @test df[:x] == [1, 3, 5] && df[:y] == [2, 4, 6]

    df = DataFrame(x=1, y=2)
    @test_throws ArgumentError push!(df, Dict(:x=>1, "y"=>2))
    @test df == DataFrame(x=1, y=2)
end

@testset "deletecols!" begin
    df = DataFrame(a=1, b=2, c=3, d=4, e=5)
    @test_throws ArgumentError deletecols!(df, 0)
    @test_throws ArgumentError deletecols!(df, 6)
    @test_throws ArgumentError deletecols!(df, [1, 1])
    @test_throws ArgumentError deletecols!(df, :f)

    d = copy(df)
    deletecols!(d, [:a, :e, :c])
    @test names(d) == [:b, :d]
    deletecols!(d, :b)
    @test d == DataFrame(d=4)

    d = copy(df)
    deletecols!(d, [2, 5, 3])
    @test names(d) == [:a, :d]
    deletecols!(d, 2)
    @test d == DataFrame(a=1)

    d = copy(df)
    deletecols!(d, 2:3)
    @test d == DataFrame(a=1, d=4, e=5)
end

@testset "deleterows!" begin
    df = DataFrame(a=[1, 2], b=[3.0, 4.0])
    @test deleterows!(df, 1) === df
    @test df == DataFrame(a=[2], b=[4.0])

    df = DataFrame(a=[1, 2], b=[3.0, 4.0])
    @test deleterows!(df, 2) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame(a=[1, 2, 3], b=[3.0, 4.0, 5.0])
    @test deleterows!(df, 2:3) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame(a=[1, 2, 3], b=[3.0, 4.0, 5.0])
    @test deleterows!(df, [2, 3]) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame(a=Union{Int, Missing}[1, 2], b=Union{Float64, Missing}[3.0, 4.0])
    @test deleterows!(df, 1) === df
    @test df == DataFrame(a=[2], b=[4.0])

    df = DataFrame(a=Union{Int, Missing}[1, 2], b=Union{Float64, Missing}[3.0, 4.0])
    @test deleterows!(df, 2) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame(a=Union{Int, Missing}[1, 2, 3], b=Union{Float64, Missing}[3.0, 4.0, 5.0])
    @test deleterows!(df, 2:3) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame(a=Union{Int, Missing}[1, 2, 3], b=Union{Float64, Missing}[3.0, 4.0, 5.0])
    @test deleterows!(df, [2, 3]) === df
    @test df == DataFrame(a=[1], b=[3.0])

    df = DataFrame()
    @test_throws BoundsError deleterows!(df, 10)
    @test_throws BoundsError deleterows!(df, [10])

    df = DataFrame(a=[])
    @test_throws BoundsError deleterows!(df, 10)
    # the exception type changed between Julia 1.0.2 and Julia 1.1
    # so we use their supertype below
    @test_throws Exception deleterows!(df, [10])

    df = DataFrame(a=[1, 2, 3], b=[3, 2, 1])
    @test_throws ArgumentError deleterows!(df, [3,2])
    @test_throws ArgumentError deleterows!(df, [2,2])
    @test deleterows!(df, [false, true, false]) === df
    @test df == DataFrame(a=[1, 3], b=[3, 1])

    x = [1, 2, 3]
    df = DataFrame(x=x)
    @test deleterows!(df, 1) == DataFrame(x=[2, 3])
    @test x == [2, 3]

    x = [1, 2, 3]
    df = DataFrame(x=x)
    @test deleterows!(df, [1]) == DataFrame(x=[2, 3])
    @test x == [2, 3]

    x = [1, 2, 3]
    df = DataFrame(x=x)
    @test deleterows!(df, 1:1) == DataFrame(x=[2, 3])
    @test x == [2, 3]

    x = [1, 2, 3]
    df = DataFrame(x=x)
    @test deleterows!(df, [true, false, false]) == DataFrame(x=[2, 3])
    @test x == [2, 3]
end

@testset "describe" begin
    # Construct the test dataframe
    df = DataFrame(number = [1, 2, 3, 4],
                   number_missing = [1,2, 3, missing],
                   string = ["a", "b", "c", "d"],
                   string_missing = ["a", "b", "c", missing],
                   dates  = Date.([2000, 2001, 2003, 2004]),
                   catarray = CategoricalArray([1,2,1,2]))

    describe_output = DataFrame(variable = [:number, :number_missing, :string,
                                            :string_missing, :dates, :catarray],
                                mean = [2.5, 2.0, nothing, nothing, nothing, nothing],
                                std = [std(df[:number]), 1.0, nothing,
                                       nothing, nothing, nothing],
                                min = [1.0, 1.0, "a", "a", Date(2000), 1],
                                q25 = [1.75, 1.5, nothing, nothing, nothing, nothing],
                                median = [2.5, 2.0, nothing, nothing, nothing, nothing],
                                q75 = [3.25, 2.5, nothing, nothing, nothing, nothing],
                                max = [4.0, 3.0, "d", "c", Date(2004), 2],
                                nunique = [nothing, nothing, 4, 3, 4, 2],
                                nmissing = [nothing, 1, nothing, 1, nothing, nothing],
                                first = [1, 1, "a", "a", Date(2000), 1],
                                last = [4, missing, "d", missing, Date(2004), 2],
                                eltype = [Int, Int, String, String, Date,
                                          eltype(df[:catarray])])

    default_fields = [:mean, :min, :median, :max, :nunique, :nmissing, :eltype]

    # Test that it works as a whole, without keyword arguments
    @test describe_output[[:variable; default_fields]] == describe(df)

    # Test that it works with one stats argument
    @test describe_output[[:variable, :mean]] == describe(df, :mean)

    # Test that it works with all keyword arguments
    @test describe_output ≅ describe(df, :all)

    # Test that it works on a custom function
    describe_output.test_std = describe_output.std
    # Test that describe works with a Pair and a symbol
    @test describe_output[[:variable, :mean, :test_std]] ≅ describe(df, :mean, :test_std => std)

    # Test that describe works with a dataframe with no observations
    df = DataFrame(a = Int[], b = String[], c = [])
    @test describe(df, :mean) ≅ DataFrame(variable = [:a, :b, :c],
                                          mean = [NaN, nothing, nothing])

    @test_throws ArgumentError describe(df, :mean, :all)
end

#Check the output of unstack
df = DataFrame(Fish = CategoricalArray{Union{String, Missing}}(["Bob", "Bob", "Batman", "Batman"]),
               Key = CategoricalArray{Union{String, Missing}}(["Mass", "Color", "Mass", "Color"]),
               Value = Union{String, Missing}["12 g", "Red", "18 g", "Grey"])
# Check that reordering levels does not confuse unstack
levels!(df[1], ["XXX", "Bob", "Batman"])
levels!(df[2], ["YYY", "Color", "Mass"])
#Unstack specifying a row column
df2 = unstack(df, :Fish, :Key, :Value)
@test levels(df[1]) == ["XXX", "Bob", "Batman"] # make sure we did not mess df[1] levels
@test levels(df[2]) == ["YYY", "Color", "Mass"] # make sure we did not mess df[2] levels
#Unstack without specifying a row column
df3 = unstack(df, :Key, :Value)
#The expected output, XXX level should be dropped as it has no rows with this key
df4 = DataFrame(Fish = Union{String, Missing}["Bob", "Batman"],
                Color = Union{String, Missing}["Red", "Grey"],
                Mass = Union{String, Missing}["12 g", "18 g"])
@test df2 ≅ df4
@test typeof(df2[:Fish]) <: CategoricalVector{Union{String, Missing}}
# first column stays as CategoricalArray in df3
@test df3 == df4
#Make sure unstack works with missing values at the start of the value column
df[1,:Value] = missing
df2 = unstack(df, :Fish, :Key, :Value)
#This changes the expected result
df4[1,:Mass] = missing
@test df2 ≅ df4

#The same as above but without CategoricalArray
df = DataFrame(Fish = ["Bob", "Bob", "Batman", "Batman"],
               Key = ["Mass", "Color", "Mass", "Color"],
               Value = ["12 g", "Red", "18 g", "Grey"])
#Unstack specifying a row column
df2 = unstack(df, :Fish, :Key, :Value)
#Unstack without specifying a row column
df3 = unstack(df, :Key, :Value)
#The expected output, XXX level should be dropped as it has no rows with this key
df4 = DataFrame(Fish = ["Batman", "Bob"],
                Color = ["Grey", "Red"],
                Mass = ["18 g", "12 g"])
@test df2 ≅ df4
@test typeof(df2[:Fish]) <: Vector{String}
# first column stays as CategoricalArray in df3
@test df3 == df4
#Make sure unstack works with missing values at the start of the value column
allowmissing!(df, :Value)
df[1,:Value] = missing
df2 = unstack(df, :Fish, :Key, :Value)
#This changes the expected result
allowmissing!(df4, :Mass)
df4[2,:Mass] = missing
@test df2 ≅ df4

# test empty set of grouping variables
@test_throws ArgumentError unstack(df, Int[], :Key, :Value)
@test_throws ArgumentError unstack(df, Symbol[], :Key, :Value)
@test_throws ArgumentError unstack(stack(DataFrame(rand(10, 10))),
                              :id, :variable, :value)

# test missing value in grouping variable
mdf = DataFrame(id=[missing,1,2,3], a=1:4, b=1:4)
@test unstack(melt(mdf, :id), :id, :variable, :value)[1:3,:] == sort(mdf)[1:3,:]
@test unstack(melt(mdf, :id), :id, :variable, :value)[2:3] == sort(mdf)[2:3]

# test more than one grouping column
wide = DataFrame(id = 1:12,
                 a  = repeat([1:3;], inner = [4]),
                 b  = repeat([1:4;], inner = [3]),
                 c  = randn(12),
                 d  = randn(12))

long = stack(wide)
wide3 = unstack(long, [:id, :a], :variable, :value)
@test wide3 == wide[[1, 2, 4, 5]]

df = DataFrame(A = 1:10, B = 'A':'J')
@test !(df[:] === df)
@test !(df[:,:] === df)

@testset "append!" begin
    df = DataFrame(A = 1:2, B = 1:2)
    df2 = DataFrame(A = 1:4, B = 1:4)
    @test append!(df, DataFrame(A = 3:4, B = [3.0, 4.0])) == df2
    @test_throws InexactError append!(df, DataFrame(A = 3:4, B = [3.5, 4.5]))
    @test df == df2
    @test_throws MethodError append!(df, DataFrame(A = 3:4, B = ["a", "b"]))
    @test df == df2

    dfx = DataFrame()
    df3 = append!(dfx, df)
    @test dfx === df3
    @test df3 == df
    @test df3[1] !== df[1]
    @test df3[2] !== df[2]

    df4 = append!(df3, DataFrame())
    @test df4 === df3
    @test df4 == df
end

df = DataFrame(A = Vector{Union{Int, Missing}}(1:3), B = Vector{Union{Int, Missing}}(4:6))
DRT = CategoricalArrays.DefaultRefType
@test all(c -> isa(c, Vector{Union{Int, Missing}}), columns(categorical!(deepcopy(df))))
@test all(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
          columns(categorical!(deepcopy(df), [1,2])))
@test all(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
          columns(categorical!(deepcopy(df), [:A,:B])))
@test findfirst(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
                _columns(categorical!(deepcopy(df), [:A]))) == 1
@test findfirst(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
                _columns(categorical!(deepcopy(df), :A))) == 1
@test findfirst(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
                _columns(categorical!(deepcopy(df), [1]))) == 1
@test findfirst(c -> typeof(c) <: CategoricalVector{Union{Int, Missing}},
                _columns(categorical!(deepcopy(df), 1))) == 1

@testset "categorical!" begin
    df = DataFrame([["a", "b"], ['a', 'b'], [true, false], 1:2, ["x", "y"]])
    @test all(map(<:, eltypes(categorical!(deepcopy(df))),
                  [CategoricalArrays.CategoricalString{UInt32},
                   Char, Bool, Int,
                   CategoricalArrays.CategoricalString{UInt32}]))
    @test all(map(<:, eltypes(categorical!(deepcopy(df), compress=true)),
                  [CategoricalArrays.CategoricalString{UInt8},
                   Char, Bool, Int,
                   CategoricalArrays.CategoricalString{UInt8}]))
    @test all(map(<:, eltypes(categorical!(df, names(df))),
                  [CategoricalArrays.CategoricalString{UInt32},
                   CategoricalArrays.CategoricalValue{Char,UInt32},
                   CategoricalArrays.CategoricalValue{Bool,UInt32},
                   CategoricalArrays.CategoricalValue{Int,UInt32},
                   CategoricalArrays.CategoricalString{UInt32}]))
    @test all(map(<:, eltypes(categorical!(df, names(df), compress=true)),
                  [CategoricalArrays.CategoricalString{UInt8},
                   CategoricalArrays.CategoricalValue{Char,UInt8},
                   CategoricalArrays.CategoricalValue{Bool,UInt8},
                   CategoricalArrays.CategoricalValue{Int,UInt8},
                   CategoricalArrays.CategoricalString{UInt8}]))
end

@testset "unstack promotion to support missing values" begin
    df = DataFrame([repeat(1:2, inner=4), repeat('a':'d', outer=2), collect(1:8)],
                   [:id, :variable, :value])
    udf = unstack(df, :variable, :value)
    @test udf == unstack(df, :variable, :value) == unstack(df, :id, :variable, :value)
    @test udf == DataFrame([Union{Int, Missing}[1, 2], Union{Int, Missing}[1, 5],
                            Union{Int, Missing}[2, 6], Union{Int, Missing}[3, 7],
                            Union{Int, Missing}[4, 8]], [:id, :a, :b, :c, :d])
    @test isa(udf[1], Vector{Int})
    @test all(isa.(columns(udf)[2:end], Vector{Union{Int, Missing}}))
    df = DataFrame([categorical(repeat(1:2, inner=4)),
                       categorical(repeat('a':'d', outer=2)), categorical(1:8)],
                   [:id, :variable, :value])
    udf = unstack(df, :variable, :value)
    @test udf == unstack(df, :variable, :value) == unstack(df, :id, :variable, :value)
    @test udf == DataFrame([Union{Int, Missing}[1, 2], Union{Int, Missing}[1, 5],
                            Union{Int, Missing}[2, 6], Union{Int, Missing}[3, 7],
                            Union{Int, Missing}[4, 8]], [:id, :a, :b, :c, :d])
    @test isa(udf[1], CategoricalVector{Int})
    @test all(isa.(columns(udf)[2:end], CategoricalVector{Union{Int, Missing}}))
end

@testset "duplicate entries in unstack warnings" begin
    df = DataFrame(id=Union{Int, Missing}[1, 2, 1, 2],
                   id2=Union{Int, Missing}[1, 2, 1, 2],
                   variable=["a", "b", "a", "b"], value=[3, 4, 5, 6])
    @test_logs (:warn, "Duplicate entries in unstack at row 3 for key 1 and variable a.") unstack(df, :id, :variable, :value)
    @test_logs (:warn, "Duplicate entries in unstack at row 3 for key (1, 1) and variable a.") unstack(df, :variable, :value)
    a = unstack(df, :id, :variable, :value)
    @test a ≅ DataFrame(id = [1, 2], a = [5, missing], b = [missing, 6])
    b = unstack(df, :variable, :value)
    @test b ≅ DataFrame(id = [1, 2], id2 = [1, 2], a = [5, missing], b = [missing, 6])

    df = DataFrame(id=1:2, variable=["a", "b"], value=3:4)
    @test_nowarn unstack(df, :id, :variable, :value)
    @test_nowarn unstack(df, :variable, :value)
    a = unstack(df, :id, :variable, :value)
    b = unstack(df, :variable, :value)
    @test a ≅ b ≅ DataFrame(id = [1, 2], a = [3, missing], b = [missing, 4])

    df = DataFrame(variable=["x", "x"], value=[missing, missing], id=[1,1])
    @test_logs (:warn, "Duplicate entries in unstack at row 2 for key 1 and variable x.") unstack(df, :variable, :value)
    @test_logs (:warn, "Duplicate entries in unstack at row 2 for key 1 and variable x.") unstack(df, :id, :variable, :value)
end

@testset "missing values in colkey" begin
    df = DataFrame(id=[1, 1, 1, missing, missing, missing, 2, 2, 2],
                   variable=["a", "b", missing, "a", "b", "missing", "a", "b", "missing"],
                   value=[missing, 2.0, 3.0, 4.0, 5.0, missing, 7.0, missing, 9.0])
    @test_logs (:warn, "Missing value in variable variable at row 3. Skipping.") unstack(df, :variable, :value)
    udf = unstack(df, :variable, :value)
    @test names(udf) == [:id, :a, :b, :missing]
    @test udf[:missing] ≅ [missing, 9.0, missing]
    df = DataFrame(id=[1, 1, 1, missing, missing, missing, 2, 2, 2],
                   id2=[1, 1, 1, missing, missing, missing, 2, 2, 2],
                   variable=["a", "b", missing, "a", "b", "missing", "a", "b", "missing"],
                   value=[missing, 2.0, 3.0, 4.0, 5.0, missing, 7.0, missing, 9.0])
    @test_logs (:warn, "Missing value in variable variable at row 3. Skipping.") unstack(df, 3, 4)
    udf = unstack(df, 3, 4)
    @test names(udf) == [:id, :id2, :a, :b, :missing]
    @test udf[:missing] ≅ [missing, 9.0, missing]
end

@testset "stack-unstack correctness" begin
    x = DataFrame(rand(100, 50))
    x[:id] = [1:99; missing]
    x[:id2] = string.("a", x[:id])
    x[:s] = [i % 2 == 0 ? randstring() : missing for i in 1:100]
    allowmissing!(x, :x1)
    x[1, :x1] = missing
    y = melt(x, [:id, :id2])
    z = unstack(y, :id, :variable, :value)
    @test all(isequal(z[n], x[n]) for n in names(z))
    z = unstack(y, :variable, :value)
    @test all(isequal(z[n], x[n]) for n in names(x))
end

@testset "rename" begin
    df = DataFrame(A = 1:3, B = 'A':'C')
    @test names(rename(df, :A => :A_1)) == [:A_1, :B]
    @test names(df) == [:A, :B]
    @test names(rename(df, :A => :A_1, :B => :B_1)) == [:A_1, :B_1]
    @test names(df) == [:A, :B]
    @test names(rename(df, [:A => :A_1, :B => :B_1])) == [:A_1, :B_1]
    @test names(df) == [:A, :B]
    @test names(rename(df, Dict(:A => :A_1, :B => :B_1))) == [:A_1, :B_1]
    @test names(df) == [:A, :B]
    @test names(rename(x->Symbol(lowercase(string(x))), df)) == [:a, :b]
    @test names(df) == [:A, :B]

    @test rename!(df, :A => :A_1) === df
    @test names(df) == [:A_1, :B]
    @test rename!(df, :A_1 => :A_2, :B => :B_2) === df
    @test names(df) == [:A_2, :B_2]
    @test rename!(df, [:A_2 => :A_3, :B_2 => :B_3]) === df
    @test names(df) == [:A_3, :B_3]
    @test rename!(df, Dict(:A_3 => :A_4, :B_3 => :B_4)) === df
    @test names(df) == [:A_4, :B_4]
    @test rename!(x->Symbol(lowercase(string(x))), df) === df
    @test names(df) == [:a_4, :b_4]
end

@testset "size" begin
    df = DataFrame(A = 1:3, B = 'A':'C')
    @test_throws ArgumentError size(df, 3)
    @test ndims(df) == 2
    @test ndims(typeof(df)) == 2
    @test (nrow(df), ncol(df)) == (3, 2)
    @test size(df) == (3, 2)
    @inferred nrow(df)
    @inferred ncol(df)
end

@testset "description" begin
    df = DataFrame(A = 1:10)

    @test first(df) == df[1, :]
    @test last(df) == df[end, :]
    @test_throws BoundsError first(DataFrame(x=[]))
    @test_throws BoundsError last(DataFrame(x=[]))

    @test first(df, 6) == DataFrame(A = 1:6)
    @test first(df, 1) == DataFrame(A = 1)
    @test last(df, 6) == DataFrame(A = 5:10)
    @test last(df, 1) == DataFrame(A = 10)
end

@testset "misc" begin
    df = DataFrame([collect('A':'C')])
    @test sprint(dump, df) == "DataFrame  3 observations of 1 variables\n  x1: ['A', 'B', 'C']\n\n"
    df = DataFrame(A = 1:12, B = repeat('A':'C', inner=4))
    # @test DataFrames.without(df, 1) == DataFrame(B = repeat('A':'C', inner=4))
end

@testset "column conversions" begin
    df = DataFrame([collect(1:10), collect(1:10)])
    @test !isa(df[1], Vector{Union{Int, Missing}})
    allowmissing!(df, 1)
    @test isa(df[1], Vector{Union{Int, Missing}})
    @test !isa(df[2], Vector{Union{Int, Missing}})
    df[1,1] = missing
    @test_throws MethodError disallowmissing!(df, 1)
    df[1,1] = 1
    disallowmissing!(df, 1)
    @test isa(df[1], Vector{Int})

    df = DataFrame([collect(1:10), collect(1:10)])
    allowmissing!(df, [1,2])
    @test isa(df[1], Vector{Union{Int, Missing}}) && isa(df[2], Vector{Union{Int, Missing}})
    disallowmissing!(df, [1,2])
    @test isa(df[1], Vector{Int}) && isa(df[2], Vector{Int})

    df = DataFrame([collect(1:10), collect(1:10)])
    allowmissing!(df)
    @test isa(df[1], Vector{Union{Int, Missing}}) && isa(df[2], Vector{Union{Int, Missing}})
    disallowmissing!(df)
    @test isa(df[1], Vector{Int}) && isa(df[2], Vector{Int})

    df = DataFrame([CategoricalArray(1:10),
                    CategoricalArray(string.('a':'j'))])
    allowmissing!(df)
    @test all(x->x <: CategoricalVector, typeof.(columns(df)))
    @test eltypes(df)[1] <: Union{CategoricalValue{Int}, Missing}
    @test eltypes(df)[2] <: Union{CategoricalString, Missing}
    df[1,2] = missing
    @test_throws MissingException disallowmissing!(df)
    df[1,2] = "a"
    disallowmissing!(df)
    @test all(x->x <: CategoricalVector, typeof.(columns(df)))
    @test eltypes(df)[1] <: CategoricalValue{Int}
    @test eltypes(df)[2] <: CategoricalString
end

@testset "similar" begin
    df = DataFrame(a = ["foo"],
                   b = CategoricalArray(["foo"]),
                   c = [0.0],
                   d = CategoricalArray([0.0]))
    @test eltypes(similar(df)) == eltypes(df)
    @test size(similar(df)) == size(df)

    rows = size(df, 1) + 5
    @test size(similar(df, rows)) == (rows, size(df, 2))
    @test eltypes(similar(df, rows)) == eltypes(df)

    @test size(similar(df, 0)) == (0, size(df, 2))
    @test eltypes(similar(df, 0)) == eltypes(df)

    e = @test_throws ArgumentError similar(df, -1)
    @test e.value.msg == "the number of rows must be non-negative"
end

@testset "setindex! special cases" begin
    df = DataFrame(rand(3,2), [:x3, :x3_1])
    @test_throws ArgumentError df[3] = [1, 2]
    @test_throws ArgumentError df[4] = [1, 2, 3]
    df[3] = [1,2,3]
    df[4] = [1,2,3]
    @test names(df) == [:x3, :x3_1, :x3_2, :x4]
    df = DataFrame()
    @test_throws MethodError df[true] = 1
    @test_throws MethodError df[true] = [1,2,3]
    @test_throws MethodError df[1:2, true] = [1,2]
    @test_throws MethodError df[1, true] = 1
    @test_throws ArgumentError df[1, 100] = 1
    @test_throws ArgumentError df[1:2, 100] = [1,2]
end

@testset "passing range to a DataFrame" begin
    df = DataFrame(a=1:3, b='a':'c')
    df[:c] = 1:3
    df[:d] = 'a':'c'
    @test all(typeof(df[i]) <: Vector for i in 1:ncol(df))
end

@testset "test corner case of getindex" begin
    df = DataFrame(x=[1], y=[1])
    @test_throws MethodError df[true, 1:2]
end

@testset "empty data frame getindex" begin
    @test_throws BoundsError DataFrame(x=[])[1, :]
    @test_throws BoundsError DataFrame()[1, :]
    @test_throws BoundsError DataFrame()[1:2, :]
    @test_throws BoundsError DataFrame()[1, Bool[]]
    @test_throws BoundsError DataFrame()[1:2, Bool[]]
    @test_throws BoundsError DataFrame(x=[1])[1:2, [false]]
    @test_throws BoundsError DataFrame(x=[1])[2, [false]]
    #but this is OK:
    @test DataFrame(x=[1])[1:1, [false]] == DataFrame()
end

@testset "handling of end in indexing" begin
    z = DataFrame(rand(4,5))
    for x in [z, view(z, 1:4, :)]
        y = deepcopy(x)
        @test x[end] == x[5]
        @test x[end:end] == x[5:5]
        @test x[end, :] == x[4, :]
        @test x[end:end, :] == x[4:4, :]
        @test x[end, end] == x[4,5]
        @test x[2:end, 2:end] == x[2:4,2:5]
        x[end] = 1:4
        y[5] = 1:4
        @test x == y
        x[4:end] = DataFrame([11:14, 21:24])
        y[4] = [11:14;]
        y[5] = [21:24;]
        @test x == y
        x[end, :] = 111
        y[4, :] = 111
        @test x == y
        x[end,end] = 1000
        y[4,5] = 1000
        @test x == y
        x[2:end, 2:end] = 0
        y[2:4, 2:5] = 0
        @test x == y
    end
end

@testset "aliasing in indexing" begin
    # columns should not alias if scalar broadcasted
    df = DataFrame(A=[0], B=[0])
    df[1:end] = 0.0
    df[1, :A] = 1.0
    @test df[1, :B] === 0

    df = DataFrame(A=[0], B=[0])
    df[:, 1:end] = 0.0
    df[1, :A] = 1.0
    @test df[1, :B] === 0

    # columns should not alias if vector assigned
    df = DataFrame(A=[0], B=[0])
    x = [0.0]
    df[1:end] = x
    x[1] = 1.0
    @test df[1, :A] === 0.0
    @test df[1, :B] === 0.0
    df[1, :A] = 1.0
    @test df[1, :B] === 0.0

    df = DataFrame(A=[0], B=[0])
    x = [0.0]
    df[:, 1:end] = x
    x[1] = 1.0
    @test df[1, :A] === 0.0
    @test df[1, :B] === 0.0
    df[1, :A] = 1.0
    @test df[1, :B] === 0.0
end

@testset "permutecols!" begin
    a, b, c = 1:5, 2:6, 3:7
    original = DataFrame(a=a, b=b, c=c)

    df = deepcopy(original)
    expected = deepcopy(original)
    @test permutecols!(df, [:a, :b, :c]) === df
    @test df == expected
    @test permutecols!(df, 1:3) === df
    @test df == expected

    df = deepcopy(original)
    expected = DataFrame(b=b, c=c, a=a)
    permutecols!(df, [:b, :c, :a]) === df
    @test df == expected
    df = deepcopy(original)
    permutecols!(df, [2, 3, 1]) === df
    @test df == expected

    df = deepcopy(original)
    expected = DataFrame(c=c, a=a, b=b)
    permutecols!(df, [:c, :a, :b]) === df
    @test df == expected
    df = deepcopy(original)
    permutecols!(df, [3, 1, 2]) === df
    @test df == expected

    df = deepcopy(original)
    expected = DataFrame(a=a, c=c, b=b)
    permutecols!(df, [:a, :c, :b]) === df
    @test df == expected
    df = deepcopy(original)
    permutecols!(df, [1, 3, 2]) === df
    @test df == expected

    df = deepcopy(original)
    expected = DataFrame(b=b, a=a, c=c)
    permutecols!(df, [:b, :a, :c]) === df
    @test df == expected
    df = deepcopy(original)
    permutecols!(df, [2, 1, 3]) === df
    @test df == expected

    df = deepcopy(original)
    expected = DataFrame(c=c, b=b, a=a)
    permutecols!(df, [:c, :b, :a]) === df
    @test df == expected
    df = deepcopy(original)
    permutecols!(df, [3, 2, 1]) === df
    @test df == expected

    # Invalid
    df = DataFrame(a=a, b=b, c=c)
    @test_throws ArgumentError permutecols!(df, [:a, :b])
    @test_throws ArgumentError permutecols!(df, 1:4)
    @test_throws ArgumentError permutecols!(df, [:a, :b, :c, :d])
    @test_throws ArgumentError permutecols!(df, [1, 3])
    @test_throws ArgumentError permutecols!(df, [:a, :c])
    @test_throws ArgumentError permutecols!(df, [1, 2, 3, 1])
    @test_throws ArgumentError permutecols!(df, [:a, :b, :c, :a])
end

@testset "getproperty, setproperty! and propertynames" begin
    x = collect(1:10)
    y = collect(1.0:10.0)
    z = collect(10:-1:1)
    df = DataFrame(x = x, y = y)

    @test Base.propertynames(df) == names(df)

    @test df.x === x
    @test df.y === y
    @test_throws ArgumentError df.z

    df.x = 2:11
    @test df.x == 2:11
    @test x == 1:10
    df.y = 1
    @test df.y == [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
    @test df.y === y
    df.z = z
    @test df.z === z
    df.zz = 1
    @test df.zz == df.y
end

@testset "duplicate column names" begin
    x = DataFrame(a = [1, 2, 3], b = [4, 5, 6])
    v = DataFrame(a = [5, 6, 7], b = [8, 9, 10])
    z = vcat(v, x)
    @test_throws ArgumentError z[:, [1, 1, 2]]
    @test_throws ArgumentError z[[1, 1, 2]]
end

@testset "parent, size and axes" begin
    x = DataFrame(a = [1, 2, 3], b = [4, 5, 6])
    @test parent(x) === x
    @test parentindices(x) === (Base.OneTo(3), Base.OneTo(2))
    @test size(x) == (3,2)
    @test size(x, 1) == 3
    @test size(x, 2) == 2
    @test_throws ArgumentError size(x, 3)
    @test axes(x) === (Base.OneTo(3), Base.OneTo(2))
    @test axes(x, 1) === Base.OneTo(3)
    @test axes(x, 2) === Base.OneTo(2)
    @test_throws ArgumentError axes(x, 3)
    @test size(DataFrame()) == (0,0)
end

end # module


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

struct Offsets{T <: Union{Int32, Int64}} <: ArrowVector{Tuple{T, T}}
    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
    offsets::Vector{T}
end

Base.size(o::Offsets) = (length(o.offsets) - 1,)

@propagate_inbounds function Base.getindex(o::Offsets, i::Integer)
    @boundscheck checkbounds(o, i)
    @inbounds lo = o.offsets[i] + 1
    @inbounds hi = o.offsets[i + 1]
    return lo, hi
end

"""
    Arrow.List

An `ArrowVector` where each element is a variable sized list of some kind, like an `AbstractVector` or `AbstractString`.
"""
struct List{T, O, A} <: ArrowVector{T}
    arrow::Vector{UInt8} # need to hold a reference to arrow memory blob
    validity::ValidityBitmap
    offsets::Offsets{O}
    data::A
    ℓ::Int
    metadata::Union{Nothing, Dict{String, String}}
end

Base.size(l::List) = (l.ℓ,)

@propagate_inbounds function Base.getindex(l::List{T}, i::Integer) where {T}
    @boundscheck checkbounds(l, i)
    @inbounds lo, hi = l.offsets[i]
    if ArrowTypes.isstringtype(T)
        if Base.nonmissingtype(T) !== T
            return l.validity[i] ? ArrowTypes.arrowconvert(T, unsafe_string(pointer(l.data, lo), hi - lo + 1)) : missing
        else
            return ArrowTypes.arrowconvert(T, unsafe_string(pointer(l.data, lo), hi - lo + 1))
        end
    elseif Base.nonmissingtype(T) !== T
        return l.validity[i] ? ArrowTypes.arrowconvert(T, view(l.data, lo:hi)) : missing
    else
        return ArrowTypes.arrowconvert(T, view(l.data, lo:hi))
    end
end

# @propagate_inbounds function Base.setindex!(l::List{T}, v, i::Integer) where {T}

# end

# an AbstractVector version of Iterators.flatten
# code based on SentinelArrays.ChainedVector
struct ToList{T, stringtype, A, I} <: AbstractVector{T}
    data::Vector{A} # A is AbstractVector or AbstractString
    inds::Vector{I}
end

function ToList(input; largelists::Bool=false)
    AT = eltype(input)
    ST = Base.nonmissingtype(AT)
    stringtype = ArrowTypes.isstringtype(ST)
    T = stringtype ? UInt8 : eltype(ST)
    len = stringtype ? ncodeunits : length
    data = AT[]
    I = largelists ? Int64 : Int32
    inds = I[0]
    sizehint!(data, length(input))
    sizehint!(inds, length(input))
    totalsize = I(0)
    for x in input
        if x === missing
            push!(data, missing)
        else
            push!(data, x)
            totalsize += len(x)
            if I === Int32 && totalsize > 2147483647
                I = Int64
                inds = convert(Vector{Int64}, inds)
            end
        end
        push!(inds, totalsize)
    end
    return ToList{T, stringtype, AT, I}(data, inds)
end

Base.IndexStyle(::Type{<:ToList}) = Base.IndexLinear()
Base.size(x::ToList) = (length(x.inds) == 0 ? 0 : x.inds[end],)

function Base.pointer(A::ToList{UInt8}, i::Integer)
    chunk = searchsortedfirst(A.inds, i)
    return pointer(A.data[chunk - 1])
end

@inline function index(A::ToList, i::Integer)
    chunk = searchsortedfirst(A.inds, i)
    return chunk - 1, i - (@inbounds A.inds[chunk - 1])
end

Base.@propagate_inbounds function Base.getindex(A::ToList{T, stringtype}, i::Integer) where {T, stringtype}
    @boundscheck checkbounds(A, i)
    chunk, ix = index(A, i)
    @inbounds x = A.data[chunk]
    return @inbounds stringtype ? codeunits(x)[ix] : x[ix]
end

Base.@propagate_inbounds function Base.setindex!(A::ToList{T, stringtype}, v, i::Integer) where {T, stringtype}
    @boundscheck checkbounds(A, i)
    chunk, ix = index(A, i)
    @inbounds x = A.data[chunk]
    if stringtype
        codeunits(x)[ix] = v
    else
        x[ix] = v
    end
    return v
end

# efficient iteration
@inline function Base.iterate(A::ToList{T, stringtype}) where {T, stringtype}
    length(A) == 0 && return nothing
    i = 1
    chunk = 2
    chunk_i = 1
    chunk_len = A.inds[chunk]
    while i > chunk_len
        chunk += 1
        chunk_len = A.inds[chunk]
    end
    val = A.data[chunk - 1]
    x = stringtype ? codeunits(val)[1] : val[1]
    # find next valid index
    i += 1
    if i > chunk_len
        while true
            chunk += 1
            chunk > length(A.inds) && break
            chunk_len = A.inds[chunk]
            i <= chunk_len && break
        end
    else
        chunk_i += 1
    end
    return x, (i, chunk, chunk_i, chunk_len, length(A))
end

@inline function Base.iterate(A::ToList{T, stringtype}, (i, chunk, chunk_i, chunk_len, len)) where {T, stringtype}
    i > len && return nothing
    @inbounds val = A.data[chunk - 1]
    @inbounds x = stringtype ? codeunits(val)[chunk_i] : val[chunk_i]
    i += 1
    if i > chunk_len
        chunk_i = 1
        while true
            chunk += 1
            chunk > length(A.inds) && break
            @inbounds chunk_len = A.inds[chunk]
            i <= chunk_len && break
        end
    else
        chunk_i += 1
    end
    return x, (i, chunk, chunk_i, chunk_len, len)
end

arrowvector(::ListType, x::List, i, nl, fi, de, ded, meta; kw...) = x

function arrowvector(::ListType, x, i, nl, fi, de, ded, meta; largelists::Bool=false, kw...)
    len = length(x)
    validity = ValidityBitmap(x)
    flat = ToList(x; largelists=largelists)
    offsets = Offsets(UInt8[], flat.inds)
    if eltype(flat) == UInt8 # binary or utf8string
        data = flat
    else
        data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; lareglists=largelists, kw...)
    end
    return List{eltype(x), eltype(flat.inds), typeof(data)}(UInt8[], validity, offsets, data, len, meta)
end

function compress(Z::Meta.CompressionType, comp, x::List{T, O, A}) where {T, O, A}
    len = length(x)
    nc = nullcount(x)
    validity = compress(Z, comp, x.validity)
    offsets = compress(Z, comp, x.offsets.offsets)
    buffers = [validity, offsets]
    children = Compressed[]
    if eltype(A) == UInt8
        push!(buffers, compress(Z, comp, x.data))
    else
        push!(children, compress(Z, comp, x.data))
    end
    return Compressed{Z, typeof(x)}(x, buffers, len, nc, children)
end

ext: jl

max_stars_repo_name: timkpaine/arrow

max_issues_repo_name: timkpaine/arrow

max_forks_repo_name: XpressAI/arrow


text:
module Rasterization
  using CG;
  using PyPlot;

  export FrameBuffer, clear!, colorPixel!, line!, triangle!, plot;
  type FrameBuffer
    nx::Int;
    ny::Int;
    data::Array{UInt8};
    FrameBuffer(nx::Int, ny::Int) = new(nx, ny, zeros(nx, ny));
  end

  function clear!(buffer::FrameBuffer)
    buffer.data = zeros(buffer.nx, buffer.ny);
  end

  function colorPixel!(buffer::FrameBuffer, x::Int, y::Int, color::UInt8)
    buffer.data[x, y] = color;
  end

  function plot(buffer::FrameBuffer)
    figure()
    gray()
  	clf()
    imshow(buffer.data', interpolation="nearest");
    show();
  end

  function swap!(a, b)
    tmp = a;
    a = b;
    b = tmp;
  end

  function line!(buffer::FrameBuffer, x0::Int, y0::Int, x1::Int, y1::Int, color::UInt8)
    # reassign input variables:
    if (x1 < x0)
      tmp = x0;
      x0 = x1;
      x1 = tmp;
    end

    if (y0 > y1)
      tmp = y0;
      y0 = y1;
      y1 = tmp;
    end

    m = (y1 - y0) / (x1 - x0);
    if m < 0

    elseif m >= 1
      tmp = y0;
      y0 = x0;
      x0 = tmp;

      tmp = y1;
      y1 = x1;
      x1 = tmp;
    end
    # Bresenham
    A = y1 - y0;
    B = -(x1 - x0);
    C = (x1 - x0) * y0 - (y1 - y0) * x0;

    F = function(x, y)
      return A*x + B*y + C
    end

    x = x0;
    y = y0;
    while x < x1
      colorPixel!(buffer, x, y, color);

      if (F(x + 1, y + 0.5) > 0)
        y = y + 1;
      end

      x = x + 1;
    end
  end

  function triangle!(s::FrameBuffer, x0::Int, y0::Int, x1::Int, y1::Int, x2::Int, y2::Int, ca::UInt8, cb::UInt8, cc::UInt8)
    # calculate bounding box
    xmax = min(max(x0, x1, x2), s.nx);
    ymax = min(max(y0, y1, y2), s.ny);
    xmin = max(min(x0, x1, x2), 1);
    ymin = max(min(y0, y1, y2), 1);

    for x = xmin:xmax
      for y = ymin:ymax
        gamma = ((y0 - y1) * x + (x1 - x0) * y + x0*y1 - x1*y0) / ((y0 - y1) * x2 + (x1 - x0) * y2 + x0*y1 - x1*y0);
        beta = ((y0 - y2) * x + (x2 - x0) * y + x0*y2 - x2*y0) / ((y0 - y2) * x1 + (x2 - x0) * y1 + x0*y2 - x2*y0);
        alpha = 1 - beta - gamma;

        if (0 < alpha && alpha < 1 && 0 < beta && beta < 1 && 0 < gamma && gamma < 1)
          color = UInt8(round(alpha * ca + beta * cb + gamma * cc));
          colorPixel!(s, x, y, color);
        end
      end
    end
  end
end

max_stars_repo_name: fpoppinga/cg

max_issues_repo_name: fpoppinga/cg

max_forks_repo_name: fpoppinga/cg


text:
using LinearAlgebra

N=2000
const A=Diagonal(randn(N))
const x = randn(N,N)
function testmat_A(A,x)

    x*A
end
const c=zeros(N,N)
const B=randn(N)

function testmat_B(c,B)

    for j in 1:N
        c[:,j]=B[j]*x[:,j] ;
    end
end

K=1000

@time for i in 1:K; testmat_A(A,x) ; end
@time for i in 1:K;  testmat_B(c,B) ; end

max_stars_repo_name: yosefcap/MC

max_issues_repo_name: yosefcap/MC

max_forks_repo_name: yosefcap/MC


text:
#function to produce matrix of hmm chain parameter coordinate evolution- selects 3 matched emission parameters from 3+ state HMMs, one per state for 3 states. Matching is performed by minimizing euclidean distance between the state emission probability vectors B for the states to be matched (as EM-optimised replicates will have similar states organized in different orders)

function chain_3devo_coords(chains::Vector{Vector{Any}})
    length(chains)<=0 && throw(ArgumentError, "Argument must be vector of more than one hmm chain")
    coords=[Vector{Tuple{AbstractFloat,AbstractFloat,AbstractFloat}}() for i in 1:length(chains)]

    for step in chains[1]
        hmm=step[2]
        length(hmm.B)<3 && throw(ArgumentError, "3- or greater state hmms required")
        push!(coords[1], (hmm.B[1].p[1],hmm.B[2].p[1],hmm.B[3].p[1]))
    end

    for (c, chain) in enumerate(chains[2:end])
        ref_idxs=Vector{Integer}()
        ref_vecs=[chains[1][end][2].B[1].p,chains[1][end][2].B[2].p,chains[1][end][2].B[3].p]
        end_hmm=chain[end][2]
        length(end_hmm.B)<3 && throw(ArgumentError, "3- or greater state hmms required")

        for vec in ref_vecs
            euclideans=Vector{AbstractFloat}()
            for d in 1:length(end_hmm.B)
                push!(euclideans, euclidean(vec, end_hmm.B[d].p))
            end
            min_euc_idx = findmin(euclideans)[2]
            while min_euc_idx in ref_idxs
                euclideans[min_euc_idx]=1.
                min_euc_idx = findmin(euclideans)[2]
            end
            push!(ref_idxs,findmin(euclideans)[2])
        end

        for step in chain
            hmm=step[2]
            push!(coords[c+1], (hmm.B[ref_idxs[1]].p[1],hmm.B[ref_idxs[2]].p[1],hmm.B[ref_idxs[3]].p[1]))
        end
    end

    return coords
end


text:
module OptimJulia
      using SymPy

      export Newton
      export QuasiNewton

      include("Newton.jl")
      include("QuasiNewton.jl")

end


text:
module iJR904

    import BSON
    import ..Chemostat_Folsom2014
    const ChF = Chemostat_Folsom2014
    import ..BegData
    const Bd = BegData
    import ..FolsomData
    const Fd = FolsomData
    import Chemostat
    const ChU = Chemostat.Utils

    using ProjAssistant
    @gen_sub_proj

    include("const.jl")
    include("load_data.jl")
    include("beg_enz_cost.jl")
    include("load_model.jl")
    include("ME_MODES.jl")

    function __init__()
        @create_proj_dirs
    end

end


text:
import PyCall

if !isdefined(:gym)
    global const gym = PyCall.pywrap(PyCall.pyimport("gym"))
end

max_stars_repo_name: dmitrijsc/practical-rl

max_issues_repo_name: dmitrijsc/practical-rl

max_forks_repo_name: dmitrijsc/practical-rl


text:
abstract type LBM_state end
abstract type LBM_state_2D <: LBM_state end
abstract type LBM_state_1D <: LBM_state end

"""
    SysConst{T}

Struct that contains all run time constants, e.g. lattice size, surface tension `γ` and so on.

# Arguments

- `Lx :: Int`: Length of the lattice sides in x-direction
- `Ly :: Int`: Length of the lattice sides in y-direction
- `Tmax :: Int`: Number of lattice Boltzmann time iterations
- `tdump :: Int`: Dumping interval for e.g. data output
- `τ :: T`: BGK relaxation rate
- `cₛ :: T`: Lattice speed of sound, every physical velocity needs to be smaller than this!
- `μ :: T`: Kinematic fluid viscosity
- `δ :: T`: Slip length, defines how far the **no-slip** condition is interpolated into the substrate
- `kbt :: T`: Thermal energy of the film, works with small values ≈ 10^(-7)
- `γ :: T`: Surface tension
- `n :: Int`: Greater exponent of the two used for the powerlaw in the disjoining pressure
- `m :: Int`: Smaller exponent of the two used for the powerlaw in the disjoining pressure
- `hmin :: T`: Height value at which the disjoining pressure functional vanishes
- `hcrit :: T`: Numerical stabilizer for the disjoining pressure term
- `θ :: T`: Contact angle in multiples of π
- `g :: T`: gravitational acceleration, usually neglected in thin film simulations

"""
Base.@kwdef struct SysConst{T}
    # Lattice
    Lx :: Int = 256
    Ly :: Int = 256
    Tmax :: Int = 1000
    tdump :: Int = Tmax÷10
    # Collision related
    τ :: T = 1.0
    cₛ :: T = 1/sqrt(3.0)
    μ :: T =  cₛ^2 *( τ - 0.5)
    # Force related
    δ :: T = 1.0
    kbt :: T = 0.0
    γ :: T = 0.01
    n :: Int = 9
    m :: Int = 3
    hmin :: T = 0.1
    hcrit :: T = 0.05
    θ :: T = 1/9
    g :: T = 0.0
end

"""
    SysConst_1D{T}

Struct that contains all run time constants, e.g. lattice size, surface tension `γ` and so on.

# Arguments

- `L :: Int`: Number of lattice points
- `Tmax :: Int`: Number of lattice Boltzmann time iterations
- `tdump :: Int`: Dumping interval for e.g. data output
- `τ :: T`: BGK relaxation rate
- `cₛ :: T`: Lattice speed of sound, every physical velocity needs to be smaller than this!
- `μ :: T`: Kinematic fluid viscosity
- `δ :: T`: Slip length, defines how far the **no-slip** condition is interpolated into the substrate
- `kbt :: T`: Thermal energy of the film, works with small values ≈ 10^(-7)
- `γ :: T`: Surface tension
- `n :: Int`: Greater exponent of the two used for the powerlaw in the disjoining pressure
- `m :: Int`: Smaller exponent of the two used for the powerlaw in the disjoining pressure
- `hmin :: T`: Height value at which the disjoining pressure functional vanishes
- `hcrit :: T`: Numerical stabilizer for the disjoining pressure term
- `θ :: T`: Contact angle in multiples of π
- `g :: T`: gravitational acceleration, usually neglected in thin film simulations

"""
Base.@kwdef struct SysConst_1D{T}
    # Lattice
    L :: Int = 256
    Tmax :: Int = 1000
    tdump :: Int = Tmax÷10
    # Collision related
    τ :: T = 1.0
    cₛ :: T = 1/sqrt(3.0)
    μ :: T =  cₛ^2 *( τ - 0.5)
    # Force related
    δ :: T = 1.0
    kbt :: T = 0.0
    γ :: T = 0.01
    n :: Int = 9
    m :: Int = 3
    hmin :: T = 0.1
    hcrit :: T = 0.05
    θ :: T = 1/9
    g :: T = 0.0
end

"""
    State{T, N}

Data structure that stores all arrays for a given simulation.

# Arguments

- `fout :: Array{T,N}`: Output distribution function
- `ftemp :: Array{T,N}`: Temporary distribution function, only used if `sys.τ ≠ 1`
- `feq :: Array{T,N}`: Equilibrium distribution function
- `height :: Matrix{T}`: Field that stores the scalar height values
- `velx :: Matrix{T}`: Field that stores the x-component of the velocity vector
- `vely :: Matrix{T}`: Field that stores the y-component of the velocity vector
- `vsq :: Matrix{T}`: Field that stores the velocity squared, used in `equilibrium!`
- `pressure :: Matrix{T}`: Pressure distribution computed using the `filmpressure!` function
- `Fx :: Matrix{T}`: Total force acting on the fluid, x-component
- `Fy :: Matrix{T}`: Total force acting on the fluid, y-component
- `slipx :: Matrix{T}`: Friction force due to substrate slip, x-component
- `slipy :: Matrix{T}`: Friction force due to substrate slip, y-component
- `h∇px :: Matrix{T}`: Pressure gradient times the height, x-component
- `h∇py :: Matrix{T}`: Pressure gradient times the height, y-component
- `dgrad :: Array{T,N}`: Dummy allocation to store shifted arrays using `circshift!`

"""
Base.@kwdef struct State{T, N} <: LBM_state_2D
    # Distribution functions
    fout :: Array{T, N}
    ftemp :: Array{T, N}
    feq :: Array{T, N}
    # Macroscopic variables and moments
    height :: Matrix{T}
    velx :: Matrix{T}
    vely :: Matrix{T}
    vsq :: Matrix{T}
    pressure :: Matrix{T}
    # Forces and a dummy for the gradient
    Fx :: Matrix{T}
    Fy :: Matrix{T}
    slipx :: Matrix{T}
    slipy :: Matrix{T}
    h∇px :: Matrix{T}
    h∇py :: Matrix{T}
    dgrad :: Array{T,N}
end

Base.@kwdef struct State_thermal{T, N} <: LBM_state_2D
    # Distribution functions
    fout :: Array{T, N}
    ftemp :: Array{T, N}
    feq :: Array{T, N}
    # Macroscopic variables and moments
    height :: Matrix{T}
    velx :: Matrix{T}
    vely :: Matrix{T}
    vsq :: Matrix{T}
    pressure :: Matrix{T}
    # Forces and a dummy for the gradient
    Fx :: Matrix{T}
    Fy :: Matrix{T}
    slipx :: Matrix{T}
    slipy :: Matrix{T}
    h∇px :: Matrix{T}
    h∇py :: Matrix{T}
    kbtx :: Matrix{T}
    kbty :: Matrix{T}
    dgrad :: Array{T,N}
end

"""
    CuState

Data structure that stores all arrays for a given simulation.
Specific for GPU computing using `CUDA.jl`

# Arguments

- `fout :: Array{T,N}`: Output distribution function
- `ftemp :: Array{T,N}`: Temporary distribution function, only used if `sys.τ ≠ 1`
- `feq :: Array{T,N}`: Equilibrium distribution function
- `height :: Matrix{T}`: Field that stores the scalar height values
- `velx :: Matrix{T}`: Field that stores the x-component of the velocity vector
- `vely :: Matrix{T}`: Field that stores the y-component of the velocity vector
- `vsq :: Matrix{T}`: Field that stores the velocity squared, used in `equilibrium!`
- `pressure :: Matrix{T}`: Pressure distribution computed using the `filmpressure!` function
- `Fx :: Matrix{T}`: Total force acting on the fluid, x-component
- `Fy :: Matrix{T}`: Total force acting on the fluid, y-component
- `slipx :: Matrix{T}`: Friction force due to substrate slip, x-component
- `slipy :: Matrix{T}`: Friction force due to substrate slip, y-component
- `h∇px :: Matrix{T}`: Pressure gradient times the height, x-component
- `h∇py :: Matrix{T}`: Pressure gradient times the height, y-component
- `dgrad :: Array{T,N}`: Dummy allocation to store shifted arrays using `circshift!`

"""
struct CuState <: LBM_state_2D
    # Distribution functions
    fout :: CuArray
    ftemp :: CuArray
    feq :: CuArray
    # Macroscopic variables and moments
    height :: CuArray
    velx :: CuArray
    vely :: CuArray
    vsq :: CuArray
    pressure :: CuArray
    # Forces and a dummy for the gradient
    Fx :: CuArray
    Fy :: CuArray
    slipx :: CuArray
    slipy :: CuArray
    h∇px :: CuArray
    h∇py :: CuArray
    dgrad :: CuArray
end

struct CuState_thermal <: LBM_state_2D
    # Distribution functions
    fout :: CuArray
    ftemp :: CuArray
    feq :: CuArray
    # Macroscopic variables and moments
    height :: CuArray
    velx :: CuArray
    vely :: CuArray
    vsq :: CuArray
    pressure :: CuArray
    # Forces and a dummy for the gradient
    Fx :: CuArray
    Fy :: CuArray
    slipx :: CuArray
    slipy :: CuArray
    h∇px :: CuArray
    h∇py :: CuArray
    kbtx :: CuArray
    kbty :: CuArray
    dgrad :: CuArray
end

"""
    State_1D{T, N}

Data structure that stores all arrays for a given simulation.

# Arguments

- `fout :: Matrix{T}`: Output distribution function
- `ftemp :: Matrix{T}`: Temporary distribution function, only used if `sys.τ ≠ 1`
- `feq :: Matrix{T}`: Equilibrium distribution function
- `height :: Vector{T}`: Field that stores the scalar height values
- `vel :: Vector{T}`: Field that stores the velocity
- `pressure :: Vector{T}`: Pressure distribution computed using the `filmpressure!` function
- `F :: Vector{T}`: Total force acting on the fluid
- `slip :: Vector{T}`: Friction force due to substrate slip
- `h∇p :: Vector{T}`: Pressure gradient times the height
- `dgrad :: Matrix{T}`: Dummy allocation to store shifted arrays using `circshift!`
"""
Base.@kwdef struct State_1D{T} <: LBM_state_1D
    # Distribution functions
    fout :: Matrix{T}
    ftemp :: Matrix{T}
    feq :: Matrix{T}
    # Macroscopic variables and moments
    height :: Vector{T}
    vel :: Vector{T}
    pressure :: Vector{T}
    # Forces and a dummy for the gradient
    F :: Vector{T}
    slip :: Vector{T}
    h∇p :: Vector{T}
    dgrad :: Matrix{T}
end

Base.@kwdef struct State_thermal_1D{T} <: LBM_state_1D
    # Distribution functions
    fout :: Matrix{T}
    ftemp :: Matrix{T}
    feq :: Matrix{T}
    # Macroscopic variables and moments
    height :: Vector{T}
    vel :: Vector{T}
    pressure :: Vector{T}
    # Forces and a dummy for the gradient
    F :: Vector{T}
    slip :: Vector{T}
    h∇p :: Vector{T}
    kbt :: Vector{T}
    dgrad :: Matrix{T}
end

"""
    Sys(sysc, device, exotic)

Mostly allocations of arrays used to run a simulation, but all within one function :)

# Arguments

- `sysc :: SysConst`: Needed for the lattice dimensions, `Lx` and `Ly`
- `device :: String`: Use either `CPU` for computation of a CPU or `GPU` for computation on the GPU
- `exotic :: Bool`: If true thermal fluctuations can be computed and saved to the `fthermalx` and `fthermaly` field
- `T <: Number`: Numerical type, it is strongly suggested to use `Float64`
"""
function Sys(sysc::SysConst, device::String, exotic::Bool, T)
    if device == "CPU"
        # Meso
        fout = zeros(sysc.Lx, sysc.Ly, 9)
        ftemp = zeros(sysc.Lx, sysc.Ly, 9)
        feq = zeros(sysc.Lx, sysc.Ly, 9)

        # Macro
        height = ones(sysc.Lx, sysc.Ly)
        velx = zeros(sysc.Lx, sysc.Ly)
        vely = zeros(sysc.Lx, sysc.Ly)
        vsq = zeros(sysc.Lx, sysc.Ly)
        pressure = zeros(sysc.Lx, sysc.Ly)
        dgrad = zeros(sysc.Lx, sysc.Ly, 8)

        # Forces
        Fx = zeros(sysc.Lx, sysc.Ly)
        Fy = zeros(sysc.Lx, sysc.Ly)
        slipx = zeros(sysc.Lx, sysc.Ly)
        slipy = zeros(sysc.Lx, sysc.Ly)
        h∇px = zeros(sysc.Lx, sysc.Ly)
        h∇py = zeros(sysc.Lx, sysc.Ly)
        if exotic
            # Probably many more forces here in the future
            fthermalx = zeros(sysc.Lx, sysc.Ly)
            fthermaly = zeros(sysc.Lx, sysc.Ly)
            return fout, ftemp, feq, height, velx, vely, vsq, pressure, dgrad, Fx, Fy, slipx, slipy, h∇px, h∇py, fthermalx, fthermaly
        end
        return fout, ftemp, feq, height, velx, vely, vsq, pressure, dgrad, Fx, Fy, slipx, slipy, h∇px, h∇py
    elseif device == "GPU"
        # Meso
        fout = CUDA.zeros(T, sysc.Lx, sysc.Ly, 9)
        ftemp = CUDA.zeros(T, sysc.Lx, sysc.Ly, 9)
        feq = CUDA.zeros(T, sysc.Lx, sysc.Ly, 9)

        # Macro
        height = CUDA.ones(T, sysc.Lx, sysc.Ly)
        velx = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        vely = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        vsq = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        pressure = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        dgrad = CUDA.zeros(T, sysc.Lx, sysc.Ly, 8)

        # Forces
        Fx = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        Fy = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        slipx = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        slipy = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        h∇px = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        h∇py = CUDA.zeros(T, sysc.Lx, sysc.Ly)
        if exotic
            fthermalx = CUDA.zeros(T, sysc.Lx, sysc.Ly)
            fthermaly = CUDA.zeros(T, sysc.Lx, sysc.Ly)
            return fout, ftemp, feq, height, velx, vely, vsq, pressure, dgrad, Fx, Fy, slipx, slipy, h∇px, h∇py, fthermalx, fthermaly
        end
        return fout, ftemp, feq, height, velx, vely, vsq, pressure, dgrad, Fx, Fy, slipx, slipy, h∇px, h∇py
    end
end

function Sys(sysc::SysConst, device::String; T=Float64, kind="simple")
    if kind == "simple"
        if device == "CPU"
            dyn = State{T, 3}(
                fout = zeros(sysc.Lx, sysc.Ly, 9),
                ftemp = zeros(sysc.Lx, sysc.Ly, 9),
                feq = zeros(sysc.Lx, sysc.Ly, 9),
                height = ones(sysc.Lx, sysc.Ly),
                velx = zeros(sysc.Lx, sysc.Ly),
                vely = zeros(sysc.Lx, sysc.Ly),
                vsq = zeros(sysc.Lx, sysc.Ly),
                pressure = zeros(sysc.Lx, sysc.Ly),
                dgrad = zeros(sysc.Lx, sysc.Ly, 8),
                Fx = zeros(sysc.Lx, sysc.Ly),
                Fy = zeros(sysc.Lx, sysc.Ly),
                slipx = zeros(sysc.Lx, sysc.Ly),
                slipy = zeros(sysc.Lx, sysc.Ly),
                h∇px = zeros(sysc.Lx, sysc.Ly),
                h∇py = zeros(sysc.Lx, sysc.Ly)
            )
            return dyn
        elseif device == "GPU"
            dyn = CuState(
                # Distribution functions
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                # Macro
                CUDA.ones(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                # Forces
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                # For gradients
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 8)
            )
            return dyn
        end
    elseif kind == "thermal"
        if device == "CPU"
            dyn = State_thermal{T, 3}(
                fout = zeros(sysc.Lx, sysc.Ly, 9),
                ftemp = zeros(sysc.Lx, sysc.Ly, 9),
                feq = zeros(sysc.Lx, sysc.Ly, 9),
                height = ones(sysc.Lx, sysc.Ly),
                velx = zeros(sysc.Lx, sysc.Ly),
                vely = zeros(sysc.Lx, sysc.Ly),
                vsq = zeros(sysc.Lx, sysc.Ly),
                pressure = zeros(sysc.Lx, sysc.Ly),
                dgrad = zeros(sysc.Lx, sysc.Ly, 8),
                Fx = zeros(sysc.Lx, sysc.Ly),
                Fy = zeros(sysc.Lx, sysc.Ly),
                slipx = zeros(sysc.Lx, sysc.Ly),
                slipy = zeros(sysc.Lx, sysc.Ly),
                h∇px = zeros(sysc.Lx, sysc.Ly),
                h∇py = zeros(sysc.Lx, sysc.Ly),
                kbtx = zeros(sysc.Lx, sysc.Ly),
                kbty = zeros(sysc.Lx, sysc.Ly)
            )
            return dyn
        elseif device == "GPU"
            dyn = CuState_thermal(
                # Distribution functions
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 9),
                # Macro
                CUDA.ones(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                # For gradients
                CUDA.zeros(T, sysc.Lx, sysc.Ly, 8),
                # Forces
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly),
                CUDA.zeros(T, sysc.Lx, sysc.Ly)
            )
            return dyn
        end
    end
end

"""
    Sys(sysc, exotic, T)

Mostly allocations of arrays used to run a simulation, but all within one function :)

# Arguments

- `sysc :: SysConst_1D`: Needed for the lattice dimensions, `L`
- `exotic :: Bool`: If true thermal fluctuations can be computed and saved to the `fthermalx` and `fthermaly` field
- `T <: Number`: Numerical type, it is strongly suggested to use `Float64`
"""
function Sys(sysc::SysConst_1D, exotic::Bool, T)
    # Meso
    fout = zeros(sysc.L, 3)
    ftemp = zeros(sysc.L, 3)
    feq = zeros(sysc.L, 3)

    # Macro
    height = ones(sysc.L)
    vel = zeros(sysc.L)
    pressure = zeros(sysc.L)
    dgrad = zeros(sysc.L, 2)
    # Forces
    F = zeros(sysc.L)
    slip = zeros(sysc.L)
    h∇p = zeros(sysc.L)
    if exotic
        # Probably many more forces here in the future
        fthermal = zeros(sysc.L)

        return fout, ftemp, feq, height, vel, pressure, dgrad, F, slip, h∇p, fthermal
    end

    return fout, ftemp, feq, height, vel, pressure, dgrad, F, slip, h∇p

end

function Sys(sysc::SysConst_1D; T=Float64, kind="simple")
    if kind == "simple"
        dyn = State_1D{T}(
            fout = zeros(sysc.L, 3),
            ftemp = zeros(sysc.L, 3),
            feq = zeros(sysc.L, 3),
            height = ones(sysc.L),
            vel = zeros(sysc.L),
            pressure = zeros(sysc.L),
            dgrad = zeros(sysc.L, 2),
            F = zeros(sysc.L),
            slip = zeros(sysc.L),
            h∇p = zeros(sysc.L),
        )
    elseif kind == "thermal"
        dyn = State_thermal_1D{T}(
            fout = zeros(sysc.L, 3),
            ftemp = zeros(sysc.L, 3),
            feq = zeros(sysc.L, 3),
            height = ones(sysc.L),
            vel = zeros(sysc.L),
            pressure = zeros(sysc.L),
            dgrad = zeros(sysc.L, 2),
            F = zeros(sysc.L),
            slip = zeros(sysc.L),
            kbt = zeros(sysc.L),
            h∇p = zeros(sysc.L),
        )
    end
    return dyn

end


text:
using Compat.Test
include("../dataset/BarStripes.jl")

@testset "barstripes" begin
    @test size(binary_basis(2,2)) == (2,2,16)

    ba = reshape(bitarray(7, num_bit=9), 3,3)
    @test is_bar(ba) == true
    @test is_stripe(ba) == false

    ba0 = reshape(bitarray(0, num_bit=9), 3,3)
    @test is_bar(ba0) == true
    @test is_stripe(ba0) == true
    @test is_bs(ba) == is_bs(ba0) == true

    @test reshape(bs_configs(2,2), 4, :) |> packbits == findn(barstripe_pdf(2,2))-1
end

max_stars_repo_name: Roger-luo/QCBM

max_issues_repo_name: Roger-luo/QCBM

max_forks_repo_name: Roger-luo/QCBM


text:
using Bridge
using StaticArrays, LinearAlgebra
import Bridge: b, σ, B, β, a, constdiff
const ℝ = SVector{N,T} where {N,T}
import Base.resize!

struct LorenzNS{T,N} <: ContinuousTimeProcess{ℝ{N,T}}
    θ::T
    σ::T
    function LorenzNS(θ::T, σ::T, N) where T
        new{T,N}(θ, σ)
    end
end

function b(t, x, P::LorenzNS{T,N}) where {T,N}
    out = copy(x)
    for i in 1:N
        out[i] = ( (x[mod1(i+1, N)]-x[mod1(i-2, N)])*x[mod1(i-1, N)]
                  - x[mod1(i, N)] + P.θ )
    end
    out
end

function σ(t, x, P::LorenzNS{T,N}) where {T,N}
    P.σ * Matrix{Float64}(I, N, N)
end

constdiff(::LorenzNS) = true
clone(P::LorenzNS, θ) = LorenzNS(θ...)
params(P::LorenzNS) = [P.θ, P.σ]

nonhypo(P::LorenzNS, x) = x
@inline hypo_a_inv(P::LorenzNS, t, x) = Matrix{Float64}(I, N, N)/P.σ^2
num_non_hypo(P::Type{<:LorenzNS{T,N}}) where {T,N} = N

P = LorenzNS(1.0, 2.0, 20)
b(0.0, [i for i in 1:20], P)


text:
# Default values shared across instruments
#const teff_solar = 5780.0
#const vrot_solar = 1800.0
#const default_line_width_mps = predict_intrinsic_stellar_line_width(teff_solar,v_rot=vrot_solar)  # m/s

default_chunk_size_factor = 3       # For default_calc_chunk_width TODO: Figure out what value to use.  Ask Alex
# Is line below still used?
#default_calc_chunk_width = ChunkWidthFixedΔlnλ(default_chunk_size_factor*default_line_width_mps/speed_of_light_mps)  # Used by read_mask_espresso and read_vald for assigning lambda_lo and lambda_hi
default_min_chunk_Δv = 20000           # m/s  for ChunkWidthFixedΔlnλ

default_Δv_to_avoid_tellurics = 15000.0  # m/s

 # TODO: OPT: Make above const once settle on good values

max_stars_repo_name: RvSpectML/RvSpectMLTemplate

max_issues_repo_name: RvSpectML/RvSpectMLTemplate

max_forks_repo_name: RvSpectML/RvSpectMLTemplate


text:
# The Mango/Cloudant Query API
#
# See
#
# * https://docs.cloudant.com/cloudant_query.html
# * https://cloudant.com/blog/cloudant-query-grows-up-to-handle-ad-hoc-queries/
#

"""
    type QueryResult
      docs::Vector{Dict{AbstractString, Any}}
      bookmark::AbstractString
    end

Returned by `query(...)`.

`bookmark` is only relevant when querying indexes of type `text`.
"""
type QueryResult
  docs::Vector{Dict{AbstractString, Any}}
  # Bookmarks only relevant to queries on Lucene indexes (type "text").
  # JSON indexes have to be limit/skipped, as are views underneath
  bookmark::AbstractString
end

"""
    result = mango_query{T<:AbstractString}(db::Database, selector::Selector;
      fields::Vector{T}          = Vector{AbstractString}(),
      sort::Vector{Dict{T, Any}} = Vector{Dict{AbstractString, Any}}(),
      limit                      = 0,
      skip                       = 0,
      bookmark                   = "")

Query database (Mango/Cloudant Query).

See the `Selector` type and the associated `q"..."` custom string literal
which implements a simplified DSL for writing selectors.

### Examples

Find all documents where "year" is greater than 2010, returning
the fields `_id`, `_rev`, `year` and `title`, sorted in ascending order
on `year`. Set the page size to 10.

    result = mango_query(db, q"year > 2010";
      fields = ["_id", "_rev", "year", "title"],
      sort   = [Dict("year" => "asc")],
      limit  = 10)

### Returns

    type QueryResult

* [API reference](https://docs.cloudant.com/cloudant_query.html)
* [Cloudant Query blog post](https://cloudant.com/blog/cloudant-query-grows-up-to-handle-ad-hoc-queries/)
"""
function mango_query{T<:AbstractString}(db::Database, selector::Selector;
  fields                     = [],
  sort::Vector{Dict{T, Any}} = Vector{Dict{AbstractString, Any}}(),
  limit                      = 0,
  skip                       = 0,
  bookmark                   = "")

  body = Dict{String, Any}("selector" => selector.dict, "skip" => skip)
  if length(fields) > 0
    body["fields"] = fields
  end

  if length(sort) > 0
    body["sort"] = sort
  end

  if limit > 0
    body["limit"] = limit
  end

  if bookmark != ""
    body["bookmark"] = bookmark
  end

  result = relax(post, endpoint(db.url, "_find"); json=body, cookies=db.client.cookies)
  QueryResult(result["docs"], haskey(result, "bookmark") ? result["bookmark"] : "")
end

"""
    result = mango_index{T<:AbstractString}(db::Database, fields::AbstractArray;
      name::T       = "",
      ddoc::T       = "",
      selector      = Selector(),
      default_field = Dict{String, Any}("analyzer" => "standard", "enabled" => true))

Create a Mango index.

All `kw` parameters are optional. The fields spec is mandatory for JSON-type indexes. For a
text index, if you give an empty vector as the fields, it will index every field, which is
occasionally convenient, but a significant performance drain. The index type will defaults to
type `"json"` and will be assumed to be `"text"` if the data in the `fields` array are `Dict`s.
Note that the `text` index type is a Cloudant-only feature.

### Examples

* Make a text index (Cloudant only)

    result = mango_index(db, [Dict("name"=>"lastname", "type"=>"string")]; ddoc="my-ddoc",
      default_field=Dict("analyzer" => "german", "enabled" => true))

* Make a json index

  result = mango_index(db, ["data", "data2"])

### Returns

`mango_index()` returns a `Dict(...)` version of the CouchDB response, of the type

    Dict(
      "name"   => "e7d18f69aa0deaa1ffcdf8f705895b61515a6bf6",
      "id"     => "_design/e7d18f69aa0deaa1ffcdf8f705895b61515a6bf6",
      "result" => "created"
    )

[API reference](https://docs.cloudant.com/cloudant_query.html#creating-an-index)
"""
function mango_index{T<:AbstractString}(db::Database, fields::AbstractArray;
  name::T       = "",
  ddoc::T       = "",
  selector      = Selector(),
  default_field = Dict{String, Any}("analyzer" => "standard", "enabled" => true))

  idxquery = fields == [] ? Dict{T, Any}("index" => Dict{T, Any}()) : Dict{T, Any}("index" => Dict{T, Any}("fields" => fields))

  indextype = "json"
  if length(fields) > 0 && isa(fields[1], Associative)
    indextype = "text"
  end

  if !isempty(selector)
    idxquery["selector"] = selector
  end

  if indextype == "json" && !isempty(selector)
    error("Indextype 'json' does not support a selector.")
  end

  if indextype == "json" && fields == []
    error("Indextype 'json' requires a fields specification.")
  end

  if indextype == "text"
    idxquery["index"]["default_field"] = default_field
  end

  if name != ""
    idxquery["name"] = name
  end

  if ddoc != ""
    idxquery["ddoc"] = ddoc
  end

  if indextype != "json"
    idxquery["type"] = "text"
  end

  relax(post, endpoint(db.url, "_index"); json=idxquery, cookies=db.client.cookies)
end

"""
    result = listindexes(db::Database)

List all existing indexes for the database. This includes views, mango and geo indexes in
addition to the primary index.

### Returns

`listindexes()` returns a `Dict(...)` version of the CouchDB response:

    Dict(
      "indexes" => [
        Dict(
          "name" => "_all_docs",
          "def"  => Dict(
            "fields" => [Dict("_id" => "asc")]
          ),
          "ddoc" => nothing,
          "type" => "special"
        ),
        Dict(
          "ddoc" => "_design/cc79a71f562af7ef36deafe511fea9a857b05bcc",
          "name" => "cc79a71f562af7ef36deafe511fea9a857b05bcc",
          "type" => "text",
          "def"  => Dict(
            "index_array_lengths" => true,
            "fields" => [Dict("cust" => "string"), Dict("value" => "string")],
            "default_field" => Dict(
              "analyzer" => "standard",
              "enabled" => true
            ),
            "selector" => Dict(),
            "default_analyzer" => "keyword"
          )
        ),
        # ...
      ]
    )

[API reference](https://docs.cloudant.com/cloudant_query.html#list-all-cloudant-query-indexes)
"""
function listindexes(db::Database)
  relax(get, endpoint(db.url, "_index"); cookies=db.client.cookies)
end

"""
    result = mango_deleteindex(db::Database; ddoc="", name="", indextype="")

Delete a query index given its ddoc, index name and index type.

Indextype is either "text" or "json".

### Returns

`mango_deleteindex()` returns a `Dict(...)` version of the CouchDB response:

    Dict("ok" => true)

[API reference](https://docs.cloudant.com/cloudant_query.html#deleting-an-index)
"""
function mango_deleteindex(db::Database; ddoc="", name="", indextype="")
  if indextype ∉ ["json", "text"]
    error("Bad indextype: $indextype")
  end
  if ddoc == "" || name == ""
    error("Expected a ddoc and index name")
  end

  relax(delete, endpoint(db.url, "_index/$ddoc/$indextype/$name"); cookies=db.client.cookies)
end


text:
# simple ticks tests

xticks = 2:2:8

x_ticks_1 = plot(1:9)
plot!(xticks=xticks)

x_ticks_2 = plot(1:9)
xticks!(x_ticks_2, xticks)

test_is_same_object(x_ticks_1, x_ticks_2)

# labeled ticks tests

yticks = (-1:+1, ["min", "zero", "max"])

x_range = -1:0.25:+1
y_range = x_range .^ 3

y_ticks_1 = plot(x_range, y_range, yticks=yticks)

y_ticks_2 = plot(x_range, y_range)
yticks!(yticks)

test_is_same_object(y_ticks_1, y_ticks_2)


text:
# Advent of code 2021, day 7

posvec = [
    1101,
    1,
    29,
    67,
    1102,
    0,
    1,
    65,
    1008,
    65,
    35,
    66,
    1005,
    66,
    28,
    1,
    67,
    65,
    20,
    4,
    0,
    1001,
    65,
    1,
    65,
    1106,
    0,
    8,
    99,
    35,
    67,
    101,
    99,
    105,
    32,
    110,
    39,
    101,
    115,
    116,
    32,
    112,
    97,
    115,
    32,
    117,
    110,
    101,
    32,
    105,
    110,
    116,
    99,
    111,
    100,
    101,
    32,
    112,
    114,
    111,
    103,
    114,
    97,
    109,
    10,
    775,
    180,
    473,
    1346,
    1189,
    1553,
    196,
    427,
    788,
    25,
    159,
    204,
    851,
    24,
    1404,
    91,
    308,
    1096,
    747,
    278,
    1185,
    36,
    752,
    57,
    54,
    583,
    570,
    260,
    735,
    1192,
    72,
    552,
    103,
    693,
    383,
    202,
    78,
    1050,
    453,
    116,
    333,
    182,
    765,
    60,
    372,
    201,
    291,
    1642,
    49,
    80,
    918,
    98,
    1443,
    36,
    577,
    696,
    1289,
    56,
    220,
    56,
    51,
    550,
    1666,
    651,
    652,
    1508,
    70,
    40,
    1473,
    57,
    1065,
    32,
    6,
    537,
    1053,
    315,
    711,
    163,
    476,
    1006,
    3,
    1092,
    18,
    1304,
    237,
    358,
    457,
    888,
    36,
    639,
    39,
    1051,
    723,
    590,
    1242,
    210,
    1217,
    473,
    488,
    1554,
    729,
    776,
    307,
    375,
    243,
    186,
    436,
    94,
    451,
    1230,
    495,
    861,
    480,
    28,
    323,
    529,
    92,
    65,
    43,
    564,
    143,
    183,
    81,
    965,
    82,
    168,
    303,
    331,
    99,
    921,
    583,
    1349,
    182,
    353,
    626,
    150,
    475,
    1388,
    381,
    539,
    1190,
    664,
    923,
    1579,
    564,
    31,
    186,
    171,
    415,
    69,
    82,
    621,
    579,
    636,
    787,
    154,
    384,
    463,
    124,
    213,
    270,
    318,
    16,
    21,
    429,
    1285,
    1052,
    755,
    248,
    67,
    1021,
    20,
    165,
    789,
    7,
    456,
    18,
    1432,
    1379,
    3,
    108,
    96,
    40,
    42,
    1148,
    665,
    526,
    392,
    616,
    405,
    633,
    399,
    152,
    388,
    9,
    1078,
    159,
    454,
    945,
    330,
    2,
    455,
    288,
    288,
    72,
    313,
    827,
    521,
    939,
    186,
    680,
    253,
    386,
    917,
    317,
    346,
    1897,
    520,
    662,
    558,
    919,
    31,
    1141,
    1025,
    29,
    80,
    601,
    1001,
    199,
    128,
    1721,
    1221,
    367,
    29,
    722,
    186,
    344,
    136,
    415,
    718,
    122,
    7,
    879,
    195,
    1430,
    250,
    65,
    391,
    1296,
    154,
    39,
    647,
    861,
    175,
    18,
    448,
    131,
    568,
    322,
    20,
    290,
    828,
    844,
    135,
    622,
    409,
    236,
    40,
    341,
    767,
    31,
    917,
    5,
    335,
    27,
    205,
    63,
    18,
    1262,
    615,
    1819,
    291,
    1263,
    179,
    206,
    686,
    102,
    18,
    483,
    525,
    331,
    303,
    932,
    664,
    337,
    774,
    1080,
    22,
    177,
    891,
    780,
    523,
    362,
    38,
    219,
    281,
    72,
    861,
    48,
    1382,
    1497,
    249,
    343,
    404,
    292,
    1252,
    1,
    532,
    370,
    421,
    155,
    1728,
    110,
    1778,
    804,
    944,
    142,
    1508,
    155,
    967,
    1201,
    1066,
    706,
    613,
    1222,
    40,
    1000,
    558,
    616,
    693,
    378,
    124,
    35,
    91,
    514,
    1445,
    7,
    280,
    775,
    744,
    421,
    236,
    597,
    143,
    380,
    91,
    1564,
    111,
    1359,
    711,
    453,
    73,
    247,
    427,
    119,
    182,
    508,
    598,
    514,
    543,
    297,
    182,
    397,
    24,
    1317,
    107,
    766,
    428,
    877,
    580,
    1135,
    13,
    266,
    568,
    369,
    570,
    5,
    214,
    1222,
    150,
    225,
    93,
    1168,
    7,
    793,
    346,
    17,
    70,
    127,
    734,
    1428,
    1513,
    274,
    409,
    1291,
    498,
    958,
    535,
    37,
    268,
    994,
    165,
    662,
    59,
    125,
    267,
    557,
    28,
    259,
    77,
    1226,
    588,
    499,
    105,
    15,
    238,
    272,
    272,
    100,
    329,
    642,
    68,
    356,
    782,
    90,
    674,
    608,
    431,
    1,
    442,
    835,
    3,
    780,
    51,
    394,
    146,
    71,
    231,
    582,
    81,
    595,
    921,
    913,
    398,
    831,
    1107,
    491,
    801,
    371,
    407,
    746,
    1337,
    196,
    7,
    86,
    427,
    72,
    217,
    3,
    98,
    717,
    1268,
    991,
    187,
    103,
    256,
    616,
    172,
    125,
    831,
    1380,
    935,
    281,
    1534,
    435,
    868,
    1291,
    51,
    894,
    91,
    527,
    443,
    868,
    1300,
    72,
    1108,
    259,
    641,
    381,
    1103,
    580,
    422,
    310,
    953,
    97,
    8,
    198,
    1249,
    1069,
    342,
    953,
    464,
    66,
    648,
    683,
    184,
    702,
    1488,
    440,
    389,
    408,
    0,
    47,
    1023,
    45,
    63,
    999,
    131,
    73,
    135,
    256,
    1586,
    798,
    61,
    43,
    708,
    138,
    500,
    952,
    10,
    170,
    1287,
    956,
    1454,
    886,
    1117,
    405,
    1064,
    252,
    1218,
    334,
    316,
    1116,
    485,
    63,
    336,
    1218,
    528,
    309,
    86,
    833,
    168,
    955,
    45,
    993,
    841,
    480,
    336,
    512,
    835,
    296,
    285,
    340,
    81,
    1291,
    61,
    51,
    5,
    284,
    1114,
    120,
    794,
    1444,
    889,
    28,
    0,
    327,
    134,
    71,
    1040,
    230,
    48,
    49,
    837,
    1191,
    558,
    3,
    26,
    760,
    1038,
    104,
    320,
    87,
    464,
    270,
    115,
    357,
    277,
    285,
    488,
    1649,
    477,
    972,
    423,
    524,
    657,
    20,
    395,
    958,
    528,
    13,
    125,
    391,
    90,
    334,
    314,
    1022,
    1147,
    200,
    1357,
    1092,
    559,
    610,
    193,
    296,
    60,
    188,
    0,
    2,
    277,
    540,
    365,
    79,
    278,
    199,
    1327,
    573,
    615,
    95,
    677,
    285,
    143,
    182,
    226,
    12,
    661,
    492,
    189,
    526,
    75,
    1358,
    923,
    228,
    59,
    417,
    535,
    544,
    270,
    1040,
    197,
    2,
    216,
    1217,
    372,
    1034,
    84,
    16,
    725,
    1798,
    352,
    147,
    290,
    528,
    1091,
    105,
    476,
    725,
    137,
    474,
    75,
    1313,
    644,
    92,
    43,
    286,
    47,
    118,
    11,
    865,
    1316,
    1464,
    1189,
    673,
    67,
    612,
    183,
    379,
    309,
    464,
    207,
    31,
    64,
    1375,
    34,
    413,
    618,
    131,
    1459,
    178,
    179,
    81,
    245,
    316,
    223,
    230,
    697,
    337,
    977,
    188,
    1335,
    811,
    163,
    592,
    181,
    93,
    108,
    865,
    112,
    20,
    497,
    986,
    1124,
    73,
    128,
    96,
    107,
    1288,
    179,
    229,
    145,
    1293,
    1224,
    1308,
    748,
    768,
    143,
    38,
    33,
    1842,
    64,
    45,
    1209,
    984,
    269,
    371,
    1451,
    876,
    1372,
    65,
    275,
    173,
    1569,
    298,
    187,
    91,
    522,
    133,
    39,
    709,
    878,
    2,
    123,
    195,
    1435,
    1569,
    482,
    1047,
    322,
    382,
    796,
    38,
    903,
    24,
    950,
    387,
    510,
    460,
    570,
    499,
    545,
    561,
    158,
    383,
    213,
    978,
    1329,
    380,
    938,
    280,
    267,
    762,
    841,
    713,
    111,
    357,
    71,
    19,
    121,
    581,
    91,
    177,
    869,
    1138,
    173,
    14,
    145,
    155,
    21,
    353,
    340,
    1145,
    113,
    594,
    685,
    91,
    781,
    558,
    500,
    10,
    33,
    300,
    270,
    457,
    675,
    850,
    64,
    49,
    81,
    311,
    906,
    404,
    207,
    176,
    309,
    45,
    855,
    16,
    9,
    881,
    428,
    194,
    300,
    329,
    715,
    985,
    559,
    656,
    66,
    184,
    1529,
    8,
    1131,
    610,
    78,
    522,
    338,
    492,
    1378,
    47,
    163,
    448,
    111,
    700,
    3,
    19,
    796,
    876,
    224,
    212,
    51,
    524,
    273,
    597,
    980,
    0,
    10,
    205,
    8,
    985,
    38,
    876,
    6,
    91,
    435,
    1273,
    38,
    147,
    214,
    362,
    1,
    95,
    87,
    724,
    1126,
    807,
    378,
    105,
    89,
    276,
    1076,
    107,
    552,
    1082,
    32,
    896,
    202,
    177,
    946,
    753,
    1106,
    464,
    72,
    61,
    225,
    55,
]

movementcosts = [sum([abs(n - p) for p in posvec]) for n in 1:maximum(posvec)]
movementcosts2 = [sum([sum(1:abs(n - p)) for p in posvec]) for n in 1:maximum(posvec)]
println("Part 1: ", first(findmin(movementcosts)))
println("Part2: ", first(findmin(movementcosts2)))


text:
module GEAS

using LinearAlgebra, Dates, HTTP, DataFrames, Random, Distributions,
    Serialization, Statistics, StatsPlots, JLD2, LaTeXStrings
import Octavian: matmul, matmul!

const prj_dir = begin
    t = splitdir(pathof(GEAS))[1]
    l = findlast('/', t) - 1
    t[1:l]
end

dat_dir = joinpath(prj_dir, "dat")
bin_dir = joinpath(prj_dir, "bin")
plink   = joinpath(bin_dir, "plink")
beagle  = joinpath(bin_dir, "beagle.jar")
macs    = joinpath(bin_dir, "macs")
qmsim   = joinpath(bin_dir, "qmsim")

include("breeding.jl")
include("b-Update.jl")
include("dropping.jl")
include("evaluation.jl")
include("gedit.jl")
include("genotypes.jl")
include("inbreeding.jl")
include("phenotypes.jl")
include("sim-base.jl")
include("sim-QTL.jl")
include("structs.jl")
include("summarize.jl")
include("workflow.jl")
include("check-par.jl")
include("utils.jl")
include("gwas.jl")
include("bayes-alphabet.jl")

copyright()

end # module


text:
for ty = (:Primitive, :Mesh, :Skin, :Node, :Scene,
          :Indices, :Values, :Sparse, :Accessor,
          :Orthographic, :Perspective, :Camera,
          :Target, :Channel, :AnimationSampler, :Animation,
          :Object, :Asset, :Buffer, :BufferView, :Image, :Texture, :Sampler,
          :TextureInfo, :NormalTextureInfo, :OcclusionTextureInfo, :PBRMetallicRoughness, :Material)
    @eval begin
        function Base.show(io::IO, ::MIME"text/plain", x::$ty)
            print(io, $ty, ":")
            for name in fieldnames($ty)
                isdefined(x, name) || continue
                v = getfield(x, name)
                v != nothing && print(io, "\n  $name: ", v)
            end
        end
    end
end


text:
# Convolutional Neural Network
# Andrew Bannerman 5/2/2019

# RelU activation functions variants https://medium.com/tinymind/a-practical-guide-to-relu-b83ca804f1f7 + Julia: https://int8.io/neural-networks-in-julia-hyperbolic-tangent-and-relu/
# Convolutional Neural Network: https://www.youtube.com/watch?v=YRhxdVk_sIs
# CNN - ground up https://towardsdatascience.com/convolutional-neural-networks-from-the-ground-up-c67bb41454e1
# CNN - http://cs231n.github.io/convolutional-networks/
# CNN - https://adeshpande3.github.io/A-Beginner%27s-Guide-To-Understanding-Convolutional-Neural-Networks/
# CNN - https://github.com/wiseodd/hipsternet/blob/master/hipsternet/neuralnet.py
# CNN layer julia = https://github.com/FluxML/Flux.jl/blob/master/src/layers/conv.jl

#----------------------
# max pooling expalined: https://www.youtube.com/watch?v=ZjM_XQa5s6s
# - Zero padding = https://www.youtube.com/watch?v=qSTv_m-KFk0
#----------------------

# Needed packages
using DataFrames
using CSV
using Distributions
using Plots
using Random
using Images, CoordinateTransformations, TestImages, OffsetArrays, CoordinateTransformations # For data augmentation
using PaddedViews # for cropping images

# Train and test set data preparation

# Load MNIST Data set from .csv format
train_set = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/My_First_Neural_Network/Data/mnist_train.csv",header=false,types=fill(Float64,785))
test_set = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/My_First_Neural_Network/Data/mnist_test.csv",header=false,types=fill(Float64,785))

# Plot Example image
example = Float64.(train_set[2,2:size(train_set,2)])
example = rotl90(reshape(example,28,28))
heatmap(example)

# Extract labels
train_label = train_set[:,1]
test_label = test_set[:,1]

# Extract Data and exclude the label in the first column
train_set = train_set[:,2:size(train_set,2)]
test_set = test_set[:,2:size(test_set,2)]

# Pre process - Scale the inputs to a range of 0.01 to 1.00
m_train = Array{Float64}(zeros(nrow(train_set),size(train_set,2))) # empty matrix
@inbounds for i in 1:nrow(train_set)
    @inbounds for j in 1:size(train_set,2)
    m_train[i,j] = (train_set[i,j] / 255.0 * 0.99) + 0.01
    end
end

m_test = Array{Float64}(zeros(nrow(test_set),size(test_set,2))) # empty matrix
@inbounds for i in 1:nrow(test_set)
    @inbounds for j in 1:size(test_set,2)
    m_test[i,j] = (test_set[i,j] / 255.0 * 0.99) + 0.01
    end
end

# Convert matrix to DataFrame and append the labels back to the train and test set
scaled_train_inputs = DataFrame(hcat(train_label,m_train))
scaled_test_inputs = DataFrame(hcat(test_label,m_test))

# Plot the new scale example
example = reshape(Float64.(scaled_train_inputs[5,2:size(scaled_train_inputs,2)]),784,1)
example = rotl90(reshape(example,28,28))
print(example)
heatmap(example,c = :greys)

function train(wih::Array{Float64,2}, who::Array{Float64,2},inputs::Array{Float64,2}, targets::Array{Float64,2}; lr::Float64=0.1, drop_out::Bool=true, drop_out_p::Float64=.8) #wih, who, lr, inputs, targets)

  A = example

# Set weights and filter
# Initialize filter using a normal distribution with mean 0
# standard deviation inversely proportional the square root of the number of units
# Resource: https://stats.stackexchange.com/questions/200513/how-to-initialize-the-elements-of-the-filter-matrix
scale = 1.0
stdev = scale/sqrt(size(A,2)*(size(A,1)))
filt = reshape(rand(Normal(0, stdev), 9),3,3)

# Build the convolution layer
"""
    conv(A::Array{Float64,2}; m::Int64=3,n::Int64=3, s::Int64=1, p::Int64=1)
m x n = image convolution block size
s = stride
p = padding
padding = "valid" = no padding, "same" = keep conv dim output same as input dim, "custom" = custom p value
Input should be divisible by 2
"""
m=3
n=3
s=1
p=3
function conv_layer(A::Array{Float64,2}; m::Int64=3,n::Int64=3, s::Int64=1, p::Int64=0, padding::String="valid") # mxn: block_size / p::String="valid"
# Note need to add a check so that the full conv box with stride fits into the image
# https://github.com/JuliaArrays/PaddedViews.jl/issues/9
  M,N = size(A)
  mc = M+(p*2)-m+1          # no. vertical blocks (rows) # adjusted for the padding dim
  nc = N+(p*2)-n+1          # no. horizontal blocks (cols)
  if padding == "same" # want output dim of conv to be same as input dims
  p=0 # reset p
  #Calculate the first conv layer output dimensions
  w = size(A,2)
  f = size(filt,2)
  #M_out,N_out = (M-f+1) , (N-f+1)
  conv_out_dim = Int64.((w-f+2*p) / s+1) # final convoltion output dim
  p = Int64.((f-1) / 2) # Calculate correct padding value to keep the output the same dims as the input
  pad = p+1 # this is first element position within pad in A for PaddedView() PaddedView.jl
    A_pad = Float64.(PaddedView(0.0, A,(M+(p*2),N+(p*2)), (pad,pad)))
elseif padding == "valid"
  p=0 # reset p
  pad = p+1 # this is first element position within pad in A
    A_pad = Float64.(PaddedView(0.0, A,(M+(p*2),N+(p*2)), (pad,pad)))
elseif padding == "custom"
    p = p # keep as function input
    pad = p+1 # this is first element position within pad in A
      A_pad = Float64.(PaddedView(0.0, A,(M+(p*2),N+(p*2)), (pad,pad)))
  end
  #A_pad = Float64.(PaddedView(0.0, A,(M+(p*2),N+(p*2)), (pad,pad))) # p*2 to increase dim in both directions pad,pad = position of the first element to pad around
  #B = Array{eltype(A)}(undef, m*n, mc*nc)
  M,N = size(A_pad)
  mc = M-m+1          # no. vertical blocks (rows)
  nc = N-n+1          # no. horizontal blocks (cols)
  #B = zeros(m*n, mc*nc)
  B = zeros(m*n, 0)
  # Create custom index to allow for stride
  stride_col_index = collect(1:s:nc)
  stride_row_index = collect(1:s:mc)
  #B = Array{eltype(A)}(undef)
  # split the image into small convolution blocks
  # This function is similar to im2col
  i = 1
  j = 1
  @inbounds for j = 1:size(stride_col_index,1) # Loop over columns horizontal
    @inbounds for i = 1:size(stride_row_index,1) # loop over rows vertical
      @views block = A_pad[stride_row_index[i]:stride_row_index[i]+m-1, stride_col_index[j]:stride_col_index[j]+n-1]
      # Consider adding extra for loop to populate output memory wise...
        shapes = reshape(block,m*n,1)
       B = hcat(B, shapes) # Append to output matrix
    #B[:, j] = shapes
      end
  end
    # multiply the filter by the convolution block
    # Calculate spatial size of the volume
    # w = Input volume size
    # f = size of the Conv Layer neurons
    # s = Stride
    # p = Amount of Zero padding
    # Neuron fit = (W−F+2P)/S+1
    w = size(A_pad,2)
    f = size(filt,2)
    conv_out_dim = Int64.((w-f) / s+1) # This is conv output dimension (excluding padding)
    conv_out = zeros(conv_out_dim,conv_out_dim) # initialize output
    @inbounds for j in 1:size(B,2) # loop over column dim (horz.)
      conv_out[j] = sum(filt .* reshape(B[:,j],m,n)) #+ bias # element wise multiplication
    end
         return conv_out
  end

      @time out = conv_layer(A,s=1,p=3,padding="same")

  A = Float64.(reshape(1.0:81.0,9,9))
  # initialize output
  B = zeros(Int64, 2, 0)
  j = 1
  for j in 1:size(A,2) # loop cols
    out = [sum(A[:,j]),j]
    out = reshape(out,2,1)
    # append out to the B
    global B = hcat(B,out) # this grows...
  end

  function f2()
      A = reshape(1:81,9,9)
      B = zeros(Int64, 2, 9)
      for j in 1:size(A,2) # loop cols
          B[:, j] .= (sum(view(A, :, j)), j)
      end
      B
  end

    @time out = conv_layer(A,s=1,p=3,padding="same")

  heatmap(conv_out,c = :greys)
m=2
n=2
s=2
pool_type = "max"
"""
    pooling_layer(A::Array{Float64,2}; m::Int64=2,n::Int64=2, s::Int64=1, pool_type::String="max")
m x n = image pooling block size
s = stride
pool_type = "max" | "avg"
"""
  function pooling_layer(A::Array{Float64,2}; m::Int64=2,n::Int64=2, s::Int64=1, pool_type::String="max")
    M,N = size(A)
    mc = M-m+1          # no. vertical blocks (rows)
    nc = N-n+1          # no. horizontal blocks (cols)
    B = zeros(Int8, m*n, 0) # output
    # Create custom index to allow for stride
    stride_col_index = collect(1:s:nc)
    stride_row_index = collect(1:s:mc)
    @inbounds for j = 1:size(stride_col_index,1) # Loop over columns horizontal
      @inbounds for i = 1:size(stride_row_index,1) # loop over rows vertical
          @views block = A[stride_row_index[i]:stride_row_index[i]+m-1, stride_col_index[j]:stride_col_index[j]+n-1]
          shapes = rotr90(reshape(block,1,m*n))
            B = hcat(B, shapes) # Append to output matrix
        end
    end
    h = size(A,1) # rows
    w = size(A,2) # cols
    f = m
    pooling_out_h = Int64.((h-f)/s+1) # rows
    pooling_out_w = Int64.((w-f)/s+1) # cols
    pooling_out = zeros(pooling_out_h,pooling_out_w) # initialize output
    if pool_type == "max"
    for j in 1:size(B,2) # loop over column dim (horz.)
      @inbounds pooling_out[j] = maximum(B[:,j]) #+ bias # element wise multiplication
    end
  elseif pool_type == "avg"
    for j in 1:size(B,2) # loop over column dim (horz.)
      @inbounds pooling_out[j] = mean(B[:,j]) #+ bias # element wise multiplication
    end
  end
    return pooling_out
  end

  @time pool_out = pooling_layer(A,m=2,n=2,s=2)
    heatmap(testing, c = :greys)

    conv2 = conv_layer(pool_out,s=1,p=3,padding="same")
heatmap(conv2, c = :greys)
  pool_out2 = pooling_layer(conv2,m=2,n=2,s=1)
heatmap(pool_out2, c = :greys)

"""
input ---> conv_layer ---> pooling ---> conv_layer ---> pooling ---> fully connected layer
"""
# Create Fully-Connected Layer
pooled_shape = size(pool_out2)[1] * size(pool_out2)[2]
flatten = reshape(pool_out2,pooled_shape,1) # flatten pool layer

# inpout for a normal NN below

       @time A = reshape(collect(1.0:1.0:81.0),9,:)
       @time B = DataFrame(conv(A,s=1,p=0))

       plots = conv(A,s=1,p=0)
       heatmap(plots)

       testing = DataFrame(A_pad)

       CSV.write("C:/Users/Andrew.Bannerman/Desktop/Julia/test_outB1.csv", testing;delim=',')