text:
module tf

    using RobotOS
    using PyCall
    using Rotations
    using StaticArrays
    using LinearAlgebra: inv
    import Base.==
    import Base.*
    import Base.rand
    import Base.inv

    const __tf__ = PyCall.PyNULL()

    function __init__()
        copy!(__tf__, pyimport("tf"))
    end

    export Transform
    export rand
    export TransformBroadcaster, sendTransform
    export TransformListener, lookupTransform, waitForTransform
    export transformWrench, poseToTransform, poseStampedToTransform

    """
       Transform(trans, rot)
    Create transform object.
    """
    struct Transform
        trans::Array{Float64}
        rot::Array{Float64}
    end

# =========================================================================== #
#                      Useful functions on Transform type
# =========================================================================== #
    function Base.:(==)(tf1::Transform, tf2::Transform)
        (all(tf1.trans .== tf2.trans) && all(tf1.rot .== tf2.rot))
    end

    function Base.:(*)(tf1::Transform, tf2::Transform)::Transform
        # Apply transform
        tf3Mat = twist2RBT(tf1) * twist2RBT(tf2)
        tf3 = RBT2twist(tf3Mat)
        return tf3
    end

    function Base.inv(tf::Transform)::Transform
        # Apply transform
        tf_inv = RBT2twist(inv(twist2RBT(tf)))
        return tf_inv
    end

    function twist2RBT(tf::Transform)::Matrix
        x, y, z, w = tf.rot
        rotMat = Matrix(UnitQuaternion(w, x, y, z))

        tfMat = [rotMat   tf.trans;
                 [0 0 0]  1];
        return tfMat
    end

    function RBT2twist(tfMat::Matrix)::Transform
        # Translations
        trans = tfMat[1:3, 4]
        # Rotations
        rotMat = RotMatrix(SMatrix{3,3}(tfMat[1:3, 1:3]))
        quat = UnitQuaternion(rotMat)
        rot = [quat.x, quat.y, quat.z, quat.w]

        tf = Transform(trans, rot)
        return tf
    end

    function Base.rand(::Type{Transform})::Transform
        trans = rand(3)
        quat = rand(UnitQuaternion)
        rot = [quat.x, quat.y, quat.z, quat.w]
        return Transform(trans, rot)
    end

# ============================================================================#
#                 tf TransformBroadcaster object and functions
# ============================================================================#
    """
        TransformBroadcaster()
    Create a transform broadcaster object.
    """
    struct TransformBroadcaster
        o::PyObject
        function TransformBroadcaster()
            new(__tf__.TransformBroadcaster())
        end
    end

    """
        sendTransformMessage(tb, translation, rotation, time, child, parent)
    Broadcast the transformation from tf frame child to parent on ROS topic "/tf".
    """
    function sendTransform(tb::TransformBroadcaster,
                           translation::Vector,
                           rotation::Vector,
                           time::Time,
                           child::AbstractString,
                           parent::AbstractString)
        trans = translation
        rot = rotation
        pytime = convert(PyObject, time)
        pycall(tb.o.sendTransform, PyAny, trans, rot, pytime, child, parent)
    end

    """
        sendTransformMessage(tb::TransformBroadcaster, transform)
    Broadcast the transformation from tf frame child to parent on ROS topic "/tf".
    """
    function sendTransformMessage(tb::TransformBroadcaster,
                                  transform::Any)
        pytransform = convert(PyObject, transform)
        pycall(tb.o.sendTransformMessage, PyAny, pytransform)
    end

# ============================================================================#
#                 tf TransformListener object and functions
# ============================================================================#
    """
        TransformListener()
    Create a transform listener object.
    """
    struct TransformListener
        o::PyObject
        function TransformListener()
            new(__tf__.TransformListener())
        end
    end

    """
        generate_error_message(err)
    Retrun error message string which includes both exception type and error massage information.
    """
    function generate_error_message(err)
        exception_type = err.T.__name__
        error_massage = exception_type * ": $(err.val.args[1])"
    end

    """
        lookupTransform(tf_listener_obj, target, source, time)
    Return tuple of (position, quaternion).
    """
    function lookupTransform(tl::TransformListener,
                             target_frame::AbstractString,
                             source_frame::AbstractString,
                             pytime::Time)::Transform
        time = convert(PyObject, pytime)
        try
            trans, rot = pycall(tl.o.lookupTransform, PyAny, target_frame,
                                source_frame, time)
            return Transform(trans, rot)
        catch err
            if isa(err, PyCall.PyError)
                error_massage = generate_error_message(err)
                error(error_massage)
            else
                rethrow(err)
            end
        end
    end

    """
        waitForTransform(tf_listener_obj, target, source, time, timeout, pypolling_sleep_duration)
    Waits for the given transformation to become available. If the timeout occurs before the transformation becomes available, raises an exception.
    """
    function waitForTransform(tl::TransformListener,
                              target_frame::AbstractString,
                              source_frame::AbstractString,
                              pytime::Time,
                              pytimeout::Duration;
                              pypolling_sleep_duration = Duration(0.01))
        time = convert(PyObject, pytime)
        timeout = convert(PyObject, pytimeout)
        polling_sleep_duration = convert(PyObject, pypolling_sleep_duration)
        try
            pycall(tl.o.waitForTransform, PyAny, target_frame, source_frame,
                   time, timeout, polling_sleep_duration)
        catch err
            if isa(err, PyCall.PyError)
                error_massage = generate_error_message(err)
                error(error_massage)
            else
                rethrow(err)
            end
        end
    end

# ============================================================================#
#                           Helpful functions
# ============================================================================#
    """
    """
    function toSkew(vect::Vector)
        all(size(vect) .== (3,)) || error("vect must be 3 vector")
        return [0       -vect[3]  vect[2];
                vect[3]  0       -vect[1];
               -vect[2]  vect[1]  0];
    end

    """
    """
    function transformWrench(transform::Transform,
                             wrench::Any,
                             )
        x, y, z, w = transform.rot
        𝑞ₛₑᵀ = inv(UnitQuaternion(w, x, y, z))
        p̂ₛₑ = toSkew(transform.trans)

        Adᵀ = [𝑞ₛₑᵀ   zeros(3, 3);
               -𝑞ₛₑᵀ*p̂ₛₑ  𝑞ₛₑᵀ]
        𝑓ₛ = [wrench.force.x, wrench.force.y, wrench.force.z,
             wrench.torque.x, wrench.torque.y, wrench.torque.z];
        𝑓ₑ = Adᵀ * 𝑓ₛ

        ret_wrench = deepcopy(wrench)
        ret_wrench.force.x = 𝑓ₑ[1]
        ret_wrench.force.y = 𝑓ₑ[2]
        ret_wrench.force.z = 𝑓ₑ[3]
        ret_wrench.torque.x = 𝑓ₑ[4]
        ret_wrench.torque.y = 𝑓ₑ[5]
        ret_wrench.torque.z = 𝑓ₑ[6]

        return ret_wrench
    end

    """
    """
    function poseToTransform(pose::Any)::Transform
        trans = [pose.position.x, pose.position.y, pose.position.z]
        rot = [pose.orientation.x, pose.orientation.y,
               pose.orientation.z, pose.orientation.w]

        return Transform(trans, rot)
    end

    """
    """
    function poseStampedToTransform(poseStamped::Any)::Transform
        return poseToTransform(poseStamped.pose)
    end

    """
    """
    function transformPose(transform::Transform,
                           pose::Any,
                           )
        x, y, z, w = transform.rot
        𝑞ₛₑᵀ = inv(UnitQuaternion(w, x, y, z))
        p̂ₛₑ = toSkew(transform.trans)
        Adᵀ = [𝑞ₛₑᵀ   zeros(3, 3);
               -𝑞ₛₑᵀ*p̂ₛₑ  𝑞ₛₑᵀ]
        𝑓ₛ = [wrench.force.x, wrench.force.y, wrench.force.z,
             wrench.torque.x, wrench.torque.y, wrench.torque.z];
        𝑓ₑ = Adᵀ * 𝑓ₛ
    end
end


text:
function fn(x::Array{Float32})
    println("array size: $(size(x))");

    sleep(5)
    x[1:3] .= -1
end

ext: jl

max_stars_repo_name: paulmelis/blender-julia-test

max_issues_repo_name: paulmelis/blender-julia-test

max_forks_repo_name: paulmelis/blender-julia-test


text:
function repr_symbol( s::Symbol )
    v = utf8(string(s))
    if length(v) == 0
        v = ":\"\""
    elseif match( r"^[a-zA-Z_][0-9a-zA-Z_]*$", v ) != nothing
        v = ":" * v
    else
        v = "sym\""*escape_string(v)*"\""
    end
    v
end

function delete_char_before( s::ASCIIString, p::Int )
    return (s[1:p-2] * s[p:end], max(p-1,1) )
end

#delete a code_point before the p "width" position
function delete_char_before( s::UTF8String, p::Int )
    local totalskip::Int = 0
    local lastj::Int = 0
    local lastcw::Int = 0
    if p == 1
        return (s,p)
    end
    for (j,c) in enumerate( s )
        cw = charwidth( c )
        if totalskip + cw < p
            totalskip += cw
            lastj = j
            lastcw = cw
            continue
        else
            if lastj <= 1
                return (s[chr2ind(s,j):end], p-lastcw)
            else
                return (s[1:chr2ind(s,lastj-1)] * s[chr2ind(s,j):end], p-lastcw)
            end
        end
    end
    if lastj == 0
        return (s,p)
    end
    if lastj == 1
        return ("", p-lastcw)
    end
    return (s[1:chr2ind(s,lastj-1)], p-lastcw)
end

function delete_char_at( s::ASCIIString, p::Int )
    return s[1:p-1] * s[p+1:end]
end

# delete at least 1 code point, could be more if there
# are trailing zero-width codepoints.
function delete_char_at( s::UTF8String, p::Int )
    local totalskip::Int = 0
    local lastj::Int = 0
    for (j,c) in enumerate( s )
        cw = charwidth( c )
        if totalskip + cw < p
            totalskip += cw
            lastj = j
            continue
        else
            if lastj == 0
                return substr_by_width( s, p, -1 )
            else
                return s[1:chr2ind(s,lastj)] * substr_by_width( s, p, -1 )
            end
        end
    end
    s
end

# TODO: test this thoroughly!!
# Insert a (short) string at the "p" position
# p is interpreted as the width position
function insertstring{T<:AbstractString}( s::UTF8String, ch::T, p::Int, overwrite::Bool )
    wskip = p-1
    local totalskip::Int = 0
    chwidth = strwidth( ch )
    if chwidth == 0 && p == 1
        return s
    end
    for (j,c) in enumerate( s )
        cw = charwidth( c )
        if totalskip + cw <= wskip
            totalskip += cw
            continue
        else
            if j == 1
                out = ch
            else
                out = s[1:chr2ind(s,j-1)] * ch
            end
            if overwrite
                out *= substr_by_width( s, wskip+chwidth, -1 )
            else
                out *= substr_by_width( s, wskip, -1 )
            end
            return out
        end
    end
    return s * ch
end

# get w charwidths worth of string, after skipping up to wskip widths
# greedy-skip: all trailing 0 width chars will be skipped
# greedy-include: all trailing 0-width chars will be included
# if w is -1, it would take all the rest of the string
function substr_by_width{T<:AbstractString}( s::T, wskip::Int, w::Int )
    local totalskip::Int = 0
    local totalwidth::Int = 0
    local startidx::Int = -1
    local endidx::Int = 0
    for (j,c) in enumerate( s )
        cw = charwidth( c )
        if startidx == -1
            if totalskip + cw <= wskip
                totalskip += cw
                continue
            else
                if w == -1 # just take until the end
                    return s[chr2ind(s,j):end]
                end
                startidx = j
                endidx = j
                if totalwidth + cw <= w
                    totalwidth += cw
                    continue
                else
                    return convert( T, "" )
                end
            end
        else
            if totalwidth + cw <= w
                totalwidth += cw
                endidx = j
                continue
            else
                break
            end
        end
    end
    if startidx == -1
        if w == -1
            return convert( T,"" )
        end
        startidx = 1
    end
    if w == -1
        return s[chr2ind(s,startidx):end]
    end
    if endidx < startidx
        return convert( T, "" )
    end
    return s[chr2ind(s,startidx):chr2ind(s,endidx)]
end

function ensure_length{T<:AbstractString}( s::T, w::Int, pad::Bool = true )
    t = replace( s, "\n", "\\n" )
    t = replace( t, "\t", " " )
    if w <= 0
        return ""
    end
    len = strwidth( t )
    if w ==1
        if len > 1
            return string( @compat Char( 0x2026 ) )
        elseif len==1
            return t
        else
            if pad
                return " "
            else
                return ""
            end
        end
    end

    if len <= w
        if pad
            return t * repeat( " ", w - len )
        else
            return t
        end
    else # ellipsis
        return substr_by_width( t, 0, w-1 ) * string( @compat Char( 0x2026 ) )
    end
end

function wordwrap{T<:AbstractString}( x::T, width::Int )
    spaceleft = width
    lines = UTF8String[]
    currline = convert( T,"" )
    words = @compat split( x, " ", keep=true ) # don't keep empty words
    for w in words
        wlen = strwidth(w)
        if wlen>width && spaceleft == width
            push!( lines, substr_by_width( w, 0, width-1 ) * string( @compat Char( 0x2026 ) ) )
        elseif wlen+1 > spaceleft
            push!( lines, currline )
            currline = w * " "
            spaceleft = width - wlen - 1
        else
            currline = currline * w * " "
            spaceleft = spaceleft - wlen -1
        end
    end
    if currline != ""
        push!(lines, currline )
    end
    return lines
end

# edit distance on signatures, should work on utf-8 strings too.
function levenstein_distance( s1, s2 )
    if s1==s2
        return 0
    end
    if length(s1)==0
        return length(s2)
    end
    if length(s2)==0
        return length(s1)
    end

    v0 = [0:length(s2)]
    v1 = Array( Int, length(s2)+1)
    for (i,c1) in enumerate(s1)
        v1[1] = i
        for (j,c2) in enumerate( s2 )
            cost = ( c1 == c2 ) ? 0 : 1
            v1[j+1] = min( v1[j] + 1, v0[ j+1 ] + 1, v0[j]+cost )
        end

        for j in 1:length( v0 )
            v0[j]=v1[j]
        end
    end
    return v1[end]
end

function longest_common_prefix( s1::UTF8String, s2::UTF8String )
    m = min( length( s1 ), length( s2 ) )
    lcpidx = 0
    for i in 1:m
        if s1[i] != s2[i]
            break
        end
        lcpidx = i
    end
    return s1[ 1:lcpidx ]
end


text:
function testSampler(vkDevice::vk.VkDevice)
    info1 = lava.SamplerCreateInfo(unnormalizedCoordinates = VkExt.VK_TRUE)
    info2 = lava.depth(lava.SamplerCreateInfo)
    info3 = lava.shadow(lava.SamplerCreateInfo)
    lava.@scope begin
        lava.@autodestroy s1 = lava.Sampler(vkDevice, info1)
        lava.@autodestroy s2 = lava.Sampler(vkDevice, info2)
        lava.@autodestroy s3 = lava.Sampler(vkDevice, info3)
    end
    return true
end


text:
using Oxygen

println("Testing Periodic Table")

@testset "Periodic Table" begin
    @test Oxygen.periodic_table["H"] == 1
    @test Oxygen.periodic_table["C"] == 6
    @test Oxygen.periodic_table["Si"] == 14
    @test Oxygen.periodic_table["Cl"] == 17
    @test Oxygen.periodic_table["Ti"] == 22
    @test Oxygen.periodic_table["Fe"] == 26
    @test Oxygen.periodic_table["Ge"] == 32
    @test Oxygen.periodic_table["Kr"] == 36
    @test Oxygen.periodic_table["Nb"] == 41
    @test Oxygen.periodic_table["Pd"] == 46
    @test Oxygen.periodic_table["Sb"] == 51
    @test Oxygen.periodic_table["Ba"] == 56
    @test Oxygen.periodic_table["Pm"] == 61
    @test Oxygen.periodic_table["Dy"] == 66
    @test Oxygen.periodic_table["Lu"] == 71
    @test Oxygen.periodic_table["Os"] == 76
    @test Oxygen.periodic_table["Tl"] == 81
    @test Oxygen.periodic_table["Rn"] == 86
    @test Oxygen.periodic_table["Pa"] == 91
    @test Oxygen.periodic_table["Cm"] == 96
    @test Oxygen.periodic_table["Md"] == 101
    @test Oxygen.periodic_table["Sg"] == 106
    @test_throws KeyError Oxygen.periodic_table["LL"]
end

@testset "Reverse Periodic Table" begin
    @test Oxygen.periodic_table_reverse[1] == "H"
    @test Oxygen.periodic_table_reverse[6] == "C"
    @test Oxygen.periodic_table_reverse[14] == "Si"
    @test Oxygen.periodic_table_reverse[17] == "Cl"
    @test Oxygen.periodic_table_reverse[22] == "Ti"
    @test Oxygen.periodic_table_reverse[26] == "Fe"
    @test Oxygen.periodic_table_reverse[32] == "Ge"
    @test Oxygen.periodic_table_reverse[36] == "Kr"
    @test Oxygen.periodic_table_reverse[41] == "Nb"
    @test Oxygen.periodic_table_reverse[46] == "Pd"
    @test Oxygen.periodic_table_reverse[51] == "Sb"
    @test Oxygen.periodic_table_reverse[56] == "Ba"
    @test Oxygen.periodic_table_reverse[61] == "Pm"
    @test Oxygen.periodic_table_reverse[66] == "Dy"
    @test Oxygen.periodic_table_reverse[71] == "Lu"
    @test Oxygen.periodic_table_reverse[76] == "Os"
    @test Oxygen.periodic_table_reverse[81] == "Tl"
    @test Oxygen.periodic_table_reverse[86] == "Rn"
    @test Oxygen.periodic_table_reverse[91] == "Pa"
    @test Oxygen.periodic_table_reverse[96] == "Cm"
    @test Oxygen.periodic_table_reverse[101] == "Md"
    @test Oxygen.periodic_table_reverse[106] == "Sg"
    @test_throws KeyError Oxygen.periodic_table_reverse[109]
end

@testset "Periodic Table Mass" begin
    @test Oxygen.periodic_table_mass[1] == 1.0080
    @test Oxygen.periodic_table_mass[6] == 12.011
    @test Oxygen.periodic_table_mass[8] == 15.999
    @test Oxygen.periodic_table_mass[17] == 35.45
    @test Oxygen.periodic_table_mass[22] == 47.87
    @test Oxygen.periodic_table_mass[35] == 79.9
end

@testset "Periodic Table Electronegativity" begin
    @test Oxygen.periodic_table_electronegativity[1] == 2.2
    @test Oxygen.periodic_table_electronegativity[6] == 2.55
    @test Oxygen.periodic_table_electronegativity[8] == 3.44
    @test Oxygen.periodic_table_electronegativity[17] == 3.16
    @test Oxygen.periodic_table_electronegativity[22] == 1.54
    @test Oxygen.periodic_table_electronegativity[35] == 2.96
    @test ismissing(Oxygen.periodic_table_electronegativity[2])
end


text:
# TODO:
# - refactor, simplify branching, unify duplications
# - (maybe) export latex completions into a separate package

struct CompletionState
    offset::Int
    completions::Dict{String,CompletionItem}
    range::Range
    x::EXPR
    doc::Document
    server::LanguageServerInstance
    using_stmts::Dict{String,Any}
end

function add_completion_item(state::CompletionState, completion::CompletionItem)
    if haskey(state.completions, completion.label) && ismissing(state.completions[completion.label].data)
        # For the above statement: we've (1) already got a completion which (2) doesn't require adding an explicit import statement.
        return
    end
    state.completions[completion.label] = completion
end

StaticLint.getenv(state::CompletionState) = getenv(state.doc, state.server)

using REPL

"""
    is_completion_match(s::AbstractString, prefix::AbstractString, cutoff=3)

Returns true if `s` starts with `prefix` or has a sufficiently high fuzzy score.
"""
function is_completion_match(s::AbstractString, prefix::AbstractString, cutoff=3)
    starter = if all(islowercase, prefix)
        startswith(lowercase(s), prefix)
    else
        startswith(s, prefix)
    end
    starter || REPL.fuzzyscore(prefix, s) >= cutoff
end

function textDocument_completion_request(params::CompletionParams, server::LanguageServerInstance, conn)
    state = let
        doc = getdocument(server, URI2(params.textDocument.uri))
        offset = get_offset(doc, params.position)
        rng = Range(doc, offset:offset)
        x = get_expr(getcst(doc), offset)
        using_stmts = server.completion_mode == :import ? get_preexisting_using_stmts(x, doc) : Dict()
        CompletionState(offset, Dict{String,CompletionItem}(), rng, x, doc, server, using_stmts)
    end

    ppt, pt, t, is_at_end  = get_partial_completion(state)

    if pt isa CSTParser.Tokens.Token && pt.kind == CSTParser.Tokenize.Tokens.BACKSLASH
        latex_completions(string("\\", CSTParser.Tokenize.untokenize(t)), state)
    elseif ppt isa CSTParser.Tokens.Token && ppt.kind == CSTParser.Tokenize.Tokens.BACKSLASH && pt isa CSTParser.Tokens.Token && pt.kind === CSTParser.Tokens.CIRCUMFLEX_ACCENT
        latex_completions(string("\\", CSTParser.Tokenize.untokenize(pt), CSTParser.Tokenize.untokenize(t)), state)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokenize.Tokens.COMMENT
        partial = is_latex_comp(t.val, state.offset - t.startbyte)
        !isempty(partial) && latex_completions(partial, state)
    elseif t isa CSTParser.Tokens.Token && (t.kind in (CSTParser.Tokenize.Tokens.STRING,
                                                       CSTParser.Tokenize.Tokens.TRIPLE_STRING,
                                                       CSTParser.Tokenize.Tokens.CMD,
                                                       CSTParser.Tokenize.Tokens.TRIPLE_CMD))
        string_completion(t, state)
    elseif state.x isa EXPR && is_in_import_statement(state.x)
        import_completions(ppt, pt, t, is_at_end, state.x, state)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.DOT && pt isa CSTParser.Tokens.Token && pt.kind == CSTParser.Tokens.IDENTIFIER
        # getfield completion, no partial
        px = get_expr(getcst(state.doc), state.offset - (1 + t.endbyte - t.startbyte))
        _get_dot_completion(px, "", state)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.IDENTIFIER && pt isa CSTParser.Tokens.Token && pt.kind == CSTParser.Tokens.DOT && ppt isa CSTParser.Tokens.Token && ppt.kind == CSTParser.Tokens.IDENTIFIER
        # getfield completion, partial
        px = get_expr(getcst(state.doc), state.offset - (1 + t.endbyte - t.startbyte) - (1 + pt.endbyte - pt.startbyte)) # get offset 2 tokens back
        _get_dot_completion(px, t.val, state)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.IDENTIFIER
        # token completion
        if is_at_end && state.x !== nothing
            if pt isa CSTParser.Tokens.Token && pt.kind == CSTParser.Tokens.AT_SIGN
                spartial = string("@", t.val)
            else
                spartial = t.val
            end
            kw_completion(spartial, state)
            rng = Range(state.doc, state.offset:state.offset)
            collect_completions(state.x, spartial, state, false)
        end
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.AT_SIGN
        # only `@` given
        state.x !== nothing && collect_completions(state.x, "@", state, false)
    elseif t isa CSTParser.Tokens.Token && Tokens.iskeyword(t.kind) && is_at_end
        kw_completion(CSTParser.Tokenize.untokenize(t), state)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.IN && is_at_end
        collect_completions(state.x, "in", state, false)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.ISA && is_at_end
        collect_completions(state.x, "isa", state, false)
    elseif t isa CSTParser.Tokens.Token && t.kind == CSTParser.Tokens.COMMA &&
           pt isa CSTParser.Tokens.Token && pt.kind == CSTParser.Tokens.IDENTIFIER &&
           ppt isa CSTParser.Tokens.Token && ppt.kind == CSTParser.Tokens.LPAREN &&
           !(parentof(state.x) isa EXPR && CSTParser.iscall(parentof(state.x)))
        # method completion for given argument
        ptlen = (1 + pt.endbyte - pt.startbyte)
        px = get_expr(getcst(state.doc), state.offset - ptlen)
        method_completion(px, state, ptlen)
    end

    return CompletionList(true, unique(values(state.completions)))
end

function get_partial_completion(state::CompletionState)
    ppt, pt, t = toks = get_toks(state.doc, state.offset)
    is_at_end = state.offset == t.endbyte + 1
    return ppt, pt, t, is_at_end
end

function latex_completions(partial::String, state::CompletionState)
    for (k, v) in REPL.REPLCompletions.latex_symbols
        if is_completion_match(string(k), partial)
            # t1 = TextEdit(Range(state.doc, (state.offset - sizeof(partial)):state.offset), v)
            add_completion_item(state, CompletionItem(k, 11, missing, v, v, missing, missing, missing, missing, missing, missing, texteditfor(state, partial, v), missing, missing, missing, missing))
        end
    end
end

function kw_completion(partial::String, state::CompletionState)
    length(partial) == 0 && return
    for (kw, comp) in snippet_completions
        if startswith(kw, partial)
            add_completion_item(state, CompletionItem(kw, 14, missing, missing, kw, missing, missing, missing, missing, missing, InsertTextFormats.Snippet, texteditfor(state, partial, comp), missing, missing, missing, missing))
        end
    end
end

const snippet_completions = Dict{String,String}(
    "abstract" => "abstract type \$0 end",
    "baremodule" => "baremodule \$1\n\t\$0\nend",
    "begin" => "begin\n\t\$0\nend",
    "break" => "break",
    "catch" => "catch",
    "const" => "const ",
    "continue" => "continue",
    "do" => "do \$1\n\t\$0\nend",
    "else" => "else",
    "elseif" => "elseif ",
    "end" => "end",
    "export" => "export ",
    "finally" => "finally",
    "for" => "for \$1 in \$2\n\t\$0\nend",
    "function" => "function \$1(\$2)\n\t\$0\nend",
    "global" => "global ",
    "if" => "if \$1\n\t\$0\nend",
    "import" => "import",
    "let" => "let \$1\n\t\$0\nend",
    "local" => "local ",
    "macro" => "macro \$1(\$2)\n\t\$0\nend",
    "module" => "module \$1\n\t\$0\nend",
    "mutable" => "mutable struct \$0\nend",
    "outer" => "outer ",
    "primitive" => "primitive type \$1 \$0 end",
    "quote" => "quote\n\t\$0\nend",
    "return" => "return",
    "struct" => "struct \$0 end",
    "try" => "try\n\t\$0\ncatch\nend",
    "using" => "using ",
    "while" => "while \$1\n\t\$0\nend"
    )

function texteditfor(state::CompletionState, partial, n)
    TextEdit(Range(Position(state.range.start.line, state.range.start.character - sizeof(partial)), state.range.stop), n)
end

function collect_completions(m::SymbolServer.ModuleStore, spartial, state::CompletionState, inclexported=false, dotcomps=false)
    for val in m.vals
        n, v = String(val[1]), val[2]
        (startswith(n, ".") || startswith(n, "#")) && continue
        !is_completion_match(n, spartial) && continue
        if v isa SymbolServer.VarRef
            v = SymbolServer._lookup(v, getsymbols(getenv(state)), true)
            v === nothing && return
        end
        if StaticLint.isexportedby(n, m) || inclexported
            add_completion_item(state, CompletionItem(n, _completion_kind(v), MarkupContent(sanitize_docstring(v.doc)), texteditfor(state, spartial, n)))
        elseif dotcomps
            push!(state.completions, CompletionItem(n, _completion_kind(v), MarkupContent(sanitize_docstring(v.doc)), texteditfor(state, spartial, string(m.name, ".", n))))
        elseif length(spartial) > 3 && !variable_already_imported(m, n, state)
            if state.server.completion_mode === :import
                # These are non-exported names and require the insertion of a :using statement.
                # We need to insert this statement at the start of the current top-level scope (e.g. Main or a module) and tag it onto existing :using statements if possible.
                cmd = Command("Apply text edit", "language-julia.applytextedit", [
                    WorkspaceEdit(missing, [textedit_to_insert_using_stmt(m, n, state)])
                ])
                ci = CompletionItem(n, _completion_kind(v), missing, "This is an unexported symbol and will be explicitly imported.", MarkupContent(sanitize_docstring(v.doc)), missing, missing, missing, missing, missing, InsertTextFormats.PlainText, texteditfor(state, spartial, n), missing, missing, cmd, "import")
                add_completion_item(state, ci)
            elseif state.server.completion_mode === :qualify
                add_completion_item(state, CompletionItem(string(m.name, ".", n), _completion_kind(v), missing, "This is an unexported symbol and will be explicitly imported.", MarkupContent(sanitize_docstring(v.doc)), missing, missing, string(n), missing, missing, InsertTextFormats.PlainText, texteditfor(state, spartial, string(m.name, ".", n)), missing, missing, missing, missing))
            end
        end
    end
end

function variable_already_imported(m, n, state)
    haskey(state.using_stmts, String(m.name.name)) && import_has_x(state.using_stmts[String(m.name.name)][1], n)
end

function import_has_x(expr::EXPR, x::String)
    if length(expr.args) == 1 && length(expr.args[1]) > 1
        for i = 2:length(expr.args[1].args)
            arg = expr.args[1].args[i]
            if CSTParser.isoperator(arg.head) && length(arg.args) == 1 && CSTParser.isidentifier(arg.args[1]) && CSTParser.valof(arg.args[1]) == x
                return true
            end
        end
    end
    return false
end

function collect_completions(x::EXPR, spartial, state::CompletionState, inclexported=false, dotcomps=false)
    if scopeof(x) !== nothing
        collect_completions(scopeof(x), spartial, state, inclexported, dotcomps)
        if scopeof(x).modules isa Dict
            for m in scopeof(x).modules
                collect_completions(m[2], spartial, state, inclexported, dotcomps)
            end
        end
    end
    if parentof(x) !== nothing && !CSTParser.defines_module(x)
        return collect_completions(parentof(x), spartial, state, inclexported, dotcomps)
    end
end

function collect_completions(x::StaticLint.Scope, spartial, state::CompletionState, inclexported=false, dotcomps=false)
    if x.names !== nothing
        for n in x.names
            if is_completion_match(n[1], spartial)
                documentation = ""
                if n[2] isa StaticLint.Binding
                    documentation = get_hover(n[2], documentation, state.server)
                    sanitize_docstring(documentation)
                end
                add_completion_item(state, CompletionItem(n[1], _completion_kind(n[2]), MarkupContent(documentation), texteditfor(state, spartial, n[1])))
            end
        end
    end
end

function is_rebinding_of_module(x)
    x isa EXPR && refof(x).type === StaticLint.CoreTypes.Module && # binding is a Module
    refof(x).val isa EXPR && CSTParser.isassignment(refof(x).val) && # binding expr is an assignment
    StaticLint.hasref(refof(x).val.args[2]) && refof(refof(x).val.args[2]).type === StaticLint.CoreTypes.Module &&
    refof(refof(x).val.args[2]).val isa EXPR && CSTParser.defines_module(refof(refof(x).val.args[2]).val)# double check the rhs points to a module
end

function _get_dot_completion(px, spartial, state::CompletionState) end
function _get_dot_completion(px::EXPR, spartial, state::CompletionState)
    if px !== nothing
        if refof(px) isa StaticLint.Binding
            if refof(px).val isa StaticLint.SymbolServer.ModuleStore
                collect_completions(refof(px).val, spartial, state, true)
            elseif refof(px).val isa EXPR && CSTParser.defines_module(refof(px).val) && scopeof(refof(px).val) isa StaticLint.Scope
                collect_completions(scopeof(refof(px).val), spartial, state, true)
            elseif is_rebinding_of_module(px)
                collect_completions(scopeof(refof(refof(px).val.args[2]).val), spartial, state, true)
            elseif refof(px).type isa SymbolServer.DataTypeStore
                for a in refof(px).type.fieldnames
                    a = String(a)
                    if is_completion_match(a, spartial)
                        add_completion_item(state, CompletionItem(a, 2, MarkupContent(a), texteditfor(state, spartial, a)))
                    end
                end
            elseif refof(px).type isa StaticLint.Binding && refof(px).type.val isa SymbolServer.DataTypeStore
                for a in refof(px).type.val.fieldnames
                    a = String(a)
                    if is_completion_match(a, spartial)
                        add_completion_item(state, CompletionItem(a, 2, MarkupContent(a), texteditfor(state, spartial, a)))
                    end
                end
            elseif refof(px).type isa StaticLint.Binding && refof(px).type.val isa EXPR && CSTParser.defines_struct(refof(px).type.val) && scopeof(refof(px).type.val) isa StaticLint.Scope
                collect_completions(scopeof(refof(px).type.val), spartial, state, true)
            end
        elseif refof(px) isa StaticLint.SymbolServer.ModuleStore
            collect_completions(refof(px), spartial, state, true)
        end
    end
end

function _completion_kind(b)
    if b isa StaticLint.Binding
        if b.type == StaticLint.CoreTypes.String
            return 1
        elseif b.type == StaticLint.CoreTypes.Function
            return 2
        elseif b.type == StaticLint.CoreTypes.Module
            return 9
        elseif b.type == Int || b.type == StaticLint.CoreTypes.Float64
            return 12
        elseif b.type == StaticLint.CoreTypes.DataType
            return 22
        else
            return 13
        end
    elseif b isa SymbolServer.ModuleStore || b isa SymbolServer.VarRef
        return 9
    elseif b isa SymbolServer.MethodStore
        return 2
    elseif b isa SymbolServer.FunctionStore
        return 3
    elseif b isa SymbolServer.DataTypeStore
        return 22
    else
        return 6
    end
end

function get_import_root(x::EXPR)
    if CSTParser.isoperator(headof(x.args[1])) && valof(headof(x.args[1])) == ":"
        return last(x.args[1].args[1].args)
    end
end

function string_completion(t, state::CompletionState)
    path_completion(t, state)
    # Need to adjust things for quotation marks
    if t.kind in (CSTParser.Tokenize.Tokens.STRING,CSTParser.Tokenize.Tokens.CMD)
        t.startbyte < state.offset <= t.endbyte || return
        relative_offset = state.offset - t.startbyte - 1
        content = t.val[2:prevind(t.val, lastindex(t.val))]
    else
        t.startbyte < state.offset <= t.endbyte - 2 || return
        relative_offset = state.offset - t.startbyte - 3
        content = t.val[4:prevind(t.val, lastindex(t.val), 3)]
    end
    partial = is_latex_comp(content, relative_offset)
    !isempty(partial) && latex_completions(partial, state)
end

function is_latex_comp(s, i)
    i0 = i
    while firstindex(s) <= i
        s[i] == '\\' && return s[i:i0]
        !is_latex_comp_char(s[i]) && return ""
        i = prevind(s, i)
    end
    return ""
end

is_latex_comp_char(c::Char) = UInt32(c) <= typemax(UInt8) ? is_latex_comp_char(UInt8(c)) : false
function is_latex_comp_char(u)
    # Checks whether a Char (represented as a UInt8) is in the set of those those used to trigger
    # latex completions.
    # from: UInt8.(sort!(unique(prod([k[2:end] for (k,_) in REPL.REPLCompletions.latex_symbols]))))
    u === 0x28 ||
    u === 0x29 ||
    u === 0x2b ||
    u === 0x2d ||
    u === 0x2f ||
    0x30 <= u <= 0x39 ||
    u === 0x3d ||
    0x41 <= u <= 0x5a ||
    u === 0x5e ||
    u === 0x5f ||
    0x61 <= u <= 0x7a
end

function path_completion(t, state::CompletionState)
    if t.kind == CSTParser.Tokenize.Tokens.STRING
        path = t.val[2:prevind(t.val, lastindex(t.val))]
        if startswith(path, "~")
            path = replace(path, '~' => homedir())
            dir, partial = _splitdir(path)
        else
            dir, partial = _splitdir(path)
            if !startswith(dir, "/")
                doc_path = getpath(state.doc)
                isempty(doc_path) && return
                dir = joinpath(_dirname(doc_path), dir)
            end
        end
        try
            fs = readdir(dir)
            for f in fs
                if startswith(f, partial)
                    try
                        if isdir(joinpath(dir, f))
                            f = string(f, "/")
                        end
                        rng1 = Range(state.doc, state.offset - sizeof(partial):state.offset)
                        add_completion_item(state, CompletionItem(f, 17, f, TextEdit(rng1, f)))
                    catch err
                        isa(err, Base.IOError) || isa(err, Base.SystemError) || rethrow()
                    end
                end
            end
        catch err
            isa(err, Base.IOError) || isa(err, Base.SystemError) || rethrow()
        end
    end
end

is_in_import_statement(x::EXPR) = is_in_fexpr(x, x -> headof(x) in (:using, :import))

function import_completions(ppt, pt, t, is_at_end, x, state::CompletionState)
    import_statement = StaticLint.get_parent_fexpr(x, x -> headof(x) === :using || headof(x) === :import)

    import_root = get_import_root(import_statement)

    if (t.kind == CSTParser.Tokens.WHITESPACE && pt.kind ∈ (CSTParser.Tokens.USING, CSTParser.Tokens.IMPORT, CSTParser.Tokens.IMPORTALL, CSTParser.Tokens.COMMA, CSTParser.Tokens.COLON)) ||
        (t.kind in (CSTParser.Tokens.COMMA, CSTParser.Tokens.COLON))
        # no partial, no dot
        if import_root !== nothing && refof(import_root) isa SymbolServer.ModuleStore
            for (n, m) in refof(import_root).vals
                n = String(n)
                if is_completion_match(n, t.val) && !startswith(n, "#")
                    add_completion_item(state, CompletionItem(n, _completion_kind(m), MarkupContent(m isa SymbolServer.SymStore ? sanitize_docstring(m.doc) : n), texteditfor(state, t.val, n)))
                end
            end
        else
            for (n, m) in StaticLint.getsymbols(getenv(state))
                n = String(n)
                (startswith(n, ".") || startswith(n, "#")) && continue
                add_completion_item(state, CompletionItem(n, 9, MarkupContent(sanitize_docstring(m.doc)), TextEdit(state.range, n)))
            end
        end
    elseif t.kind == CSTParser.Tokens.DOT && pt.kind == CSTParser.Tokens.IDENTIFIER
        # no partial, dot
        if haskey(getsymbols(getenv(state)), Symbol(pt.val))
            collect_completions(getsymbols(getenv(state))[Symbol(pt.val)], "", state)
        end
    elseif t.kind == CSTParser.Tokens.IDENTIFIER && is_at_end
        # partial
        if pt.kind == CSTParser.Tokens.DOT && ppt.kind == CSTParser.Tokens.IDENTIFIER
            if haskey(StaticLint.getsymbols(getenv(state)), Symbol(ppt.val))
                rootmod = StaticLint.getsymbols(getenv(state))[Symbol(ppt.val)]
                for (n, m) in rootmod.vals
                    n = String(n)
                    if is_completion_match(n, t.val) && !startswith(n, "#")
                        add_completion_item(state, CompletionItem(n, _completion_kind(m), MarkupContent(m isa SymbolServer.SymStore ? sanitize_docstring(m.doc) : n), texteditfor(state, t.val, n)))
                    end
                end
            end
        else
            if import_root !== nothing && refof(import_root) isa SymbolServer.ModuleStore
                for (n, m) in refof(import_root).vals
                    n = String(n)
                    if is_completion_match(n, t.val) && !startswith(n, "#")
                        add_completion_item(state, CompletionItem(n, _completion_kind(m), MarkupContent(m isa SymbolServer.SymStore ? sanitize_docstring(m.doc) : n), texteditfor(state, t.val, n)))
                    end
                end
            else
                for (n, m) in StaticLint.getsymbols(getenv(state))
                    n = String(n)
                    if is_completion_match(n, t.val)
                        add_completion_item(state, CompletionItem(n, 9, MarkupContent(m isa SymbolServer.SymStore ? m.doc : n), texteditfor(state, t.val, n)))
                    end
                end
            end
        end
    end
end

function get_preexisting_using_stmts(x::EXPR, doc::Document)
    using_stmts = Dict{String,Any}()
    tls = StaticLint.retrieve_toplevel_scope(x)
    file_level_arg = get_file_level_parent(x)

    if scopeof(getcst(doc)) == tls
        # check for :using stmts in current file
        for a in getcst(doc).args
            if headof(a) === :using
                add_using_stmt(a, using_stmts)
            end
            a == file_level_arg && break
        end
    end

    if tls !== nothing
        args = get_tls_arglist(tls)
        for a in args
            if headof(a) === :using
                add_using_stmt(a, using_stmts)
            end

        end
    end
    return using_stmts
end

function add_using_stmt(x::EXPR, using_stmts)
    if length(x.args) > 0 && CSTParser.is_colon(x.args[1].head)
        if CSTParser.is_dot(x.args[1].args[1].head) && length(x.args[1].args[1].args) == 1
            using_stmts[valof(x.args[1].args[1].args[1])] = (x, get_file_loc(x))
        end
    end
end

function get_file_level_parent(x::EXPR)
    if x.parent isa EXPR && x.parent.head === :file
        x
    else
        if x.parent === nothing
            return nothing
        end
        get_file_level_parent(x.parent)
    end
end

function textedit_to_insert_using_stmt(m::SymbolServer.ModuleStore, n::String, state::CompletionState)
    tls = StaticLint.retrieve_toplevel_scope(state.x)
    if haskey(state.using_stmts, String(m.name.name))
        (using_stmt, (using_doc, using_offset)) = state.using_stmts[String(m.name.name)]

        l, c = get_position_at(using_doc, using_offset + using_stmt.span)
        TextDocumentEdit(VersionedTextDocumentIdentifier(using_doc._uri, using_doc._version),
            [TextEdit(Range(l, c, l, c), ", $n")])
    elseif tls !== nothing
        if tls.expr.head === :file
            # Insert at the head of the file
            tlsdoc, offset1 = get_file_loc(tls.expr)
            TextDocumentEdit(VersionedTextDocumentIdentifier(tlsdoc._uri, tlsdoc._version),
            [TextEdit(Range(0, 0, 0, 0), "using $(m.name): $(n)\n")])
        elseif tls.expr.head === :module
            # Insert at start of module
            tlsdoc, offset1 = get_file_loc(tls.expr)
            offset2 = tls.expr.trivia[1].fullspan + tls.expr.args[2].fullspan
            l, c = get_position_at(tlsdoc, offset1 + offset2)

            TextDocumentEdit(VersionedTextDocumentIdentifier(tlsdoc._uri, tlsdoc._version),
            [TextEdit(Range(l, c, l, c), "using $(m.name): $(n)\n")])
        else
            error()
        end
    else
        # Fallback, add it to the start of the current file.
        TextDocumentEdit(VersionedTextDocumentIdentifier(state.doc._uri, state.doc._version),
            [TextEdit(Range(0, 0, 0, 0), "using $(m.name): $(n)\n")])
    end
end

function get_tls_arglist(tls::StaticLint.Scope)
    if tls.expr.head === :file
        tls.expr.args
    elseif tls.expr.head === :module
        tls.expr.args[3].args
    else
        error()
    end
end

function method_completion(x, state, xlen)
    scope = scopeof(parentof(parentof(state.x)))

    x_type = refof(x).type.name

    if x_type isa EXPR
        typename = x_type.val
    elseif x_type isa SymbolServer.FakeTypeName
        typename = x_type.name.name
    else
        return
    end

    for m in scope.modules
        for val in m[2].vals
            n, v = String(val[1]), val[2]
            (startswith(n, ".") || startswith(n, "#") || startswith(n, "_")) && continue
            !(typeof(v) == SymbolServer.FunctionStore) && continue
            siglen_max = 0  # maximum signature length
            for m in v.methods
                isempty(m.sig) && continue
                !(typeof(m.sig[1][2]) == SymbolServer.FakeTypeName) && continue
                !(m.sig[1][2].name.name == typename) && continue
                siglen_max = max(siglen_max, length(m.sig))
            end
            (siglen_max == 0) && continue

            prefix_edit = TextEdit(Range(
                    Position(state.range.start.line,
                    state.range.start.character - xlen - 2),
                    Position(state.range.stop.line,
                    state.range.stop.character - xlen - 2)), n)

            if siglen_max == 1  # need to close bracket right away
                additional_edits = [TextEdit(Range(
                    Position(state.range.start.line,
                    state.range.start.character - 1),
                    Position(state.range.stop.line,
                    state.range.stop.character)), ""), prefix_edit]
                inplace_text = ""
            else
                inplace_text = " "
                additional_edits = [prefix_edit]
            end

            inplace_edit = TextEdit(Range(
                Position(state.range.start.line, state.range.start.character),
                Position(state.range.stop.line, state.range.stop.character)),
                inplace_text)

            item = CompletionItem(n, 2, missing, missing, n,
                missing, missing, missing, missing, missing,
                InsertTextFormats.PlainText, inplace_edit, additional_edits,
                missing, missing, missing)
            add_completion_item(state, item)
        end
    end
end


text:
#=============================#
#== put needed modules here ==#
#=============================#
import JuliaChem
import Statistics
import HypothesisTests

#================================#
#== JuliaChem execution script ==#
#================================#
Base.@ccallable function julia_main(ARGS::Vector{String})::Cint
  #== initialize JuliaChem runtime ==#
  JuliaChem.initialize()

  #== read in input file ==#
  input_file = ARGS[1]

  input_time1_t1 = time_ns()/1e9
  molecule, driver, model, keywords = JuliaChem.JCInput.run(input_file)
  input_time1_t2 = time_ns()/1e9
  input_time1 = input_time1_t2 - input_time1_t1

  input_time2_t1 = time_ns()/1e9
  molecule, driver, model, keywords = JuliaChem.JCInput.run(input_file)
  input_time2_t2 = time_ns()/1e9
  input_time2 = input_time2_t2 - input_time2_t1

  input_jit = input_time1 - input_time2
  molecule, driver, model, keywords = JuliaChem.JCInput.run(input_file)

  #== generate basis set ==#
  basis_time1_t1 = time_ns()/1e9
  basis = JuliaChem.JCBasis.run(molecule, model)
  basis_time1_t2 = time_ns()/1e9
  basis_time1 = basis_time1_t2 - basis_time1_t1

  basis_time2_t1 = time_ns()/1e9
  basis = JuliaChem.JCBasis.run(molecule, model)
  basis_time2_t2 = time_ns()/1e9
  basis_time2 = basis_time2_t2 - basis_time2_t1

  basis_jit = basis_time1 - basis_time2
  basis = JuliaChem.JCBasis.run(molecule, model)

  #== perform scf benchmark ==#
  timeof = Vector{Float64}(undef,0)
  scf_time1 = 0.0
  if (driver == "energy")
    if (model["method"] == "RHF")
      scf_time1_t1 = time_ns()/1e9
      scf = JuliaChem.JCRHF.run(basis, molecule, keywords)
      scf_time1_t2 = time_ns()/1e9
      scf_time1 = scf_time1_t2 - scf_time1_t1

      JuliaChem.reset()

      for index in 1:10
        molecule, driver, model, keywords = JuliaChem.JCInput.run(input_file)
        basis = JuliaChem.JCBasis.run(molecule, model)

        scf_timeof_t1 = time_ns()/1e9
        scf = JuliaChem.JCRHF.run(basis, molecule, keywords) #initial run
        scf_timeof_t2 = time_ns()/1e9
        push!(timeof, scf_timeof_t2 - scf_timeof_t1)

        JuliaChem.reset()
      end
    end
  end
  scf_jit = scf_time1 - Statistics.mean(timeof)

  #== output relevant information ==#
  println("Input JIT: ", input_jit)
  println("Basis JIT: ", basis_jit)
  println("SCF JIT: ", scf_jit)
  println("Total JIT: ", input_jit + basis_jit + scf_jit)
  println("")

  #== perform t-test to compare to GAMESS ==#
  p = HypothesisTests.OneSampleTTest(timeof,2.94+0.19)
  println(p)

  #== finalize JuliaChem runtime ==#
  JuliaChem.finalize()

  #== we are done ==#
  return 0
end


text:
import JSON
using DataFrames
import CSV
using Glob
using Statistics

function pmap(f, c)
    out = Vector{typeof(f(c[1]))}(undef, length(c))
    Threads.@threads for i in 1:length(c)
        out[i] = f(c[i])
    end
    out
end

function main()
    df = pmap(glob("$(ARGS[1])/*.json")) do f
        map(readlines(f)) do l
            o = JSON.parse(l)
            config = o["i"]
            input = JSON.parse(config[1][1])
            task = input[1]
            target = config[1][2]
            cost = mean(o["r"][1])
            schedule = JSON.json(config[2][2])
            loop_shape = mapreduce(x -> x[1], *, config[2][2])
            DataFrame(input=JSON.json(input),task=task,target=target,cost=cost,schedule=schedule,loop_shape=loop_shape)
        end |> x -> vcat(x...)
    end |> x -> vcat(x...)

    CSV.write(ARGS[2], df)
end

main()

max_stars_repo_name: tkonolige/tenset

max_issues_repo_name: tkonolige/tenset

max_forks_repo_name: tkonolige/tenset


text:
export @p

#######################################
##  pipelining:  x = @p add 1 2 | minus _ 3

macro p(a...)
    # println("\n\n\ninside p")
    # println()
    # println(a...)
    #map(x->print(typeof(x)),a)
    #println()
    #a = map(x->

    #b = {}
    #println(a)
    #map(x-> typeof(x)==Expr && x.args[1]==:|?append!(b,x.args[[2,1,3]]):push!(b,x),a)
    #println(string("b:  ",b))

    # flip arguments where the order was messed up by |
    output = a
    input = Any[]
    while input!=output
        input = output
        output = Any[]
        flatten(x) = typeof(x)==Expr && x.args[1]==:|?append!(output,x.args[[2,1,3]]):push!(output,x)
        # println(string("input before: ",input))
        # println(string("output before: ",output))
        # @show flatten input
        Base.map(flatten,input)
        # println(string("input after: ",input))
        # println(string("output after:  ",output))
    end

    currying = Any[:map, :map!, :map!r, :map2!, :mapmap, :shmap, :shmap!, :shmap!r, :shmap2!, :pmap, :lmap,
        :work, :workwork, :shwork, :pwork, :lwork, :tee, :filter, :select, :reject, :takewhile, :dropwhile, :mapkeys, :mapvalues,
        :minelem, :maxelem, :extremaelem, :isany, :areall, :mapvec, :sort, :sortrev, :groupdict, :groupby, :uniq, :fold]
    currying2 = Any[:map2, :mapvec2]
    currying3 = Any[:map3, :mapvec3]
    currying4 = Any[:map4, :mapvec4]
    currying5 = Any[:map5, :mapvec5]

    parts = split(output,:|)
    # @show parts
    part = parts[1]
    if len(part) > 2 && in(part[1],currying) # ==:map
        length(part) < 3 && error("FunctionalData.@p: trying to use $(part[1]), but too few parameters given: $parts")
        x = gensym()
        f = :($x -> $(part[3])($x, $(part[4:end]...)) )
        ex = :( $(part[1])($(part[2]), $f) )
     elseif len(part) > 3 && in(part[1], currying2) # ==:map2
         length(part) < 4 && error("FunctionalData.@p: trying to use $(part[1]), but too few parameters given: $parts")
         x = gensym(); y = gensym()
         f = :(($x,$y) -> $(part[4])($x, $y, $(part[5:end]...)) )
         ex = :( $(part[1])($(part[2]), $(part[3]), $f) )
     elseif len(part) > 4 && in(part[1], currying3) # ==:map3
         length(part) < 5 && error("FunctionalData.@p: trying to use $(part[1]), but too few parameters given: $parts")
         x = gensym(); y = gensym(); z = gensym()
         f = :(($x,$y,$z) -> $(part[5])($x, $y, $z, $(part[6:end]...)) )
         ex = :( $(part[1])($(part[2]), $(part[3]),$(part[4]), $f) )
     elseif len(part) > 5 && in(part[1], currying2) # ==:map2
         length(part) < 6 && error("FunctionalData.@p: trying to use $(part[1]), but too few parameters given: $parts")
         x = gensym(); y = gensym(); z = gensym(); z2 = gensym()
         f = :(($x,$y,$z,$z2) -> $(part[6])($x, $y, $z, $z2, $(part[7:end]...)) )
         ex = :( $(part[1])($(part[2]), $(part[3]), $(part[4]), $(part[5]), $f) )
     elseif len(part) > 6 && in(part[1], currying2) # ==:map2
         length(part) < 7 && error("FunctionalData.@p: trying to use $(part[1]), but too few parameters given: $parts")
         x = gensym(); y = gensym(); z = gensym(); z2 = gensym(); z3 = gensym()
         f = :(($x,$y,$z,$z2,$z3) -> $(part[7])($x, $y, $z, $z2, $z3, $(part[8:end]...)) )
         ex = :( $(part[1])($(part[2]), $(part[3]), $(part[4]), $(part[5]), $(part[6]), $f) )
    else
        ex = :( $(part[1])($(part[2:end]...)) )
    end
    # println(ex)

    for i = 2:length(parts)
        # println("i is $i")
        part = parts[i]
        # @show part
        didreplace = false
        function replace_(a::Array)
            # @show "replacing" typeof(a) a
            r = a
            r = Base.map(replace_, r)
            if anyequal(a, :_)   # replace _ in expression
                if sum(part.==:_) > 1
                    error("Use can use '_' only once per section of a @p ... | ... | ... pipeline")
                end
                part[find(part.==:_)] = ex
                didreplace = true
            end
            r = :( $(part[1])( $(part[2:end]...)) )
        end
        function replace_(a::Expr)
            # @show "replacing" typeof(a) a
            if isa(a.args, Array)
                a.args = Base.map(replace_, a.args)
            end
            a
        end
        function replace_(a)
            # @show "replacing" typeof(a) a
            if a == :_
                didreplace = true
                ex
            else
                a
            end
        end
        newex = replace_(part)
        if didreplace
            ex = newex
        else
            #println("fany==false")
            #println("### part[1] $(part[1]))")
            if len(part) > 1 && in(part[1],currying) # ==:map
                x = gensym()
                f = :( $x-> $(part[2])($x, $(part[3:end]...)) )
                ex = :( $(part[1])($ex, $f) )
            elseif len(part) > 2 && in(part[1], currying2) # ==:map2
                x = gensym(); y = gensym()
                f = :( ($x,$y) -> $(part[3])($x, $y, $(part[4:end]...)) )
                ex = :( $(part[1])($ex, $(part[2]), $f) )
            elseif len(part) > 3 && in(part[1], currying3) # ==:map3
                x = gensym(); y = gensym(); z = gensym()
                f = :( ($x,$y,$z) -> $(part[4])($x, $y, $z, $(part[5:end]...)) )
                ex = :( $(part[1])($ex, $(part[2]), $(part[3]), $f) )
            elseif len(part) > 4 && in(part[1], currying4) # ==:map4
                x = gensym(); y = gensym(); z = gensym(); z2 = gensym()
                f = :( ($x,$y,$z,$z2) -> $(part[5])($x, $y, $z, $z2, $(part[6:end]...)) )
                ex = :( $(part[1])($ex, $(part[2]), $(part[3]),  $(part[4]), $f) )
            elseif len(part) > 5 && in(part[1], currying5) # ==:map5
                x = gensym(); y = gensym(); z = gensym(); z2 = gensym(); z3 = gensym()
                f = :( ($x,$y,$z,$z2,$z3) -> $(part[6])($x, $y, $z, $z2, $z3, $(part[7:end]...)) )
                ex = :( $(part[1])($ex, $(part[2]), $(part[3]), $(part[4]), $(part[5]), $f) )
            else
                ex = :( $(part[1])($ex, $(part[2:end]...)) )
            end
        end
        # println(ex)
    end
    #println()
    #println(ex)
    return esc(ex)
end


text:
# Using same struct type from REopt utils.jl
# Warning: I've seen notes of Base.@kwdef not being officially supported (not documented)
"""
    InputsStruct

This struct defines the inputs for the GHPGHX module
"""
Base.@kwdef struct InputsStruct
    ##### These are the exact /ghpghx POST names from the API #####
    # Parameters
    borehole_depth_ft::Float64
    ghx_header_depth_ft::Float64
    borehole_spacing_ft::Float64
    borehole_diameter_inch::Float64
    borehole_spacing_type::String  # "rectangular" or "hexagonal"
    ghx_pipe_outer_diameter_inch::Float64
    ghx_pipe_wall_thickness_inch::Float64
    ghx_pipe_thermal_conductivity_btu_per_hr_ft_f::Float64
    ghx_shank_space_inch::Float64
    ground_thermal_conductivity_btu_per_hr_ft_f::Float64
    ground_mass_density_lb_per_ft3::Float64
    ground_specific_heat_btu_per_lb_f::Float64
    grout_thermal_conductivity_btu_per_hr_ft_f::Float64
    ghx_fluid_specific_heat_btu_per_lb_f::Float64
    ghx_fluid_mass_density_lb_per_ft3::Float64
    ghx_fluid_thermal_conductivity_btu_per_hr_ft_f::Float64
    ghx_fluid_dynamic_viscosity_lbm_per_ft_hr::Float64
    ghx_fluid_flow_rate_gpm_per_ton::Float64
    ghx_pump_power_watt_per_gpm::Float64
    ghx_pump_min_speed_fraction::Float64
    ghx_pump_power_exponent::Float64
    max_eft_allowable_f::Float64
    min_eft_allowable_f::Float64

    # Array/Dict inputs
    heating_thermal_load_mmbtu_per_hr::Array{Float64,1}
    cooling_thermal_load_ton::Array{Float64,1}
    ambient_temperature_f::Array{Float64,1}
    cop_map_eft_heating_cooling::Array{Any,1}

    # Model Settings
    simulation_years::Int64  # Number of years for GHP-GHX model
    solver_eft_tolerance_f::Float64  # Tolerance for the EFT error to accept a GHX sizing solution
    solver_eft_tolerance::Float64  # Convert to degC
    ghx_model::String  # "TESS" or "DST" (DST is currently not available, pending license agreement)
    dst_ghx_timesteps_per_hour::Int64
    tess_ghx_minimum_timesteps_per_hour::Int64
    max_sizing_iterations::Int64
    init_sizing_factor_ft_per_peak_ton::Float64

    ##### These are the variable names used in the GHPGHX, kept from TESS ######
    # TODO eventually just use the API names above in GHPGHX to remove redundancy (BUT WOULD HAVE TO DEAL WITH UNITS CONVERSION STILL)
    I_Configuration::Int64 = 1  #!1=Decentralized WSHPs, 2=Decentralized WWHPs, 3=Centralized WWHPs, 4=Centralized WWHPs with HRC
    Depth_Bores::Float64 #!Depth of the boreholes (ft)
    Depth_Header::Float64  #!Depth of the ground heat exchanger headers below grade (ft)
    BoreSpacing::Float64  #!Spacing between boreholes (ft)
    SpacingType::Int64  # Spacing type (1=rectangular, 2 = hexagonal)
    Radius_Bores::Float64  #!Radius of the boreholes (in)
    Ro_Pipe::Float64  #!Outer radius of individual u-tube pipe (in)
    Ri_Pipe::Float64  #!Inner radius of individual u-tube pipe (in)
    K_Pipe::Float64  #!Thermal conductivity of pipe material (BTU/h.ft.F)
    Center_Center_Distance::Float64  #!Distance between centers of upwards and downwards u-tube legs (in)
    K_Soil::Float64  #!Thermal conductivity of the soil (BTU/h.ft.F)
    Rho_Soil::Float64  #!Density of the soil (lbm/ft3)
    Cp_Soil::Float64  #!Specific heat of the soil (BTU/lbm.F)
    K_Grout::Float64  #!Thermal conductivity of the borehole grout (BTU/h.ft.F)
    T_Ground::Float64  #!Average soil surface temperature over the year (F)
    Tamp_Ground::Float64  #!Amplitude of soil surface temperature over the year (Delta_F)
    DayMin_Surface::Float64  #!Day of minimum soil surface temperature (day)
    Cp_GHXFluid::Float64 #!Specific heat of the ground heat exchanger fluid (BTU/lbm.F)
    Rho_GHXFluid::Float64  #!Density of the ground heat exchanger fluid (lbm/ft3)
    K_GHXFluid::Float64  #!Thermal conductivity of the ground heat exchanger fluid (BTU/h.ft.F)
    Mu_GHXFluid::Float64  #!Dynamic viscosity of the ground heat exchanger fluid (lbm/ft.h)
    GPMperTon_WSHP::Float64  #!Nominal flow rate of the water source heat pumps (gpm/ton)
    WattPerGPM_GHXPump::Float64  #!Nominal ground loop pump power (Watt/gpm)
    fMin_VSP_GHXPump::Float64  #!Minimum ground heat exchanger pump speed for variable speed option (set to 1 if constant speed pumps)
    Exponent_GHXPump::Float64  #!Exponent for relationship between ground heat exchanger pump power and ground heat exchanger pump flow rate
    Tmax_Sizing::Float64  #!Maximum allowable return fluid temperature from the ground loop (F)
    Tmin_Sizing::Float64  #!Minimum allowable return fluid temperature from the ground loop (F)

    # Additional parameters and site data
    HeatingThermalLoadKW::Array{Float64, 1}  # Heating thermal load to be served by GHP
    CoolingThermalLoadKW::Array{Float64, 1}  # Cooling thermal load to be served by GHP
    AmbientTemperature::Array{Float64, 1}  # Dry-bulb outdoor air temperature in degrees Fahrenheit
    HeatPumpCOPMap::Array{Float64, 2}  # Includes heating and cooling heat pump COP versus EWT (degF)

    # These are defined based on the above inputs and processed in the function below
    TON_TO_KW::Float64 # [kw/ton]
    MMBTU_TO_KWH::Float64  # [kwh/MMBtu]
    METER_TO_FEET::Float64  # [ft/m]

    PeakTons_WSHP_H::Float64
    PeakTons_WSHP_C::Float64
    PeakTons_WSHP_GHX::Float64
    X_init::Float64
    N_Series::Int64
    N_Radial::Int64
    N_Vertical::Int64
    RhoCp_Soil::Float64
    DayMin_DST::Float64
    GPM_GHXPump::Float64
    Prated_GHXPump::Float64
    LPS_GHXPump::Float64
    Mdot_GHXPump::Float64
end

"""
    InputsProcess(d::Dict)

Performs unit conversions, name conversions, and additional processing of inputs.

"""
function InputsProcess(d::Dict)
    # Converts dictionary to allow String or Symbol for keys and Any type for values
    d = convert(Dict{Union{String, Symbol}, Any}, d)

    # Constants
    d[:TON_TO_KW] = 3.5169  # [kw/ton]
    d[:MMBTU_TO_KWH] = 293.07  # [kwh/MMBtu]
    d[:METER_TO_FEET] = 3.28084  # [ft/m]

    # Convert API inputs to GHPGHX variable names, and units from English to SI
    d[:Depth_Bores] =  d["borehole_depth_ft"] / d[:METER_TO_FEET]  # [m]
    d[:Depth_Header] = d["ghx_header_depth_ft"] / d[:METER_TO_FEET]  # [m]
    d[:BoreSpacing] = d["borehole_spacing_ft"] / d[:METER_TO_FEET]  # [m]
    if d["borehole_spacing_type"] == "rectangular"
        d[:SpacingType] = 1
    else
        d[:SpacingType] = 2
    end
    d[:Radius_Bores] = d["borehole_diameter_inch"] / 2.0 / 12.0 / d[:METER_TO_FEET]  # [m]
    d[:Ro_Pipe] = d["ghx_pipe_outer_diameter_inch"] / 2.0 / 12.0 / d[:METER_TO_FEET]  # [m]
    d[:Ri_Pipe] = (d["ghx_pipe_outer_diameter_inch"] / 2.0 - d["ghx_pipe_wall_thickness_inch"]) / 12.0 / d[:METER_TO_FEET]  # [m]
    d[:K_Pipe] = d["ghx_pipe_thermal_conductivity_btu_per_hr_ft_f"] * 1.055 * d[:METER_TO_FEET] * 1.8  # [kJ/h.m.K]
    d[:Center_Center_Distance] = d["ghx_shank_space_inch"] / 12.0 / d[:METER_TO_FEET]  # [m]
    d[:K_Soil] = d["ground_thermal_conductivity_btu_per_hr_ft_f"] * 1.055 * d[:METER_TO_FEET] * 1.8  # [kJ/h.m]
    d[:Rho_Soil] = d["ground_mass_density_lb_per_ft3"] * 35.31467 / 2.20462  # [kg/m3]
    d[:Cp_Soil] = d["ground_specific_heat_btu_per_lb_f"] * 1.8 * 2.20462 * 1.055  # [kJ/kg.K]
    d[:K_Grout] = d["grout_thermal_conductivity_btu_per_hr_ft_f"] * 1.055 * d[:METER_TO_FEET] * 1.8  # [kJ/h.m.K]
    # See processing of ambient temperature for temperature parameters
    d[:Cp_GHXFluid] = d["ghx_fluid_specific_heat_btu_per_lb_f"] * 1.8 * 2.20462 * 1.055  # [kJ/kg.K]
    d[:Rho_GHXFluid] = d["ghx_fluid_mass_density_lb_per_ft3"] * 35.31467 / 2.20462  # [kg/m3]
    d[:K_GHXFluid] = d["ghx_fluid_thermal_conductivity_btu_per_hr_ft_f"] * 1.055 * d[:METER_TO_FEET] * 1.8  # [kJ/h.m.K]
    d[:Mu_GHXFluid] = d["ghx_fluid_dynamic_viscosity_lbm_per_ft_hr"] * d[:METER_TO_FEET] / 2.20462  # [kg/h.m]
    d[:GPMperTon_WSHP] = d["ghx_fluid_flow_rate_gpm_per_ton"]
    d[:WattPerGPM_GHXPump] = d["ghx_pump_power_watt_per_gpm"]
    d[:fMin_VSP_GHXPump] = d["ghx_pump_min_speed_fraction"]
    d[:Exponent_GHXPump] = d["ghx_pump_power_exponent"]
    d[:Tmax_Sizing] = (d["max_eft_allowable_f"] - 32.0) / 1.8  # [C]
    d[:Tmin_Sizing] = (d["min_eft_allowable_f"] - 32.0) / 1.8  # [C]
    d[:solver_eft_tolerance] = d["solver_eft_tolerance_f"] / 1.8  # [C]

    # Convert array input units to SI
    d[:HeatingThermalLoadKW] = d["heating_thermal_load_mmbtu_per_hr"] * d[:MMBTU_TO_KWH]
    d[:CoolingThermalLoadKW] = d["cooling_thermal_load_ton"] * d[:TON_TO_KW]
    d[:AmbientTemperature] = (d["ambient_temperature_f"] .- 32.0) / 1.8  # [C]

    # Use AmbientTemperature to calculate other temperature parameters
    days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
    avg_temp_month = zeros(12)
    for (mo,days) in enumerate(days_in_month)
        hour_start = sum(days_in_month[1:mo-1]) * 24 + 1  # mo=1 results in 0 + 1 = 1
        hour_end = sum(days_in_month[1:mo]) * 24
        avg_temp_month[mo] = sum(d[:AmbientTemperature][hour_start:hour_end]) / (days * 24)
    end

    d[:T_Ground] = sum(d[:AmbientTemperature]) / (365 * 24)  # [C]
    d[:Tamp_Ground] =  (maximum(avg_temp_month) - minimum(avg_temp_month)) / 2  # [C]
    d[:DayMin_Surface] = convert(Int64, round(argmin(d[:AmbientTemperature]) / 24))  # day of year

    # Convert COP map list_of_dict to Array{Float64, 2}
    d[:HeatPumpCOPMap] = zeros(Float64, length(d["cop_map_eft_heating_cooling"]), 3)
    for i in 1:length(d["cop_map_eft_heating_cooling"])
        d[:HeatPumpCOPMap][i,1] = d["cop_map_eft_heating_cooling"][i]["eft"]
        d[:HeatPumpCOPMap][i,2] = d["cop_map_eft_heating_cooling"][i]["heat_cop"]
        d[:HeatPumpCOPMap][i,3] = d["cop_map_eft_heating_cooling"][i]["cool_cop"]
    end

    # Find peak heating, cooling, and combined for initial sizing guess
    d[:PeakTons_WSHP_H] = maximum(d[:HeatingThermalLoadKW]) / d[:TON_TO_KW]
    d[:PeakTons_WSHP_C] = maximum(d["cooling_thermal_load_ton"])
    d[:PeakTons_WSHP_GHX] = maximum(d[:HeatingThermalLoadKW] + d[:CoolingThermalLoadKW]) / d[:TON_TO_KW]
    d[:X_init] = d["init_sizing_factor_ft_per_peak_ton"] / d[:METER_TO_FEET] * d[:PeakTons_WSHP_GHX]  # [m]

    # Set some intermediate conditions
    d[:N_Series] = 1
    d[:N_Radial] = d[:N_Series]
    d[:N_Vertical] = 50
    d[:RhoCp_Soil] = d[:Rho_Soil] * d[:Cp_Soil]
    d[:DayMin_DST] = 270.0 - d[:DayMin_Surface]
    d[:GPM_GHXPump] = d[:GPMperTon_WSHP] * d[:PeakTons_WSHP_GHX]
    d[:Prated_GHXPump] = d[:WattPerGPM_GHXPump] * 3.6 * d[:GPM_GHXPump]
    d[:LPS_GHXPump] = d[:GPM_GHXPump] / 60 / 264.172 * 1000.0
    d[:Mdot_GHXPump] = d[:GPM_GHXPump] * 60 / 264.172 * d[:Rho_GHXFluid]

    # Convert all Dict key strings to symbols which is required for kwargs inputs of InputsStruct
    d = string_dictkeys_tosymbols(d)

    # Report which field is missing
    d = filter_dict_to_match_struct_field_names(d, InputsStruct)

    # Pass/return InputsStruct (after ";" means kwargs, "..." => "splat" Dict into individual terms)
    return InputsStruct(;d...)
end

"""
    string_dictkeys_tosymbols(d::Dict)

Assists in using a Dict as an input to instantiate the struct.

"""
function string_dictkeys_tosymbols(d::Dict)
    d2 = Dict()
    for (k, v) in d
        d2[Symbol(k)] = v
    end
    return d2
end

"""
    filter_dict_to_match_struct_field_names(d::Dict, s::DataType)

Filter the input dict to match the struct field names, and provide a warning if inputs are missing.

"""
function filter_dict_to_match_struct_field_names(d::Dict, s::DataType)
    f = fieldnames(s)
    d2 = Dict()
    for k in f
        if haskey(d, k)
            d2[k] = d[k]
        else
            @warn "inputs.jl: dict is missing InputsStruct field $(k)!"
        end
    end
    return d2
end


text:
# This file is a part of UnsafeArrays.jl, licensed under the MIT License (MIT).

const IdxUnitRange = AbstractUnitRange{<:Integer}
const DenseIdx = Union{IdxUnitRange,Integer}

# Similar to Base._indices_sub:
@inline _sub_axes() = ()
@inline _sub_axes(::Real, I...) = _sub_axes(I...)
@inline _sub_axes(i1::AbstractArray, I...) = (Base.unsafe_indices(i1)..., _sub_axes(I...)...)

@inline _sub_size(sub_idxs...) = map(n->Int(Base.unsafe_length(n)), _sub_axes(sub_idxs...))

function _require_one_based_indexing(A::AbstractArray{T,N}) where {T,N}
    typeof(axes(A)) == NTuple{N,Base.OneTo{Int}} || throw(ArgumentError("Parent array must have one-based indexing"))
    nothing
end

@noinline function _noinline_nop(x::Tuple)
    nothing
end

"""
    UnsafeArrays.@gc_preserve A B ... expr

Protect `A`, `B`, ... from garbage collection while executing `expr`. Contrary
to `GC.@preserve`, expr is executed in a new scope.

Equivalent to

```
GC.@preserve A B ... let
    expr
end
```

On Julia versions that do not provide `GC.@preserve`, a fallback
implementation is used.
"""
macro gc_preserve(args...)
    syms = args[1:end-1]
    expr = args[end]

    for s in syms
        s isa Symbol || error("@gc_preserve targets must be a symbols")
    end

    esc(:(GC.@preserve $(syms...) $(Expr(:let, Expr(:block), expr))))
end


text:
using Reexport

@reexport module CausalityTests
    using LaTeXStrings, RecipesBase
    import CausalityToolsBase: causality
    import NearestNeighbors
    import StatsBase
    import Distances
    import Distances: Chebyshev
    import CausalityToolsBase
    import CausalityToolsBase: CausalityTest, RectangularBinning
    import CrossMappings: crossmap, convergentcrossmap
    import TimeseriesSurrogates: randomshuffle
    import TransferEntropy: transferentropy,
        TransferOperatorGrid, VisitationFrequency, NearestNeighbourMI,
        BinningTransferEntropyEstimator
    import UncertainData
    import UncertainData:
        AbstractUncertainValue,
        AbstractUncertainValueDataset,
        AbstractUncertainIndexValueDataset,
        resample,
        ConstrainedResampling
    resample(v::Vector{Real}) = v

    import ..transferentropy

    include("causality.jl")

    ################################################################
    # Test definitions
    ################################################################
    # These are the basic tests
    include("tests/test_definitions.jl")

    # These meta tests apply the causality tests defined in
    # `test_definitions.jl` in some more complicated manner,
    # for example on random subsets of the data.
    include("MetaCausalityTest/MetaCausalityTest.jl")

    ################################################################
    # Test results
    ################################################################
    include("CausalAnalysis/causal_analyses.jl")

    ################################################################
    # On uncertain data with uncertainties in both index and value
    ################################################################
    include("causality_on_uncertaindata.jl")

    export causality
end

"""
	CausalityTests

A module defining causality tests.
"""
CausalityTests


text:
import .QTrees.decode
import Colors.Gray
function imageof(layer::AbstractMatrix{UInt8})
    Gray.(decode.(layer))
end
function showmask!(img, mask; highlight=ARGB(1, 0, 0, 0.3))
    mask = .!mask
    hl = convert(eltype(img), parsecolor(highlight))
    img[mask] .= Render.overlay.(img[mask], hl)
    img
end
showmask(img, args...; kargs...) = showmask!(deepcopy(img), args...; kargs...)


text:
using NNlib: DenseConvDims

@testset "convolution" begin
    a, b, c = rand(Float64, 10, 10, 3, 1), rand(Float64, 2, 2, 3, 4), rand(Float64, 9, 9, 4, 1)
    da, db, dc = CuArray(a), CuArray(b), CuArray(c)
    cdims = DenseConvDims(a, b)
    @test NNlib.conv(a, b, cdims) ≈ collect(NNlib.conv(da, db, cdims))
    @test ∇conv_data(c, b, cdims) ≈ collect(∇conv_data(dc, db, cdims))
    @test ∇conv_filter(a, c, cdims) ≈ collect(∇conv_filter(da, dc, cdims))

    # Test for agreement between CPU NNlib and CuDNN versions, across a variety of kwargs
    for num_spatial_dims in (1, 2, 3)
        # Initialize data we'll run our tests over
        C_in = 3
        C_out = 4
        batch_size = 1
        x = rand(Float64, fill(8, num_spatial_dims)..., C_in, batch_size)
        w = rand(Float64, fill(2, num_spatial_dims)..., C_in, C_out)
        b = rand(Float64, fill(1, num_spatial_dims)..., C_in, C_out)
        options = (Dict(), Dict(:dilation => 2), Dict(:flipkernel => true), Dict(:stride => 2), Dict(:padding => 1))

        # @denizyuret: algo option deprecated for nnlib, handling in cudnn
        # algos = (1, 0, 1, 1,)
        # for (opts, algo) in zip(options, algos)

        for opts in options
            cdims = DenseConvDims(x, w; opts...)
            y = NNlib.conv(x, w, cdims)

            # Test that basic convolution is equivalent across GPU/CPU
            gputest((x, w) -> NNlib.conv(x, w, cdims), x, w)
            gputest((y, w) -> NNlib.∇conv_data(y, w, cdims), y, w)
            gputest((x, y) -> NNlib.∇conv_filter(x, y, cdims), x, y, checkgrad=false) # TODO fix grad

            # Scaling factors
            gputest((x, w) -> NNlib.conv(x, w, cdims; alpha=2.0), x, w, checkgrad=false) # TODO
            gputest((y, w) -> NNlib.∇conv_data(y, w, cdims; alpha=2.0), y, w, checkgrad=false) # TODO
            gputest((x, y) -> NNlib.∇conv_filter(x, y, cdims; alpha=2.0), x, y, checkgrad=false) # TODO

            gputest((y, x, w) -> NNlib.conv!(copy(y), x, w, cdims; beta=2.0), y, x, w, checkgrad=false) # TODO
            # @test_broken gputest((x, y, w) -> NNlib.∇conv_data!(copy(x), y, w, cdims; beta=2.0), x, y, w, checkgrad=false) #TODO
            gputest((w, x, y) -> NNlib.∇conv_filter!(copy(w), x, y, cdims; beta=2.0), w, x, y, checkgrad=false) # TODO

            # Test the compatibility shims
            cy,cx,cw = CuArray{Float32}.((y,x,w))
            opts2 = Dict((k==:padding ? :pad : k)=>v for (k,v) in opts)
            @test NNlib.conv!(similar(cy),cx,cw; opts2...) ≈ NNlib.conv!(similar(cy),cx,cw,cdims)
            @test NNlib.∇conv_filter!(similar(cw),cy,cx; opts2...) ≈ NNlib.∇conv_filter!(similar(cw),cx,cy,cdims)
        end

        # CPU implementation of ∇conv_bias!
        db = zeros(Float64, 1, 1, 3, 1)
        dy = randn(Float64, 8, 8, 3, 1)
        function NNlibCUDA.∇conv_bias!(db, dy)
            db .= sum(dy, dims=(1:(ndims(dy)-2)))
            return db
        end
        gputest(NNlibCUDA.∇conv_bias!, db, dy, checkgrad=false)
    end
end


text:
module VennEuler

using Base
using IterTools
using NLopt
using Cairo

export
	EulerState,
	EulerObject,
	EulerSpec,
	make_euler_object,
	optimize,
	optimize_iteratively,
	render,
	random_state

include("types.jl")
include("states.jl")
include("optim.jl")
include("render.jl")

end # module


text:
# GitHub API related
"""
    GITHUB_REST_ENDPOINT::String = "https://api.github.com"

GitHub API v3 RESTful root endpoint.
"""
const GITHUB_REST_ENDPOINT = "https://api.github.com"
"""
    GITHUB_GRAPHQL_ENDPOINT::String = "https://api.github.com/graphql"

GitHub API v4 GraphQL API endpoint.
"""
const GITHUB_GRAPHQL_ENDPOINT = "https://api.github.com/graphql"
"""
    GITHUB_API_QUERY

GitHub GraphQL query.
"""
const GITHUB_API_QUERY =
"""
query Magic(\$id: ID!, \$path: String!, \$since: GitTimestamp!, \$first: Int!, \$cursor: String) {
  node(id: \$id) {
    ... on Repository {
      nameWithOwner
      defaultBranchRef {
        target {
          ... on Commit {
            ... History
          }
        }
      }
    }
  }
}
fragment History on Commit {
  history(path: \$path, since: \$since, first: \$first, after: \$cursor) {
    totalCount
    pageInfo {
      hasNextPage
      endCursor
    }
    edges {
      node {
        id
        tree {
          entries {
            name
            object {
              ... on Tree {
                entries {
                  name
                  oid
                }
              }
            }
          }
        }
      }
    }
  }
}
""" |>
    (obj -> replace(obj, r"\s+" => " ")) |>
    strip |>
    string
# Structs
"""
    Limits

GitHub API limits.

It includes how many remaining queries are available for the current time period and when it resets.

# Fields
- `limit::Int`
- `remaining::Int`
- `reset::ZonedDateTime`
"""
mutable struct Limits
    limit::Int
    remaining::Int
    reset::ZonedDateTime
end
"""
    API_Limits

GitHub API limits for a PersonalAccessToken.

# Fields
- `core::Limits`
- `search::Limits`
- `graphql::Limits`
"""
mutable struct API_Limits
    core::Limits
    search::Limits
    graphql::Limits
end
"""
    GitHubPersonalAccessToken(login::AbstractString,
                              token::AbstractString
                              )::GitHubPersonalAccessToken

A GitHub Personal Access Token

# Fields

- `login::String`
- `token::String`
- `client::Client`
- `limits::Limits`
"""
struct GitHubPersonalAccessToken
    login::String
    token::String
    client::Client
    limits::API_Limits
    function GitHubPersonalAccessToken(login::AbstractString, token::AbstractString)
        client = GraphQLClient(
            GITHUB_GRAPHQL_ENDPOINT,
            auth = "bearer $token",
            headers = Dict("User-Agent" => login),
        )
        # Dummy values
        limits = API_Limits(Limits(0, 0, now(utc_tz)),
                            Limits(0, 0, now(utc_tz)),
                            Limits(0, 0, now(utc_tz)))
        output = new(login, token, client, limits)
        # Update dummy values for actual ones
        update!(output)
    end
end
summary(io::IO, obj::GitHubPersonalAccessToken) =
    println(io, "GitHub Personal Access Token")
function show(io::IO, obj::GitHubPersonalAccessToken)
    print(io, summary(obj))
    println(io, "  login: $(obj.login)")
    println(io, "  core remaining: $(obj.limits.core.remaining)")
    println(io, "  core reset: $(obj.limits.core.reset)")
    println(io, "  graphql remaining: $(obj.limits.graphql.remaining)")
    println(io, "  graphql reset: $(obj.limits.graphql.reset)")
end
function update!(obj::GitHubPersonalAccessToken)
    response = request(
        "GET",
        "$GITHUB_REST_ENDPOINT/rate_limit",
        [
            "Accept" => "application/vnd.github.v3+json",
            "User-Agent" => obj.login,
            "Authorization" => "token $(obj.token)",
        ],
    )
    json = JSON3.read(response.body).resources
    obj.limits.core.remaining = json.core.remaining
    obj.limits.core.reset = ZonedDateTime(unix2datetime(json.core.reset), utc_tz)
    obj.limits.search.remaining = json.search.remaining
    obj.limits.search.reset = ZonedDateTime(unix2datetime(json.search.reset), utc_tz)
    obj.limits.graphql.remaining = json.graphql.remaining
    obj.limits.graphql.reset = ZonedDateTime(unix2datetime(json.graphql.reset), utc_tz)
    obj
end
"""
    graphql(obj::GitHubPersonalAccessToken,
            operationName::AbstractString,
            vars::Dict{String})

Return JSON of the GraphQL query.
"""
function graphql(
    obj::GitHubPersonalAccessToken,
    operationName::AbstractString,
    vars::Dict{String};
    GITHUB_API_QUERY = GITHUB_API_QUERY
)
    update!(obj)
    if iszero(obj.limits.graphql.remaining)
        w = obj.limits.graphql.reset - now(utc_tz)
        sleep(max(w, zero(w)))
        obj.limits.remaining = obj.limits.graphql.limit
    end
    result = try
        result = obj.client.Query(GITHUB_API_QUERY, operationName = operationName, vars = vars)
        @assert result.Info.status == 200
        # If the cost is higher than the current remaining, it will return a 200 with the API rate limit message
        if result.Data == "{\"errors\":[{\"type\":\"RATE_LIMITED\",\"message\":\"API rate limit exceeded\"}]}"
            w = obj.limits.graphql.reset - now(utc_tz)
            sleep(max(w, zero(w)))
            result = obj.client.Query(GITHUB_API_QUERY, operationName = operationName, vars = vars)
            @assert result.Info.status == 200
        end
        result
    catch err
        # If the query triggered an abuse behavior it will check for a retry_after
        retry_after = (x[2] for x ∈ values(err.response.headers) if x[1] == "Retry-After")
        isempty(retry_after) || sleep(parse(Int, first(retry_after)) + 1)
        # The other case is when it timeout. We try once more just in case.
        try
            obj.client.Query(GITHUB_API_QUERY, operationName = operationName, vars = vars)
        catch err
            return err
        end
    end
    update!(obj)
    if isa(result, Exception)
        println(result)
    end
    result
end
"""
    restful(obj::GitHubPersonalAccessToken,
            endpoint::AbstractString;
            method::AbstractString = "GET",
            params::Dict{String} = Dict{String,String}(),
    )

Return response.
"""
function restful(
    obj::GitHubPersonalAccessToken,
    endpoint::AbstractString;
    method::AbstractString = "GET",
    params::Dict{String} = Dict{String,String}(),
  )
    update!(obj)
    if iszero(obj.limits.core.remaining)
        w = obj.limits.core.reset - now(utc_tz)
        sleep(max(w, zero(w)))
        obj.limits.core.remaining = obj.limits.core.limit
    end
    response = request(
      method,
      "$GITHUB_REST_ENDPOINT/$endpoint",
      [
        "Accept" => "application/vnd.github.v3+json",
        "User-Agent" => obj.login,
        "Authorization" => "token $(obj.token)",
      ],
      JSON3.write(params),
      )
    update!(obj)
    response
end

# Postgres related
"""
    Opt(login::AbstractString,
        token::AbstractString;
        db_usr::AbstractString = "postgres",
        db_pwd::AbstractString = "postgres",
        host::AbstractString = "postgres",
        port::Integer = 5432,
        dbname::AbstractString = "postgres",
        schema::AbstractString = "postgres",
        role::AbstractString = "postgres"
        )::Opt

Structure for passing arguments to functions.

# Fields
- `conn::Connection`
- `schema::String`
- `role::String`
- `pat::GitHubPersonalAccessToken`

# Example
```julia-repl
julia> opt = Opt("Nosferican",
                 ENV["GITHUB_TOKEN"],
                 host = ENV["POSTGRES_HOST"],
                 port = parse(Int, ENV["POSTGRES_PORT"]));

```
"""
struct Opt
    conn::Connection
    schema::String
    role::String
    pat::GitHubPersonalAccessToken
    function Opt(
        login::AbstractString,
        token::AbstractString;
        db_usr::AbstractString = "postgres",
        db_pwd::AbstractString = "postgres",
        schema::AbstractString = "postgres",
        role::AbstractString = "postgres",
        host::AbstractString = "postgres",
        port::Integer = 5432,
        dbname::AbstractString = "postgres",
    )
        conn =
            Connection("host = $host port = $port dbname = $dbname user = $db_usr password = $db_pwd")
        pat = GitHubPersonalAccessToken(login, token)
        new(conn, schema, role, pat)
    end
end
summary(io::IO, obj::Opt) = println(io, "Options for functions")
function show(io::IO, obj::Opt)
    print(io, summary(obj))
    println(io, replace(replace(string(obj.conn), r"^" => "  "), r"\n[^$]" => "\n  "))
    println(io, "  Schema: $(obj.schema)")
    println(io, "  Role: $(obj.role)")
    print(io, replace(replace(string(obj.pat), r"^" => "  "), r"\n[^$]" => "\n  "))
end


text:
# abstract HyperRectangle{N, T}

struct HyperRectangle{T}
    mins::Vector{T}
    maxes::Vector{T}
end

# Computes a bounding box around a point cloud
function compute_bbox(data::Vector{V}) where {V <: AbstractVector}
    T = eltype(V)
    n_dim = length(V)
    maxes = Vector{T}(undef, n_dim)
    mins = Vector{T}(undef, n_dim)
    @inbounds for j in 1:length(V)
        dim_max = typemin(T)
        dim_min = typemax(T)
        for k in 1:length(data)
            dim_max = max(data[k][j], dim_max)
            dim_min = min(data[k][j], dim_min)
        end
        maxes[j] = dim_max
        mins[j] = dim_min
    end
    return HyperRectangle(mins, maxes)
end

############################################
# Rectangle - Point functions
############################################
@inline function get_min_dim(rec::HyperRectangle, point::AbstractVector, dim::Int)
    @inbounds d = abs2(max(0, max(rec.mins[dim] - point[dim], point[dim] - rec.maxes[dim])))
    d
end

@inline function get_max_dim(rec::HyperRectangle, point::AbstractVector, dim::Int)
    @inbounds d = abs2(max(rec.maxes[dim] - point[dim], point[dim] - rec.mins[dim]))
    d
end

# Max distance between rectangle and point
@inline function get_max_distance(rec::HyperRectangle, point::AbstractVector{T}) where {T}
    max_dist = zero(T)
    @inbounds @simd for dim in eachindex(point)
        max_dist += abs2(max(rec.maxes[dim] - point[dim], point[dim] - rec.mins[dim]))
    end
    return max_dist
end

# Min distance between rectangle and point
@inline function get_min_distance(rec::HyperRectangle, point::AbstractVector{T}) where {T}
    min_dist = zero(T)
    @inbounds @simd for dim in eachindex(point)
        min_dist += abs2(max(0, max(rec.mins[dim] - point[dim], point[dim] - rec.maxes[dim])))
    end
    return min_dist
end


text:
using PastaQ
using ITensors
using Test
using LinearAlgebra
using Random

""" HELPER FUNCTIONS """
function numgradslogZ(L::LPDO; accuracy=1e-8)
  M = L.X
  N = length(M)
  grad_r = []
  grad_i = []
  for j in 1:N
    push!(grad_r, zeros(ComplexF64, size(M[j])))
    push!(grad_i, zeros(ComplexF64, size(M[j])))
  end

  epsilon = zeros(ComplexF64, size(M[1]))
  # Site 1
  for i in 1:length(epsilon)
    epsilon[i] = accuracy
    eps = ITensor(epsilon, inds(M[1]))
    M[1] += eps
    loss_p = 2.0 * lognorm(M)
    M[1] -= eps
    loss_m = 2.0 * lognorm(M)
    grad_r[1][i] = (loss_p - loss_m) / (accuracy)

    epsilon[i] = im * accuracy
    eps = ITensor(epsilon, inds(M[1]))
    M[1] += eps
    loss_p = 2.0 * lognorm(M)
    M[1] -= eps
    loss_m = 2.0 * lognorm(M)
    grad_i[1][i] = (loss_p - loss_m) / (im * accuracy)

    epsilon[i] = 0.0
  end

  for j in 2:(N - 1)
    epsilon = zeros(ComplexF64, size(M[j]))
    for i in 1:length(epsilon)
      epsilon[i] = accuracy
      eps = ITensor(epsilon, inds(M[j]))
      M[j] += eps
      loss_p = 2.0 * lognorm(M)
      M[j] -= eps
      loss_m = 2.0 * lognorm(M)
      grad_r[j][i] = (loss_p - loss_m) / (accuracy)

      epsilon[i] = im * accuracy
      eps = ITensor(epsilon, inds(M[j]))
      M[j] += eps
      loss_p = 2.0 * lognorm(M)
      M[j] -= eps
      loss_m = 2.0 * lognorm(M)
      grad_i[j][i] = (loss_p - loss_m) / (im * accuracy)

      epsilon[i] = 0.0
    end
  end
  # Site N
  epsilon = zeros(ComplexF64, size(M[N]))
  for i in 1:length(epsilon)
    epsilon[i] = accuracy
    eps = ITensor(epsilon, inds(M[N]))
    M[N] += eps
    loss_p = 2.0 * lognorm(M)
    M[N] -= eps
    loss_m = 2.0 * lognorm(M)
    grad_r[N][i] = (loss_p - loss_m) / (accuracy)

    epsilon[i] = im * accuracy
    eps = ITensor(epsilon, inds(M[N]))
    M[N] += eps
    loss_p = 2.0 * lognorm(M)
    M[N] -= eps
    loss_m = 2.0 * lognorm(M)
    grad_i[N][i] = (loss_p - loss_m) / (im * accuracy)

    epsilon[i] = 0.0
  end

  return grad_r - grad_i
end

numgradslogZ(M::MPS; kwargs...) = numgradslogZ(LPDO(M); kwargs...)

function numgradsnll(L::LPDO, data::Array; accuracy=1e-8)
  M = L.X
  N = length(M)
  grad_r = []
  grad_i = []
  for j in 1:N
    push!(grad_r, zeros(ComplexF64, size(M[j])))
    push!(grad_i, zeros(ComplexF64, size(M[j])))
  end

  epsilon = zeros(ComplexF64, size(M[1]))
  # Site 1
  for i in 1:length(epsilon)
    epsilon[i] = accuracy
    eps = ITensor(epsilon, inds(M[1]))
    M[1] += eps
    loss_p = PastaQ.nll(L, data)
    M[1] -= eps
    loss_m = PastaQ.nll(L, data)
    grad_r[1][i] = (loss_p - loss_m) / (accuracy)

    epsilon[i] = im * accuracy
    eps = ITensor(epsilon, inds(M[1]))
    M[1] += eps
    loss_p = PastaQ.nll(L, data)
    M[1] -= eps
    loss_m = PastaQ.nll(L, data)
    grad_i[1][i] = (loss_p - loss_m) / (im * accuracy)

    epsilon[i] = 0.0
  end

  for j in 2:(N - 1)
    epsilon = zeros(ComplexF64, size(M[j]))
    for i in 1:length(epsilon)
      epsilon[i] = accuracy
      eps = ITensor(epsilon, inds(M[j]))
      M[j] += eps
      loss_p = PastaQ.nll(L, data)
      M[j] -= eps
      loss_m = PastaQ.nll(L, data)
      grad_r[j][i] = (loss_p - loss_m) / (accuracy)

      epsilon[i] = im * accuracy
      eps = ITensor(epsilon, inds(M[j]))
      M[j] += eps
      loss_p = PastaQ.nll(L, data)
      M[j] -= eps
      loss_m = PastaQ.nll(L, data)
      grad_i[j][i] = (loss_p - loss_m) / (im * accuracy)

      epsilon[i] = 0.0
    end
  end

  # Site N
  epsilon = zeros(ComplexF64, size(M[N]))
  for i in 1:length(epsilon)
    epsilon[i] = accuracy
    eps = ITensor(epsilon, inds(M[N]))
    M[N] += eps
    loss_p = PastaQ.nll(L, data)
    M[N] -= eps
    loss_m = PastaQ.nll(L, data)
    grad_r[N][i] = (loss_p - loss_m) / (accuracy)

    epsilon[i] = im * accuracy
    eps = ITensor(epsilon, inds(M[N]))
    M[N] += eps
    loss_p = PastaQ.nll(L, data)
    M[N] -= eps
    loss_m = PastaQ.nll(L, data)
    grad_i[N][i] = (loss_p - loss_m) / (im * accuracy)

    epsilon[i] = 0.0
  end

  return grad_r - grad_i
end

numgradsnll(M::MPS, args...; kwargs...) = numgradsnll(LPDO(M), args...; kwargs...)

""" MPS TESTS """

@testset "mps-qst: normalization" begin
  N = 10
  χ = 4
  ψ = randomstate(N; χ=χ)
  @test length(ψ) == N
  logZ = lognorm(ψ)
  localZ = []
  normalize!(ψ; (localnorms!)=localZ)
  @test logZ ≈ sum(log.(localZ))
  @test norm(ψ) ≈ 1
end

@testset "mps-qst: grad logZ" begin
  N = 5
  χ = 4

  # 1. Unnormalized
  ψ = randomstate(N; χ=χ)
  alg_grad, _ = PastaQ.gradlogZ(ψ)
  num_grad = numgradslogZ(ψ)
  for j in 1:N
    @test ITensors.array(alg_grad[j]) ≈ num_grad[j] rtol = 1e-3
  end

  # 2. Globally normalized
  ψ = randomstate(N; χ=χ)
  normalize!(ψ)
  @test norm(ψ)^2 ≈ 1
  alg_grad, _ = PastaQ.gradlogZ(ψ)
  num_grad = numgradslogZ(ψ)
  for j in 1:N
    @test ITensors.array(alg_grad[j]) ≈ num_grad[j] rtol = 1e-3
  end

  # 3. Locally normalized
  ψ = randomstate(N; χ=χ)
  num_grad = numgradslogZ(ψ)

  localnorms = []
  normalize!(ψ; (localnorms!)=localnorms)
  @test norm(ψ) ≈ 1
  alg_grad, _ = PastaQ.gradlogZ(ψ; localnorms=localnorms)
  for j in 1:N
    @test ITensors.array(alg_grad[j]) ≈ num_grad[j] rtol = 1e-3
  end
end

@testset "mps-qst: grad nll" begin
  N = 5
  χ = 4
  nsamples = 10
  Random.seed!(1234)
  data = PastaQ.convertdatapoints(randompreparations(N, nsamples))

  # 1. Unnormalized
  ψ = randomstate(N; χ=χ)
  num_grad = numgradsnll(ψ, data)
  alg_grad, loss = PastaQ.gradnll(ψ, data)
  for j in 1:N
    @test ITensors.array(alg_grad[j]) ≈ num_grad[j] rtol = 1e-3
  end

  # 2. Globally normalized
  ψ = randomstate(N; χ=χ)
  normalize!(ψ)
  num_grad = numgradsnll(ψ, data)
  alg_grad, loss = PastaQ.gradnll(ψ, data)
  for j in 1:N
    @test ITensors.array(alg_grad[j]) ≈ num_grad[j] rtol = 1e-3
  end

  # 3. Locally normalized
  ψ = randomstate(N; χ=χ)
  num_grad = numgradsnll(ψ, data)
  localnorms = []
  normalize!(ψ; (localnorms!)=localnorms)
  @test norm(ψ) ≈ 1
  alg_grad_localnorm, loss = PastaQ.gradnll(ψ, data; localnorms=localnorms)
  for j in 1:N
    @test ITensors.array(alg_grad_localnorm[j]) ≈ num_grad[j] rtol = 1e-3
  end
end

@testset "mps-qst: full gradients" begin
  N = 5
  χ = 4
  nsamples = 10
  data = PastaQ.convertdatapoints(randompreparations(N, nsamples))

  # 1. Unnormalized
  ψ = randomstate(N; χ=χ)
  logZ = 2.0 * log(norm(ψ))
  NLL = PastaQ.nll(ψ, data)
  ex_loss = logZ + NLL
  num_gradZ = numgradslogZ(ψ)
  num_gradNLL = numgradsnll(ψ, data)
  num_grads = num_gradZ + num_gradNLL

  alg_grads, loss = PastaQ.gradients(ψ, data)
  @test ex_loss ≈ loss
  for j in 1:N
    @test ITensors.array(alg_grads[j]) ≈ num_grads[j] rtol = 1e-3
  end

  # 2. Globally normalized
  ψ = randomstate(N; χ=χ)
  normalize!(ψ)
  num_gradZ = numgradslogZ(ψ)
  num_gradNLL = numgradsnll(ψ, data)
  num_grads = num_gradZ + num_gradNLL
  NLL = PastaQ.nll(ψ, data)
  ex_loss = NLL
  @test norm(ψ)^2 ≈ 1

  alg_grads, loss = PastaQ.gradients(ψ, data)
  @test ex_loss ≈ loss
  for j in 1:N
    @test ITensors.array(alg_grads[j]) ≈ num_grads[j] rtol = 1e-3
  end

  # 3. Locally normalized
  ψ = randomstate(N; χ=χ)
  num_gradZ = numgradslogZ(ψ)
  num_gradNLL = numgradsnll(ψ, data)
  num_grads = num_gradZ + num_gradNLL

  localnorms = []
  normalize!(ψ; (localnorms!)=localnorms)
  NLL = PastaQ.nll(ψ, data)
  ex_loss = NLL
  @test norm(ψ)^2 ≈ 1

  alg_grads, loss = PastaQ.gradients(ψ, data; localnorms=localnorms)
  @test ex_loss ≈ loss
  for j in 1:N
    @test ITensors.array(alg_grads[j]) ≈ num_grads[j] rtol = 1e-3
  end
end

""" LPDO TESTS """

@testset "lpdo-qst: normalization" begin
  N = 10
  χ = 4
  ξ = 3
  ρ = randomstate(N; mixed=true, ξ=ξ, χ=χ)
  @test length(ρ) == N
  logZ = logtr(ρ)
  localZ = []
  normalize!(ρ; (sqrt_localnorms!)=localZ)
  @test logZ ≈ 2.0 * sum(log.(localZ))
  @test tr(ρ) ≈ 1
end

@testset "lpdo-qst: grad logZ" begin
  N = 5
  χ = 4
  ξ = 3

  # 1. Unnormalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  alg_grad, logZ = PastaQ.gradlogZ(ρ)
  @test logZ ≈ logtr(ρ)
  num_grad = numgradslogZ(ρ)
  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [1, 3, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [2, 1, 3, 4])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [2, 1, 3])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3

  # 2. Globally normalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  normalize!(ρ)
  @test tr(ρ) ≈ 1
  alg_grad, _ = PastaQ.gradlogZ(ρ)
  num_grad = numgradslogZ(ρ)

  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [1, 3, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [2, 1, 3, 4])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [2, 1, 3])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3

  # 3. Locally normalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  num_grad = numgradslogZ(ρ)

  sqrt_localnorms = []
  normalize!(ρ; (sqrt_localnorms!)=sqrt_localnorms)
  @test tr(ρ) ≈ 1
  alg_grad, _ = PastaQ.gradlogZ(ρ; sqrt_localnorms=sqrt_localnorms)

  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [1, 3, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [2, 1, 3, 4])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [2, 1, 3])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3
end

@testset "lpdo-qst: grad nll" begin
  N = 5
  χ = 4
  ξ = 3

  nsamples = 10
  Random.seed!(1234)
  data = PastaQ.convertdatapoints(randompreparations(N, nsamples))

  # 1. Unnormalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  num_grad = numgradsnll(ρ, data)
  alg_grad, loss = PastaQ.gradnll(ρ, data)
  @test loss ≈ PastaQ.nll(ρ, data)

  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [4, 2, 3, 1])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3

  ## 2. Globally normalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  normalize!(ρ)
  num_grad = numgradsnll(ρ, data)
  alg_grad, loss = PastaQ.gradnll(ρ, data)

  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [4, 2, 3, 1])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3

  # 3. Locally normalized
  ρ = randomstate(N; mixed=true, χ=χ, ξ=ξ)
  num_grad = numgradsnll(ρ, data)
  sqrt_localnorms = []
  normalize!(ρ; (sqrt_localnorms!)=sqrt_localnorms)
  @test tr(ρ) ≈ 1
  alg_grad, loss = PastaQ.gradnll(ρ, data; sqrt_localnorms=sqrt_localnorms)
  alg_gradient = permutedims(ITensors.array(alg_grad[1]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[1] rtol = 1e-3
  for j in 2:(N - 1)
    alg_gradient = permutedims(ITensors.array(alg_grad[j]), [4, 2, 3, 1])
    @test alg_gradient ≈ num_grad[j] rtol = 1e-3
  end
  alg_gradient = permutedims(ITensors.array(alg_grad[N]), [3, 1, 2])
  @test alg_gradient ≈ num_grad[N] rtol = 1e-3
end


text:
#-------------------------------------------------------------------
#* EMSO Model Library (EML) Copyright (C) 2004 - 2007 ALSOC.
#*
#* This LIBRARY is free software; you can distribute it and/or modify
#* it under the therms of the ALSOC FREE LICENSE as available at
#* http://www.enq.ufrgs.br/alsoc.
#*
#* EMSO Copyright (C) 2004 - 2007 ALSOC, original code
#* from http://www.rps.eng.br Copyright (C) 2002-2004.
#* All rights reserved.
#*
#* EMSO is distributed under the therms of the ALSOC LICENSE as
#* available at http://www.enq.ufrgs.br/alsoc.
#*
#*----------------------------------------------------------------------
#* File containg models of columns: distillation, stripping, absorbers
#* rectifier, ....
#*
#* The default nomenclature is:
#*		Type_Column_reboilertype_condensertyper
#*
#* where:
#*	Type = refluxed or reboiled or section
#*	Column = Stripping, Absorption, Rectifier, Distillation
#*	Reboiler type (if exists) = kettle or thermosyphon
#*	Condenser type (if exists) = with subccoling or without subcooling
#*
#*-----------------------------------------------------------------------
#* Author: Paula B. Staudt
#* $Id: column.mso 511 2008-05-12 17:25:33Z paula $
#*---------------------------------------------------------------------
type InitializeSection
	InitializeSection()=begin
		NComp=outers.NComp
		new(
			DanaInteger(Dict{Symbol,Any}(
				:Brief=>"Number of components"
			)),
			temperature(Dict{Symbol,Any}(
				:Brief=>"Tray Temperature at Column Top",
				:Default=>300
			)),
			temperature(Dict{Symbol,Any}(
				:Brief=>"Tray Temperature at Column Bottom",
				:Default=>330
			)),
			fraction(Dict{Symbol,Any}(
				:Brief=>"Tray Level Fraction",
				:Default=>0.5
			)),
			fill(fraction(Dict{Symbol,Any}(
				:Brief=>"Component Molar Fraction at Column Top",
				:Default=>0.30
			)),(NComp)),
			fill(fraction(Dict{Symbol,Any}(
				:Brief=>"Component Molar Fraction at Column Bottom",
				:Default=>0.30
			)),(NComp)),
			[:NComp,:TopTemperature,:BottomTemperature,:LevelFraction,:TopComposition,:BottomComposition,],
		)
	end
	NComp::DanaInteger
	TopTemperature::temperature
	BottomTemperature::temperature
	LevelFraction::fraction
	TopComposition::Array{fraction}
	BottomComposition::Array{fraction}
	parameters::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export InitializeSection
function atributes(in::InitializeSection,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=false
	fields[:Brief]="Initial conditions for column section."
	drive!(fields,_)
	return fields
end
InitializeSection(_::Dict{Symbol,Any})=begin
	newModel=InitializeSection()
	newModel.attributes=atributes(newModel,_)
	newModel
end