text:
module EmojiSymbols

include(normpath(@__DIR__, "../gen/emoji_symbols.jl"))
include(normpath(@__DIR__, "../gen/emoji_name_table.jl"))
include(normpath(@__DIR__, "../gen/latex_name_table.jl"))

function __init__()
    REPL = Base.REPL_MODULE_REF[]
    merge!(REPL.REPLCompletions.emoji_symbols, emoji_symbols)
    REPL.symbol_latex("")
    for (k, v) in (VERSION < v"1.6.0-DEV.771" ? REPL.REPLCompletions.emoji_symbols : emoji_symbols)
        REPL.symbols_latex[v] = k
    end
end

# from julia/base/char.jl
using Base: Unicode, ismalformed, isoverlong, decode_overlong
function show_char(io::IO, ::MIME"text/plain", c::T) where {T<:AbstractChar}
    show(io, c)
    get(io, :compact, false) && return
    if !ismalformed(c)
        print(io, ": ")
        if isoverlong(c)
            print(io, "[overlong] ")
            u = decode_overlong(c)
            c = T(u)
        else
            u = codepoint(c)
        end
        h = uppercase(string(u, base = 16, pad = 4))
        print(io, (isascii(c) ? "ASCII/" : ""), "Unicode U+", h)
    else
        print(io, ": Malformed UTF-8")
    end
    abr = Unicode.category_abbrev(c)
    str = Unicode.category_string(c)
    print(io, " (category ", abr, ": ", str, ")")
end

# emoji ranges
#     0x00a9 <= n <= 0x00ae || 0x200d <= n <= 0x3299 || 0x1f004 <= n <= 0x1f251 || 0x1f300 <= n <= 0x1fad6
# latex ranges
#     0x00a1 <= n <= 0x03f6 || 0x1d2c <= n <= 0x1dbf || 0x2002 <= n <= 0x3012 || 0x1d400 <= n <= 0x1d7ff
function lookup_and_show_short_name(io::IO, ::MIME"text/plain", c::Char, s::String)
    n = UInt32(c)
    if 0x1f004 <= n <= 0x1f251 || 0x1f300 <= n <= 0x1fad6
        haskey(emoji_name_table, s) && printstyled(io, emoji_name_table[s], ' ', color=:cyan)
    elseif 0x1d2c <= n <= 0x1dbf || 0x1d400 <= n <= 0x1d7ff
        haskey(latex_name_table, s) && printstyled(io, latex_name_table[s], ' ', color=:cyan)
    elseif 0x00a9 <= n <= 0x00ae || 0x200d <= n <= 0x3299 ||   # emoji
           0x00a1 <= n <= 0x03f6 || 0x2002 <= n <= 0x3012      # latex
        if haskey(emoji_name_table, s)
            printstyled(io, emoji_name_table[s], ' ', color=:cyan)
        elseif haskey(latex_name_table, s)
            printstyled(io, latex_name_table[s], ' ', color=:cyan)
        end
    end
end

function Base.show(io::IO, mime::MIME"text/plain", c::Char)
    lookup_and_show_short_name(io, mime, c, string(c))
    show_char(io, mime, c)
end

const MAX_SYMBOL_LENGTH = 5
function Base.show(io::IO, mime::MIME"text/plain", s::String)
    len = length(s)
    if iszero(len)
    elseif isone(len)
        lookup_and_show_short_name(io, mime, first(s), s)
    elseif len <= MAX_SYMBOL_LENGTH
        if haskey(emoji_name_table, s)
            printstyled(io, emoji_name_table[s], ' ', color=:cyan)
        elseif haskey(latex_name_table, s)
            printstyled(io, latex_name_table[s], ' ', color=:cyan)
        end
    end
    print(io, repr(s))
end

end # module EmojiSymbols


text:
#!/usr/bin/env julia
include("commandUtils.jl")
activatePkg()
using ArgParse
using JLD
using Distributed
using Plots
s = ArgParseSettings(description="Plot Field From H5 Ouput, plots field from output of BIM field calculation.")
@add_arg_table s begin
    "--fieldFile", "-f"
        help = "File with calculated k's and field approximation as initial guess for x0."
    "--clamp", "-c"
        help = "Clamp max field value below this number (to reduce dynamic range)."
        arg_type = Float64
        default = 1.0
end
arguments = parse_args(s)
@debug arguments

fieldFile = arguments["fieldFile"]
clampMax = arguments["clamp"]
field = jldopen(fieldFile) do file
   read(file, "field/full")
end
gr()
@info "Max field value (abs) = $(maximum(abs.(field)))"
Plots.heatmap(clamp.(abs.(field), 0, clampMax),size=(1000,1000), cbar=false, aspect_ratio=1)
Plots.savefig("$fieldFile.png")

ext: jl


text:
# syntax: proto2
# COV_EXCL_START
using ProtoBuf
import ProtoBuf.meta
import ..hadoop

const Status = (;[
    Symbol("SUCCESS") => Int32(0),
    Symbol("ERROR") => Int32(1),
    Symbol("ERROR_CHECKSUM") => Int32(2),
    Symbol("ERROR_INVALID") => Int32(3),
    Symbol("ERROR_EXISTS") => Int32(4),
    Symbol("ERROR_ACCESS_TOKEN") => Int32(5),
    Symbol("CHECKSUM_OK") => Int32(6),
    Symbol("ERROR_UNSUPPORTED") => Int32(7),
    Symbol("OOB_RESTART") => Int32(8),
    Symbol("OOB_RESERVED1") => Int32(9),
    Symbol("OOB_RESERVED2") => Int32(10),
    Symbol("OOB_RESERVED3") => Int32(11),
    Symbol("IN_PROGRESS") => Int32(12),
]...)

const ShortCircuitFdResponse = (;[
    Symbol("DO_NOT_USE_RECEIPT_VERIFICATION") => Int32(0),
    Symbol("USE_RECEIPT_VERIFICATION") => Int32(1),
]...)

mutable struct HandshakeSecretProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function HandshakeSecretProto(; kwargs...)
        obj = new(meta(HandshakeSecretProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct HandshakeSecretProto
const __meta_HandshakeSecretProto = Ref{ProtoMeta}()
function meta(::Type{HandshakeSecretProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_HandshakeSecretProto)
            __meta_HandshakeSecretProto[] = target = ProtoMeta(HandshakeSecretProto)
            req = Symbol[:secret,:bpid]
            allflds = Pair{Symbol,Union{Type,String}}[:secret => Array{UInt8,1}, :bpid => AbstractString]
            meta(target, HandshakeSecretProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_HandshakeSecretProto[]
    end
end
function Base.getproperty(obj::HandshakeSecretProto, name::Symbol)
    if name === :secret
        return (obj.__protobuf_jl_internal_values[name])::Array{UInt8,1}
    elseif name === :bpid
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    else
        getfield(obj, name)
    end
end

const DataTransferEncryptorMessageProto_DataTransferEncryptorStatus = (;[
    Symbol("SUCCESS") => Int32(0),
    Symbol("ERROR_UNKNOWN_KEY") => Int32(1),
    Symbol("ERROR") => Int32(2),
]...)

mutable struct DataTransferEncryptorMessageProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function DataTransferEncryptorMessageProto(; kwargs...)
        obj = new(meta(DataTransferEncryptorMessageProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct DataTransferEncryptorMessageProto
const __meta_DataTransferEncryptorMessageProto = Ref{ProtoMeta}()
function meta(::Type{DataTransferEncryptorMessageProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_DataTransferEncryptorMessageProto)
            __meta_DataTransferEncryptorMessageProto[] = target = ProtoMeta(DataTransferEncryptorMessageProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32, :payload => Array{UInt8,1}, :message => AbstractString, :cipherOption => Base.Vector{CipherOptionProto}, :handshakeSecret => HandshakeSecretProto]
            meta(target, DataTransferEncryptorMessageProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_DataTransferEncryptorMessageProto[]
    end
end
function Base.getproperty(obj::DataTransferEncryptorMessageProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :payload
        return (obj.__protobuf_jl_internal_values[name])::Array{UInt8,1}
    elseif name === :message
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :cipherOption
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{CipherOptionProto}
    elseif name === :handshakeSecret
        return (obj.__protobuf_jl_internal_values[name])::HandshakeSecretProto
    else
        getfield(obj, name)
    end
end

mutable struct DataTransferTraceInfoProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function DataTransferTraceInfoProto(; kwargs...)
        obj = new(meta(DataTransferTraceInfoProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct DataTransferTraceInfoProto
const __meta_DataTransferTraceInfoProto = Ref{ProtoMeta}()
function meta(::Type{DataTransferTraceInfoProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_DataTransferTraceInfoProto)
            __meta_DataTransferTraceInfoProto[] = target = ProtoMeta(DataTransferTraceInfoProto)
            req = Symbol[:traceId,:parentId]
            allflds = Pair{Symbol,Union{Type,String}}[:traceId => UInt64, :parentId => UInt64]
            meta(target, DataTransferTraceInfoProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_DataTransferTraceInfoProto[]
    end
end
function Base.getproperty(obj::DataTransferTraceInfoProto, name::Symbol)
    if name === :traceId
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :parentId
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    else
        getfield(obj, name)
    end
end

mutable struct BaseHeaderProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function BaseHeaderProto(; kwargs...)
        obj = new(meta(BaseHeaderProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct BaseHeaderProto
const __meta_BaseHeaderProto = Ref{ProtoMeta}()
function meta(::Type{BaseHeaderProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_BaseHeaderProto)
            __meta_BaseHeaderProto[] = target = ProtoMeta(BaseHeaderProto)
            req = Symbol[:block]
            allflds = Pair{Symbol,Union{Type,String}}[:block => ExtendedBlockProto, :token => hadoop.common.TokenProto, :traceInfo => DataTransferTraceInfoProto]
            meta(target, BaseHeaderProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_BaseHeaderProto[]
    end
end
function Base.getproperty(obj::BaseHeaderProto, name::Symbol)
    if name === :block
        return (obj.__protobuf_jl_internal_values[name])::ExtendedBlockProto
    elseif name === :token
        return (obj.__protobuf_jl_internal_values[name])::hadoop.common.TokenProto
    elseif name === :traceInfo
        return (obj.__protobuf_jl_internal_values[name])::DataTransferTraceInfoProto
    else
        getfield(obj, name)
    end
end

mutable struct ClientOperationHeaderProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ClientOperationHeaderProto(; kwargs...)
        obj = new(meta(ClientOperationHeaderProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ClientOperationHeaderProto
const __meta_ClientOperationHeaderProto = Ref{ProtoMeta}()
function meta(::Type{ClientOperationHeaderProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ClientOperationHeaderProto)
            __meta_ClientOperationHeaderProto[] = target = ProtoMeta(ClientOperationHeaderProto)
            req = Symbol[:baseHeader,:clientName]
            allflds = Pair{Symbol,Union{Type,String}}[:baseHeader => BaseHeaderProto, :clientName => AbstractString]
            meta(target, ClientOperationHeaderProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ClientOperationHeaderProto[]
    end
end
function Base.getproperty(obj::ClientOperationHeaderProto, name::Symbol)
    if name === :baseHeader
        return (obj.__protobuf_jl_internal_values[name])::BaseHeaderProto
    elseif name === :clientName
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    else
        getfield(obj, name)
    end
end

mutable struct CachingStrategyProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function CachingStrategyProto(; kwargs...)
        obj = new(meta(CachingStrategyProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct CachingStrategyProto
const __meta_CachingStrategyProto = Ref{ProtoMeta}()
function meta(::Type{CachingStrategyProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_CachingStrategyProto)
            __meta_CachingStrategyProto[] = target = ProtoMeta(CachingStrategyProto)
            allflds = Pair{Symbol,Union{Type,String}}[:dropBehind => Bool, :readahead => Int64]
            meta(target, CachingStrategyProto, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_CachingStrategyProto[]
    end
end
function Base.getproperty(obj::CachingStrategyProto, name::Symbol)
    if name === :dropBehind
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :readahead
        return (obj.__protobuf_jl_internal_values[name])::Int64
    else
        getfield(obj, name)
    end
end

mutable struct OpReadBlockProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpReadBlockProto(; kwargs...)
        obj = new(meta(OpReadBlockProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpReadBlockProto
const __meta_OpReadBlockProto = Ref{ProtoMeta}()
function meta(::Type{OpReadBlockProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpReadBlockProto)
            __meta_OpReadBlockProto[] = target = ProtoMeta(OpReadBlockProto)
            req = Symbol[:header,:offset,:len]
            val = Dict{Symbol,Any}(:sendChecksums => true)
            allflds = Pair{Symbol,Union{Type,String}}[:header => ClientOperationHeaderProto, :offset => UInt64, :len => UInt64, :sendChecksums => Bool, :cachingStrategy => CachingStrategyProto]
            meta(target, OpReadBlockProto, allflds, req, ProtoBuf.DEF_FNUM, val, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpReadBlockProto[]
    end
end
function Base.getproperty(obj::OpReadBlockProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::ClientOperationHeaderProto
    elseif name === :offset
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :len
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :sendChecksums
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :cachingStrategy
        return (obj.__protobuf_jl_internal_values[name])::CachingStrategyProto
    else
        getfield(obj, name)
    end
end

mutable struct ChecksumProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ChecksumProto(; kwargs...)
        obj = new(meta(ChecksumProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ChecksumProto
const __meta_ChecksumProto = Ref{ProtoMeta}()
function meta(::Type{ChecksumProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ChecksumProto)
            __meta_ChecksumProto[] = target = ProtoMeta(ChecksumProto)
            req = Symbol[:_type,:bytesPerChecksum]
            allflds = Pair{Symbol,Union{Type,String}}[:_type => Int32, :bytesPerChecksum => UInt32]
            meta(target, ChecksumProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ChecksumProto[]
    end
end
function Base.getproperty(obj::ChecksumProto, name::Symbol)
    if name === :_type
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :bytesPerChecksum
        return (obj.__protobuf_jl_internal_values[name])::UInt32
    else
        getfield(obj, name)
    end
end

const OpWriteBlockProto_BlockConstructionStage = (;[
    Symbol("PIPELINE_SETUP_APPEND") => Int32(0),
    Symbol("PIPELINE_SETUP_APPEND_RECOVERY") => Int32(1),
    Symbol("DATA_STREAMING") => Int32(2),
    Symbol("PIPELINE_SETUP_STREAMING_RECOVERY") => Int32(3),
    Symbol("PIPELINE_CLOSE") => Int32(4),
    Symbol("PIPELINE_CLOSE_RECOVERY") => Int32(5),
    Symbol("PIPELINE_SETUP_CREATE") => Int32(6),
    Symbol("TRANSFER_RBW") => Int32(7),
    Symbol("TRANSFER_FINALIZED") => Int32(8),
]...)

mutable struct OpWriteBlockProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpWriteBlockProto(; kwargs...)
        obj = new(meta(OpWriteBlockProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpWriteBlockProto
const __meta_OpWriteBlockProto = Ref{ProtoMeta}()
function meta(::Type{OpWriteBlockProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpWriteBlockProto)
            __meta_OpWriteBlockProto[] = target = ProtoMeta(OpWriteBlockProto)
            req = Symbol[:header,:stage,:pipelineSize,:minBytesRcvd,:maxBytesRcvd,:latestGenerationStamp,:requestedChecksum]
            val = Dict{Symbol,Any}(:storageType => StorageTypeProto.DISK, :allowLazyPersist => false, :pinning => false)
            allflds = Pair{Symbol,Union{Type,String}}[:header => ClientOperationHeaderProto, :targets => Base.Vector{DatanodeInfoProto}, :source => DatanodeInfoProto, :stage => Int32, :pipelineSize => UInt32, :minBytesRcvd => UInt64, :maxBytesRcvd => UInt64, :latestGenerationStamp => UInt64, :requestedChecksum => ChecksumProto, :cachingStrategy => CachingStrategyProto, :storageType => Int32, :targetStorageTypes => Base.Vector{Int32}, :allowLazyPersist => Bool, :pinning => Bool, :targetPinnings => Base.Vector{Bool}]
            meta(target, OpWriteBlockProto, allflds, req, ProtoBuf.DEF_FNUM, val, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpWriteBlockProto[]
    end
end
function Base.getproperty(obj::OpWriteBlockProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::ClientOperationHeaderProto
    elseif name === :targets
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{DatanodeInfoProto}
    elseif name === :source
        return (obj.__protobuf_jl_internal_values[name])::DatanodeInfoProto
    elseif name === :stage
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :pipelineSize
        return (obj.__protobuf_jl_internal_values[name])::UInt32
    elseif name === :minBytesRcvd
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :maxBytesRcvd
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :latestGenerationStamp
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :requestedChecksum
        return (obj.__protobuf_jl_internal_values[name])::ChecksumProto
    elseif name === :cachingStrategy
        return (obj.__protobuf_jl_internal_values[name])::CachingStrategyProto
    elseif name === :storageType
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :targetStorageTypes
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{Int32}
    elseif name === :allowLazyPersist
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :pinning
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :targetPinnings
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{Bool}
    else
        getfield(obj, name)
    end
end

mutable struct OpTransferBlockProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpTransferBlockProto(; kwargs...)
        obj = new(meta(OpTransferBlockProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpTransferBlockProto
const __meta_OpTransferBlockProto = Ref{ProtoMeta}()
function meta(::Type{OpTransferBlockProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpTransferBlockProto)
            __meta_OpTransferBlockProto[] = target = ProtoMeta(OpTransferBlockProto)
            req = Symbol[:header]
            allflds = Pair{Symbol,Union{Type,String}}[:header => ClientOperationHeaderProto, :targets => Base.Vector{DatanodeInfoProto}, :targetStorageTypes => Base.Vector{Int32}]
            meta(target, OpTransferBlockProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpTransferBlockProto[]
    end
end
function Base.getproperty(obj::OpTransferBlockProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::ClientOperationHeaderProto
    elseif name === :targets
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{DatanodeInfoProto}
    elseif name === :targetStorageTypes
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{Int32}
    else
        getfield(obj, name)
    end
end

mutable struct OpReplaceBlockProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpReplaceBlockProto(; kwargs...)
        obj = new(meta(OpReplaceBlockProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpReplaceBlockProto
const __meta_OpReplaceBlockProto = Ref{ProtoMeta}()
function meta(::Type{OpReplaceBlockProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpReplaceBlockProto)
            __meta_OpReplaceBlockProto[] = target = ProtoMeta(OpReplaceBlockProto)
            req = Symbol[:header,:delHint,:source]
            val = Dict{Symbol,Any}(:storageType => StorageTypeProto.DISK)
            allflds = Pair{Symbol,Union{Type,String}}[:header => BaseHeaderProto, :delHint => AbstractString, :source => DatanodeInfoProto, :storageType => Int32]
            meta(target, OpReplaceBlockProto, allflds, req, ProtoBuf.DEF_FNUM, val, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpReplaceBlockProto[]
    end
end
function Base.getproperty(obj::OpReplaceBlockProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::BaseHeaderProto
    elseif name === :delHint
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :source
        return (obj.__protobuf_jl_internal_values[name])::DatanodeInfoProto
    elseif name === :storageType
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct OpCopyBlockProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpCopyBlockProto(; kwargs...)
        obj = new(meta(OpCopyBlockProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpCopyBlockProto
const __meta_OpCopyBlockProto = Ref{ProtoMeta}()
function meta(::Type{OpCopyBlockProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpCopyBlockProto)
            __meta_OpCopyBlockProto[] = target = ProtoMeta(OpCopyBlockProto)
            req = Symbol[:header]
            allflds = Pair{Symbol,Union{Type,String}}[:header => BaseHeaderProto]
            meta(target, OpCopyBlockProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpCopyBlockProto[]
    end
end
function Base.getproperty(obj::OpCopyBlockProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::BaseHeaderProto
    else
        getfield(obj, name)
    end
end

mutable struct OpBlockChecksumProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpBlockChecksumProto(; kwargs...)
        obj = new(meta(OpBlockChecksumProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpBlockChecksumProto
const __meta_OpBlockChecksumProto = Ref{ProtoMeta}()
function meta(::Type{OpBlockChecksumProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpBlockChecksumProto)
            __meta_OpBlockChecksumProto[] = target = ProtoMeta(OpBlockChecksumProto)
            req = Symbol[:header]
            allflds = Pair{Symbol,Union{Type,String}}[:header => BaseHeaderProto]
            meta(target, OpBlockChecksumProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpBlockChecksumProto[]
    end
end
function Base.getproperty(obj::OpBlockChecksumProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::BaseHeaderProto
    else
        getfield(obj, name)
    end
end

mutable struct ShortCircuitShmIdProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ShortCircuitShmIdProto(; kwargs...)
        obj = new(meta(ShortCircuitShmIdProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ShortCircuitShmIdProto
const __meta_ShortCircuitShmIdProto = Ref{ProtoMeta}()
function meta(::Type{ShortCircuitShmIdProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ShortCircuitShmIdProto)
            __meta_ShortCircuitShmIdProto[] = target = ProtoMeta(ShortCircuitShmIdProto)
            req = Symbol[:hi,:lo]
            allflds = Pair{Symbol,Union{Type,String}}[:hi => Int64, :lo => Int64]
            meta(target, ShortCircuitShmIdProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ShortCircuitShmIdProto[]
    end
end
function Base.getproperty(obj::ShortCircuitShmIdProto, name::Symbol)
    if name === :hi
        return (obj.__protobuf_jl_internal_values[name])::Int64
    elseif name === :lo
        return (obj.__protobuf_jl_internal_values[name])::Int64
    else
        getfield(obj, name)
    end
end

mutable struct ShortCircuitShmSlotProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ShortCircuitShmSlotProto(; kwargs...)
        obj = new(meta(ShortCircuitShmSlotProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ShortCircuitShmSlotProto
const __meta_ShortCircuitShmSlotProto = Ref{ProtoMeta}()
function meta(::Type{ShortCircuitShmSlotProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ShortCircuitShmSlotProto)
            __meta_ShortCircuitShmSlotProto[] = target = ProtoMeta(ShortCircuitShmSlotProto)
            req = Symbol[:shmId,:slotIdx]
            allflds = Pair{Symbol,Union{Type,String}}[:shmId => ShortCircuitShmIdProto, :slotIdx => Int32]
            meta(target, ShortCircuitShmSlotProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ShortCircuitShmSlotProto[]
    end
end
function Base.getproperty(obj::ShortCircuitShmSlotProto, name::Symbol)
    if name === :shmId
        return (obj.__protobuf_jl_internal_values[name])::ShortCircuitShmIdProto
    elseif name === :slotIdx
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct OpRequestShortCircuitAccessProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpRequestShortCircuitAccessProto(; kwargs...)
        obj = new(meta(OpRequestShortCircuitAccessProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpRequestShortCircuitAccessProto
const __meta_OpRequestShortCircuitAccessProto = Ref{ProtoMeta}()
function meta(::Type{OpRequestShortCircuitAccessProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpRequestShortCircuitAccessProto)
            __meta_OpRequestShortCircuitAccessProto[] = target = ProtoMeta(OpRequestShortCircuitAccessProto)
            req = Symbol[:header,:maxVersion]
            val = Dict{Symbol,Any}(:supportsReceiptVerification => false)
            allflds = Pair{Symbol,Union{Type,String}}[:header => BaseHeaderProto, :maxVersion => UInt32, :slotId => ShortCircuitShmSlotProto, :supportsReceiptVerification => Bool]
            meta(target, OpRequestShortCircuitAccessProto, allflds, req, ProtoBuf.DEF_FNUM, val, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpRequestShortCircuitAccessProto[]
    end
end
function Base.getproperty(obj::OpRequestShortCircuitAccessProto, name::Symbol)
    if name === :header
        return (obj.__protobuf_jl_internal_values[name])::BaseHeaderProto
    elseif name === :maxVersion
        return (obj.__protobuf_jl_internal_values[name])::UInt32
    elseif name === :slotId
        return (obj.__protobuf_jl_internal_values[name])::ShortCircuitShmSlotProto
    elseif name === :supportsReceiptVerification
        return (obj.__protobuf_jl_internal_values[name])::Bool
    else
        getfield(obj, name)
    end
end

mutable struct ReleaseShortCircuitAccessRequestProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ReleaseShortCircuitAccessRequestProto(; kwargs...)
        obj = new(meta(ReleaseShortCircuitAccessRequestProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ReleaseShortCircuitAccessRequestProto
const __meta_ReleaseShortCircuitAccessRequestProto = Ref{ProtoMeta}()
function meta(::Type{ReleaseShortCircuitAccessRequestProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ReleaseShortCircuitAccessRequestProto)
            __meta_ReleaseShortCircuitAccessRequestProto[] = target = ProtoMeta(ReleaseShortCircuitAccessRequestProto)
            req = Symbol[:slotId]
            allflds = Pair{Symbol,Union{Type,String}}[:slotId => ShortCircuitShmSlotProto, :traceInfo => DataTransferTraceInfoProto]
            meta(target, ReleaseShortCircuitAccessRequestProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ReleaseShortCircuitAccessRequestProto[]
    end
end
function Base.getproperty(obj::ReleaseShortCircuitAccessRequestProto, name::Symbol)
    if name === :slotId
        return (obj.__protobuf_jl_internal_values[name])::ShortCircuitShmSlotProto
    elseif name === :traceInfo
        return (obj.__protobuf_jl_internal_values[name])::DataTransferTraceInfoProto
    else
        getfield(obj, name)
    end
end

mutable struct ReleaseShortCircuitAccessResponseProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ReleaseShortCircuitAccessResponseProto(; kwargs...)
        obj = new(meta(ReleaseShortCircuitAccessResponseProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ReleaseShortCircuitAccessResponseProto
const __meta_ReleaseShortCircuitAccessResponseProto = Ref{ProtoMeta}()
function meta(::Type{ReleaseShortCircuitAccessResponseProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ReleaseShortCircuitAccessResponseProto)
            __meta_ReleaseShortCircuitAccessResponseProto[] = target = ProtoMeta(ReleaseShortCircuitAccessResponseProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32, :error => AbstractString]
            meta(target, ReleaseShortCircuitAccessResponseProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ReleaseShortCircuitAccessResponseProto[]
    end
end
function Base.getproperty(obj::ReleaseShortCircuitAccessResponseProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :error
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    else
        getfield(obj, name)
    end
end

mutable struct ShortCircuitShmRequestProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ShortCircuitShmRequestProto(; kwargs...)
        obj = new(meta(ShortCircuitShmRequestProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ShortCircuitShmRequestProto
const __meta_ShortCircuitShmRequestProto = Ref{ProtoMeta}()
function meta(::Type{ShortCircuitShmRequestProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ShortCircuitShmRequestProto)
            __meta_ShortCircuitShmRequestProto[] = target = ProtoMeta(ShortCircuitShmRequestProto)
            req = Symbol[:clientName]
            allflds = Pair{Symbol,Union{Type,String}}[:clientName => AbstractString, :traceInfo => DataTransferTraceInfoProto]
            meta(target, ShortCircuitShmRequestProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ShortCircuitShmRequestProto[]
    end
end
function Base.getproperty(obj::ShortCircuitShmRequestProto, name::Symbol)
    if name === :clientName
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :traceInfo
        return (obj.__protobuf_jl_internal_values[name])::DataTransferTraceInfoProto
    else
        getfield(obj, name)
    end
end

mutable struct ShortCircuitShmResponseProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ShortCircuitShmResponseProto(; kwargs...)
        obj = new(meta(ShortCircuitShmResponseProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ShortCircuitShmResponseProto
const __meta_ShortCircuitShmResponseProto = Ref{ProtoMeta}()
function meta(::Type{ShortCircuitShmResponseProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ShortCircuitShmResponseProto)
            __meta_ShortCircuitShmResponseProto[] = target = ProtoMeta(ShortCircuitShmResponseProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32, :error => AbstractString, :id => ShortCircuitShmIdProto]
            meta(target, ShortCircuitShmResponseProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ShortCircuitShmResponseProto[]
    end
end
function Base.getproperty(obj::ShortCircuitShmResponseProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :error
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :id
        return (obj.__protobuf_jl_internal_values[name])::ShortCircuitShmIdProto
    else
        getfield(obj, name)
    end
end

mutable struct PacketHeaderProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function PacketHeaderProto(; kwargs...)
        obj = new(meta(PacketHeaderProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct PacketHeaderProto
const __meta_PacketHeaderProto = Ref{ProtoMeta}()
function meta(::Type{PacketHeaderProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_PacketHeaderProto)
            __meta_PacketHeaderProto[] = target = ProtoMeta(PacketHeaderProto)
            req = Symbol[:offsetInBlock,:seqno,:lastPacketInBlock,:dataLen]
            val = Dict{Symbol,Any}(:syncBlock => false)
            wtype = Dict(:offsetInBlock => :sfixed64, :seqno => :sfixed64, :dataLen => :sfixed32)
            allflds = Pair{Symbol,Union{Type,String}}[:offsetInBlock => Int64, :seqno => Int64, :lastPacketInBlock => Bool, :dataLen => Int32, :syncBlock => Bool]
            meta(target, PacketHeaderProto, allflds, req, ProtoBuf.DEF_FNUM, val, ProtoBuf.DEF_PACK, wtype, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_PacketHeaderProto[]
    end
end
function Base.getproperty(obj::PacketHeaderProto, name::Symbol)
    if name === :offsetInBlock
        return (obj.__protobuf_jl_internal_values[name])::Int64
    elseif name === :seqno
        return (obj.__protobuf_jl_internal_values[name])::Int64
    elseif name === :lastPacketInBlock
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :dataLen
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :syncBlock
        return (obj.__protobuf_jl_internal_values[name])::Bool
    else
        getfield(obj, name)
    end
end

mutable struct PipelineAckProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function PipelineAckProto(; kwargs...)
        obj = new(meta(PipelineAckProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct PipelineAckProto
const __meta_PipelineAckProto = Ref{ProtoMeta}()
function meta(::Type{PipelineAckProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_PipelineAckProto)
            __meta_PipelineAckProto[] = target = ProtoMeta(PipelineAckProto)
            req = Symbol[:seqno]
            val = Dict{Symbol,Any}(:downstreamAckTimeNanos => 0)
            pack = Symbol[:flag]
            wtype = Dict(:seqno => :sint64)
            allflds = Pair{Symbol,Union{Type,String}}[:seqno => Int64, :reply => Base.Vector{Int32}, :downstreamAckTimeNanos => UInt64, :flag => Base.Vector{UInt32}]
            meta(target, PipelineAckProto, allflds, req, ProtoBuf.DEF_FNUM, val, pack, wtype, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_PipelineAckProto[]
    end
end
function Base.getproperty(obj::PipelineAckProto, name::Symbol)
    if name === :seqno
        return (obj.__protobuf_jl_internal_values[name])::Int64
    elseif name === :reply
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{Int32}
    elseif name === :downstreamAckTimeNanos
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :flag
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{UInt32}
    else
        getfield(obj, name)
    end
end

mutable struct ReadOpChecksumInfoProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ReadOpChecksumInfoProto(; kwargs...)
        obj = new(meta(ReadOpChecksumInfoProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ReadOpChecksumInfoProto
const __meta_ReadOpChecksumInfoProto = Ref{ProtoMeta}()
function meta(::Type{ReadOpChecksumInfoProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ReadOpChecksumInfoProto)
            __meta_ReadOpChecksumInfoProto[] = target = ProtoMeta(ReadOpChecksumInfoProto)
            req = Symbol[:checksum,:chunkOffset]
            allflds = Pair{Symbol,Union{Type,String}}[:checksum => ChecksumProto, :chunkOffset => UInt64]
            meta(target, ReadOpChecksumInfoProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ReadOpChecksumInfoProto[]
    end
end
function Base.getproperty(obj::ReadOpChecksumInfoProto, name::Symbol)
    if name === :checksum
        return (obj.__protobuf_jl_internal_values[name])::ChecksumProto
    elseif name === :chunkOffset
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    else
        getfield(obj, name)
    end
end

mutable struct ClientReadStatusProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ClientReadStatusProto(; kwargs...)
        obj = new(meta(ClientReadStatusProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ClientReadStatusProto
const __meta_ClientReadStatusProto = Ref{ProtoMeta}()
function meta(::Type{ClientReadStatusProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ClientReadStatusProto)
            __meta_ClientReadStatusProto[] = target = ProtoMeta(ClientReadStatusProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32]
            meta(target, ClientReadStatusProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ClientReadStatusProto[]
    end
end
function Base.getproperty(obj::ClientReadStatusProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct DNTransferAckProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function DNTransferAckProto(; kwargs...)
        obj = new(meta(DNTransferAckProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct DNTransferAckProto
const __meta_DNTransferAckProto = Ref{ProtoMeta}()
function meta(::Type{DNTransferAckProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_DNTransferAckProto)
            __meta_DNTransferAckProto[] = target = ProtoMeta(DNTransferAckProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32]
            meta(target, DNTransferAckProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_DNTransferAckProto[]
    end
end
function Base.getproperty(obj::DNTransferAckProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct OpBlockChecksumResponseProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpBlockChecksumResponseProto(; kwargs...)
        obj = new(meta(OpBlockChecksumResponseProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpBlockChecksumResponseProto
const __meta_OpBlockChecksumResponseProto = Ref{ProtoMeta}()
function meta(::Type{OpBlockChecksumResponseProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpBlockChecksumResponseProto)
            __meta_OpBlockChecksumResponseProto[] = target = ProtoMeta(OpBlockChecksumResponseProto)
            req = Symbol[:bytesPerCrc,:crcPerBlock,:md5]
            allflds = Pair{Symbol,Union{Type,String}}[:bytesPerCrc => UInt32, :crcPerBlock => UInt64, :md5 => Array{UInt8,1}, :crcType => Int32]
            meta(target, OpBlockChecksumResponseProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpBlockChecksumResponseProto[]
    end
end
function Base.getproperty(obj::OpBlockChecksumResponseProto, name::Symbol)
    if name === :bytesPerCrc
        return (obj.__protobuf_jl_internal_values[name])::UInt32
    elseif name === :crcPerBlock
        return (obj.__protobuf_jl_internal_values[name])::UInt64
    elseif name === :md5
        return (obj.__protobuf_jl_internal_values[name])::Array{UInt8,1}
    elseif name === :crcType
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct BlockOpResponseProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function BlockOpResponseProto(; kwargs...)
        obj = new(meta(BlockOpResponseProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct BlockOpResponseProto
const __meta_BlockOpResponseProto = Ref{ProtoMeta}()
function meta(::Type{BlockOpResponseProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_BlockOpResponseProto)
            __meta_BlockOpResponseProto[] = target = ProtoMeta(BlockOpResponseProto)
            req = Symbol[:status]
            allflds = Pair{Symbol,Union{Type,String}}[:status => Int32, :firstBadLink => AbstractString, :checksumResponse => OpBlockChecksumResponseProto, :readOpChecksumInfo => ReadOpChecksumInfoProto, :message => AbstractString, :shortCircuitAccessVersion => UInt32]
            meta(target, BlockOpResponseProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_BlockOpResponseProto[]
    end
end
function Base.getproperty(obj::BlockOpResponseProto, name::Symbol)
    if name === :status
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :firstBadLink
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :checksumResponse
        return (obj.__protobuf_jl_internal_values[name])::OpBlockChecksumResponseProto
    elseif name === :readOpChecksumInfo
        return (obj.__protobuf_jl_internal_values[name])::ReadOpChecksumInfoProto
    elseif name === :message
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :shortCircuitAccessVersion
        return (obj.__protobuf_jl_internal_values[name])::UInt32
    else
        getfield(obj, name)
    end
end

mutable struct OpCustomProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function OpCustomProto(; kwargs...)
        obj = new(meta(OpCustomProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct OpCustomProto
const __meta_OpCustomProto = Ref{ProtoMeta}()
function meta(::Type{OpCustomProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_OpCustomProto)
            __meta_OpCustomProto[] = target = ProtoMeta(OpCustomProto)
            req = Symbol[:customId]
            allflds = Pair{Symbol,Union{Type,String}}[:customId => AbstractString]
            meta(target, OpCustomProto, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_OpCustomProto[]
    end
end
function Base.getproperty(obj::OpCustomProto, name::Symbol)
    if name === :customId
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    else
        getfield(obj, name)
    end
end

export Status, ShortCircuitFdResponse, DataTransferEncryptorMessageProto_DataTransferEncryptorStatus, DataTransferEncryptorMessageProto, HandshakeSecretProto, BaseHeaderProto, DataTransferTraceInfoProto, ClientOperationHeaderProto, CachingStrategyProto, OpReadBlockProto, ChecksumProto, OpWriteBlockProto_BlockConstructionStage, OpWriteBlockProto, OpTransferBlockProto, OpReplaceBlockProto, OpCopyBlockProto, OpBlockChecksumProto, ShortCircuitShmIdProto, ShortCircuitShmSlotProto, OpRequestShortCircuitAccessProto, ReleaseShortCircuitAccessRequestProto, ReleaseShortCircuitAccessResponseProto, ShortCircuitShmRequestProto, ShortCircuitShmResponseProto, PacketHeaderProto, PipelineAckProto, ReadOpChecksumInfoProto, BlockOpResponseProto, ClientReadStatusProto, DNTransferAckProto, OpBlockChecksumResponseProto, OpCustomProto
# COV_EXCL_STOP


text:
#!/usr/bin/env julia

const clones_dir = "clones"
const static_dir = "static"
const get_old_package_artifacts = false

# Wait one day before attempting to download bad artifacts/packages again
const blacklist_dir = "blacklist"
const blacklist_timeout = 60*60*24*1

import Dates: DateTime, now
import Pkg
import Pkg.TOML
import Pkg.Artifacts: download_artifact, artifact_path, artifact_names
import Pkg.PlatformEngines: download_verify_unpack, probe_platform_engines!
import Tar
import TranscodingStreams: TranscodingStream
import CodecZlib: GzipCompressor, GzipDecompressor

probe_platform_engines!()
mkpath(clones_dir)
mkpath(static_dir)
mkpath(blacklist_dir)

compress(io::IO) = TranscodingStream(GzipCompressor(level=9), io)
decompress(io::IO) = TranscodingStream(GzipDecompressor(), io)

function print_exception(e)
    eio = IOBuffer()
    Base.showerror(eio, e, catch_backtrace())
    @error String(take!(eio))
end

function blacklist(args...)
    black_path = joinpath(blacklist_dir, args...)
    mkpath(dirname(black_path))
    touch(black_path)
end

# Returns true if the file has been blacklisted and should not be downloaded
function is_blacklisted(args...)
    black_path = joinpath(blacklist_dir, args...)
    st = stat(black_path)
    if isfile(st) && mtime(st) > time() - blacklist_timeout
        return true
    end
    return false
end

function prune_empty_blacklist_dirs(path)
    if !isdir(path)
        return
    end
    if abspath(path) == blacklist_dir
        return
    end
    if isempty(readdir(path))
        rm(path; force=true)
        prune_empty_blacklist_dirs(dirname(path))
    end
end

function clear_blacklist(args...)
    black_path = joinpath(blacklist_dir, args...)
    if isfile(black_path)
        rm(black_path; force=true)
        prune_empty_blacklist_dirs(dirname(black_path))
    end
end

function make_tarball(
    tarball::AbstractString,
    tree_path::AbstractString,
)
    open(tarball, write=true) do io
        close(Tar.create(tree_path, compress(io)))
    end
    return tarball
end

function create_git_tarball(
    tarball::AbstractString,
    repo_path::AbstractString,
    tree_hash::AbstractString,
)
    # If this download is blacklisted, don't even bother.
    if is_blacklisted(tree_hash)
        return
    end
    mktempdir() do tree_path
        run(`git -C $(repo_path) --work-tree=$(tree_path) checkout -f $(tree_hash) -- .`)
        make_tarball(tarball, tree_path)
        try
            nskip_hash, yskip_hash = verify_tarball_hash(tarball, tree_hash)
            clear_blacklist(tree_hash)

            # If we've been calling it by the wrong hash, make sure to store it at the right name too
            if nskip_hash != tree_hash
                nskip_tarball = joinpath(dirname(tarball), nskip_hash)
                if !isfile(nskip_tarball)
                    cp(tarball, nskip_tarball)
                end
            end
        catch err
            @warn err repo_path tarball tree_path
            ondisk_hash = bytes2hex(Pkg.GitTools.tree_hash(tree_path))
            @warn "verifying on-disk hash" ondisk_hash
            blacklist(tree_hash)
            rm(tarball, force=true)
        end
    end
    return
end

function verify_tarball_hash(
    tarball::AbstractString,
    tree_hash::AbstractString,
)
    nskip_hash = open(io -> Tar.tree_hash(decompress(io)), tarball)
    yskip_hash = open(io -> Tar.tree_hash(decompress(io); skip_empty=true), tarball)
    if nskip_hash != tree_hash && yskip_hash != tree_hash
        error("""
            tree hash mismatch:
            - expected: $tree_hash
            - computed: $nskip_hash/$yskip_hash
        """)
    end
    return nskip_hash, yskip_hash
end

function process_artifact(info::Dict)
    local tree_hash, tree_path, tarball
    try
        tree_hash = info["git-tree-sha1"]
        if is_blacklisted(tree_hash)
            return
        end
        tree_sha1 = Pkg.Types.SHA1(tree_hash)
        tarball = joinpath(static_dir, "artifact", tree_hash)
        isfile(tarball) && return
        haskey(info, "download") || return
        downloads = info["download"]
        downloads isa Array || (downloads = [downloads])
        tree_path = artifact_path(tree_sha1, honor_overrides=false)
        for download in downloads
            url = download["url"]
            hash = download["sha256"]
            download_verify_unpack(url, hash, tree_path; verbose=true, force=true) && break
        end
        if !isdir(tree_path)
            blacklist(tree_hash)
            error("artifact install failed")
        else
            clear_blacklist(tree_hash)
        end
    catch err
        @warn err
        blacklist(tree_hash)
        return
    end
    mkpath(dirname(tarball))
    make_tarball(tarball, tree_path)
    try
        nskip_hash, yskip_hash = verify_tarball_hash(tarball, tree_hash)
        clear_blacklist(tree_hash)

        if nskip_hash != tree_hash
            nskip_tarball = joinpath(dirname(tarball), nskip_hash)
            if !isfile(nskip_tarball)
                cp(tarball, nskip_tarball)
            end
        end
    catch err
        @warn err tree_path=tree_path tarball=tarball
        blacklist(tree_hash)
        rm(tarball, force=true)
    finally
        chmod(tree_path, 0o700, recursive=true)
        rm(tree_path, force=true, recursive=true)
    end
    return
end

registries = Dict{String,String}()

# These functions are very linux-only
function getsid(pid::Cint)
    return ccall(:getsid, Cint, (Cint,), pid)
end
getsid(p::Base.Process) = getsid(getpid(p))

function killpg(pgid::Cint, sig=Base.SIGTERM)
    return ccall(:killpg, Cint, (Cint, Cint), pgid, sig)
end
killpg(p::Base.Process, sig=Base.SIGTERM) = killpg(getsid(p), sig)

"""
    run_with_timeout(cmd, timeout = 600, term_timeout=10)

Run a command.  After `timeout` seconds, sends `SIGTERM` to the process.
After `term_timeout` seconds, sends `SIGKILL` to the process.  If the process
is not successful, throws an error.
"""
function run_with_timeout(cmd::Cmd, timeout = 600.0, term_timeout = 10.0)
    out = Pipe()
    err = Pipe()
    process = run(pipeline(detach(cmd), stdout=out, stderr=err); wait=false)
    close(out.in)
    close(err.in)

    # Start asynchronous task to get all stdout and stderr data
    out_task = @async String(read(out))
    err_task = @async String(read(err))

    timeout_start = time()
    while process_running(process)
        sleep(.1)
        elapsed = (time() - timeout_start)
        if elapsed > timeout
            @warn("Terminating long-running command", cmd, elapsed)
            killpg(process)

            # Wait for the SIGTERM to get responded to
            term_start = time()
            while process_running(process)
                sleep(.1)
                term_elapsed = time() - term_start
                if time() - term_start > term_timeout
                    @warn("Killing long-running command", cmd, elapsed, term_elapsed)
                    killpg(process, Base.SIGKILL)
                    wait(process)
                    break
                end
            end
        end
    end
    if !success(process)
        stdout = fetch(out_task)
        stderr = fetch(err_task)
        @warn("Failed command", cmd, stdout, stderr)
        error("Failed command")
    end
    return nothing
end

for depot in DEPOT_PATH
    depot_regs = joinpath(depot, "registries")
    isdir(depot_regs) || continue
    @info "Begin Sync...", string(DateTime(now()))
    for reg_dir in readdir(depot_regs, join=true)
        isdir(reg_dir) || continue
        reg_file = joinpath(reg_dir, "Registry.toml")
        isfile(reg_file) || continue
        reg_data = TOML.parsefile(reg_file)
        # generate registry tarball
        run(`git -C $reg_dir pull`)
        let tree_hash = readchomp(`git -C $reg_dir rev-parse 'HEAD^{tree}'`)
            uuid = reg_data["uuid"]
            tarball = joinpath(static_dir, "registry", uuid, tree_hash)
            if !isfile(tarball)
                mkpath(dirname(tarball))
                create_git_tarball(tarball, reg_dir, tree_hash)
            end
            registries[reg_data["uuid"]] = tree_hash
        end
        total_packages = length(reg_data["packages"])
        @debug "Packages to be processed:", total_packages
        pc = 0
        for (uuid, info) in reg_data["packages"]
            pc += 1
            @debug "processing $(info["name"]) .. $pc/$total_packages"
            name = info["name"]
            path = info["path"]
            pkg_dir = joinpath(reg_dir, path)
            pkg_info = TOML.parsefile(joinpath(pkg_dir, "Package.toml"))
            versions = TOML.parsefile(joinpath(pkg_dir, "Versions.toml"))
            pkg_repo = pkg_info["repo"]
            # generate archive of each version
            static_pkg_dir = joinpath(static_dir, "package", uuid)
            isdir(static_pkg_dir) || mkpath(static_pkg_dir)
            clone_dir = joinpath(clones_dir, uuid)
            updated = false
            if !isdir(clone_dir)
                is_clone_failure = false
                try
                    @info("($pc/$total_packages) Cloning in process...", name, pkg_repo)
                    run_with_timeout(`git clone --mirror $pkg_repo $clone_dir`)
                catch e
                    if isa(e, InterruptException)
                        rethrow(e)
                    end
                    print_exception(e)
                    @warn "Unable to clone $name [$uuid] from $pkg_repo"
                    is_clone_failure = true
                end
                is_clone_failure && continue
                updated = true
            end
            for (ver, info) in versions
                tree_hash = info["git-tree-sha1"]
                # If this tree_hash is blacklisted, skip it.
                if is_blacklisted(uuid, tree_hash)
                    continue
                end

                tarball = joinpath(static_pkg_dir, tree_hash)
                if (is_new_tarball = !isfile(tarball))
                    pkg_repo = pkg_info["repo"]
                    @label again
                    try create_git_tarball(tarball, clone_dir, tree_hash)
                    catch err
                        @info("create_git_tarball() failed", err)
                        if updated
                            println(stderr, "Cannot checkout $name [$uuid] $tree_hash")
                            blacklist(uuid, tree_hash)
                            rm(tarball, force=true)
                            continue
                        end
                        updated = true
                        try
                            run_with_timeout(`git -C $clone_dir remote update`)
                        catch e
                            if isa(e, InterruptException)
                                rethrow(e)
                            end
                            print_exception(e)
                            @warn "Unable to update $name [$uuid]"
                            continue
                        end
                        @goto again
                    end
                    isfile(tarball) || continue
                end
                is_new_tarball || get_old_package_artifacts || continue
                # Clear the blacklist on this treehash
                clear_blacklist(uuid, tree_hash)

                # look for artifact files
                tmp_dir, paths = open(tarball) do io
                    paths = String[]
                    Tar.extract(decompress(io)) do hdr
                        if split(hdr.path, '/')[end] in artifact_names
                            push!(paths, hdr.path)
                            return true
                        else
                            return false
                        end
                    end, paths
                end
                for path in paths
                    sys_path = joinpath(tmp_dir, path)
                    try
                        artifacts = TOML.parse(read(sys_path, String))
                        for (key, val) in artifacts
                            if val isa Dict
                                process_artifact(val)
                            elseif val isa Vector
                                foreach(process_artifact, val)
                            else
                                @warn "invalid artifact file entry: $val" name path
                            end
                        end
                    catch err
                        @warn "error processing artifact file" error=err name path
                    end
                end
                chmod(tmp_dir, 0o700, recursive=true)
                rm(tmp_dir, recursive=true)
            end
            isempty(readdir(static_pkg_dir)) && rm(static_pkg_dir)
        end
    end
    @debug "Sync completed", string(DateTime(now()))
end

# generate current registries file
open(joinpath(static_dir, "registries"), write=true) do io
    for (uuid, tree_hash) in sort!(collect(registries))
        println(io, "/registry/$uuid/$tree_hash")
    end
end

max_stars_repo_name: StefanKarpinski/PkgServer

max_issues_repo_name: StefanKarpinski/PkgServer

max_forks_repo_name: StefanKarpinski/PkgServer


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder, Pkg

name = "tblis"
version = v"1.2.0"

# Collection of sources required to complete build
sources = [
    GitSource("https://github.com/devinamatthews/tblis.git", "3e4c4b82943726c443b6f408c9c9791dcad7a847")
]

# Bash recipe for building across all platforms
script = raw"""
cd $WORKSPACE/srcdir
cd tblis/
./configure --prefix=${prefix} --build=${MACHTYPE} --host=${target}
make -j${nproc}
cp lib/.libs/libtblis.so.0.0.0 ${libdir}/libtblis.so.0
cp src/external/tci/lib/.libs/libtci.so.0.0.0 ${libdir}/libtci.so.0
"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line
platforms = [
    Linux(:x86_64, libc=:glibc),
    Linux(:x86_64, libc=:musl)
]
platforms = expand_cxxstring_abis(platforms)

# The products that we will ensure are always built
products = [
    LibraryProduct("libtci", :tci),
    LibraryProduct("libtblis", :tblis)
]

# Dependencies that must be installed before this package can be built
dependencies = [
    Dependency(PackageSpec(name="Hwloc_jll", uuid="e33a78d0-f292-5ffc-b300-72abe9b543c8"))
    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"))
]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; preferred_gcc_version = v"7.1.0")


text:
#################### LengthDistributions ####################

"""
    exponentialBL(scale::Float64) <: ContinuousUnivariateDistribution

This structure implememts an exponential prior on the branch lengths of a tree.
"""
mutable struct exponentialBL <: ContinuousUnivariateDistribution
    scale::Float64
    constraints::Union{Dict, Missing}

    exponentialBL(scale::Float64) =
        new(scale, missing)
    exponentialBL(scale::Float64, c) =
            new(scale, c)
end

"""
    CompoundDirichlet(alpha::Float64, a::Float64, beta::Float64, c::Float64, nterm::Float64)

This structure implememts the CompoundDirichlet distribution described
in Zhang, Rannala and Yang 2012. (DOI:10.1093/sysbio/sys030)
"""
mutable struct CompoundDirichlet <: ContinuousUnivariateDistribution
    alpha::Float64
    a::Float64
    beta::Float64
    c::Float64
    constraints::Union{Dict, Missing}

    CompoundDirichlet(alpha::Float64, a::Float64, beta::Float64, c::Float64) =
        new(alpha, a, beta, c, missing)
    CompoundDirichlet(alpha::Float64, a::Float64, beta::Float64, c::Float64, constraints::Dict) =
        new(alpha, a, beta, c, constraints)
end # CompoundDirichlet

"""
    UniformBranchLength()

Fallback struct that is used when no length distribution is given

"""
struct UniformBranchLength <: ContinuousUnivariateDistribution end

const LengthDistribution = Union{CompoundDirichlet, exponentialBL, UniformBranchLength}

#################### TopologyDistributions ####################

"""
    UniformConstrained(
        constraint_dict::Dict{Symbol,
                              Union{Vector{Vector{S}}, Vector{Tuple{Vector{S}, Vector{S}}}}
                              } where S <: AbstractString)

Wrapper struct for the topology distribution. Contains a dictionary with constraints.
Empty constructor results in an empty dictionary as a fallback.
"""
mutable struct UniformConstrained <: ContinuousUnivariateDistribution
    constraint_dict::ConstraintDict

    UniformConstrained() =
        new(ConstraintDict())
end

"""
    UniformTopology()

Fallback struct that is used when no topology distribution is given.

"""
struct UniformTopology <: ContinuousUnivariateDistribution end

const TopologyDistribution = Union{UniformConstrained, UniformTopology}

"""
    TreeDistribution(length_distr::LengthDistribution, topology_distr::TopologyDistribution)

Wrapper struct for the TreeDistribution. Contains a length distribution and a topology
distribution. Can be constructed with no arguments, one of the distributions, or both of
them. Fallbacks are UniformBranchLength for the length distribution & and UniformTopology
for the topology distribution.
"""
mutable struct TreeDistribution <: ContinuousUnivariateDistribution
    length_distr::LengthDistribution
    topology_distr::TopologyDistribution

    TreeDistribution(l::LengthDistribution, c::ConstraintDict) =
        new(l, UniformConstrained(c))
    TreeDistribution(l::LengthDistribution) =
        new(l, UniformTopology())
    TreeDistribution(c::ConstraintDict) =
        new(UniformBranchLength(), UniformConstrained(c))
    TreeDistribution() =
        new(UniformBranchLength(), UniformTopology())
end

function insupport(d::TreeDistribution, x::GeneralNode)
    insupport(d.length_distr, x) && insupport(d.topology_distr, x)
end

function logpdf(d::TreeDistribution, x::GeneralNode, transform::Bool)
    logpdf(d.length_distr, x) + logpdf(d.topology_distr, x)
end

function gradlogpdf(d::TreeDistribution, x::GeneralNode)
    rl1, rl2 = gradlogpdf(d.length_distr, x)
    rt1, rt2 = gradlogpdf(d.topology_distr, x)
    return rl1+rt1, rl2 .+ rt2
end

# placeholder, can be made more elegant using the topology_distr
function relistlength(d::TreeDistribution, x::AbstractArray)
    #n = length(x)
    #(Array(x), n)
    x[1], 1
  end

max_stars_repo_name: erathorn/MCPhylo

max_issues_repo_name: erathorn/MCPhylo

max_forks_repo_name: erathorn/MCPhylo


text:
module EcologicalNetworks

# Dependencies
using StatsBase
using Combinatorics
using Distributions

using Random
using DelimitedFiles
using LinearAlgebra
using DataStructures

using Requires

# Various utilities for probabilities
include(joinpath(".", "misc/probabilities.jl"))

# Tests to define what can be used in base types
include(joinpath(".", "misc/init_tests.jl"))

# Types
include(joinpath(".", "types/declarations.jl"))
include(joinpath(".", "types/constructors.jl"))
include(joinpath(".", "types/conversions.jl"))
export AbstractEcologicalNetwork,
   # General types for all bipartite / unipartite
   AbstractBipartiteNetwork, AbstractUnipartiteNetwork,
   # Types
   BipartiteNetwork, UnipartiteNetwork,
   BipartiteProbabilisticNetwork, UnipartiteProbabilisticNetwork,
   BipartiteQuantitativeNetwork, UnipartiteQuantitativeNetwork,
   # Union types for all proba or deterministic
   ProbabilisticNetwork, DeterministicNetwork, QuantitativeNetwork,
   BinaryNetwork

# Datasets
include(joinpath(".", "misc/data.jl"))
export web_of_life, nz_stream_foodweb, pajek

function __init__()
   @require Mangal="b8b640a6-63d9-51e6-b784-5033db27bef2" begin
      function check_species_validity(::Mangal.MangalReferenceTaxon)
      end
      function check_species_validity(::Mangal.MangalNode)
      end
   end
   @require GBIF="ee291a33-5a6c-5552-a3c8-0f29a1181037" begin
      function check_species_validity(::GBIF.GBIFTaxon)
      end
   end
end

# General useful manipulations
include(joinpath(".", "types/utilities.jl"))
include(joinpath(".", "types/comparisons.jl"))
include(joinpath(".", "types/iteration.jl"))
export species, interactions, has_interaction, richness, nodiagonal, nodiagonal!

# Degree
include(joinpath(".", "links/degree.jl"))
export degree, degree_var

include(joinpath(".", "links/specificity.jl"))
export specificity

include(joinpath(".", "links/connectance.jl"))
export links, L, links_var, connectance, connectance_var,
   linkage_density, link_number

include(joinpath(".", "links/degenerate.jl"))
export isdegenerate, simplify, simplify!
   #, species_has_no_successors, species_has_no_predecessors,
   #species_is_free, free_species

# Random networks and permutations
include(joinpath(".", "rand/draws.jl"))
include(joinpath(".", "rand/sample.jl"))
export sample
include(joinpath(".", "rand/shuffle.jl"))
include(joinpath(".", "rand/null.jl"))
include(joinpath(".", "rand/linfilter.jl"))
export linearfilter, linearfilterzoo
export null1, null2, null3, null4

# Random networks from structural models
include(joinpath(".", "structuralmodels/cascademodel.jl"))
export cascademodel
include(joinpath(".", "structuralmodels/mpnmodel.jl"))
export mpnmodel
include(joinpath(".", "structuralmodels/nestedhierarchymodel.jl"))
export nestedhierarchymodel
include(joinpath(".", "structuralmodels/nichemodel.jl"))
export nichemodel

# Nestedness
include(joinpath(".", "community/nestedness.jl"))
export η, nodf

# Spectral radius
include(joinpath(".", "community/spectralradius.jl"))
export ρ

# Paths
include(joinpath(".", "community/paths.jl"))
export number_of_paths, shortest_path, bellman_ford, dijkstra

# Centrality
include(joinpath(".", "community/centrality.jl"))
export centrality_katz, centrality_closeness, centrality_degree

# Motifs
include(joinpath(".", "community/motifs.jl"))
export find_motif, expected_motif_count, unipartitemotifs

# Overlap
include(joinpath(".", "community/overlap.jl"))
export overlap
export AJS, EAJS

# Overlap
include(joinpath(".", "community/resilience.jl"))
export resilience
export symmetry, heterogeneity
export s
export σ_in, σ_out

# Modularity
include(joinpath(".", "modularity/utilities.jl"))
export Q, Qr

include(joinpath(".", "modularity/labelpropagation.jl"))
export lp, salp

include(joinpath(".", "modularity/starters.jl"))
export n_random_modules, each_species_its_module

include(joinpath(".", "modularity/brim.jl"))
export brim

include(joinpath(".", "modularity/roles.jl"))
export functional_cartography

# Beta-diversity
include(joinpath(".", "betadiversity/operations.jl"))
include(joinpath(".", "betadiversity/partitions.jl"))
export βs, βos, βwn

include(joinpath(".", "betadiversity/measures.jl"))
export KGL01, KGL02, KGL03, KGL04, KGL05, KGL06, KGL07, KGL08, KGL09, KGL10,
   KGL11, KGL12, KGL13, KGL14, KGL15, KGL16, KGL17, KGL18, KGL19, KGL20, KGL21,
   KGL22, KGL23, KGL24

# Food webs
include(joinpath(".", "foodwebs/trophiclevels.jl"))
export fractional_trophic_level, trophic_level

include(joinpath(".", "foodwebs/omnivory.jl"))
export omnivory

include(joinpath(".", "information/entropy.jl"))
export entropy, make_joint_distribution, mutual_information, conditional_entropy,
   variation_information, diff_entropy_uniform, information_decomposition,
   convert2effective, potential_information

end


text:
@safetestset "rastrigin" begin
    using Test, DifferentialEvolutionMCMC, Random, Distributions
    import DifferentialEvolutionMCMC: minimize!

    Random.seed!(78454111)

    function sample_prior()
       return [rand(Uniform(-5, 5), 2)]
    end

    bounds = ((-5.0,5.0),)
    names = (:x,)

    function rastrigin(data, x)
        A = 10.0
        n = length(x)
        y = A * n
        for  i in 1:n
            y +=  + x[i]^2 - A * cos(2 * π * x[i])
        end
        return y
    end

    model = DEModel(;
        sample_prior,
        loglike = rastrigin,
        data = nothing,
        names
    )

    de = DE(;
        sample_prior,
        bounds,
        Np = 6,
        n_groups = 1,
        update_particle! = minimize!,
        evaluate_fitness! = evaluate_fun!
    )

    n_iter = 10_000
    particles = optimize(model, de, n_iter, progress=true);
    results = get_optimal(de, model, particles)
    @test results[2] ≈ 0.0 atol = 1e-8
end

@safetestset "maximum likelihood estimation" begin
    using Test, DifferentialEvolutionMCMC, Random, Distributions
    import DifferentialEvolutionMCMC: maximize!

    Random.seed!(50514)

    # bounds of parameters, same order as above
    bounds = (
        (-Inf, Inf),
        (0.1, Inf),
    )
    # a function to optimize, data argument is required but ignored if not needed
    function loglike(data, μ, σ)
        return logpdf.(Normal(μ, σ), data) |> sum
    end

    data = rand(Normal(0, 1), 100)

    function prior_loglike(μ, σ)
        LL = 0.0
        LL += logpdf(Normal(0, 1), μ)
        LL += logpdf(truncated(Cauchy(0, 1), 0, Inf), σ)
        return LL
    end

    function sample_prior()
        μ = rand(Normal(0, 1))
        σ = rand(truncated(Cauchy(0, 1), 0, Inf))
        return [μ,σ]
    end

    names = (:μ,:σ)

    model = DEModel(;
        sample_prior,
        loglike,
        data,
        names
    )

    de = DE(;
        sample_prior,
        bounds,
        burnin = 1000,
        Np = 6,
        n_groups = 1,
        update_particle! = maximize!,
        evaluate_fitness! = evaluate_fun!
    )

    n_iter = 10000
    # # run the optimization algorithm
    particles = optimize(model, de, MCMCThreads(), n_iter, progress=true);
    # extract the optimal parameters
    parms,LL = get_optimal(de, model, particles)
    @test mean(data) ≈ parms.μ atol = .0001
    @test std(data; corrected=false) ≈ parms.σ atol = .0001
end


text:
"""
    truncated(d, l, u):

Truncate a distribution between `l` and `u`.
Builds the most appropriate distribution for the type of `d`,
the fallback is constructing a `Truncated` wrapper.

To implement a specialized truncated form for a distribution `D`,
the method `truncate(d::D, l::T, u::T) where {T <: Real}`
should be implemented.

# Arguments
- `d::UnivariateDistribution`: The original distribution.
- `l::Real`: The lower bound of the truncation, which can be a finite value or `-Inf`.
- `u::Real`: The upper bound of the truncation, which can be a finite value of `Inf`.

Throws an error if `l >= u`.
"""
function truncated(d::UnivariateDistribution, l::Real, u::Real)
    return truncated(d, promote(l, u)...)
end

function truncated(d::UnivariateDistribution, l::T, u::T) where {T <: Real}
    l < u || error("lower bound should be less than upper bound.")
    T2 = promote_type(T, eltype(d))
    lcdf = isinf(l) ? zero(T2) : T2(cdf(d, l))
    ucdf = isinf(u) ? one(T2) : T2(cdf(d, u))
    tp = ucdf - lcdf
    Truncated(d, promote(l, u, lcdf, ucdf, tp, log(tp))...)
end

truncated(d::UnivariateDistribution, l::Integer, u::Integer) = truncated(d, float(l), float(u))

"""
    Truncated(d, l, u):

Create a generic wrapper for a truncated distribution.
Prefer calling the function `truncated(d, l, u)`, which can choose the appropriate
representation of the truncated distribution.

# Arguments
- `d::UnivariateDistribution`: The original distribution.
- `l::Real`: The lower bound of the truncation, which can be a finite value or `-Inf`.
- `u::Real`: The upper bound of the truncation, which can be a finite value of `Inf`.
"""
struct Truncated{D<:UnivariateDistribution, S<:ValueSupport, T <: Real} <: UnivariateDistribution{S}
    untruncated::D      # the original distribution (untruncated)
    lower::T      # lower bound
    upper::T      # upper bound
    lcdf::T       # cdf of lower bound
    ucdf::T       # cdf of upper bound

    tp::T         # the probability of the truncated part, i.e. ucdf - lcdf
    logtp::T      # log(tp), i.e. log(ucdf - lcdf)
    function Truncated(d::UnivariateDistribution, l::T, u::T, lcdf::T, ucdf::T, tp::T, logtp::T) where {T <: Real}
        new{typeof(d), value_support(typeof(d)), T}(d, l, u, lcdf, ucdf, tp, logtp)
    end
end

### Constructors
function Truncated(d::UnivariateDistribution, l::T, u::T) where {T <: Real}
    l < u || error("lower bound should be less than upper bound.")
    T2 = promote_type(T, eltype(d))
    lcdf = isinf(l) ? zero(T2) : T2(cdf(d, l))
    ucdf = isinf(u) ? one(T2) : T2(cdf(d, u))
    tp = ucdf - lcdf
    Truncated(d, promote(l, u, lcdf, ucdf, tp, log(tp))...)
end

Truncated(d::UnivariateDistribution, l::Integer, u::Integer) = Truncated(d, float(l), float(u))

@deprecate Truncated(d::UnivariateDistribution, l::Real, u::Real) truncated(d, l, u)

params(d::Truncated) = tuple(params(d.untruncated)..., d.lower, d.upper)
partype(d::Truncated) = partype(d.untruncated)
### range and support

islowerbounded(d::Truncated) = islowerbounded(d.untruncated) || isfinite(d.lower)
isupperbounded(d::Truncated) = isupperbounded(d.untruncated) || isfinite(d.upper)

minimum(d::Truncated) = max(minimum(d.untruncated), d.lower)
maximum(d::Truncated) = min(maximum(d.untruncated), d.upper)

function insupport(d::Truncated{D,<:Union{Discrete,Continuous}}, x::Real) where {D<:UnivariateDistribution}
    return d.lower <= x <= d.upper && insupport(d.untruncated, x)
end

### evaluation

quantile(d::Truncated, p::Real) = quantile(d.untruncated, d.lcdf + p * d.tp)

function _pdf(d::Truncated, x::T) where {T<:Real}
    if d.lower <= x <= d.upper
        pdf(d.untruncated, x) / d.tp
    else
        zero(T)
    end
end

function pdf(d::Truncated{<:ContinuousUnivariateDistribution}, x::T) where {T<:Real}
    _pdf(d, float(x))
end

function pdf(d::Truncated{D}, x::T) where {D<:DiscreteUnivariateDistribution, T<:Real}
    isinteger(x) || return zero(float(T))
    _pdf(d, x)
end

function pdf(d::Truncated{D}, x::T) where {D<:DiscreteUnivariateDistribution, T<:Integer}
    _pdf(d, float(x))
end

function _logpdf(d::Truncated, x::T) where {T<:Real}
    if d.lower <= x <= d.upper
        logpdf(d.untruncated, x) - d.logtp
    else
        TF = float(T)
        -TF(Inf)
    end
end

function logpdf(d::Truncated{D}, x::T) where {D<:DiscreteUnivariateDistribution, T<:Real}
    TF = float(T)
    isinteger(x) || return -TF(Inf)
    return _logpdf(d, x)
end

function logpdf(d::Truncated{D}, x::Integer) where {D<:DiscreteUnivariateDistribution}
    _logpdf(d, x)
end

function logpdf(d::Truncated{D, Continuous}, x::T) where {D<:ContinuousUnivariateDistribution, T<:Real}
    _logpdf(d, x)
end

# fallback to avoid method ambiguities
_cdf(d::Truncated, x::T) where {T<:Real} =
    x <= d.lower ? zero(T) :
    x >= d.upper ? one(T) :
    (cdf(d.untruncated, x) - d.lcdf) / d.tp

cdf(d::Truncated, x::Real) = _cdf(d, x)
cdf(d::Truncated, x::Integer) = _cdf(d, float(x)) # float conversion for stability

function _logcdf(d::Truncated, x::T) where {T<:Real}
    TF = float(T)
    if x <= d.lower
        -TF(Inf)
    elseif x >= d.upper
        zero(TF)
    else
        log(cdf(d.untruncated, x) - d.lcdf) - d.logtp
    end
end

logcdf(d::Truncated, x::Real) = _logcdf(d, x)
logcdf(d::Truncated, x::Integer) = _logcdf(d, x)

_ccdf(d::Truncated, x::T) where {T<:Real} =
    x <= d.lower ? one(T) :
    x >= d.upper ? zero(T) :
    (d.ucdf - cdf(d.untruncated, x)) / d.tp

ccdf(d::Truncated, x::Real) = _ccdf(d, x)
ccdf(d::Truncated, x::Integer) = _ccdf(d, float(x))

function _logccdf(d::Truncated, x::T) where {T<:Real}
    TF = float(T)
    if x <= d.lower
        zero(TF)
    elseif x >= d.upper
        -TF(Inf)
    else
        log(d.ucdf - cdf(d.untruncated, x)) - d.logtp
    end
end

logccdf(d::Truncated, x::Real) = _logccdf(d, x)
logccdf(d::Truncated, x::Integer) = _logccdf(d, x)

## random number generation

function _rand!(rng::AbstractRNG, d::Truncated)
    d0 = d.untruncated
    if d.tp > 0.25
        while true
            r = _rand!(rng, d0)
            if d.lower <= r <= d.upper
                return r
            end
        end
    else
        return quantile(d0, d.lcdf + rand(rng) * d.tp)
    end
end

## show

function show(io::IO, d::Truncated)
    print(io, "Truncated(")
    d0 = d.untruncated
    uml, namevals = _use_multline_show(d0)
    uml ? show_multline(io, d0, namevals) :
          show_oneline(io, d0, namevals)
    print(io, ", range=($(d.lower), $(d.upper)))")
    uml && println(io)
end

_use_multline_show(d::Truncated) = _use_multline_show(d.untruncated)

### specialized truncated distributions

include(joinpath("truncated", "normal.jl"))
include(joinpath("truncated", "exponential.jl"))
include(joinpath("truncated", "uniform.jl"))


text: for n = 1:10-2 3 end

max_stars_repo_name: gabriel-doriath-dohler/pjulia

max_issues_repo_name: gabriel-doriath-dohler/pjulia

max_forks_repo_name: gabriel-doriath-dohler/pjulia


text:
# Vector of (eclassid, position_of_literal_in_eclass_nodes)
struct Sub
    sourcenode::Union{Nothing, ENode}
    ids::Vector{EClassId}
    nodes::Vector{Union{Nothing,ENode}}
end

haseclassid(sub::Sub, p::PatVar) = sub.ids[p.idx] >= 0
geteclassid(sub::Sub, p::PatVar) = sub.ids[p.idx]

hasliteral(sub::Sub, p::PatVar) = sub.nodes[p.idx] !== nothing
getliteral(sub::Sub, p::PatVar) = sub.nodes[p.idx]

## ====================== Instantiation =======================

function instantiate(g::EGraph, pat::PatVar, sub::Sub, rule::AbstractRule)
    if haseclassid(sub, pat)
        ec = geteclass(g, geteclassid(sub, pat))
        if hasliteral(sub, pat)
            node = getliteral(sub, pat)
            @assert arity(node) == 0
            return node.head
        end
        return ec
    else
        error("unbound pattern variable $pat in rule $rule")
    end
end

function instantiate(g::EGraph, pat::PatLiteral{T}, sub::Sub, rule::AbstractRule) where T
    pat.val
end

function instantiate(g::EGraph, pat::PatTypeAssertion, sub::Sub, rule::AbstractRule)
    instantiate(g, pat.name, sub, rule)
end

function instantiate(g::EGraph, pat::PatTerm, sub::Sub, rule::AbstractRule)
    f = pat.head
    ar = arity(pat)
    if pat.head == :call
        @assert pat.args[1] isa PatLiteral
        f = pat.args[1].val
        ar = ar-1
    end

    T = gettermtype(g, f, ar)
    children = map(x -> instantiate(g, x, sub, rule), pat.args)
    similarterm(T, pat.head, children)
end


text:
using PyPlot
using Convex
using PyCall
using HDF5, JLD
@pyimport matplotlib2tikz

#using CenterOfMass
include("CenterOfMass.jl")
include("setup_parameters.jl")

n_trial = 100
n_measurement = 20
#n_trial = 5
#n_measurement = 10

plt[:close]("all")

prior = initialize_prior(circle_ps, resolution, interior_q, masses)

if false
@time critical_forces_by_point = map(attachment_ws) do wrench
  get_critical_values(circle_ws, wrench, prior, interior_q)
end
end

if false
@time normals_matrices = map(critical_forces_by_point) do forces
  normal_matrix(forces[:], sigma^2)
end
end

rand_com() = r_attach * rand_in_circle()[1:2] # hack
rand_mass() = min_mass + rand() * (max_mass - min_mass)

rand_parameters() = [rand_com(); rand_mass()]

thetas = hcat(map(x->rand_parameters(), 1:n_trial)...)

select_random(prior) = rand(1:length(attachment_ps))

function select_csqmi(prior)
  n = length(critical_forces_by_point)
  csqmis = map(1:n) do ii
    compute_mutual_information(get_data(prior)[:], normals_matrices[ii])
  end

  indmax(csqmis)
end

selection_functions = [select_csqmi, select_random]
nf = length(selection_functions)

estimates = cell(n_trial, nf, n_measurement)
errors = zeros(n_trial, nf, n_measurement)

if false
for ii = 1:n_trial
  theta = thetas[:,ii]
  com_p = [theta[1:2];0]
  mass = theta[3]

  println("trial $(ii)")

  for jj = 1:nf
    println("function $(jj)")
    selection_function = selection_functions[jj]
    belief = deepcopy(prior)

    for kk = 1:n_measurement
      index = selection_function(belief)
      applied_p = attachment_ps[index]

      applied_f = critical_force_from_points(circle_ps, com_p, applied_p, mass)

      f_hat = applied_f + sigma * randn()

      critical_forces = critical_forces_by_point[index]

      update_prior!(belief, critical_forces, f_hat, sigma)

      estimate = weighted_average(belief)

      estimates[ii,jj,kk] = estimate
      errors[ii,jj,kk] = normalized_error(estimate - theta)
    end
  end
end
else
@load "greedy_vs_random/data.old.good" errors thetas estimates
for ii = 1:n_trial
  theta = thetas[:,ii]
  for jj = 1:nf
    for kk = 1:n_measurement
      estimate = estimates[ii,jj,kk]
      errors[ii,jj,kk] = normalized_error(estimate - theta)
    end
  end
end
end

colors = ["b" "g"]

mean_error = mean(errors, 1)
plot(1:n_measurement, mean_error[:,1,:][:], color = "b", linewidth = 3.0)
plot(1:n_measurement, mean_error[:,2,:][:], color = "g", linewidth = 3.0)
legend(["csqmi", "random"])

indices = collect(1:n_measurement)
for ii = 1:size(errors,2)
  stds = std(errors[:,ii,:],1)[:]/sqrt(size(errors,1))
  es = mean_error[:,ii,:][:]
  fill([indices;reverse(indices)], [es+stds;reverse(es-stds)], color = colors[ii],
    alpha=0.2, linewidth=0.0)
end

if false
for ii = 1:size(errors, 2)
  for jj = 1:size(errors, 1)
    plot(1:n_measurement, errors[jj, ii,:][:], color = colors[ii], alpha = 0.1,
    linewidth = 1.0)
  end
end
end
plot([0;20],[0.1;0.1],linestyle="--", color="k")
plot([7;7],[0.0;0.1],linestyle="--", color="b")
plot([13;13],[0.0;0.1],linestyle="--", color="g")

xlabel("Iteration")
ylabel("Normalized error")

#@save "greedy_vs_random/data" errors thetas estimates

matplotlib2tikz.save("greedy_vs_random/convergence.tex", figureheight="\\figureheight",
  figurewidth="\\figurewidth")

max_stars_repo_name: mcorah/CenterOfMass

max_issues_repo_name: mcorah/CenterOfMass

max_forks_repo_name: mcorah/CenterOfMass