text:
include("table/support.jl")
include("table/residues.jl")
include("table/RSparseBitSet.jl")

function init_constraint_struct(::TableSetInternal, internals)
    TableConstraint(
        internals,
        RSparseBitSet(),
        TableSupport(), # will be filled in init_constraint!
        Int[], # will be changes later as it needs the number of words
        TableResidues(),
        Vector{TableBacktrackInfo}(),
        Int[], # changed_vars
        Int[], # unfixed_vars
        Int[], # sum_min
        Int[],  # sum_max
    )
end

"""
    init_constraint!(com::CS.CoM, constraint::TableConstraint, fct::MOI.VectorOfVariables, set::TableSetInternal)

"""
function init_constraint!(
    com::CS.CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal
)
    table = set.table
    num_pos_rows = size(table, 1)
    constraint.last_sizes = zeros(Int, length(constraint.indices))

    possible_rows = trues(num_pos_rows)
    search_space = com.search_space

    indices = constraint.indices
    for row_id in 1:size(table)[1]
        for (i, vidx) in enumerate(indices)
            if !has(search_space[vidx], table[row_id, i])
                possible_rows[row_id] = false
                num_pos_rows -= 1
                break
            end
        end
    end
    num_pos_rows == 0 && return false
    pos_rows_idx = findall(possible_rows)
    row_sums = nothing

    # if there is an lp model we compute bounds for the sum
    if com.lp_model !== nothing
        row_sums = sum(table[possible_rows, :]; dims = 2)[:, 1]
        local_sort_perm = sortperm(row_sums)
        row_sums = row_sums[local_sort_perm]
        pos_rows_idx = pos_rows_idx[local_sort_perm]
        # initial bounds for sum(variables[indices])

        # currently the constraint can be inactive. Bounds will be set in activate_constraint!
        table_min = typemin(Int)
        table_max = typemax(Int)

        lp_backend = backend(com.lp_model)
        lp_vidx =
            create_lp_variable!(com.lp_model, com.lp_x; lb = table_min, ub = table_max)
        # create == constraint with sum of all variables equal the newly created variable
        sats = [MOI.ScalarAffineTerm(1.0, MOI.VariableIndex(vidx)) for vidx in indices]
        push!(sats, MOI.ScalarAffineTerm(-1.0, MOI.VariableIndex(lp_vidx)))
        saf = MOI.ScalarAffineFunction(sats, 0.0)
        MOI.add_constraint(lp_backend, saf, MOI.EqualTo(0.0))
        constraint.bound_rhs = [BoundRhsVariable(lp_vidx, table_min, table_max)]
    end

    support = constraint.supports

    support.var_start = cumsum([length(search_space[vidx].init_vals) for vidx in indices])
    support.var_start .+= 1
    num_supports = support.var_start[end] - 1

    pushfirst!(support.var_start, 1)
    # println("var_start: $(support.var_start)")
    num_64_words = cld(num_pos_rows, 64)
    # println("num_64_words: $num_64_words")

    # define RSparseBitSet
    rsbs = constraint.current
    rsbs.words = fill(~zero(UInt64), num_64_words)
    rsbs.indices = 1:num_64_words
    rsbs.last_ptr = num_64_words
    rsbs.mask = fill(~zero(UInt64), num_64_words)
    ending_ones = (num_pos_rows - 1) % 64 + 1
    rsbs.words[end] = rsbs.words[end] .⊻ ((UInt64(1) << (64 - ending_ones)) - 1)

    support.values = zeros(UInt64, (num_64_words, num_supports))
    # if we have an LP model we store sum_min and sum_max for each 64 block
    if row_sums !== nothing
        constraint.sum_min = zeros(Int64, num_64_words)
        constraint.sum_max = fill(row_sums[end], num_64_words)
    end

    num_64_idx = 0
    rp_idx = 0
    for r in pos_rows_idx
        rp_idx += 1
        if rp_idx % 64 == 1
            num_64_idx += 1
        end
        if row_sums !== nothing
            # store the smallest number in the block
            if rp_idx % 64 == 1
                constraint.sum_min[num_64_idx] = row_sums[rp_idx]
            elseif rp_idx % 64 == 0 # store the biggest number in the block
                constraint.sum_max[num_64_idx] = row_sums[rp_idx]
            end
        end
        for (c, vidx) in enumerate(indices)
            variable = search_space[vidx]
            var_offset = variable.offset
            table_val = table[r, c]
            val_idx = variable.init_val_to_index[table_val + var_offset]
            @assert variable.init_vals[val_idx] == table_val
            support_col_idx = support.var_start[c] + val_idx - 1
            # println("support_col_idx: $support_col_idx")
            cell_64 = support.values[num_64_idx, support_col_idx]
            pos_in_64 = (rp_idx - 1) % 64 + 1
            support.values[num_64_idx, support_col_idx] =
                cell_64 .| (UInt64(1) << (64 - pos_in_64))
        end
    end

    # define residues
    residues = constraint.residues
    residues.var_start = copy(support.var_start)
    residues.values = zeros(Int, num_supports)
    for sc in 1:num_supports
        idx = support.values[:, sc]
        residues.values[sc] = intersect_index(rsbs, idx)
        @assert residues.values[sc] <= num_64_words
    end

    # define last_sizes
    constraint.last_sizes = [CS.nvalues(com.search_space[vidx]) for vidx in indices]
    return true
end

function activate_constraint!(
    com::CS.CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal
)
    search_space = com.search_space
    indices = constraint.indices
    support = constraint.supports

    num_supports = support.var_start[end] - 1
    # check if a support column is completely zero
    # that means that the variable corresponding to that column can't have the value corresponding to the column
    feasible = true
    for c in 1:num_supports
        if all(i -> i == UInt64(0), support.values[:, c])
            # getting the correct index in indices
            var_i = 1
            while support.var_start[var_i] <= c
                var_i += 1
            end
            var_i -= 1
            val_i = c - support.var_start[var_i] + 1
            var = indices[var_i]
            val = search_space[var].init_vals[val_i]
            if has(search_space[var], val)
                feasible = rm!(com, search_space[var], val)
                if !feasible
                    break
                end
            end
        end
    end

    # vidx = 0 such that the current best bound is computed not the one when limiting vidx to something
    update_best_bound_constraint!(com, constraint, fct, set, 0, 0, 0)
    return feasible
end

function update_table(com::CoM, constraint::TableConstraint)
    current = constraint.current
    supports = constraint.supports
    indices = constraint.indices
    variables = com.search_space
    backtrack_idx = com.c_backtrack_idx
    for local_vidx in constraint.changed_vars
        vidx = indices[local_vidx]
        var = variables[vidx]
        clear_mask(current)
        if num_removed(var) < nvalues(var) && !isfixed(var)
            for value in view_removed_values(var)
                support = get_view(supports, com, vidx, local_vidx, value)
                add_to_mask(current, support)
            end
            invert_mask(current)
        else
            # reset based update
            for value in view_values(var)
                support = get_view(supports, com, vidx, local_vidx, value)
                add_to_mask(current, support)
            end
        end

        intersect_with_mask(current)
        is_empty(current) && break
    end
end

function filter_domains(com::CoM, constraint::TableConstraint)
    current = constraint.current
    supports = constraint.supports
    residues = constraint.residues
    indices = constraint.indices
    variables = com.search_space
    feasible = true
    changed = false
    for local_vidx in constraint.unfixed_vars
        vidx = indices[local_vidx]
        for value in CS.values(variables[vidx])
            idx = residues[com, vidx, local_vidx, value]
            # residues is 0 when the constraint was inactive at the beginning
            if idx == 0
                if has(variables[vidx], value)
                    changed = true
                    if !rm!(com, variables[vidx], value)
                        feasible = false
                        break
                    end
                    @assert !has(variables[vidx], value)
                end
            elseif current.words[idx] & supports[com, vidx, local_vidx, value, idx] ==
                   UInt64(0)
                support = get_view(supports, com, vidx, local_vidx, value)
                idx = intersect_index(current, support)
                if idx != 0
                    residues[com, vidx, local_vidx, value] = idx
                else
                    if has(variables[vidx], value)
                        changed = true
                        if !rm!(com, variables[vidx], value)
                            feasible = false
                            break
                        end
                        @assert !has(variables[vidx], value)
                    end
                end
            end
        end
        constraint.last_sizes[local_vidx] = CS.nvalues(variables[vidx])
        !feasible && break
    end
    return feasible, changed
end

"""
    prune_constraint!(com::CS.CoM, constraint::TableConstraint, fct::MOI.VectorOfVariables, set::TableSetInternal; logs = true)

Reduce the number of possibilities given the `TableConstraint`.
Return whether still feasible and throws a warning if infeasible and `logs` is set to `true`
"""
function prune_constraint!(
    com::CS.CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal;
    logs = true,
)
    current = constraint.current
    indices = constraint.indices
    variables = com.search_space

    changed = true
    feasible = true
    while changed
        # All local indices (1 corresponds to indices[1]) which have changed
        constraint.changed_vars = findall(
            x -> CS.nvalues(variables[indices[x]]) != constraint.last_sizes[x],
            1:length(indices),
        )
        for local_vidx in constraint.changed_vars
            constraint.last_sizes[local_vidx] = CS.nvalues(variables[indices[local_vidx]])
        end
        constraint.unfixed_vars =
            findall(x -> constraint.last_sizes[x] > 1, 1:length(indices))
        if length(constraint.changed_vars) != 0
            update_table(com, constraint)
            if is_empty(current)
                feasible = false
                break
            end
        end

        feasible, changed = filter_domains(com, constraint)
        !feasible && break
    end
    empty!(constraint.changed_vars)
    return feasible
end

function finished_pruning_constraint!(
    com::CS.CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
)

    @assert com.c_backtrack_idx > 0

    backtrack = constraint.backtrack
    while length(backtrack) < com.c_backtrack_idx
        push!(backtrack, TableBacktrackInfo(UInt64[], zero(UInt64), Int[]))
    end

    backtrack[com.c_backtrack_idx].words = copy(constraint.current.words)
    backtrack[com.c_backtrack_idx].last_ptr = constraint.current.last_ptr
    backtrack[com.c_backtrack_idx].indices = copy(constraint.current.indices)
    for (i, vidx) in enumerate(constraint.indices)
        constraint.last_sizes[i] = CS.nvalues(com.search_space[vidx])
    end
end

"""
    still_feasible(com::CoM, constraint::TableConstraint, fct::MOI.VectorOfVariables, set::TableSetInternal, vidx::Int, value::Int)

Return whether the constraint can be still fulfilled when setting a variable with index `vidx` to `value`.
"""
function still_feasible(
    com::CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    vidx::Int,
    value::Int,
)
    current = constraint.current
    supports = constraint.supports
    indices = constraint.indices
    full_mask(current)
    for i in 1:length(indices)
        if indices[i] == vidx
            support = get_view(supports, com, vidx, i, value)
            intersect_mask_with_mask(current, support)
        elseif isfixed(com.search_space[indices[i]])
            support = get_view(
                supports,
                com,
                indices[i],
                i,
                CS.value(com.search_space[indices[i]]),
            )
            intersect_mask_with_mask(current, support)
        end
    end
    feasible = intersect_with_mask_feasible(current)

    return feasible
end

"""
    update_best_bound_constraint!(com::CS.CoM,
        constraint::TableConstraint,
        fct::MOI.VectorOfVariables,
        set::TableSetInternal,
        vidx::Int,
        lb::Int,
        ub::Int
    )

Update the bound constraint associated with this constraint. This means that the `bound_rhs` bounds will be changed according to
the possible values the table constraint allows. `vidx`, `lb` and `ub` are not considered atm.
Additionally only a rough estimated bound is used which can be computed relatively fast.
"""
function update_best_bound_constraint!(
    com::CS.CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    vidx::Int,
    vlb::Int,
    vub::Int,
)
    isempty(constraint.bound_rhs) && return
    sum_min = constraint.sum_min
    sum_max = constraint.sum_max
    bitset = constraint.current

    bound_rhs = constraint.bound_rhs[1]

    lb = typemax(Int)
    ub = typemin(Int)

    @inbounds for i in 1:(bitset.last_ptr)
        idx = bitset.indices[i]
        lb = min(lb, sum_min[idx])
        ub = max(ub, sum_max[idx])
    end
    bound_rhs.lb = lb
    bound_rhs.ub = ub
end

"""
    single_reverse_pruning_constraint!(
        com::CoM,
        constraint::TableConstraint,
        fct::MOI.VectorOfVariables,
        set::TableSetInternal,
        var::Variable,
        backtrack_idx::Int
    )

It gets called after the variables returned to their state after backtracking.
A single reverse pruning step for the TableConstraint.
Add the removed values to the mask and in `reverse_pruning_constraint` the corresponding table rows
will be reactivated.
"""
function single_reverse_pruning_constraint!(
    com::CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    var::Variable,
    backtrack_idx::Int,
)
    current = constraint.current
    variables = com.search_space
    supports = constraint.supports
    residues = constraint.residues
    indices = constraint.indices
    loc_vidx = 1
    vidx = var.idx
    changes = var.changes[backtrack_idx]
    while loc_vidx <= length(indices)
        if vidx == indices[loc_vidx]
            break
        end
        loc_vidx += 1
    end
    @assert loc_vidx <= length(indices)
    @assert indices[loc_vidx] == vidx

    constraint.last_sizes[loc_vidx] = CS.nvalues(variables[vidx])

    push!(constraint.changed_vars, loc_vidx)
end

"""
    reset_residues!(com, constraint::TableConstraint)

Reset residues for constraint.changed_vars
"""
function reset_residues!(com, constraint::TableConstraint)
    supports = constraint.supports
    residues = constraint.residues
    current = constraint.current
    indices = constraint.indices
    variables = com.search_space
    num_residues = length(residues.values)
    for local_vidx in constraint.changed_vars
        vidx = indices[local_vidx]
        var = variables[vidx]
        for val_idx in (var.first_ptr):(var.last_ptr)
            support = get_view(supports, com, vidx, local_vidx, var.values[val_idx])
            new_residue = intersect_index(current, support)
            if new_residue != 0
                residues[com, vidx, local_vidx, var.values[val_idx]] = new_residue
            end
        end
    end
end

"""
    reverse_pruning_constraint!(
        com::CoM,
        constraint::TableConstraint,
        fct::MOI.VectorOfVariables,
        set::TableSetInternal,
    )

Is called after `single_reverse_pruning_constraint!`.
"""
function reverse_pruning_constraint!(
    com::CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    backtrack_id::Int,
)
    isempty(constraint.changed_vars) && return
    current = constraint.current
    if backtrack_id == 1
        current.last_ptr = length(current.words)
        current.words = fill(~zero(UInt64), current.last_ptr)
        current.indices = 1:(current.last_ptr)
    else
        parent = com.backtrack_vec[backtrack_id].parent_idx
        if parent <= length(constraint.backtrack) &&
           !isempty(constraint.backtrack[parent].words)
            current.last_ptr = constraint.backtrack[parent].last_ptr
            current.words = copy(constraint.backtrack[parent].words)
            current.indices = copy(constraint.backtrack[parent].indices)
        end
        # otherwise there is nothing to reverse
    end
    reset_residues!(com, constraint)
    empty!(constraint.changed_vars)
end

"""
    restore_pruning_constraint!(
        com::CoM,
        constraint::TableConstraint,
        fct::MOI.VectorOfVariables,
        set::TableSetInternal,
    )

Is called after `restore_prune!`.
"""
function restore_pruning_constraint!(
    com::CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    prune_steps::Union{Int,Vector{Int}},
)
    # revert to the last of prune steps
    constraint.changed_vars = collect(1:length(constraint.indices))
    current = constraint.current
    backtrack_id = last(prune_steps)
    while backtrack_id > length(constraint.backtrack) ||
        isempty(constraint.backtrack[backtrack_id].words)
        backtrack_id = com.backtrack_vec[backtrack_id].parent_idx
    end
    current.last_ptr = constraint.backtrack[backtrack_id].last_ptr
    current.words = copy(constraint.backtrack[backtrack_id].words)
    current.indices = copy(constraint.backtrack[backtrack_id].indices)
    for (i, vidx) in enumerate(constraint.indices)
        constraint.last_sizes[i] = CS.nvalues(com.search_space[vidx])
    end
    reset_residues!(com, constraint)
    empty!(constraint.changed_vars)
end

function is_constraint_solved(
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
    values::Vector{Int},
)

    table = set.table
    return findfirst(ri -> table[ri, :] == values, 1:size(table)[1]) !== nothing
end

"""
    is_constraint_violated(
        com::CoM,
        constraint::TableConstraint,
        fct::MOI.VectorOfVariables,
        set::TableSetInternal,
    )

Checks if the constraint is violated as it is currently set. This can happen inside an
inactive reified or indicator constraint.
"""
function is_constraint_violated(
    com::CoM,
    constraint::TableConstraint,
    fct::MOI.VectorOfVariables,
    set::TableSetInternal,
)
    if all(isfixed(var) for var in com.search_space[constraint.indices])
        return !is_constraint_solved(
            constraint,
            fct,
            set,
            [CS.value(var) for var in com.search_space[constraint.indices]],
        )
    end
    return false
end


text:
# This is a slightly modified version of ProgressMeter.jl
#
# The MIT License (MIT)
# Copyright (c) 2013 Timothy E. Holy
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

module ProgressMeter

using Printf: @sprintf

export Progress, ProgressUnknown, next!, update!, cancel, finish!

"""
`ProgressMeter` contains a suite of utilities for displaying progress
in long-running computations. The major functions/types in this module
are:

- `@showprogress`: an easy interface for straightforward situations
- `Progress`: an object for managing progress updates with a predictable number of iterations
- `ProgressThresh`: an object for managing progress updates where termination is governed by a threshold
- `next!` and `update!`: report that progress has been made
- `cancel` and `finish!`: early termination
"""
ProgressMeter

abstract type AbstractProgress end

"""
Holds the five characters that will be used to generate the progress bar.
"""
mutable struct BarGlyphs
    leftend::Char
    fill::Char
    front::Union{Vector{Char},Char}
    empty::Char
    rightend::Char
end
"""
String constructor for BarGlyphs - will split the string into 5 chars
"""
function BarGlyphs(s::AbstractString)
    glyphs = (s...,)
    if !isa(glyphs, NTuple{5,Char})
        error("""
            Invalid string in BarGlyphs constructor.
            You supplied "$s".
            Note: string argument must be exactly 5 characters long, e.g. "[=> ]".
        """)
    end
    return BarGlyphs(glyphs...)
end

"""
`prog = Progress(n; dt=0.1, desc="Progress: ", color=:green,
output=stdout, barlen=tty_width(desc))` creates a progress meter for a
task with `n` iterations or stages. Output will be generated at
intervals at least `dt` seconds apart, and perhaps longer if each
iteration takes longer than `dt`. `desc` is a description of
the current task.
"""
mutable struct Progress <: AbstractProgress
    n::Int
    dt::Float64
    counter::Int
    tfirst::Float64
    tlast::Float64
    printed::Bool           # true if we have issued at least one status update
    desc::AbstractString    # prefix to the percentage, e.g.  "Computing..."
    barlen::Int             # progress bar size (default is available terminal width)
    barglyphs::BarGlyphs    # the characters to be used in the bar
    color::Symbol           # default to green
    output::IO              # output stream into which the progress is written
    offset::Int             # position offset of progress bar (default is 0)
    numprintedvalues::Int   # num values printed below progress in last iteration
    clear_output_ijulia::Bool  # flush current display in IJulia possibly overwriting other printed things

    function Progress(
        n::Integer;
        dt::Real = 0.1,
        desc::AbstractString = "Progress: ",
        color::Symbol = :green,
        output::IO = stdout,
        delay::Real = 0.0,
        barlen::Integer = tty_width(desc),
        barglyphs::BarGlyphs = BarGlyphs(
            '|',
            '█',
            Sys.iswindows() ? '█' : ['▏', '▎', '▍', '▌', '▋', '▊', '▉'],
            ' ',
            '|',
        ),
        offset::Int = 0,
        clear_output_ijulia::Bool = false,
    )
        counter = 0
        tfirst = time()
        tlast = tfirst + delay
        printed = false
        new(
            n,
            dt,
            counter,
            tfirst,
            tlast,
            printed,
            desc,
            barlen,
            barglyphs,
            color,
            output,
            offset,
            0,
            clear_output_ijulia,
        )
    end
end

Progress(
    n::Integer,
    dt::Real,
    desc::AbstractString = "Progress: ",
    barlen::Integer = tty_width(desc),
    color::Symbol = :green,
    output::IO = stdout;
    kwargs...,
) = Progress(
    n;
    dt = dt, desc = desc, barlen = barlen, color = color, output = output, kwargs...,
)

Progress(n::Integer, desc::AbstractString, offset::Integer = 0) =
    Progress(n, desc = desc, offset = offset)

"""
`prog = ProgressUnknown(; dt=0.1, desc="Progress: ",
color=:green, output=stdout)` creates a progress meter for a task
which has a non-deterministic termination criterion.
Output will be generated at intervals at least `dt` seconds
apart, and perhaps longer if each iteration takes longer than
`dt`. `desc` is a description of the current task.
"""
mutable struct ProgressUnknown <: AbstractProgress
    done::Bool
    dt::Float64
    counter::Int
    triggered::Bool
    tfirst::Float64
    tlast::Float64
    printed::Bool        # true if we have issued at least one status update
    desc::AbstractString # prefix to the percentage, e.g.  "Computing..."
    color::Symbol        # default to green
    output::IO           # output stream into which the progress is written
    numprintedvalues::Int   # num values printed below progress in last iteration
    clear_output_ijulia::Bool      # flush current display in IJulia possibly overwriting other printed things
end

function ProgressUnknown(;
    dt::Real = 0.1,
    desc::AbstractString = "Progress: ",
    color::Symbol = :green,
    output::IO = stdout,
    delay::Real = 0.0,
    clear_output_ijulia::Bool = false,
)
    tfirst = time()
    tlast = tfirst + delay
    printed = false
    ProgressUnknown(
        false,
        dt,
        0,
        false,
        tfirst,
        tlast,
        printed,
        desc,
        color,
        output,
        0,
        clear_output_ijulia,
    )
end

ProgressUnknown(
    dt::Real,
    desc::AbstractString = "Progress: ",
    color::Symbol = :green,
    output::IO = stdout;
    kwargs...,
) = ProgressUnknown(dt = dt, desc = desc, color = color, output = output; kwargs...)

ProgressUnknown(desc::AbstractString; kwargs...) = ProgressUnknown(; desc = desc, kwargs...)

#...length of percentage and ETA string with days is 29 characters
tty_width(desc) = max(0, displaysize(stdout)[2] - (length(desc) + 29))

flush_display(p) = p.clear_output_ijulia && isdefined(Main, :IJulia)

# update progress display
function updateProgress!(
    p::Progress;
    showvalues = Any[],
    valuecolor = :blue,
    offset::Integer = p.offset,
    keep = (offset == 0),
)
    p.offset = offset
    t = time()
    if p.counter >= p.n
        if p.counter == p.n && p.printed
            percentage_complete = 100.0 * p.counter / p.n
            bar = barstring(p.barlen, percentage_complete, barglyphs = p.barglyphs)
            dur = durationstring(t - p.tfirst)
            msg =
                @sprintf "%s%3u%%%s Time: %s" p.desc round(Int, percentage_complete) bar dur
            !flush_display(p) && print(p.output, "\n"^(p.offset + p.numprintedvalues))
            move_cursor_up_while_clearing_lines(
                p.output,
                p.numprintedvalues,
                flush_display(p),
            )
            printover(p.output, msg, p.color, flush_display(p))
            printvalues!(p, showvalues; color = valuecolor)
            if keep
                println(p.output)
            else
                !flush_display(
                    p,
                ) && print(p.output, "\r\u1b[A"^(p.offset + p.numprintedvalues))
            end
            flush(p.output)
        end
        return nothing
    end

    if t > p.tlast + p.dt
        percentage_complete = 100.0 * p.counter / p.n
        bar = barstring(p.barlen, percentage_complete, barglyphs = p.barglyphs)
        elapsed_time = t - p.tfirst
        est_total_time = 100 * elapsed_time / percentage_complete
        if 0 <= est_total_time <= typemax(Int)
            eta_sec = round(Int, est_total_time - elapsed_time)
            eta = durationstring(eta_sec)
        else
            eta = "N/A"
        end
        msg = @sprintf "%s%3u%%%s  ETA: %s" p.desc round(Int, percentage_complete) bar eta
        !flush_display(p) && print(p.output, "\n"^(p.offset + p.numprintedvalues))
        move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues, flush_display(p))
        printover(p.output, msg, p.color, flush_display(p))
        printvalues!(p, showvalues; color = valuecolor)
        !flush_display(p) && print(p.output, "\r\u1b[A"^(p.offset + p.numprintedvalues))
        flush(p.output)
        # Compensate for any overhead of printing. This can be
        # especially important if you're running over a slow network
        # connection.
        p.tlast = t + 2 * (time() - t)
        p.printed = true
    end
    return nothing
end

function updateProgress!(p::ProgressUnknown; showvalues = Any[], valuecolor = :blue)
    t = time()
    if p.done
        if p.printed
            dur = durationstring(t - p.tfirst)
            msg = @sprintf "%s %d \t Time: %s" p.desc p.counter dur
            move_cursor_up_while_clearing_lines(
                p.output,
                p.numprintedvalues,
                flush_display(p),
            )
            printover(p.output, msg, p.color, flush_display(p))
            printvalues!(p, showvalues; color = valuecolor)
            println(p.output)
            flush(p.output)
        end
        return
    end

    if t > p.tlast + p.dt
        dur = durationstring(t - p.tfirst)
        msg = @sprintf "%s %d \t Time: %s" p.desc p.counter dur
        move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues, flush_display(p))
        printover(p.output, msg, p.color, flush_display(p))
        printvalues!(p, showvalues; color = valuecolor)
        flush(p.output)
        # Compensate for any overhead of printing. This can be
        # especially important if you're running over a slow network
        # connection.
        p.tlast = t + 2 * (time() - t)
        p.printed = true
        return
    end
end

# update progress display
"""
`next!(prog, [color])` reports that one unit of progress has been
made. Depending on the time interval since the last update, this may
or may not result in a change to the display.

You may optionally change the color of the display. See also `update!`.
"""
function next!(p::Union{Progress,ProgressUnknown}; options...)
    p.counter += 1
    updateProgress!(p; options...)
end

function next!(p::Union{Progress,ProgressUnknown}, color::Symbol; options...)
    p.color = color
    next!(p; options...)
end

"""
`update!(prog, counter, [color])` sets the progress counter to
`counter`, relative to the `n` units of progress specified when `prog`
was initialized.  Depending on the time interval since the last
update, this may or may not result in a change to the display.

If `prog` is a `ProgressThresh`, `update!(prog, val, [color])` specifies
the current value.

You may optionally change the color of the display. See also `next!`.
"""
function update!(p::Union{Progress,ProgressUnknown}, counter::Int; options...)
    p.counter = counter
    updateProgress!(p; options...)
end

function update!(
    p::Union{Progress,ProgressUnknown},
    counter::Int,
    color::Symbol;
    options...,
)
    p.color = color
    update!(p, counter; options...)
end

"""
`cancel(prog, [msg], [color=:red])` cancels the progress display
before all tasks were completed. Optionally you can specify the
message printed and its color.

See also `finish!`.
"""
function cancel(
    p::AbstractProgress,
    msg::AbstractString = "Aborted before all tasks were completed",
    color = :red;
    showvalues = Any[],
    valuecolor = :blue,
    offset = p.offset,
    keep = (offset == 0),
)
    p.offset = offset
    if p.printed
        print(p.output, "\n"^(p.offset + p.numprintedvalues))
        move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues, flush_display(p))
        printover(p.output, msg, color, flush_display(p))
        printvalues!(p, showvalues; color = valuecolor)
        if keep
            println(p.output)
        else
            print(p.output, "\r\u1b[A"^(p.offset + p.numprintedvalues))
        end
    end
    return
end

"""
`finish!(prog)` indicates that all tasks have been completed.

See also `cancel`.
"""
function finish!(p::Progress; options...)
    while p.counter < p.n
        next!(p; options...)
    end
end

function finish!(p::ProgressUnknown; options...)
    p.done = true
    updateProgress!(p; options...)
end

# Internal method to print additional values below progress bar
function printvalues!(p::AbstractProgress, showvalues; color = false)
    length(showvalues) == 0 && return
    maxwidth = maximum(Int[length(string(name)) for (name, _) in showvalues])
    for (name, value) in showvalues
        msg = "\n  " * rpad(string(name) * ": ", maxwidth + 2 + 1) * string(value)
        (color == false) ? print(p.output, msg) : printstyled(p.output, msg; color = color)
    end
    p.numprintedvalues = length(showvalues)
end

function move_cursor_up_while_clearing_lines(io, numlinesup, clear_output_ijulia)
    if numlinesup > 0 &&
       clear_output_ijulia && isdefined(Main, :IJulia) && Main.IJulia.inited
        Main.IJulia.clear_output(true)
    else
        for _ = 1:numlinesup
            print(io, "\r\u1b[K\u1b[A")
        end
    end
end

function printover(
    io::IO,
    s::AbstractString,
    color::Symbol = :color_normal,
    clear_output_ijulia = false,
)
    print(io, "\r")
    printstyled(io, s; color = color)
    if isdefined(Main, :IJulia)
        Main.IJulia.stdio_bytes[] = 0 # issue #76: circumvent IJulia I/O throttling
    elseif isdefined(Main, :ESS) || isdefined(Main, :Atom)
    else
        print(io, "\u1b[K")     # clear the rest of the line
    end
end

function compute_front(barglyphs::BarGlyphs, frac_solid::AbstractFloat)
    barglyphs.front isa Char && return barglyphs.front
    idx = round(Int, frac_solid * (length(barglyphs.front) + 1))
    return idx > length(barglyphs.front) ? barglyphs.fill :
           idx == 0 ? barglyphs.empty : barglyphs.front[idx]
end

function barstring(barlen, percentage_complete; barglyphs)
    bar = ""
    if barlen > 0
        if percentage_complete == 100 # if we're done, don't use the "front" character
            bar = string(
                barglyphs.leftend,
                repeat(string(barglyphs.fill), barlen),
                barglyphs.rightend,
            )
        else
            n_bars = barlen * percentage_complete / 100
            nsolid = trunc(Int, n_bars)
            frac_solid = n_bars - nsolid
            nempty = barlen - nsolid - 1
            bar = string(
                barglyphs.leftend,
                repeat(string(barglyphs.fill), max(0, nsolid)),
                compute_front(barglyphs, frac_solid),
                repeat(string(barglyphs.empty), max(0, nempty)),
                barglyphs.rightend,
            )
        end
    end
    bar
end

function durationstring(nsec)
    days = div(nsec, 60 * 60 * 24)
    r = nsec - 60 * 60 * 24 * days
    hours = div(r, 60 * 60)
    r = r - 60 * 60 * hours
    minutes = div(r, 60)
    seconds = floor(r - 60 * minutes)

    hhmmss = @sprintf "%u:%02u:%02u" hours minutes seconds
    if days > 9
        return @sprintf "%.2f days" nsec / (60 * 60 * 24)
    elseif days > 0
        return @sprintf "%u days, %s" days hhmmss
    end
    hhmmss
end

end

ext: jl


text:
module UnionArrays

# Use README as the docstring of the module:
@doc let path = joinpath(dirname(@__DIR__), "README.md")
    include_dependency(path)
    replace(read(path, String), "```julia" => "```jldoctest README")
end UnionArrays

function unionof end
function buffereltypefor end
function buffereltypeof end
function eltypebyid end

include("abstract/abstract.jl")
include("impl/impl.jl")

Abstract.UnionVector(args...; kwargs...) = Impl.UnionVector(args...; kwargs...)
Abstract.UnionArray(args...; kwargs...) = Impl.UnionArray(args...; kwargs...)

using .Abstract: UnionArray, UnionVector, UnionMatrix
export UnionArray, UnionVector, UnionMatrix

end # module


text:
# extract estimates from jld file

using Caesar, Distributions, RoME

const KDE = KernelDensityEstimate

datetimestamp = "2018-08-09T00:04:36.373"
datadir = "/home/dehann/Pictures/racecarimgs/$(datetimestamp)/"
filename = datadir*"racecar_fg_x237.jld"

fg, = loadjld(file=filename)

# using RoMEPlotting
# drawPosesLandms(fg, spscale=0.1)

xx,ll = ls(fg)

fid = open(datadir*"results_$(datetimestamp).csv","w")
for x in xx
  # @show x,
  mx = KDE.getKDEMean(getBelief(getVariable(fg, x)))
  println(fid, "$x, $(mx[1]), $(mx[2]), $(mx[3]), $(mx[4]), $(mx[5])")
end
for x in ll
  # @show x,
  mx = KDE.getKDEMean(getBelief(getVariable(fg, x)))
  println(fid, "$x, $(mx[1]), $(mx[2])")
end
close(fid)

#


text:
# ------------------------------------------------------------------
# Licensed under the ISC License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    NearestNeighborSearcher(object, k; locations=all, metric=Euclidean())

A method for searching `k` nearest neighbors in spatial `object`
`locations` according to `metric`.
"""
struct NearestNeighborSearcher{O,K} <: AbstractBoundedNeighborSearcher
  # input fields
  object::O
  k::Int
  locations::Vector{Int}

  # state fields
  kdtree::K
end

function NearestNeighborSearcher(object::O, k::Int; locations=1:npoints(object), metric=Euclidean()) where {O}
  @assert 1 ≤ k ≤ length(locations) "number of neighbors must be smaller than number of locations"
  @assert locations ⊆ 1:npoints(object) "invalid locations for spatial object"
  kdtree = KDTree(coordinates(object, locations), metric)
  NearestNeighborSearcher{O,typeof(kdtree)}(object, k, locations, kdtree)
end

maxneighbors(searcher::NearestNeighborSearcher) = searcher.k

function search!(neighbors::AbstractVector{Int}, xₒ::AbstractVector,
                 searcher::NearestNeighborSearcher; mask=nothing)
  k       = searcher.k
  inds, _ = knn(searcher.kdtree, xₒ, k, true)
  locs    = view(searcher.locations, inds)

  if mask ≠ nothing
    nneigh = 0
    @inbounds for i in 1:k
      if mask[locs[i]]
        nneigh += 1
        neighbors[nneigh] = locs[i]
      end
    end
  else
    nneigh = k
    @inbounds for i in 1:k
      neighbors[i] = locs[i]
    end
  end

  nneigh
end


text:
using SeisIO, SeisNoise, JLD2, Distributed

@everywhere include("../../src/reference.jl")
include("../../src/convergence.jl")
include("../../src/utils.jl")

# project directory
base_dir = "/Users/jared/SCECintern2019/data"
dir = "BPnetwork_2004"

# input cross-correlations
corrname  = "$base_dir/xcorrs/$dir/BPnetwork_2004_xcorrs"
# reference output file
base_dir = "/Users/jared/SCECintern2019/data"
# input references
refname = "$base_dir/reference/$dir/BPnetwork_2004_ref.jld2"

# input file holds metadata (stationlist, timestamplist, etc.)
f = jldopen(corrname*".jld2")
tslist = f["info/timestamplist"]
close(f) # base xcorr file

# generate reference by stacking all cross-correlations
ref_dicts = pmap(t->compute_reference_xcorr(t, corrname, phase_smoothing=0., stack="linear"), tslist)

# collect all references into one dictionary
ref_dict_out = Dict()
for i=1:length(ref_dicts)
    for stnpair in keys(ref_dicts[i])
        if haskey(ref_dict_out, stnpair)
            ref_dict_out[stnpair].corr .+= ref_dicts[i][stnpair].corr
        else
            ref_dict_out[stnpair] = copy(ref_dicts[i][stnpair])
        end
    end
end

f_out = jldopen(refname, "w")
for stnpair in keys(ref_dict_out)
    f_out[stnpair] = ref_dict_out[stnpair]
end
close(f_out)
rmprocs(workers())

# compute convergence of successively long stacks to the reference xcorr
# coda wave
convname_cc = "$base_dir/convergence/$dir/BPnet_2004_coda_sel.jld2"
convergence(corrname, refname2, convname_cc, ntimes=90, metric="snr", slice=[10.0, 50.0], thresh=0.0)
# ballistic wave
convname_cc = "$base_dir/convergence/$dir/BPnet_2004_bal_sel.jld2"
convergence(corrname, refname2, convname_cc, ntimes=90, metric="snr", slice=10.0, thresh=0.0)

max_stars_repo_name: kura-okubo/SeisXcorrelation

max_issues_repo_name: kura-okubo/SeisXcorrelation

max_forks_repo_name: kura-okubo/SeisXcorrelation


text:
# Mirror of C PyObject struct (for non-debugging Python builds).
# We won't actually access these fields directly; we'll use the Python
# C API for everything.  However, we need to define a unique Ptr type
# for PyObject*, and we might as well define the actual struct layout
# while we're at it.
struct CPyObject
    ob_refcnt::Int
    ob_type::Ptr{Cvoid}
end

const PyPtr = Ptr{CPyObject} # type for PythonObject* in ccall
const PyPtr_NULL = PyPtr(C_NULL)

const sizeof_CPyObject_HEAD = sizeof(Int) + sizeof(PyPtr)

################################################################
# complex

struct CPy_complex
	real :: Cdouble
	imag :: Cdouble
end

################################################################
# buffer

struct CPy_buffer
    buf::Ptr{Cvoid}
    obj::PyPtr
    len::Cssize_t
    itemsize::Cssize_t

    readonly::Cint
    ndim::Cint
    format::Ptr{Cchar}
    shape::Ptr{Cssize_t}
    strides::Ptr{Cssize_t}
    suboffsets::Ptr{Cssize_t}

    # some opaque padding fields to account for differences between
    # Python versions (the structure changed in Python 2.7 and 3.3)
    internal0::Ptr{Cvoid}
    internal1::Ptr{Cvoid}
    internal2::Ptr{Cvoid}
end

const CPy_buffer_NULL = CPy_buffer(C_NULL, C_NULL, 0, 0, 0, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)

const PyBUF_SIMPLE    = convert(Cint, 0)
const PyBUF_WRITABLE  = convert(Cint, 0x0001)
const PyBUF_FORMAT    = convert(Cint, 0x0004)
const PyBUF_ND        = convert(Cint, 0x0008)
const PyBUF_STRIDES        = convert(Cint, 0x0010) | PyBUF_ND
const PyBUF_C_CONTIGUOUS   = convert(Cint, 0x0020) | PyBUF_STRIDES
const PyBUF_F_CONTIGUOUS   = convert(Cint, 0x0040) | PyBUF_STRIDES
const PyBUF_ANY_CONTIGUOUS = convert(Cint, 0x0080) | PyBUF_STRIDES
const PyBUF_INDIRECT       = convert(Cint, 0x0100) | PyBUF_STRIDES
const PyBUF_ND_STRIDED    = Cint(PyBUF_WRITABLE | PyBUF_FORMAT | PyBUF_ND |
                                 PyBUF_STRIDES)
const PyBUF_ND_CONTIGUOUS = PyBUF_ND_STRIDED | PyBUF_ANY_CONTIGUOUS

################################################################
# mirror of Python API types and constants from methodobject.h

struct CPyMethodDef
    ml_name::Ptr{UInt8}
    ml_meth::Ptr{Cvoid}
    ml_flags::Cint
    ml_doc::Ptr{UInt8}
end

const CPyMethodDef_NULL = CPyMethodDef(C_NULL, C_NULL, 0, C_NULL)

# A PyCFunction is a C function of the form
#     PyObject *func(PyObject *self, PyObject *args)
# or
#     PyObject *func(PyObject *self, PyObject *args, PyObject *kwargs)
# The first parameter is the "self" function for method, or
# for module functions it is the module object.  The second
# parameter is either a tuple of args (for METH_VARARGS),
# a single arg (for METH_O), or NULL (for METH_NOARGS).  func
# must return non-NULL (Py_None is okay) unless there was an
# error, in which case an exception must have been set.

# ml_flags should be one of:
const METH_VARARGS = 0x0001 # args are a tuple of arguments
const METH_KEYWORDS = 0x0002  # two arguments: the varargs and the kwargs
const METH_NOARGS = 0x0004  # no arguments (NULL argument pointer)
const METH_O = 0x0008       # single argument (not wrapped in tuple)

# not sure when these are needed:
const METH_CLASS = 0x0010 # for class methods
const METH_STATIC = 0x0020 # for static methods

################################################################
# mirror of Python API types and constants from descrobject.h

struct CPyGetSetDef
    name::Ptr{UInt8}
    get::Ptr{Cvoid}
    set::Ptr{Cvoid} # may be NULL for read-only members
    doc::Ptr{UInt8} # may be NULL
    closure::Ptr{Cvoid} # pass-through thunk, may be NULL
end

const CPyGetSetDef_NULL = CPyGetSetDef(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)

################################################################
# from Python structmember.h:

# declare immutable because we need a C-like array of these
struct CPyMemberDef
    name::Ptr{UInt8}
    typ::Cint
    offset::Int # warning: was Cint for Python <= 2.4
    flags::Cint
    doc::Ptr{UInt8}
end

const CPyMemberDef_NULL = CPyMemberDef(C_NULL, 0, 0, 0, C_NULL)

# types:
const T_SHORT        =0
const T_INT          =1
const T_LONG         =2
const T_FLOAT        =3
const T_DOUBLE       =4
const T_STRING       =5
const T_OBJECT       =6
const T_CHAR         =7
const T_BYTE         =8
const T_UBYTE        =9
const T_USHORT       =10
const T_UINT         =11
const T_ULONG        =12
const T_STRING_INPLACE       =13
const T_BOOL         =14
const T_OBJECT_EX    =16
const T_LONGLONG     =17 # added in Python 2.5
const T_ULONGLONG    =18 # added in Python 2.5
const T_PYSSIZET     =19 # added in Python 2.6
const T_NONE         =20 # added in Python 3.0

# flags:
const READONLY = 1
const READ_RESTRICTED = 2
const PY_WRITE_RESTRICTED = 4
const RESTRICTED = (READ_RESTRICTED | PY_WRITE_RESTRICTED)

################################################################
# type-flag constants, from Python object.h:

# Python 2.7
const Py_TPFLAGS_HAVE_GETCHARBUFFER  = (0x00000001<<0)
const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1)
const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1
const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3)
const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4)
const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5)
const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6)
const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7)
const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8)
const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17)
const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21)
const Py_TPFLAGS_STRING_SUBCLASS       = (0x00000001<<27)

# Python 3.0+ has only these:
const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9)
const Py_TPFLAGS_BASETYPE = (0x00000001<<10)
const Py_TPFLAGS_READY = (0x00000001<<12)
const Py_TPFLAGS_READYING = (0x00000001<<13)
const Py_TPFLAGS_HAVE_GC = (0x00000001<<14)
const Py_TPFLAGS_HAVE_VERSION_TAG   = (0x00000001<<18)
const Py_TPFLAGS_VALID_VERSION_TAG  = (0x00000001<<19)
const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20)
const Py_TPFLAGS_INT_SUBCLASS         = (0x00000001<<23)
const Py_TPFLAGS_LONG_SUBCLASS        = (0x00000001<<24)
const Py_TPFLAGS_LIST_SUBCLASS        = (0x00000001<<25)
const Py_TPFLAGS_TUPLE_SUBCLASS       = (0x00000001<<26)
const Py_TPFLAGS_BYTES_SUBCLASS       = (0x00000001<<27)
const Py_TPFLAGS_UNICODE_SUBCLASS     = (0x00000001<<28)
const Py_TPFLAGS_DICT_SUBCLASS        = (0x00000001<<29)
const Py_TPFLAGS_BASE_EXC_SUBCLASS    = (0x00000001<<30)
const Py_TPFLAGS_TYPE_SUBCLASS        = (0x00000001<<31)

# only use this if we have the stackless extension
const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15)

################################################################
# Mirror of PyNumberMethods in Python object.h

struct CPyNumberMethods
     nb_add::Ptr{Cvoid}
     nb_subtract::Ptr{Cvoid}
     nb_multiply::Ptr{Cvoid}
     nb_remainder::Ptr{Cvoid}
     nb_divmod::Ptr{Cvoid}
     nb_power::Ptr{Cvoid}
     nb_negative::Ptr{Cvoid}
     nb_positive::Ptr{Cvoid}
     nb_absolute::Ptr{Cvoid}
     nb_bool::Ptr{Cvoid}
     nb_invert::Ptr{Cvoid}
     nb_lshift::Ptr{Cvoid}
     nb_rshift::Ptr{Cvoid}
     nb_and::Ptr{Cvoid}
     nb_xor::Ptr{Cvoid}
     nb_or::Ptr{Cvoid}
     nb_int::Ptr{Cvoid}
     nb_reserved::Ptr{Cvoid}
     nb_float::Ptr{Cvoid}
     nb_inplace_add::Ptr{Cvoid}
     nb_inplace_subtract::Ptr{Cvoid}
     nb_inplace_multiply::Ptr{Cvoid}
     nb_inplace_remainder::Ptr{Cvoid}
     nb_inplace_power::Ptr{Cvoid}
     nb_inplace_lshift::Ptr{Cvoid}
     nb_inplace_rshift::Ptr{Cvoid}
     nb_inplace_and::Ptr{Cvoid}
     nb_inplace_xor::Ptr{Cvoid}
     nb_inplace_or::Ptr{Cvoid}
     nb_floordivide::Ptr{Cvoid}
     nb_truedivide::Ptr{Cvoid}
     nb_inplace_floordivide::Ptr{Cvoid}
     nb_inplace_truedivide::Ptr{Cvoid}
     nb_index::Ptr{Cvoid}
     nb_matrixmultiply::Ptr{Cvoid}
     nb_imatrixmultiply::Ptr{Cvoid}
end

const CPyNumberMethods_NULL = CPyNumberMethods(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)

################################################################
# Mirror of PySequenceMethods in Python object.h

struct CPySequenceMethods
    sq_length::Ptr{Cvoid}
    sq_concat::Ptr{Cvoid}
    sq_repeat::Ptr{Cvoid}
    sq_item::Ptr{Cvoid}
    was_sq_item::Ptr{Cvoid}
    sq_ass_item::Ptr{Cvoid}
    was_sq_ass_slice::Ptr{Cvoid}
    sq_contains::Ptr{Cvoid}
    sq_inplace_concat::Ptr{Cvoid}
    sq_inplace_repeat::Ptr{Cvoid}
end

const CPySequenceMethods_NULL = CPySequenceMethods(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)

################################################################
# Mirror of PyMappingMethods in Python object.h

struct CPyMappingMethods
    mp_length::Ptr{Cvoid}
    mp_subscript::Ptr{Cvoid}
    mp_ass_subscript::Ptr{Cvoid}
end

const CPyMappingMethods_NULL = CPyMappingMethods(C_NULL, C_NULL, C_NULL)

################################################################
# Mirror of PyTypeObject in Python object.h
#  -- assumes non-debugging Python build (no Py_TRACE_REFS)
#  -- most fields can default to 0 except where noted

struct CPyTypeObject
    # PyObject_HEAD (for non-Py_TRACE_REFS build):
    ob_refcnt::Int
    ob_type::PyPtr
    ob_size::Int # PyObject_VAR_HEAD, C_NULL

    # PyTypeObject fields:
    tp_name::Ptr{UInt8} # required, should be in format "<module>.<name>"

    # warning: these two were Cint for Python <= 2.4
    tp_basicsize::Int # required, = sizeof(instance)
    tp_itemsize::Int

    tp_dealloc::Ptr{Cvoid}
    tp_print::Ptr{Cvoid}
    tp_getattr::Ptr{Cvoid}
    tp_setattr::Ptr{Cvoid}
    tp_compare::Ptr{Cvoid}
    tp_repr::Ptr{Cvoid}

    tp_as_number::Ptr{CPyNumberMethods}
    tp_as_sequence::Ptr{CPySequenceMethods}
    tp_as_mapping::Ptr{CPyMappingMethods}

    tp_hash::Ptr{Cvoid}
    tp_call::Ptr{Cvoid}
    tp_str::Ptr{Cvoid}
    tp_getattro::Ptr{Cvoid}
    tp_setattro::Ptr{Cvoid}

    tp_as_buffer::Ptr{Cvoid}

    tp_flags::Clong # Required, should default to Py_TPFLAGS_DEFAULT

    tp_doc::Ptr{UInt8} # normally set in example code, but may be NULL

    tp_traverse::Ptr{Cvoid}

    tp_clear::Ptr{Cvoid}

    tp_richcompare::Ptr{Cvoid}

    tp_weaklistoffset::Int

    # added in Python 2.2:
    tp_iter::Ptr{Cvoid}
    tp_iternext::Ptr{Cvoid}

    tp_methods::Ptr{CPyMethodDef}
    tp_members::Ptr{CPyMemberDef}
    tp_getset::Ptr{CPyGetSetDef}
    tp_base::Ptr{Cvoid}

    tp_dict::PyPtr
    tp_descr_get::Ptr{Cvoid}
    tp_descr_set::Ptr{Cvoid}
    tp_dictoffset::Int

    tp_init::Ptr{Cvoid}
    tp_alloc::Ptr{Cvoid}
    tp_new::Ptr{Cvoid}
    tp_free::Ptr{Cvoid}
    tp_is_gc::Ptr{Cvoid}

    tp_bases::PyPtr
    tp_mro::PyPtr
    tp_cache::PyPtr
    tp_subclasses::PyPtr
    tp_weaklist::PyPtr
    tp_del::Ptr{Cvoid}

    # added in Python 2.6:
    tp_version_tag::Cuint

    # only used for COUNT_ALLOCS builds of Python
    tp_allocs::Int
    tp_frees::Int
    tp_maxalloc::Int
    tp_prev::Ptr{Cvoid}
    tp_next::Ptr{Cvoid}
end

const CPyTypeObject_NULL = CPyTypeObject(0, C_NULL, 0, C_NULL, 0, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, 0, 0, 0, C_NULL, C_NULL)

################################################################
# datetime

struct CPyDateTime_CAPI
    # type objects:
    DateType::PyPtr
    DateTimeType::PyPtr
    TimeType::PyPtr
    DeltaType::PyPtr
    TZInfoType::PyPtr

    # singletons:
    @static if pyversion >= v"3.7"
        TimeZone_UTC::PyPtr
    end

    # function pointers:
    Date_FromDate::Ptr{Cvoid}
    DateTime_FromDateAndTime::Ptr{Cvoid}
    Time_FromTime::Ptr{Cvoid}
    Delta_FromDelta::Ptr{Cvoid}
    @static if pyversion >= v"3.7"
        TimeZone_FromTimeZone::Ptr{Cvoid}
    end
    DateTime_FromTimestamp::Ptr{Cvoid}
    Date_FromTimestamp::Ptr{Cvoid}
end

struct CPyDateTime_Delta
    # PyObject_HEAD (for non-Py_TRACE_REFS build):
    ob_refcnt::Int
    ob_type::PyPtr
    hashcode::Py_hash_t
    days::Cint
    seconds::Cint
    microseconds::Cint
end

struct CPyDateTime_Date
	ob_refcnt::Int
	ob_type::PyPtr
	hashcode::Py_hash_t
	hastzinfo::Cchar
	data::NTuple{4,Cuchar}
end

struct CPyDateTime_Time
	ob_refcnt::Int
	ob_type::PyPtr
	hashcode::Py_hash_t
	hastzinfo::Cchar
	data::NTuple{6,Cuchar}
	fold::Cuchar
	tzinfo::PyPtr
end

struct CPyDateTime_BaseDateTime
	ob_refcnt::Int
	ob_type::PyPtr
	hashcode::Py_hash_t
	hastzinfo::Cchar
	data::NTuple{10,Cuchar}
end

struct CPyDateTime_DateTime
	ob_refcnt::Int
	ob_type::PyPtr
	hashcode::Py_hash_t
	hastzinfo::Cchar
	data::NTuple{10,Cuchar}
	fold::Cuchar
	tzinfo::PyPtr
end


text:
using AMQPClient
const VIRTUALHOST = "/"
const HOST = "127.0.0.1"

function receive()
    # 1. Create a connection to the localhost or 127.0.0.1 of virtualhost '/'
    connection(; virtualhost=VIRTUALHOST, host=HOST) do conn
        # 2. Create a channel to send messages
        channel(conn, AMQPClient.UNUSED_CHANNEL, true) do chan
            # 3. Declare a exchange and queue
            exchange = "topic_logs"
            exchange_declare(chan, exchange, EXCHANGE_TYPE_TOPIC)
            result, queue_name, _, _  = queue_declare(chan, "", exclusive=true)

            if length(Base.ARGS) <= 0
                println(Base.stdout, "Usage: [binding_key] \n")
                Base.exit(1)
            end

            # 3.1 Bind all queues
            for binding_key in Base.ARGS[1:end]
                queue_bind(chan, queue_name, exchange,
                           binding_key)
            end

            println(" [*] Waiting for messages. To exit press CTRL+C")

            # 4. Receive messages
            on_receive = (msg) -> begin
                data = String(msg.data)
                println("Received the message: $data")
                basic_ack(chan, msg.delivery_tag)
            end

            success, consumer_tag = basic_consume(chan, queue_name, on_receive)

            while true
                sleep(1)
            end
        end
    end
end

# Don't exit on Ctrl-C
Base.exit_on_sigint(false)
try
    receive()
catch ex
    if ex isa InterruptException
        println("Interrupted")
    else
        println("Exception: $ex")
    end
end

max_forks_repo_name: tomashq/rabbitmq-tutorials


text:
using Compat
using .PyCall

const PyFlowGraphs = pyimport("flowgraph.core.record")
const PyAnnotationDBs = pyimport("flowgraph.core.annotation_db")

function record_file(inpath::String, outpath::String, args::Dict, ::Val{:python})
  if isnothing(args["annotations"])
    db = nothing
  else
    db = PyAnnotationDBs.AnnotationDB()
    db.load_file(abspath(args["annotations"]))
  end
  PyFlowGraphs.record_script(
    inpath, out=outpath, cwd=dirname(inpath), db=db,
    graph_outputs=args["graph-outputs"], store_slots=false)
end

max_stars_repo_name: IBM/semanticflowgraph

max_issues_repo_name: epatters/semanticflowgraph

max_forks_repo_name: IBM/semanticflowgraph


text:
module EPDilutedClassifier

using Random, LinearAlgebra, Distances, Distributions

using GaussianEP: EPState, SpikeSlabPrior, ThetaMixturePrior, ThetaPrior, expectation_propagation, Term, @extract

include("generate_instances.jl")
include("run_instances.jl")
include("epwrapper.jl")

export run_instance, run_instance_gaussian_iid, run_instance_gaussian_cor
export run_instance_cor_from_recnet
export EP_Regress_ZeroTemp, initialize, Lambda_prior
export generate_weights
export generate_gaussian_iid_patterns, generate_gaussian_cor_patterns
export generate_weights_recnet
export generate_patterns_recnet_dynamics
export Reporter

end # end module SparsePerceptron

max_stars_repo_name: mpieropan/EPDilutedClassifier

max_issues_repo_name: mpieropan/EPDilutedClassifier

max_forks_repo_name: mpieropan/EPDilutedClassifier


text:
"""
    DiscreteSystemModel{T, N}

An abstract type that represents a set of model parameters of type T for a discrete dynamical system
consisting of `N` variables.

As a minimum, such types always include a field `dt` (an integer giving the sampling time step, which
is the `dt` argument to `DynamicalSystems.trajectory`) and a field `ui` field (giving the initial
condition). Hence, we define a discrete model system by (1) its parameters, (2) its initial condition,
(3) its sampling step (always an integer).

Concrete subtypes must implement the following methods:

- **[`DiscreteDynamicalSystem(x::DiscreteSystemModel)`](@ref)**: Create a
    `DiscreteDynamicalSystem` from the model.
- **[`get_dt`](@ref)**: Returns the time step.
- **[`get_ui`](@ref)**: Returns the initial condition.
- **[`get_nvars`](@ref)**: Returns the number of variables.
- **[`interaction_matrix`](@ref)**: Returns the interaction matrix for the model (respecting the
    coupling parameter for that particular model).

Concrete subtypes *may* implement the following methods:

- **[`rand`](@ref)**. If implemented for a discrete system model of type `SM`,
    returns an instance of the model with randomised parameters.
"""
abstract type DiscreteSystemModel{T, N} <: AbstractSystemModel{T, N} end

"""
    DiscreteDynamicalSystem(x::ContinuousSystemModel) -> DiscreteDynamicalSystem

Convert a system model to a `ContinuousDynamicalSystem` by connecting
its equations of motion to it parameters.
"""
DiscreteDynamicalSystem(x::DiscreteSystemModel)


text:
export SVD

"""
    SVD(
        data::DataAccessor,
        k::Int
    )

Recommendation based on SVD of a user-item matrix ``R \\in \\mathbb{R}^{|\\mathcal{U}| \\times |\\mathcal{I}|}``, which was originally studied by [Sarwar et al.](http://files.grouplens.org/papers/webKDD00.pdf) Rank ``k`` is configured by `k`.

In a context of recommendation, ``U_k \\in \\mathbb{R}^{|\\mathcal{U}| \\times k}``, ``V \\in \\mathbb{R}^{|\\mathcal{I}| \\times k}`` and ``\\Sigma \\in \\mathbb{R}^{k \\times k}`` are respectively seen as ``k`` user/item feature vectors and corresponding weights. The idea of low-rank approximation that discards lower singular values intuitively works as *compression* or *denoising* of the original matrix; that is, each element in a rank-``k`` matrix ``A_k`` holds the best *compressed* (or *denoised*) value of the original element in ``A``. Thus, ``R_k = \\mathrm{SVD}_k(R)``, the best rank-``k`` approximation of ``R``, captures as much as possible of underlying users' preferences. Once ``R`` is decomposed into ``U, \\Sigma`` and ``V``, a ``(u, i)`` element of ``R_k`` calculated by ``\\sum^k_{j=1} \\sigma_j u_{u, j} v_{i, j}`` could be a prediction for the user-item pair.
"""
struct SVD <: Recommender
    data::DataAccessor
    k::Int
    U::AbstractMatrix
    S::AbstractVector
    Vt::AbstractMatrix

    function SVD(data::DataAccessor, k::Int)
        n_user, n_item = size(data.R)
        U = matrix(n_user, k)
        S = vector(k)
        Vt = matrix(k, n_item)
        new(data, k, U, S, Vt)
    end
end

SVD(data::DataAccessor) = SVD(data, 20)

isbuilt(recommender::SVD) = isfilled(recommender.U)

function build!(recommender::SVD)
    # NaNs are filled by zeros for now
    R = copy(recommender.data.R)
    R[isnan.(R)] .= 0

    res = svd(R)
    recommender.U[:] = res.U[:, 1:recommender.k]
    recommender.S[:] = res.S[1:recommender.k]
    recommender.Vt[:] = res.Vt[1:recommender.k, :]
end

function predict(recommender::SVD, u::Int, i::Int)
    check_build_status(recommender)
    dot(recommender.U[u, :] .* recommender.S, recommender.Vt[:, i])
end