text:
import Statistics: mean, std
import Dierckx: Spline1D, evaluate
import Optim: BFGS, optimize

"""
`rm_events!(s::SFSpectrum; width=2, minstd=5)`
Remove spikes from a spectrum and return the number of events removed.

Takes a single spectrum or an array of spectra. `width` should be equal to the
width of the expected spikes (i.e. if a single spike takes up 4 pixels `width`
should be 4). `minstd` denotes the factor of which the peak has to be above
the standard deviation of the spectrum counts to be recognized as an event.
"""
function rm_events!(s::SFSpectrum; width=3, minstd=5)
  num_removed_events = rm_events!(s.s; width=width, minstd=minstd)
end

function rm_events!(s::Array{Float64}; width=3, minstd=5)

# Behaves like `diff` but is twice as fast
function mydiff(a)
    b = typeof(a)(undef, length(a)-1)
        for i = 1:length(a)-1
           b[i] = a[i+1] - a[i]
        end
    b
end

  r = reshape(s, :)
  dr = mydiff(r)
  num_removed_events = 0

  threshold = minstd * std(dr)
  startidx = width + 1
  endidx = length(dr)
  lowidx = 0

  for i = startidx:endidx
    lowidx = i - width

    if any(dr[j] > threshold for j in lowidx:i) && dr[i] < -threshold
      tmp = dr[lowidx:i]
      width_real = width - argmax(tmp) + 1
      r[i-width_real+1:i] .= (r[i-width_real] + r[i+1])/2
      num_removed_events += 1
    end
  end

  s .= reshape(r, size(s))

  num_removed_events
end

"""
`pixelshift!(a::AbstractArray{T,N}, dim::Int, shift::Real, amplification::Real=1.0)`
Shift the array a the number of pixels `shift` alongside dimension `dim` and
amplify the values by `amplification`.

Example:
```
julia> a = [2.0, 4.0, 6.0, 2.0];
julia> pixelshift!(a, 1, 0.5, 1.0)
4-element Array{Float64,1}:
 3.0
 5.0
 4.0
 2.0
```
"""
function pixelshift!(a::AbstractArray{T,N}, dim::Int, shift::Real, amplification::Real=1.0) where {T <: Real, N}

    N > 2 && error("pixelshift not defined for the case of $N dimensions")

    pixels = 1:size(a, dim)
    pixels_shifted = pixels .+ shift

    if N == 1
        spl = Spline1D(pixels, a, k=1)
        a .= evaluate(spl, pixels_shifted) .* amplification
    elseif N >= 2
        for i = 1:size(a', dim)
            dim == 1 && (spl = Spline1D(pixels, a[:,i], k=1))
            dim == 1 && (a[:,i] .= evaluate(spl, pixels_shifted) .* amplification)
            dim == 2 && (spl = Spline1D(pixels, a[i,:], k=1))
            dim == 2 && (a[i,:] .= evaluate(spl, pixels_shifted) .* amplification)
        end
    end

    a

end

function pixelshift(a::AbstractArray{T,N}, dim::Int, shift::Real, amplification::Real=1.0) where {T <: Real, N}

    b = deepcopy(a)
    pixelshift!(b, dim, shift, amplification)

end

"""
`findpixelshift(a::AbstractArray{T}, b::AbstractArray{T}, dim::Int)`
Find the optimal `shift` and `amplification` values for the `pixelshift`
function.

Array `b` is shifted alongside `a` in dimension `dim` to find the optimal
shift parameters. The result is on `Optim.jl` result. The shift can be accessed
by `result.minimizer[1]` and the amplitude by `result.minimizer[2]`.
"""
function findpixelshift(a::AbstractArray{T}, b::AbstractArray{T}, dim::Int) where T <: Number
    minfun(x) = mean((a .- pixelshift(b, dim, x[1], x[2])).^2)
    result = optimize(minfun, [0.0, 1.0], BFGS())
end

"""
`fieldcorrection!(spectrum::AbstractArray{T,3}; bias=[]::AbstractArray{T,3}, dark=[]::AbstractArray{T,3}, flat=[]::AbstractArray{T,3}, darkflat=[]::AbstractArray{T,3})`
Remove bias and dark counts from spectrum and normalize it with a flat spectrum.

All spectra provided have to be 3D (including bias etc.).

The algorithm works as follows where ``s`` is the spectrum ``b`` the bias,
``d`` the dark spectrum, ``f`` the flat spectrum and ``l`` the dark flat
spectrum. The mean indicated by a ``< >`` is taken along the third dimension
of the spectra.

``s_b = s - <b>``

``d_b = d - <b>``

``f_b = f - <b>``

``l_b = l - <b>``

``s_{b,d} = s_b - <d_b>``

``f_{b,d} - f_b - <l_b>``

``f_{b,d,norm} = <f_{b,d}> / <f_{b,d}>_{max}``

``s_{b,d,f} = s_{b,d} / f_{b,d,norm}``

If no bias is provided ``<b>`` equals zero.
"""
function fieldcorrection!(spectrum::AbstractArray{T,3};
                          bias=Array{Float64}(undef,0,0,0)::AbstractArray{T,3},
                          dark=Array{Float64}(undef,0,0,0)::AbstractArray{T,3},
                          flat=Array{Float64}(undef,0,0,0)::AbstractArray{T,3},
                          darkflat=Array{Float64}(undef,0,0,0)::AbstractArray{T,3}) where T

    function rm_offset!(s, offset)
        offset_m = mean(offset, dims=3)
        for i = 1:size(s, 3)
            s[:,:,i] .-= offset_m[:,:,1]
        end
        s
    end

    function rm_offset(s, offset)
        offset_m = mean(offset, dims=3)
        n = deepcopy(s)
        for i = 1:size(s, 3)
            n[:,:,i] .-= offset_m[:,:,1]
        end
        n
    end

    function flatcorrection!(s, flat)
        flatmean = mean(flat, dims=3)
        flatmean ./= maximum(flatmean)
        for i = 1:size(s, 3)
            s[:,:,i] ./= flatmean[:,:,1]
        end
        s
    end

    # For readability make same length as all
    dafl = darkflat
    spec = spectrum

    !isempty(bias) &&            rm_offset!(spec, bias)

    if !isempty(bias)
        !isempty(dark) && (dark_ub = rm_offset( dark, bias))
        !isempty(flat) && (flat_ub = rm_offset( flat, bias))
        !isempty(dafl) && (dafl_ub = rm_offset( dafl, bias))
    elseif isempty(bias) && !isempty(dark)
        !isempty(dark) && (dark_ub = dark)
        !isempty(flat) && (flat_ub = flat)
        !isempty(dafl) && (dafl_ub = dafl)
    end

    !isempty(dark) && rm_offset!(spec,    dark_ub)
    !isempty(dafl) && rm_offset!(flat_ub, dafl_ub)

    !isempty(flat) && flatcorrection!(spec, flat_ub)

    return spec
end

"""
    get_exposure_time(s)

    Returns exposure time for the spectrum s.

"""
function get_exposure_time(s::SFSpectrum{T,N}) where {T,N}
    get_attribute(s, "ccd_exposure_time")[1]
end

"""
    average(s)

Combine all spectra of a series in `s` to a single one. Returns the mean counts
per second. In more detail we take the mean of the third dimension and
divide that by the exposure time.

Note that there is also a bias of around
710 count for each readout that has to be subtracted first to get the true
mean counts per second.

# Example
```julia-repl
julia> s
3×2×2 SFSpectrum{Float64,3}:
[:, :, 1] =
 0.6  1.0
 0.2  0.2
 0.2  0.2

[:, :, 2] =
 0.6  0.6
 1.0  0.8
 0.2  0.4

julia> average(s)
3×2 SFSpectrum{Float64,2}:
 0.6  0.8
 0.6  0.5
 0.2  0.3

julia> get_attribute(s, "ccd_exposure_time")[1]
1.0
```
"""
function average(s::SFSpectrum{T,N}; combine = true) where {T,N}
  N == 3 || error("The number of dimensions of the spectrum has to be 3.")
  exptime = get_attribute(s, "ccd_exposure_time")[1]
  if size(s, 2) == 1
      if combine == true
          n = SFSpectrum(s.id, Array{T,1}(undef, size(s, 1)))
          n.s = mean(s, dims=3)[:,1,1] / exptime
      else
          n = SFSpectrum(s.id, s.s)
          n.s = s.s ./ exptime
      end
  else
      if combine == true
          n = SFSpectrum(s.id, Array{T,2}(undef, (size(s, 1), size(s, 2)) ))
          n.s = mean(s, dims=3)[:,:,1] / exptime
      else
          n = SFSpectrum(s.id, s.s)
          n.s = s.s ./ exptime
      end
  end
  n
end

"""
Return the wavelength of the detected light in nm.

The appropriate calibration curve of the spectrometer is automatically selected via the timestamp of the
spectrum. If you want to select a specific calibration set the `date` keyword argument to something like
`date=DateTime("2011-11-11")` to select the calibration curve that was valid on that specific date.
"""
function get_wavelength(s::SFSpectrum;
                        date = DateTime(get_attribute(s, "timestamp")[1]))

    λ0 = get_attribute(s, "spectrometer_wavelength")[1]
    x_binning = get_attribute(s, "x_binning")[1]
    num_points = N_PIXEL ÷ x_binning

    if date < DateTime("2018-05-14")
        # Calibration Parameters were determined in 2017-12-13_SpectrometerCalibration.ipynb
        # Pixel offset
        Δp = -0.000338165 * λ0^2 + 0.250245 * λ0 - 32.0725
        # Wavelenths per pixel
        pλ = -3.36321e-8 * λ0^2 + 4.9358e-6 * λ0 + 0.0192305
    else
        # Calibration parameters were determined on 2018-05-14T17:35:18.995
        # Pixel offset
        Δp = -0.0002511567068034343 * λ0^2 + 0.17459249760169204 * λ0 - -12.204165364873228
        # Wavelenths per pixel
        pλ = -2.8767121934619073e-8 * λ0^2 + 3.532087562855135e-7 * λ0 + 0.02024248616034501
    end

    # Central Wavelength
    λc = λ0 - Δp * pλ

    # First Pixel Wavelength:
    first_pixel_wl = λc - (num_points/2 - 1/2) * pλ * x_binning
    last_pixel_wl  = λc + (num_points/2 - 1/2) * pλ * x_binning

    wl_range = range(first_pixel_wl, stop=last_pixel_wl, length=num_points)
    wl = collect(wl_range)

end

"""
Return the wavenumbers in inverse centimeters

The appropriate calibration curve of the spectrometer is automatically selected via the timestamp of the
spectrum. If you want to select a specific calibration set the `date` keyword argument to something like
`date=DateTime("2011-11-11")` to select the calibration curve that was valid on that specific date.

You can change the default wavelength for the visible light by passing a value to the `vis` keyword argument.
"""
function get_wavenumber(s::SFSpectrum; date=DateTime(get_attribute(s, "timestamp")[1]))
    1 ./ get_wavelength(s; date=date) * 1e7
end

"""
Return the wavelength of the corresponding infrared light in nm.
"""
function get_ir_wavelength(s::SFSpectrum; vis=VIS_WAVELENGTH, date=DateTime(get_attribute(s, "timestamp")[1]))

    function sf2ir(sf, vis)
        1 ./ (1 ./ sf .- 1 ./ vis)
    end

    sf_wavelength = get_wavelength(s; date=date)
    ir_wavelength = sf2ir(sf_wavelength, vis)
end

"""
Return the wavenumber of the corresponding infrared light in inverse centimeters.

The appropriate calibration curve of the spectrometer is automatically selected via the timestamp of the
spectrum. If you want to select a specific calibration set the `date` keyword argument to something like
`date=DateTime("2011-11-11")` to select the calibration curve that was valid on that specific date.

You can change the default wavelength for the visible light by passing a value to the `vis` keyword argument.
"""
function get_ir_wavenumber(s::SFSpectrum; kwargs...)#vis=VIS_WAVELENGTH, date=DateTime(get_attribute(s, "timestamp")))
    1 ./ get_ir_wavelength(s; kwargs...) * 1e7
end

"""
Get the parameters where there is change during the measurement as a dict
"""
function get_variables(d::Array{SFSpectrum})

    dict = Dict{String, AbstractArray}()

    # Check if there is data
    length(d) == 0 && return dict

    keylist = d[1] |> get_metadata |> keys

    for k in keylist
        vals = get_attribute(d, k)
        uvals = unique(vals)
        if length(uvals) > 1
            dict[k] = uvals
        end
    end

    return dict

end

"""
Get the pump delay time in ps for a given SFSpectrum
"""
function get_pump_delay(s::SFSpectrum)
    p = get_attribute(s, "pump_dl_position")[1]
    dlpos2t(p)
end

"""
Return the delay time for the infrared pump in ps
"""
function dlpos2t(p)
    c = 299792458.0
    l = 0.61
    n = 125000.0
    p * l / (n * c) * 1e12
end


text:
export Ready

"""
Sent when the [`Client`](@ref) has authenticated, and contains the initial state.
"""
struct Ready <: AbstractEvent
    v::Int
    user::User
    private_channels::Vector{DiscordChannel}
    guilds::Vector{UnavailableGuild}
    session_id::String
    _trace::Vector{String}
end
@boilerplate Ready :dict :docs

ext: jl


text:
#
# Autogenerated by Thrift Compiler (1.0.0-dev)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING

type _enum_PartialKey
  ROW::Int32
  ROW_COLFAM::Int32
  ROW_COLFAM_COLQUAL::Int32
  ROW_COLFAM_COLQUAL_COLVIS::Int32
  ROW_COLFAM_COLQUAL_COLVIS_TIME::Int32
  ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL::Int32
end
const PartialKey = _enum_PartialKey(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5))

type _enum_TablePermission
  READ::Int32
  WRITE::Int32
  BULK_IMPORT::Int32
  ALTER_TABLE::Int32
  GRANT::Int32
  DROP_TABLE::Int32
end
const TablePermission = _enum_TablePermission(Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7))

type _enum_SystemPermission
  GRANT::Int32
  CREATE_TABLE::Int32
  DROP_TABLE::Int32
  ALTER_TABLE::Int32
  CREATE_USER::Int32
  DROP_USER::Int32
  ALTER_USER::Int32
  SYSTEM::Int32
  CREATE_NAMESPACE::Int32
  DROP_NAMESPACE::Int32
  ALTER_NAMESPACE::Int32
  OBTAIN_DELEGATION_TOKEN::Int32
end
const SystemPermission = _enum_SystemPermission(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7), Int32(8), Int32(9), Int32(10), Int32(11))

type _enum_NamespacePermission
  READ::Int32
  WRITE::Int32
  ALTER_NAMESPACE::Int32
  GRANT::Int32
  ALTER_TABLE::Int32
  CREATE_TABLE::Int32
  DROP_TABLE::Int32
  BULK_IMPORT::Int32
  DROP_NAMESPACE::Int32
end
const NamespacePermission = _enum_NamespacePermission(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7), Int32(8))

type _enum_ScanType
  SINGLE::Int32
  BATCH::Int32
end
const ScanType = _enum_ScanType(Int32(0), Int32(1))

type _enum_ScanState
  IDLE::Int32
  RUNNING::Int32
  QUEUED::Int32
end
const ScanState = _enum_ScanState(Int32(0), Int32(1), Int32(2))

type _enum_ConditionalStatus
  ACCEPTED::Int32
  REJECTED::Int32
  VIOLATED::Int32
  UNKNOWN::Int32
  INVISIBLE_VISIBILITY::Int32
end
const ConditionalStatus = _enum_ConditionalStatus(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4))

type _enum_Durability
  DEFAULT::Int32
  NONE::Int32
  LOG::Int32
  FLUSH::Int32
  SYNC::Int32
end
const Durability = _enum_Durability(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4))

type _enum_CompactionType
  MINOR::Int32
  MERGE::Int32
  MAJOR::Int32
  FULL::Int32
end
const CompactionType = _enum_CompactionType(Int32(0), Int32(1), Int32(2), Int32(3))

type _enum_CompactionReason
  USER::Int32
  SYSTEM::Int32
  CHOP::Int32
  IDLE::Int32
  CLOSE::Int32
end
const CompactionReason = _enum_CompactionReason(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4))

type _enum_IteratorScope
  MINC::Int32
  MAJC::Int32
  SCAN::Int32
end
const IteratorScope = _enum_IteratorScope(Int32(0), Int32(1), Int32(2))

type _enum_TimeType
  LOGICAL::Int32
  MILLIS::Int32
end
const TimeType = _enum_TimeType(Int32(0), Int32(1))

type Key
  row::Vector{UInt8}
  colFamily::Vector{UInt8}
  colQualifier::Vector{UInt8}
  colVisibility::Vector{UInt8}
  timestamp::Int64
  Key() = (o=new(); fillunset(o); o)
end # type Key
meta(t::Type{Key}) = meta(t, Symbol[:timestamp], Int[], Dict{Symbol,Any}(:timestamp => Int64(9223372036854775807)))

type ColumnUpdate
  colFamily::Vector{UInt8}
  colQualifier::Vector{UInt8}
  colVisibility::Vector{UInt8}
  timestamp::Int64
  value::Vector{UInt8}
  deleteCell::Bool
  ColumnUpdate() = (o=new(); fillunset(o); o)
end # type ColumnUpdate
meta(t::Type{ColumnUpdate}) = meta(t, Symbol[:colVisibility,:timestamp,:value,:deleteCell], Int[], Dict{Symbol,Any}())

type DiskUsage
  tables::Vector{String}
  usage::Int64
  DiskUsage() = (o=new(); fillunset(o); o)
end # type DiskUsage

type KeyValue
  key::Key
  value::Vector{UInt8}
  KeyValue() = (o=new(); fillunset(o); o)
end # type KeyValue

type ScanResult
  results::Vector{KeyValue}
  more::Bool
  ScanResult() = (o=new(); fillunset(o); o)
end # type ScanResult

type Range
  start::Key
  startInclusive::Bool
  stop::Key
  stopInclusive::Bool
  Range() = (o=new(); fillunset(o); o)
end # type Range

type ScanColumn
  colFamily::Vector{UInt8}
  colQualifier::Vector{UInt8}
  ScanColumn() = (o=new(); fillunset(o); o)
end # type ScanColumn
meta(t::Type{ScanColumn}) = meta(t, Symbol[:colQualifier], Int[], Dict{Symbol,Any}())

type IteratorSetting
  priority::Int32
  name::String
  iteratorClass::String
  properties::Dict{String,String}
  IteratorSetting() = (o=new(); fillunset(o); o)
end # type IteratorSetting

type ScanOptions
  authorizations::Set{Vector{UInt8}}
  range::Range
  columns::Vector{ScanColumn}
  iterators::Vector{IteratorSetting}
  bufferSize::Int32
  ScanOptions() = (o=new(); fillunset(o); o)
end # type ScanOptions
meta(t::Type{ScanOptions}) = meta(t, Symbol[:authorizations,:range,:columns,:iterators,:bufferSize], Int[], Dict{Symbol,Any}())

type BatchScanOptions
  authorizations::Set{Vector{UInt8}}
  ranges::Vector{Range}
  columns::Vector{ScanColumn}
  iterators::Vector{IteratorSetting}
  threads::Int32
  BatchScanOptions() = (o=new(); fillunset(o); o)
end # type BatchScanOptions
meta(t::Type{BatchScanOptions}) = meta(t, Symbol[:authorizations,:ranges,:columns,:iterators,:threads], Int[], Dict{Symbol,Any}())

type KeyValueAndPeek
  keyValue::KeyValue
  hasNext::Bool
  KeyValueAndPeek() = (o=new(); fillunset(o); o)
end # type KeyValueAndPeek

type KeyExtent
  tableId::String
  endRow::Vector{UInt8}
  prevEndRow::Vector{UInt8}
  KeyExtent() = (o=new(); fillunset(o); o)
end # type KeyExtent

type Column
  colFamily::Vector{UInt8}
  colQualifier::Vector{UInt8}
  colVisibility::Vector{UInt8}
  Column() = (o=new(); fillunset(o); o)
end # type Column

type Condition
  column::Column
  timestamp::Int64
  value::Vector{UInt8}
  iterators::Vector{IteratorSetting}
  Condition() = (o=new(); fillunset(o); o)
end # type Condition
meta(t::Type{Condition}) = meta(t, Symbol[:timestamp,:value,:iterators], Int[], Dict{Symbol,Any}())

type ConditionalUpdates
  conditions::Vector{Condition}
  updates::Vector{ColumnUpdate}
  ConditionalUpdates() = (o=new(); fillunset(o); o)
end # type ConditionalUpdates
meta(t::Type{ConditionalUpdates}) = meta(t, Symbol[], Int[2,3], Dict{Symbol,Any}())

type ConditionalWriterOptions
  maxMemory::Int64
  timeoutMs::Int64
  threads::Int32
  authorizations::Set{Vector{UInt8}}
  durability::Int32
  ConditionalWriterOptions() = (o=new(); fillunset(o); o)
end # type ConditionalWriterOptions
meta(t::Type{ConditionalWriterOptions}) = meta(t, Symbol[:maxMemory,:timeoutMs,:threads,:authorizations,:durability], Int[], Dict{Symbol,Any}())

type ActiveScan
  client::String
  user::String
  table::String
  age::Int64
  idleTime::Int64
  _type::Int32
  state::Int32
  extent::KeyExtent
  columns::Vector{Column}
  iterators::Vector{IteratorSetting}
  authorizations::Vector{Vector{UInt8}}
  ActiveScan() = (o=new(); fillunset(o); o)
end # type ActiveScan

type ActiveCompaction
  extent::KeyExtent
  age::Int64
  inputFiles::Vector{String}
  outputFile::String
  _type::Int32
  reason::Int32
  localityGroup::String
  entriesRead::Int64
  entriesWritten::Int64
  iterators::Vector{IteratorSetting}
  ActiveCompaction() = (o=new(); fillunset(o); o)
end # type ActiveCompaction

type WriterOptions
  maxMemory::Int64
  latencyMs::Int64
  timeoutMs::Int64
  threads::Int32
  durability::Int32
  WriterOptions() = (o=new(); fillunset(o); o)
end # type WriterOptions
meta(t::Type{WriterOptions}) = meta(t, Symbol[:durability], Int[], Dict{Symbol,Any}())

type CompactionStrategyConfig
  className::String
  options::Dict{String,String}
  CompactionStrategyConfig() = (o=new(); fillunset(o); o)
end # type CompactionStrategyConfig

type UnknownScanner <: Exception
  msg::String
  UnknownScanner() = (o=new(); fillunset(o); o)
end # type UnknownScanner

type UnknownWriter <: Exception
  msg::String
  UnknownWriter() = (o=new(); fillunset(o); o)
end # type UnknownWriter

type NoMoreEntriesException <: Exception
  msg::String
  NoMoreEntriesException() = (o=new(); fillunset(o); o)
end # type NoMoreEntriesException

type AccumuloException <: Exception
  msg::String
  AccumuloException() = (o=new(); fillunset(o); o)
end # type AccumuloException

type AccumuloSecurityException <: Exception
  msg::String
  AccumuloSecurityException() = (o=new(); fillunset(o); o)
end # type AccumuloSecurityException

type TableNotFoundException <: Exception
  msg::String
  TableNotFoundException() = (o=new(); fillunset(o); o)
end # type TableNotFoundException

type TableExistsException <: Exception
  msg::String
  TableExistsException() = (o=new(); fillunset(o); o)
end # type TableExistsException

type MutationsRejectedException <: Exception
  msg::String
  MutationsRejectedException() = (o=new(); fillunset(o); o)
end # type MutationsRejectedException

type NamespaceExistsException <: Exception
  msg::String
  NamespaceExistsException() = (o=new(); fillunset(o); o)
end # type NamespaceExistsException

type NamespaceNotFoundException <: Exception
  msg::String
  NamespaceNotFoundException() = (o=new(); fillunset(o); o)
end # type NamespaceNotFoundException

type NamespaceNotEmptyException <: Exception
  msg::String
  NamespaceNotEmptyException() = (o=new(); fillunset(o); o)
end # type NamespaceNotEmptyException

@compat abstract type AccumuloProxyClientBase end


text:
module MaxHelpingHandCustomSummitCheckpoint

using ..Ahorn, Maple

@mapdef Entity "MaxHelpingHand/CustomSummitCheckpoint" CustomSummitCheckpoint(x::Integer, y::Integer, firstDigit::String="zero", secondDigit::String="zero",
    spriteDirectory::String="MaxHelpingHand/summitcheckpoints", confettiColors::String="fe2074,205efe,cefe20")

const placements = Ahorn.PlacementDict(
    "Custom Summit Checkpoint (max480's Helping Hand)" => Ahorn.EntityPlacement(
        CustomSummitCheckpoint
    )
)

const numberlist = String["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "minus", "questionmark"]

Ahorn.editingOptions(entity::CustomSummitCheckpoint) = Dict{String, Any}(
    "firstDigit" => numberlist,
    "secondDigit" => numberlist
)

function Ahorn.selection(entity::CustomSummitCheckpoint)
    directory = get(entity.data, "spriteDirectory", "MaxHelpingHand/summitcheckpoints")
    x, y = Ahorn.position(entity)

    return Ahorn.getSpriteRectangle("$directory/base02.png", x, y)
end

function Ahorn.render(ctx::Ahorn.Cairo.CairoContext, entity::CustomSummitCheckpoint, room::Maple.Room)
    directory = get(entity.data, "spriteDirectory", "MaxHelpingHand/summitcheckpoints")
    digit1 = get(entity.data, "firstDigit", 0)
    digit2 = get(entity.data, "secondDigit", 0)

    Ahorn.drawSprite(ctx, "$directory/base02.png", 0, 0)
    Ahorn.drawSprite(ctx, "$directory/$digit1/numberbg.png", -2, 4)
    Ahorn.drawSprite(ctx, "$directory/$digit1/number.png", -2, 4)
    Ahorn.drawSprite(ctx, "$directory/$digit2/numberbg.png", 2, 4)
    Ahorn.drawSprite(ctx, "$directory/$digit2/number.png", 2, 4)
end

end

max_stars_repo_name: Kitty-Cats/MaxHelpingHand

max_issues_repo_name: Kitty-Cats/MaxHelpingHand

max_forks_repo_name: Kitty-Cats/MaxHelpingHand


text:
mutable struct MDALMesh
    mdalm::Ptr{Nothing}
    datasets::Int
end

function MDALMesh(p::Ptr{Nothing})
    p == C_NULL && error("Null pointer")
    n = MDAL_M_datasetGroupCount(p)
    m = MDALMesh(p, n)
    finalizer(MDAL_CloseMesh, m)
    m
end

MDAL_CloseMesh(m::MDALMesh) = MDAL_CloseMesh(m.mdalm)
load(fn::AbstractString) = MDALMesh(MDAL_LoadMesh(fn))

struct MDALNodeIterator
    p::Ptr{Nothing}
    l::Int
end
Base.length(i::MDALNodeIterator) = i.l

function Base.iterate(i::MDALNodeIterator, _state=nothing; buffer=1)
    coords = repeat([0., 0., 0.], buffer)
    count = MDAL_VI_next(i.p, buffer, coords)
    if count == 0
        MDAL_VI_close(i.p)
        return nothing
    else
        return (coords[1:(count*buffer*3)], nothing)
    end
end

function vertices(m::MDALMesh)
    MDALNodeIterator(MDAL_M_vertexIterator(m.mdalm), MDAL_M_vertexCount(m.mdalm))
end

struct MDALFaceIterator
    p::Ptr{Nothing}
    l::Int
end
Base.length(i::MDALFaceIterator) = i.l

function Base.iterate(i::MDALFaceIterator, _state=nothing; buffer=1)
    coords = repeat([0., 0., 0.], buffer)
    count = MDAL_VI_next(i.p, buffer, coords)
    if count == 0
        MDAL_VI_close(i.p)
        return nothing
    else
        return (coords[1:(count*buffer*3)], nothing)
    end
end

function faces(m::MDALMesh)
    MDALNodeIterator(MDAL_M_faceIterator(m.mdalm), MDAL_M_faceCount(m.mdalm))
end


text:
#GObject not defined
#Error while running choosing color reason: module could not be found.
#=
https://github.com/JuliaGraphics/Gtk.jl/issues/530

=#
chooser = ccall((:gtk_color_chooser_dialog_new, Gtk.libgtk), Ptr{GObject}, (Cstring, Ptr{GObject}), "pick color", GtkNullContainer())
ccall((:gtk_dialog_run, Gtk.libgtk), Cint, (Ptr{GtkDialog},), chooser)

#=solution #merged
https://github.com/JuliaGraphics/Gtk.jl/commit/7c8802a6fa5ce0002b529d5017ffe72365ea4b4a

=#

max_stars_repo_name: adamwillisMastery/MiniGtk

max_issues_repo_name: adamwillisMastery/MiniGtk

max_forks_repo_name: adamwillisMastery/MiniGtk


text:
"""
    abstract type AbstractGrid{T,N} <: AbstractArray{T,N}

Grids are arrays of points.
"""
abstract type AbstractGrid{T,N} <: AbstractArray{T,N}
end

const AbstractGrid1d{T <: Real} = AbstractGrid{T,1}

prectype(::Type{G}) where {G<:AbstractGrid} = prectype(eltype(G))
numtype(::Type{G}) where {G<:AbstractGrid} = numtype(eltype(G))
dimension(::AbstractGrid{T}) where {T} = DomainSets.euclideandimension(T)

# Do a bounds check and invoke unsafe_grid_getindex.
# Concrete subtypes can specialize unsafe_grid_getindex without checking bounds.
@propagate_inbounds function getindex(grid::AbstractGrid{T,1}, i::Int) where {T}
    checkbounds(grid, i)
    unsafe_grid_getindex(grid, i)
end

@propagate_inbounds function getindex(grid::AbstractGrid{T,N}, i::Vararg{Int,N}) where {T,N}
    checkbounds(grid, i...)
    unsafe_grid_getindex(grid, i...)
end

@deprecate support(grid::AbstractGrid) covering(grid) false

convert(::Type{AbstractGrid{T}}, grid::AbstractGrid{T,N}) where {T,N} = grid
convert(::Type{AbstractGrid{T}}, grid::AbstractGrid{S,N}) where {S,T,N} = similargrid(grid, S, size(grid))

similargrid(grid::AbstractGrid1d, ::Type{T}, dims::Tuple{Int}) where {T} = similargrid(grid, T, dims[1])

"""
    resize(grid::AbstractGrid, dims...)

Create a grid of the same structure (such as product structure),
but with different points and different dimensions.
"""
resize(grid::AbstractGrid{T}, dims...) where {T} = similargrid(grid, T, dims...)

"""
    hasextension(grid::AbstractGrid)

Is it possible to use the `resize` function?
See also [`resize`](@ref)
"""
hasextension(::AbstractGrid) = false

"Return the canonical grid, if any, of the family of grids."
canonicalgrid(g::AbstractGrid) = g

Display.displaysymbol(g::AbstractGrid) = 'g'


text:
_print_indent(io::IO, s, indent::Int = 0, isfirst::Bool = true) = print(io,  repeat(" ", 4indent), s)
_print_indent(io::IO, s::Pair, indent::Int = 0, isfirst::Bool = true) = print(io,  repeat(" ", 4indent), "(", s, ")")
Base.show(io::IO, s::AbstractMultiSelection) = _print_indent(io, s)

for (op, MS) in ((:|, :OrSelection), (:&, :AndSelection), (:-, :SubSelection))
    @eval begin
        struct $(MS){U,V} <: AbstractMultiSelection
            s1::U
            s2::V
            b::Bool
            function $(MS)(s1, s2, b::Bool)
                s1 = translate_types(s1)
                s2 = translate_types(s2)
                new{typeof(s1), typeof(s2)}(s1, s2, b)
            end
            function $(MS)(s1::T, s2::T, b::Bool) where T<:AbstractContextSelection
                throw(ArgumentError("Cannot chain two `s1`s"))
            end
        end
        $(MS)(s1, s2) = $(MS)(s1, s2, true)
        (-)(s::$(MS)) = $(MS)(s.s1, s.s2, !bool(s))
        function _print_indent(io::IO, s::$(MS), indent::Int = 0, isfirst::Bool = true)
            _print_indent(io, first(s), indent, false)
            println(io, " ", $(op))
            _print_indent(
                io,
                last(s),
                indent + Int(!isfirst && last(s) isa AbstractMultiSelection),
                false
            )
        end
        function _print_indent(io::IO, s::Pair{<:$(MS),<:Any}, indent::Int = 0, isfirst::Bool = true)
            _print_indent(io, first(first(s)), indent, false)
            println(io, " ", $(op))
            s2 = first(s).s2
            _print_indent(io, s2, indent + Int(!isfirst && s2 isa AbstractMultiSelection), false)
        end
        Base.show(io::IO, s::$(MS)) = _print_indent(io, s)
    end
    let _T = Union{AbstractSelection, SelectionQuery, Pair{<:AbstractSelection,<:Any}},
        _S = SelectionQuery{<:AbstractContextSelection}
        # Since operators like & and | have high precendence, we need to make sure
        # the inputs got selectioned / combined
        @eval ($(op))(s1, s2::S) where S<:$(_T) = $(MS)(s1, s2)
        @eval ($(op))(s1::S, s2) where S<:$(_T) = $(MS)(s1, s2)
        @eval ($(op))(s1::S, s2::T) where {S<:$(_T), T<:$(_T)} = $(MS)(s1, s2)
        # Negating a selection that contains all the columns results to an empty selection
        @eval (-)(s::$(MS){S,<:Any}) where {S<:$(_S)} =
            throw(ArgumentError(string("Cannot negate ", $(MS) , " containing a ", S)))
        @eval (-)(s::$(MS){<:Any,S}) where {S<:$(_S)} =
            throw(ArgumentError(string("Cannot negate ", $(MS) , " containing a ", S)))
        @eval (-)(s::$(MS){S,T}) where {S<:$(_S), T<:$(_S)} =
            throw(ArgumentError(string("Cannot negate ", $(MS) , " containing a ", S)))
    end
end

AndSelection(s1::T, s2::S, ::Bool) where {T<:AbstractSelection, S<:OtherSelection} =
    throw(ArgumentError("Cannot intersect with an `other_cols()`"))
AndSelection(s1::S, s2::T, ::Bool) where {T<:AbstractSelection, S<:OtherSelection} =
    throw(ArgumentError("Cannot intersect with an `other_cols()`"))
AndSelection(s1::T, s2::S, ::Bool) where {T<:AbstractSelection, S<:ElseSelection} =
    throw(ArgumentError("Cannot intersect with an `else_cols()`"))
AndSelection(s1::S, s2::T, ::Bool) where {T<:AbstractSelection, S<:ElseSelection} =
    throw(ArgumentError("Cannot intersect with an `else_cols()`"))


text:
function bottleneck(agent, model)
  # @info "User bottleneck function"
  return false
end


text:
struct MPSState
    N::Int
    precision::Int
    optim_prec::Int
    norm::arb
    maximum::arb
    minimizer::arb
    enclosure::arb
    metadata::Dict
end

struct MPSTrace
    states::Vector{MPSState}
end

MPSTrace() = MPSTrace(MPSState[])

function Base.show(io::IO, st::MPSState)
    @printf io "%4d    %4d    %9d    %8d    " st.N st.precision st.optim_prec ArbTools.rel_accuracy_bits(
        st.enclosure,
    )
    @printf io "%.5f    %26s    %s\n" Float64(st.norm) ArbTools.format_arb(st.maximum/st.norm, 5) string(
        st.enclosure,
    )
    return
end

function Base.show(io::IO, tr::MPSTrace)
    @printf io "   N    Prec     Opt prec    Enc prec    "
    @printf io "   Norm                  Maximum/Norm    Enclosure\n"
    @printf io "----    ----     --------    --------    "
    @printf io "-------    --------------------------    ---------\n"
    for st in tr.states
        show(io, st)
    end
    return
end

Base.push!(tr::MPSTrace, st::MPSState) = push!(tr.states, st)

function update!(
    tr::MPSTrace,
    st::MPSState,
    store_trace::Bool,
    show_trace::Bool,
    callback = nothing,
)
    if store_trace
        push!(tr, st)
    end
    if show_trace
        show(st)
    end
    if !isnothing(callback)
        callback(st)
    end
    return
end


text: println(sum([x%3 == 0 || x%5 == 0 ? x : 0 for x = 1:999]))

max_stars_repo_path: 001 - Multiples of 3 and 5/001 - Multiples_of_3_and_5.jl

max_stars_repo_name: LanguageAdventures/Project-Euler-Julia

max_issues_repo_name: LanguageAdventures/Project-Euler-Julia

max_forks_repo_name: LanguageAdventures/Project-Euler-Julia


text:
import Base.GC: gc

# dynamic memory pool allocator
#
# this allocator sits between CuArray constructors
# and the actual memory allocation in CUDAdrv.Mem
#
# the core design is a pretty simple:
# - bin allocations into multiple pools according to their size (see `poolidx`)
# - when requested memory, check the pool for unused memory, or allocate dynamically
# - conversely, when released memory, put it in the appropriate pool for future use
#
# to avoid memory hogging and/or trashing the Julia GC:
# - keep track of used and available memory, in order to determine the usage of each pool
# - keep track of each pool's usage, as well as a window of previous usages
# - regularly release memory from underused pools (see `reclaim(false)`)
#
# possible improvements:
# - pressure: have the `reclaim` background task reclaim more aggressively,
#             and call it from the failure cascade in `alloc`
# - context management: either switch contexts when performing memory operations,
#                       or just use unified memory for all allocations.
# - per-device pools

const pool_lock = ReentrantLock()

## infrastructure

const pools_used = Vector{Set{Mem.Buffer}}()
const pools_avail = Vector{Vector{Mem.Buffer}}()

poolidx(n) = ceil(Int, log2(n))+1
poolsize(idx) = 2^(idx-1)

function create_pools(idx)
  if length(pool_usage) >= idx
    # fast-path without taking a lock
    return
  end

  lock(pool_lock) do
    while length(pool_usage) < idx
      push!(pool_usage, 1)
      push!(pool_history, initial_usage)
      push!(pools_used, Set{Mem.Buffer}())
      push!(pools_avail, Vector{Mem.Buffer}())
    end
  end
end

## management

const USAGE_WINDOW = 5
const initial_usage = Tuple(1 for _ in 1:USAGE_WINDOW)

const pool_usage = Vector{Float64}()
const pool_history = Vector{NTuple{USAGE_WINDOW,Float64}}()

# min and max time between successive background task iterations.
# when the pool usages don't change, scan less regularly.
#
# together with USAGE_WINDOW, this determines how long it takes for objects to get reclaimed
const MIN_DELAY = 1.0
const MAX_DELAY = 5.0

# debug stats
mutable struct PoolStats
  # allocation requests
  req_nalloc::Int
  req_nfree::Int
  ## in bytes
  req_alloc::Int
  user_free::Int

  # actual allocations
  actual_nalloc::Int
  actual_nfree::Int
  ## in bytes
  actual_alloc::Int
  actual_free::Int

  cuda_time::Float64
  total_time::Float64

  # internal stats
  alloc_1::Int
  alloc_2::Int
  alloc_3::Int
  alloc_4::Int
end
const stats = PoolStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
Base.copy(stats::PoolStats) =
  PoolStats((getfield(stats, field) for field in fieldnames(PoolStats))...)

# allocation traces
const pool_traces = Dict{Mem.Buffer, Tuple{Int, Base.StackTraces.StackTrace}}()
const tracing = parse(Bool, get(ENV, "CUARRAYS_TRACE_POOL", "false"))

function __init_memory__()
  create_pools(30) # up to 512 MiB

  managed = parse(Bool, get(ENV, "CUARRAYS_MANAGED_POOL", "true"))
  if managed
    delay = MIN_DELAY
    @async begin
      while true
        if scan()
          delay = MIN_DELAY
        else
          delay = min(delay*2, MAX_DELAY)
        end

        reclaim()

        sleep(delay)
      end
    end
  end

  verbose = haskey(ENV, "CUARRAYS_MANAGED_POOL")
  if verbose
    atexit(()->begin
      Core.println("""
        Pool statistics (managed: $(managed ? "yes" : "no")):
         - requested alloc/free: $(stats.req_nalloc)/$(stats.req_nfree) ($(Base.format_bytes(stats.req_nalloc))/$(Base.format_bytes(stats.req_free)))
         - actual alloc/free: $(stats.actual_nalloc)/$(stats.actual_nfree) ($(Base.format_bytes(stats.actual_alloc))/$(Base.format_bytes(stats.actual_free)))
         - alloc types: $(stats.alloc_1) $(stats.alloc_2) $(stats.alloc_3) $(stats.alloc_4)""")
    end)
  end
end

# scan every pool and manage the usage history
#
# returns a boolean indicating whether any pool is active (this can be a false negative)
function scan()
  gc(false) # quick, incremental collection

  lock(pool_lock) do
    active = false

    @inbounds for pid in 1:length(pool_history)
      nused = length(pools_used[pid])
      navail = length(pools_avail[pid])
      history = pool_history[pid]

      if nused+navail > 0
        usage = pool_usage[pid]
        current_usage = nused / (nused + navail)

        if any(usage->usage != current_usage, history)
          # shift the history window with the recorded usage
          history = pool_history[pid]
          pool_history[pid] = (Base.tail(pool_history[pid])..., usage)

          # reset the usage with the current one
          pool_usage[pid] = current_usage
        end

        if usage != current_usage
          active = true
        end
      else
        pool_usage[pid] = 1
        pool_history[pid] = initial_usage
      end
    end

    active
  end
end

# reclaim free objects
function reclaim(full::Bool=false)
  lock(pool_lock) do
    stats.total_time += Base.@elapsed begin
      if full
        # reclaim all currently unused buffers
        for (pid, pl) in enumerate(pools_avail)
          for buf in pl
            stats.actual_nfree += 1
            stats.cuda_time += Base.@elapsed Mem.free(buf)
            stats.actual_free += poolsize(pid)
          end
          empty!(pl)
        end
      else
        # only reclaim really unused buffers
        @inbounds for pid in 1:length(pool_usage)
          nused = length(pools_used[pid])
          navail = length(pools_avail[pid])
          recent_usage = (pool_history[pid]..., pool_usage[pid])

          if navail > 0
            # reclaim as much as the usage allows
            reclaimable = floor(Int, (1-maximum(recent_usage))*(nused+navail))
            @assert reclaimable <= navail

            while reclaimable > 0
              buf = pop!(pools_avail[pid])
              stats.actual_nfree += 1
              stats.cuda_time += Base.@elapsed Mem.free(buf)
              stats.actual_free += poolsize(pid)
              reclaimable -= 1
            end
          end
        end
      end
    end
  end
end

const MAX_POOL = 100*1024^2 # 100 MiB

## interface

function alloc(bytes)
  # 0-byte allocations shouldn't hit the pool
  bytes == 0 && return Mem.alloc(0)
  buf = Ref{Mem.Buffer}()

  stats.req_nalloc += 1
  stats.req_alloc += bytes
  stats.total_time += Base.@elapsed begin
    # do we even consider pooling?
    pooling = bytes <= MAX_POOL
    if pooling
      pid = poolidx(bytes)
      create_pools(pid)
      alloc_bytes = poolsize(pid)

      @inbounds used = pools_used[pid]
      @inbounds avail = pools_avail[pid]
    else
      alloc_bytes = bytes
    end

    lock(pool_lock) do
      # 1. find an unused buffer in our pool
      if pooling && !isempty(avail)
        stats.alloc_1 += 1
        buf[] = pop!(avail)
      else
        try
          # 2. didn't have one, so allocate a new buffer
          stats.cuda_time += Base.@elapsed begin
            buf[] = Mem.alloc(alloc_bytes)
          end
          stats.alloc_2 += 1
          stats.actual_nalloc += 1
          stats.actual_alloc += alloc_bytes
        catch ex
          ex == CUDAdrv.ERROR_OUT_OF_MEMORY || rethrow()
          # 3. that failed; make Julia collect objects and check 1. again
          gc(true) # full collection
          if pooling && !isempty(avail)
            stats.alloc_3 += 1
            buf[] = pop!(avail)
          else
            # 4. didn't have one, so reclaim all other unused buffers and do 2. again
            reclaim(true)
            try
              stats.cuda_time += Base.@elapsed begin
                buf[] = Mem.alloc(alloc_bytes)
              end
              stats.alloc_4 += 1
              stats.actual_nalloc += 1
              stats.actual_alloc += alloc_bytes
            catch ex
              ex == CUDAdrv.ERROR_OUT_OF_MEMORY || rethrow()
              if tracing
                @error "Failed to allocate $(Base.format_bytes(bytes)) (requires $(Base.format_bytes(alloc_bytes)) buffer)"
                for buf in keys(pool_traces)
                  bytes, bt = pool_traces[buf]
                  @warn "Outstanding allocation of $(Base.format_bytes(bytes)) (requires $(Base.format_bytes(buf.bytesize)) buffer)" exception=(ex,bt)
                end
              end
              rethrow()
            end
          end
        end
      end

      if pooling
        # mark the buffer as used
        push!(used, buf[])

        # update pool usage
        current_usage = length(used) / (length(avail) + length(used))
        pool_usage[pid] = max(pool_usage[pid], current_usage)
      end
    end
  end

  if tracing
    pool_traces[buf[]] = (bytes, stacktrace())
  end

  buf[]
end

function dealloc(buf, bytes)
  # 0-byte allocations shouldn't hit the pool
  bytes == 0 && return Mem.alloc(0)

  stats.req_nfree += 1
  stats.user_free += bytes
  stats.total_time += Base.@elapsed begin
    # was this a pooled buffer?
    pooling = bytes <= MAX_POOL
    if pooling
      pid = poolidx(bytes)
      @assert pid <= length(pools_used)

      @inbounds used = pools_used[pid]
      @inbounds avail = pools_avail[pid]

      lock(pool_lock) do
        # mark the buffer as available
        delete!(used, buf)
        push!(avail, buf)

        # update pool usage
        current_usage = length(used) / (length(used) + length(avail))
        pool_usage[pid] = max(pool_usage[pid], current_usage)
      end
    else
      Mem.free(buf)
    end
  end

  if tracing
    delete!(pool_traces, buf)
  end

  return
end

## utility macros

using Printf

macro allocated(ex)
    quote
        let
            local f
            function f()
                b0 = stats.req_alloc
                $(esc(ex))
                stats.req_alloc - b0
            end
            f()
        end
    end
end

macro time(ex)
    quote
        local gpu_mem_stats0 = copy(stats)
        local cpu_mem_stats0 = Base.gc_num()
        local cpu_time0 = time_ns()

        local val = $(esc(ex))

        local cpu_time1 = time_ns()
        local cpu_mem_stats1 = Base.gc_num()
        local gpu_mem_stats1 = copy(stats)

        local cpu_time = (cpu_time1 - cpu_time0) / 1e9
        local gpu_gc_time = gpu_mem_stats1.cuda_time - gpu_mem_stats0.cuda_time
        local gpu_lib_time = gpu_mem_stats1.cuda_time - gpu_mem_stats0.cuda_time
        local gpu_alloc_count = gpu_mem_stats1.req_nalloc - gpu_mem_stats0.req_nalloc
        local gpu_alloc_size = gpu_mem_stats1.req_alloc - gpu_mem_stats0.req_alloc
        local cpu_mem_stats = Base.GC_Diff(cpu_mem_stats1, cpu_mem_stats0)
        local cpu_gc_time = cpu_mem_stats.total_time / 1e9
        local cpu_alloc_count = Base.gc_alloc_count(cpu_mem_stats)
        local cpu_alloc_size = cpu_mem_stats.allocd

        Printf.@printf("%10.6f seconds", cpu_time)
        for (typ, gctime, libtime, bytes, allocs) in
            (("CPU", cpu_gc_time, 0, cpu_alloc_size, cpu_alloc_count),
             ("GPU", gpu_gc_time, gpu_lib_time, gpu_alloc_size, gpu_alloc_count))
          if bytes != 0 || allocs != 0
              allocs, ma = Base.prettyprint_getunits(allocs, length(Base._cnt_units), Int64(1000))
              if ma == 1
                  Printf.@printf(" (%d%s %s allocation%s: ", allocs, Base._cnt_units[ma], typ, allocs==1 ? "" : "s")
              else
                  Printf.@printf(" (%.2f%s %s allocations: ", allocs, Base._cnt_units[ma], typ)
              end
              print(Base.format_bytes(bytes))
              if gctime > 0
                  Printf.@printf(", %.2f%% gc time", 100*gctime/cpu_time)
                if libtime > 0
                    Printf.@printf(" of which %.2f%% spent allocating", 100*libtime/gctime)
                end
              end
              print(")")
          elseif gctime > 0
              Printf.@printf(", %.2f%% %s gc time", 100*gctime/cpu_time, typ)
          end
        end
        println()

        val
    end
end


text:
using LinearAlgebra
include("householder.jl");

function tridiagonalizacao(T)

    n = size(T,1);

    if n >= 3
        for i = 2:n-1

            x = T[ i:n, i-1];
            sizeX = size(x,1);
            y = zeros(sizeX,1);
            y[1] = norm(x);

            Q = householder(x,y);

            T[ i:n, i-1] = y;
            T[i-1, i:n] = y;
            T[ i:n , i:n] = Q * T[i:n,i:n] * Q;

        end
    end

    return T;

end


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule MUMPS_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("MUMPS")
JLLWrappers.@generate_main_file("MUMPS", UUID("ca64183c-ec4f-5579-95d5-17e128c21291"))
end  # module MUMPS_jll


text:
#Computes the force and energy of every particles of the system

function force_energy!(p,f,data)
    ut = 0.
    for x in 1:length(f)
      f[x] .= 0.
    end
      for i in 1:(data.N)-1, j in i+1:(data.N)
        dist = distpbc(p[i],p[j],data.side)
          if dist <= data.cutoff
          upair, fpair = force_pair(p[i],p[j],dist,data)
          ut += upair
            f[i] .= f[i] .+ fpair
            f[j] .= f[j] .- fpair
          end
      end
    return ut, f
  end


text:
using Documenter, ODBC

makedocs(
    modules = [ODBC],
    sitename = "ODBC.jl",
    pages = ["Home" => "index.md"]
)

deploydocs(
    repo = "github.com/JuliaDatabases/ODBC.jl.git",
    target = "build"
)


text:
text = "the input x must be a vector of length in, or a batch of vectors"
alphabet = (c for c ∈ unique(text))
m = CharEmbedding(alphabet)
m(text)
m(['c','e'])


text:
#Additional constants for repeated structs
const TGOV1DU = SteamTurbineGov1

"""
Parse .dyr file into a dictionary indexed by bus number.
Each bus number key has a dictionary indexed by component type and id.

"""
function _parse_dyr_file(file::AbstractString)
    dyr_text = read(file, String)
    start = 1
    parsed_values = Dict{Int, Dict}()
    models = Vector{String}()
    while start < length(dyr_text)
        val = findnext('/', dyr_text, start)
        if isnothing(val)
            break
        end
        line = replace(dyr_text[start:(val - 1)], "\'" => "")
        val_array = strip.(split(line))
        bus = parse(Int, val_array[1])
        model = string(val_array[2])
        id = parse(Int, val_array[3])
        component_dict = get!(parsed_values, bus, Dict{Tuple{String, Int}, Array}())
        component_dict[(model, id)] = parse.(Float64, val_array[4:end])
        start = val + 1
    end
    return parsed_values
end

"""
Populate arguments in a vector for each dynamic component (except Shafts).
Returns a vector with the parameter values of the argument of each component.

"""
function _populate_args(param_map, val)
    struct_args = Vector{Any}(undef, length(param_map))
    for (ix, _v) in enumerate(param_map)
        #If the parameter is a Float64, then use the value directly as the argument.
        #Typically uses for the resistance that is not available in .dyr files.
        if isa(_v, Float64)
            struct_args[ix] = _v
            #If the parameter is an Int, then use the integer as the key of the value in the dictionary.
        elseif isa(_v, Int)
            struct_args[ix] = val[_v]
            #If the parameter is a tuple (as a string), then construct the tuple directly.
        else
            _t = strip(_v, ['(', ')'])
            _t2int = parse.(Int, split(_t, ','))
            struct_args[ix] = (val[_t2int[1]], val[_t2int[2]])
        end
    end
    return struct_args
end

"""
Create a SingleMass shaft struct directly using the parameter mapping.

"""
function _make_shaft(param_map, val)
    constructor_shaft =
        (args...) -> InteractiveUtils.getfield(PowerSystems, :SingleMass)(args...)
    shaft_args = _populate_args(param_map, val)
    return constructor_shaft(shaft_args...)
end

function _make_source(g::StaticInjection, r::Float64, x::Float64)
    return Source(
        name = get_name(g),
        available = true,
        bus = get_bus(g),
        active_power = get_active_power(g),
        reactive_power = get_reactive_power(g),
        R_th = r,
        X_th = x,
    )
end

function _assign_missing_components!(bus_dict, component_table)
    for va in values(bus_dict)
        if !isassigned(va, component_table["AVR"])
            va[component_table["AVR"]] = AVRFixed(1.0)
        end
        if !isassigned(va, component_table["TurbineGov"])
            va[component_table["TurbineGov"]] = TGFixed(1.0)
        end
        if !isassigned(va, component_table["PSS"])
            va[component_table["PSS"]] = PSSFixed(0.0)
        end
    end
end

"""
Parse a .dyr file directly from its name by constructing its dictionary of dictionaries.

"""
function _parse_dyr_components(dyr_file::AbstractString)
    ext = splitext(dyr_file)[2]
    if lowercase(ext) in [".dyr"]
        data = _parse_dyr_file(dyr_file)
        return _parse_dyr_components(data)
    else
        throw(DataFormatError("$dyr_file is not a .dyr file type"))
    end
end

"""
Parse dictionary of dictionaries of data (from `_parse_dyr_file`) into a dictionary of struct components.
The function receives the parsed dictionary and construct a dictionary indexed by bus, that containts a
dictionary with each dynamic generator components (indexed via its id).

Each dictionary indexed by id contains a vector with 5 of its components:
* Machine
* Shaft
* AVR
* TurbineGov
* PSS

"""
function _parse_dyr_components(data::Dict)
    yaml_mapping = open(PSSE_DYR_MAPPING_FILE) do io
        aux = YAML.load(io)
        return aux
    end
    #param_map contains the mapping between struct params and dyr file.
    param_map = yaml_mapping["parameter_mapping"][1]
    #dic will contain the dictionary index by bus.
    #Each entry will be a dictionary, with id as keys, that contains the vector of components
    dic = Dict{Int, Any}()
    component_table =
        Dict("Machine" => 1, "Shaft" => 2, "AVR" => 3, "TurbineGov" => 4, "PSS" => 5)
    for (bus_num, bus_data) in data
        bus_dict = Dict{Int, Any}()
        for (componentID, componentValues) in bus_data
            #Fill array of 5 components per generator
            temp = get!(bus_dict, componentID[2], Vector{Any}(undef, 5))
            #Only create if name is in the supported keys in the mapping
            if componentID[1] in keys(yaml_mapping)
                #Get the component dictionary
                components_dict = yaml_mapping[componentID[1]][1]
                for (gen_field, struct_as_str) in components_dict
                    #Get the parameter indices w/r to componentValues
                    params_ix = param_map[struct_as_str]
                    #Construct Shaft if it appears
                    if occursin("Shaft", struct_as_str)
                        temp[2] = _make_shaft(params_ix, componentValues)
                        continue
                    end
                    #Construct Components
                    component_constructor =
                        (args...) ->
                            InteractiveUtils.getfield(PowerSystems, Symbol(struct_as_str))(args...)
                    struct_args = _populate_args(params_ix, componentValues)
                    if struct_as_str == "BaseMachine"
                        struct_args = [
                            0.0,    #R
                            1e-5,   #X
                            1.0,    #eq_p
                        ]
                    end
                    temp[component_table[gen_field]] = component_constructor(struct_args...)
                end
            else
                @warn "$(componentID[1]) at bus $bus_num, id $(componentID[2]), not supported in PowerSystems.jl. Skipping data."
            end
        end
        #Assign generic components if there were not provided in Dynamic Data
        _assign_missing_components!(bus_dict, component_table)
        #Store dictionary of components in a dictionary indexed by bus
        dic[bus_num] = bus_dict
    end
    return dic
end

"""
Parse dictionary of dictionaries of data (from `_parse_dyr_file`) into a dictionary of struct components.
The function receives the parsed dictionary and construct a dictionary indexed by bus, that containts a
dictionary with each dynamic generator components (indexed via its id).

Each dictionary indexed by id contains a vector with 5 of its components:
* Machine
* Shaft
* AVR
* TurbineGov
* PSS

Files must be parsed from a .raw file (PTI data format) and a .dyr file.

## Examples:
```julia
raw_file = "Example.raw"
dyr_file = "Example.dyr"
sys = System(raw_file, dyr_file)
```

"""
function System(sys_file::AbstractString, dyr_file::AbstractString; kwargs...)
    ext = splitext(sys_file)[2]
    if lowercase(ext) in [".raw"]
        pm_kwargs = Dict(k => v for (k, v) in kwargs if !in(k, SYSTEM_KWARGS))
        sys = System(PowerModelsData(sys_file; pm_kwargs...); kwargs...)
    else
        throw(DataFormatError("$sys_file is not a .raw file type"))
    end
    bus_dict_gen = _parse_dyr_components(dyr_file)
    add_dyn_injectors!(sys, bus_dict_gen)
    return sys
end

"""
Add to a system already created the dynamic components.
The system should already be parsed from a .raw file.

## Examples:
```julia
dyr_file = "Example.dyr"
add_dyn_injectors!(sys, dyr_file)
```

"""
function add_dyn_injectors!(sys::System, dyr_file::AbstractString)
    bus_dict_gen = _parse_dyr_components(dyr_file)
    add_dyn_injectors!(sys, bus_dict_gen)
end

function add_dyn_injectors!(sys::System, bus_dict_gen::Dict)
    @info "Generators provided in .dyr, without a generator in .raw file will be skipped."
    for g in collect(get_components(ThermalStandard, sys))
        _num = get_number(get_bus(g))
        _name = get_name(g)
        _id = parse(Int, split(_name, "-")[end])
        temp_dict = get(bus_dict_gen, _num, nothing)
        if isnothing(temp_dict)
            @warn "Generator at bus $(_num), id $(_id), not found in Dynamic Data.\nVoltage Source will be used to model it."
            r, x = get_ext(g)["z_source"]
            if x == 0.0
                @warn "No series reactance found. Setting it to 1e-6"
                x = 1e-6
            end
            s = _make_source(g, r, x)
            remove_component!(typeof(g), sys, _name)
            add_component!(sys, s)
        else
            #Obtain Machine from Dictionary
            machine = temp_dict[_id][1]
            #Update R,X from RSORCE and XSORCE from RAW file
            r, x = get_ext(g)["z_source"]
            set_R!(machine, r)
            #Obtain Shaft from dictionary
            shaft = temp_dict[_id][2]
            if typeof(machine) == BaseMachine
                if x == 0.0
                    @warn "No series reactance found. Setting it to 1e-6"
                    x = 1e-6
                end
                set_Xd_p!(machine, x)
                if get_H(shaft) == 0.0
                    @info "Machine at bus $(_num), id $(_id) has zero inertia. Modeling it as Voltage Source"
                    s = _make_source(g, r, x)
                    remove_component!(typeof(g), sys, _name)
                    add_component!(sys, s)
                    #Don't add DynamicComponent in case of adding Source
                    continue
                end
            end
            #Add Dynamic Generator
            dyn_gen = DynamicGenerator(get_name(g), 1.0, temp_dict[_id]...)
            add_component!(sys, dyn_gen, g)
        end
    end
end


text:
@testset "Basis functions" begin
    rₘₐₓ = 10
    ρ = rand()
    N = ceil(Int, rₘₐₓ/ρ + 1/2)

    for B in [StaggeredFiniteDifferences(N, ρ), FiniteDifferences(N, 1.0)]
        x = locs(B)
        χ = B[x, :]
        @test χ isa AbstractSparseMatrix
        @test χ ≈ Diagonal(ones(N))
    end

    @testset "Basis functions in restricted bases" begin
        N = 100
        L = 20.0

        B = FiniteDifferences(range(-L, stop=L, length=N+2)[2:end-1])
        xx = axes(B,1)
        a = xx.domain
        x = range(a.left, stop=a.right, length=max(N,101))
        χ = B[x,:]
        @test χ isa AbstractSparseMatrix

        sell = 1:round(Int, 0.6N)
        selv = round(Int, 0.4N):N

        Bl = B[:,sell]
        Bv = B[:,selv]

        χl = Bl[x, :]
        @test χl isa AbstractSparseMatrix
        @test χl == χ[:,sell]
        χv = Bv[x, :]
        @test χv == χ[:,selv]

        @test Bv[x, 4:8] == χ[:,selv[4:8]]
    end
end


text:
# ------------------------------------------------------------------------------------------
# # Introduction to DataFrames
# **[Bogumił Kamiński](http://bogumilkaminski.pl/about/), Apr 21, 2017**
# ------------------------------------------------------------------------------------------

using DataFrames # load package

# ------------------------------------------------------------------------------------------
# ## Joining DataFrames
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ### Preparing DataFrames for a join
# ------------------------------------------------------------------------------------------

x = DataFrame(ID=[1,2,3,4,missing], name = ["Alice", "Bob", "Conor", "Dave","Zed"])
y = DataFrame(id=[1,2,5,6,missing], age = [21,22,23,24,99])
x,y

rename!(x, :ID=>:id) # names of columns on which we want to join must be the same

# ------------------------------------------------------------------------------------------
# ### Standard joins: inner, left, right, outer, semi, anti
# ------------------------------------------------------------------------------------------

join(x, y, on=:id) # :inner join by default, missing is joined

join(x, y, on=:id, kind=:left)

join(x, y, on=:id, kind=:right)

join(x, y, on=:id, kind=:outer)

join(x, y, on=:id, kind=:semi)

join(x, y, on=:id, kind=:anti)

# ------------------------------------------------------------------------------------------
# ### Cross join
# ------------------------------------------------------------------------------------------

# cross-join does not require on argument
# it produces a Cartesian product or arguments
function expand_grid(;xs...) # a simple replacement for expand.grid in R
    reduce((x,y) -> join(x, DataFrame(Pair(y...)), kind=:cross),
           DataFrame(Pair(xs[1]...)), xs[2:end])
end

expand_grid(a=[1,2], b=["a","b","c"], c=[true,false])

# ------------------------------------------------------------------------------------------
# ### Complex cases of joins
# ------------------------------------------------------------------------------------------

x = DataFrame(id1=[1,1,2,2,missing,missing],
              id2=[1,11,2,21,missing,99],
              name = ["Alice", "Bob", "Conor", "Dave","Zed", "Zoe"])
y = DataFrame(id1=[1,1,3,3,missing,missing],
              id2=[11,1,31,3,missing,999],
              age = [21,22,23,24,99, 100])
x,y

join(x, y, on=[:id1, :id2]) # joining on two columns

join(x, y, on=[:id1], makeunique=true) # with duplicates all combinations are produced (here :inner join)

join(x, y, on=[:id1], kind=:semi) # but not by :semi join (as it would duplicate rows)

max_stars_repo_name: grenkoca/JuliaTutorials

max_issues_repo_name: grenkoca/JuliaTutorials

max_forks_repo_name: grenkoca/JuliaTutorials


text:
for arg in ARGS
    println(arg)
end


text:
abstract type AbstractKernel{F,TT} end

struct Kernel{F,TT}
    device::MtlDevice
    mod::MtlLibrary
    fun::MtlFunction
end

#launchKernel

##########################################
# Blocking call to a kernel
# Should implement somethjing better
mtlcall(f::MtlFunction, types::Tuple, args...; kwargs...) =
    mtlcall(f, Base.to_tuple_type(types), args...; kwargs...)

function mtlcall(f::MtlFunction, types::Type, args...; kwargs...)
    convert_arguments(types, args...) do pointers...
        launch(f, pointers...; kwargs...)
    end
end

##############################################
# Enqueue a function for execution
#
function enqueue_function(f::MtlFunction, args...;
                blocks::MtlDim=1, threads::MtlDim=1,
                cce::MtlComputeCommandEncoder)
    blocks = MtlDim3(blocks)
    threads = MtlDim3(threads)
    (blocks.x>0 && blocks.y>0 && blocks.z>0)    || throw(ArgumentError("Grid dimensions should be non-null"))
    (threads.x>0 && threads.y>0 && threads.z>0) || throw(ArgumentError("Thread dimensions should be non-null"))
    all(threads .< f.maxTotalThreadsPerThreadgroup) || throw(ArgumentError("Max Thread dimension is $(f.maxTotalThreadsPerThreadgroup)"))

    # Set the function that we are currently encoding
    MTL.set_function!(cce, f)
    # Encode all arguments
    MTL.encode_arguments!(cce, f, args...)
    # Flush everything
    MTL.append_current_function!(cce, blocks, threads)
    return nothing
end

#####
function encode_arguments!(cce::MtlComputeCommandEncoder, f::MtlFunction, args...)
    for (i, a) in enumerate(args)
        encode_argument!(cce, f, i, a)
    end
end

function encode_argument!(cce::MtlComputeCommandEncoder, f::MtlFunction, idx::Integer, arg::Nothing)
    @assert idx > 0
    #@check api.clSetKernelArg(k.id, cl_uint(idx-1), sizeof(CL_mem), C_NULL)
    set_bytes!(cce, sizeof(C_NULL), C_NULL, idx-1)
    return cce
end

function encode_argument!(cce::MtlComputeCommandEncoder, f::MtlFunction, idx::Integer, arg::MtlBuffer)
    @assert idx > 0
    set_buffer!(cce, buf, 0, idx-1)
    return cce
end

function encode_argument!(enc::MTL.MtlComputeCommandEncoder, f::MtlFunction, idx::Integer, val::T) where T
    @assert idx > 0 "Kernel idx must be bigger 0"
    #if does not contain a buffer we can use setbytes
    if !contains_mtlbuffer(T)
        ref, tsize = to_mtl_ref(val)
        set_bytes!(cce, ref, tsize, idx-1)
    else
        #otherwise, we need an argument buffer
        throw("Not implemented: If an argument contains a mtlbuffer automatic aragument encoding is not yet supported.")

        # create an encoder to write into the argument buffer
        argbuf_enc = MtlArgumentEncoder(f, idx)
        # allocate the argument buffer
        argbuf = alloc(Cchar, device(enc), sizeof(argbuf_enc), storage=Shared)
        # assign the argument buffer to the encoder
        MTL.assign_argument_buffer!(argbuf_enc, argbuf, 1)

        # TODO Implement automatic conversion of struct into a assign_argument_buffer

        #for field in val
        #    MTL.set_field!(argbuf_enc, size(val), 1)
        #    set_buffer!(argbuf_enc, pointer(val), 0, 2)
        #end

        # set argubuf_enc into cce
    end
    return cce
end

function encode_argument!(enc::MTL.MtlComputeCommandEncoder, f::MtlFunction, idx::Integer, val::MtlDeviceArray)
    @assert contains_mtlbuffer(typeof(val))

    # create an encoder to write into the argument buffer
    argbuf_enc = MtlArgumentEncoder(f, idx)
    # allocate the argument buffer
    argbuf = alloc(Cchar, device(enc), sizeof(argbuf_enc), storage=Shared)
    # assign the argument buffer to the encoder
    MTL.assign_argument_buffer!(argbuf_enc, argbuf, 1)

    #
    MTL.set_field!(argbuf_enc, size(val), 1)
    set_buffer!(argbuf_enc, pointer(val), 0, 2)

    MTL.use!(enc, pointer(val), MTL.ReadWriteUsage)

    set_buffer!(enc, argbuf, 0, idx)
    @info "Leaked temporary argument buffer $(argbuf.handle) for argument #$idx"
    #TODO memmgmt
    return argbuf
end


text:
@testset "sdp problems" begin
    @testset "sdpt1_test" begin
        #=
        primal
            min Tr(X)
        s.t.
            X[2,1] = 1 : y_1
            X in PSD : y_2, y_3, y_4

        dual
            max y_1
        s.t.
            y_2 == 1
            y_1 + 2y_3 == 0
            y_4 == 1
            [y_2   y_3
             y_3   y_4] in PSD
        =#
        primal_model = sdpt1_test()
        dual_model, primal_dual_map = dual_model_and_map(primal_model)

        @test MOI.get(dual_model, MOI.NumberOfVariables()) == 1
        list_of_cons = MOI.get(dual_model, MOI.ListOfConstraints())
        @test Set(list_of_cons) ==
              Set([(VAF{Float64}, MOI.PositiveSemidefiniteConeTriangle)],)
        @test MOI.get(
            dual_model,
            MOI.NumberOfConstraints{
                VAF{Float64},
                MOI.PositiveSemidefiniteConeTriangle,
            }(),
        ) == 1
        obj_type = MOI.get(dual_model, MOI.ObjectiveFunctionType())
        @test obj_type == SAF{Float64}
        obj = MOI.get(dual_model, MOI.ObjectiveFunction{obj_type}())
        @test MOI.get(dual_model, MOI.ObjectiveSense()) == MOI.MAX_SENSE
        @test MOI.constant(obj) == 0.0
        @test MOI.coefficient.(obj.terms) == [1.0]
    end

    @testset "sdpt2_test" begin
        #=
        primal
            min Tr(X)
        s.t.
            X[2,1] = 1 : y_1
            X in PSD : y_2, y_3, y_4

        dual
            max y_1
        s.t.
            y_2 == 1
            y_1 + 2y_3 == 0
            y_4 == 1
            [y_2   y_3
             y_3   y_4] in PSD
        =#
        primal_model = sdpt2_test()
        dual_model, primal_dual_map = dual_model_and_map(primal_model)

        @test MOI.get(dual_model, MOI.NumberOfVariables()) == 4
        list_of_cons = MOI.get(dual_model, MOI.ListOfConstraints())
        @test Set(list_of_cons) == Set(
            [
                (SAF{Float64}, MOI.EqualTo{Float64})
                (VVF, MOI.PositiveSemidefiniteConeTriangle)
            ],
        )
        @test MOI.get(
            dual_model,
            MOI.NumberOfConstraints{VVF,MOI.PositiveSemidefiniteConeTriangle}(),
        ) == 1
        @test MOI.get(
            dual_model,
            MOI.NumberOfConstraints{SAF{Float64},MOI.EqualTo{Float64}}(),
        ) == 3
        obj_type = MOI.get(dual_model, MOI.ObjectiveFunctionType())
        @test obj_type == SAF{Float64}
        obj = MOI.get(dual_model, MOI.ObjectiveFunction{obj_type}())
        @test MOI.get(dual_model, MOI.ObjectiveSense()) == MOI.MAX_SENSE
        @test MOI.constant(obj) == 0.0
        @test MOI.coefficient.(obj.terms) == [1.0]

        eq_con1, eq_con2, eq_con3 = MOI.get(
            dual_model,
            MOI.ListOfConstraintIndices{SAF{Float64},MOI.EqualTo{Float64}}(),
        )
        spd_con = MOI.get(
            dual_model,
            MOI.ListOfConstraintIndices{
                VVF,
                MOI.PositiveSemidefiniteConeTriangle,
            }(),
        )

        eq_con1_fun = MOI.get(dual_model, MOI.ConstraintFunction(), eq_con1)
        eq_con1_set = MOI.get(dual_model, MOI.ConstraintSet(), eq_con1)
        @test MOI.coefficient.(eq_con1_fun.terms) == [1.0]
        @test MOI.constant.(eq_con1_fun) == 0.0
        @test MOI.constant(eq_con1_set) == 1.0
        eq_con2_fun = MOI.get(dual_model, MOI.ConstraintFunction(), eq_con2)
        eq_con2_set = MOI.get(dual_model, MOI.ConstraintSet(), eq_con2)
        @test MOI.coefficient.(eq_con2_fun.terms) == [1.0; 2.0]
        @test MOI.constant.(eq_con2_fun) == 0.0
        @test MOI.constant(eq_con2_set) == 0.0
        eq_con3_fun = MOI.get(dual_model, MOI.ConstraintFunction(), eq_con3)
        eq_con3_set = MOI.get(dual_model, MOI.ConstraintSet(), eq_con3)
        @test MOI.coefficient.(eq_con3_fun.terms) == [1.0]
        @test MOI.constant.(eq_con3_fun) == 0.0
        @test MOI.constant(eq_con3_set) == 1.0

        sdp_con = MOI.get(dual_model, MOI.ConstraintFunction(), spd_con)

        primal_con_dual_var = primal_dual_map.primal_con_dual_var
        @test primal_con_dual_var[eq_con1] == [VI(1)]
        @test primal_con_dual_var[CI{
            VAF{Float64},
            MOI.PositiveSemidefiniteConeTriangle,
        }(
            1,
        )] == VI.(2:4)

        primal_var_dual_con = primal_dual_map.primal_var_dual_con
        @test primal_var_dual_con[VI(1)] == eq_con1
        @test primal_var_dual_con[VI(2)] == eq_con2
        @test primal_var_dual_con[VI(3)] == eq_con3
    end
end


text:
function cleanQflx2atm!(ocn::Ocean)

    @loop_hor ocn i j let

        ocn.qflx2atm[i, j] = 0.0

    end

end

function calLatentHeatReleaseOfFreezing!(ocn::Ocean; Δt::Float64, do_convadjust::Bool)

    @loop_hor ocn i j let

        #OC_updateB!(ocn, i, j)

        #=
        old_FLDO=ocn.FLDO[i,j]
        Δb = (old_FLDO == -1 ) ? 0.0 : ocn.b_ML[i, j] - ocn.bs[old_FLDO, i, j]

        if Δb < -3e-6
            println(format("[Qflx2ATM] [BEFORE] At {:d}, {:d}: Δb = {:f}. Even after adjustment.", i, j, Δb))
        end
        =#

        ocn.qflx2atm[i, j] = (T_sw_frz - ocn.T_ML[i, j]) * ρc_sw * ocn.h_ML[i, j] / Δt
        ocn.qflx2atm_pos[i, j] = max(ocn.qflx2atm[i,j], 0.0)
        ocn.qflx2atm_neg[i, j] = min(ocn.qflx2atm[i,j], 0.0)

        if ocn.T_ML[i, j] < T_sw_frz
            ocn.T_ML[i, j] = T_sw_frz
            ocn.Ts[1:((ocn.FLDO[i, j] == -1) ? ocn.Nz[i, j] : ocn.FLDO[i, j]-1 ), i, j] .= T_sw_frz
            OC_updateB!(ocn, i, j)
        end

    end

    if do_convadjust
        @loop_hor ocn i j let
            OC_doConvectiveAdjustment!(ocn, i, j;)
        end
    end

end

max_stars_repo_name: meteorologytoday/IOM

max_issues_repo_name: meteorologytoday/IOM

max_forks_repo_name: meteorologytoday/IOM


text:
function ITensor(::Type{ElT},
                 flux::QN,
                 inds::IndexSet) where {ElT<:Number}
  blocks = nzblocks(flux,inds)
  T = BlockSparseTensor(ElT,blocks,inds)
  return itensor(T)
end

function ITensor(inds::QNIndex...)
  T = BlockSparseTensor(IndexSet(inds))
  return itensor(T)
end

function ITensor(::Type{ELT},inds::QNIndex...) where {ELT<:Number}
  T = BlockSparseTensor(ELT,inds)
  return itensor(T)
end

ITensor(::Type{T},
        flux::QN,
        inds::Index...) where {T<:Number} = ITensor(T,flux,IndexSet(inds...))

ITensor(flux::QN,inds::IndexSet) = ITensor(Float64,flux::QN,inds...)

ITensor(flux::QN,
        inds::Index...) = ITensor(flux,IndexSet(inds...))

function randomITensor(::Type{ElT},
                       flux::QN,
                       inds::IndexSet) where {ElT<:Number}
  T = ITensor(ElT,flux,inds)
  randn!(T)
  return T
end

function randomITensor(::Type{ElT},
                       flux::QN,
                       inds::Index...) where {ElT<:Number}
  return randomITensor(ElT,flux,IndexSet(inds...))
end

randomITensor(flux::QN,inds::IndexSet) = randomITensor(Float64,flux::QN,inds...)

randomITensor(flux::QN,
              inds::Index...) = randomITensor(flux,IndexSet(inds...))

# Throw error if flux is not specified
randomITensor(::Type{<:Number},
              inds::QNIndexSet) = error("In randomITensor constructor, must specify desired flux when using QN Indices")

function combiner(inds::QNIndex...; kwargs...)
  # TODO: support combining multiple set of indices
  tags = get(kwargs, :tags, "CMB,Link")
  new_ind = ⊗(inds...)
  if all(i->dir(i)!=Out,inds) && all(i->dir(i)!=In,inds)
    new_ind = dag(new_ind)
    new_ind = replaceqns(new_ind,-qnblocks(new_ind))
  end
  new_ind = settags(new_ind,tags)
  comb_ind,perm,comb = combineblocks(new_ind)
  return itensor(Combiner(perm,comb),
                 IndexSet(comb_ind, dag.(inds)...))
end

combiner(inds::Tuple{Vararg{QNIndex}};
         kwargs...) = combiner(inds...; kwargs...)

#
# DiagBlockSparse ITensor constructors
#

"""
    diagITensor(::Type{T}, flux::QN, is::IndexSet)
    diagITensor(::Type{T}, flux::QN, is::Index...)

Make a sparse ITensor of element type T with non-zero elements
only along the diagonal. Defaults to having `zero(T)` along the diagonal.
The storage will have Diag type.
"""
function diagITensor(::Type{ElT},
                     flux::QN,
                     is::IndexSet{N}) where {ElT <: Number, N}
  blocks = nzdiagblocks(flux, is)
  T = DiagBlockSparseTensor(ElT, blocks, is)
  return itensor(T)
end

diagITensor(::Type{ElT},
            flux::QN,
            inds::Index...) where {ElT <: Number} = diagITensor(ElT,
                                                                flux,
                                                                IndexSet(inds...))

diagITensor(::Type{<:Number},inds::QNIndexSet) = error("Must specify flux")

"""
    diagITensor(flux::QN, is::IndexSet)
    diagITensor(flux::QN, is::Index...)

Make a sparse ITensor of element type Float64 with non-zero elements
only along the diagonal. Defaults to storing zeros along the diagonal.
The storage will have Diag type.
"""
diagITensor(flux::QN,
            is::IndexSet) = diagITensor(Float64,flux,is)

diagITensor(flux::QN,
            inds::Index...) = diagITensor(flux, IndexSet(inds...))

diagITensor(inds::QNIndexSet) = error("Must specify flux")

#"""
#    diagITensor(v::Vector{T<:Number}, flux::QN, is::IndexSet)
#    diagITensor(v::Vector{T<:Number}, flux::QN, is::Index...)
#
#Make a sparse ITensor with non-zero elements only along the diagonal.
#The diagonal elements will be set to the values stored in `v` and
#the ITensor will have element type `float(T)`.
#The storage will have DiagBlockSparse type.
#"""
#function diagITensor(v::Vector{<:Number},
#                     flux::QN,
#                     is::IndexSet)
#  # TODO: check that the diagonal blocks all have the same flux
#  length(v) ≠ mindim(is) && error("Length of vector for diagonal must equal minimum of the dimension of the input indices")
#  return ITensor(DiagBlockSparse(float(v)),is)
#end
#
#function diagITensor(v::Vector{<:Number},
#                     flux::QN,
#                     inds::Index...)
#  return diagITensor(v,flux,IndexSet(inds...))
#end
#
#diagITensor(v::Vector{<:Number},
#            inds::QNIndexSet) = error("Must specify flux")

#"""
#    diagITensor(x::Number, flux::QN, is::IndexSet)
#    diagITensor(x::Number, flux::QN, is::Index...)
#
#Make a sparse ITensor with non-zero elements only along the diagonal.
#The diagonal elements will be set to the value `x` and
#the ITensor will have element type `float(T)`.
#The storage will have DiagBlockSparse type.
#"""
#function diagITensor(x::Number,
#                     flux::QN,
#                     is::IndexSet)
#  return ITensor(Diag(fill(float(x),mindim(is))),is)
#end
#
#function diagITensor(x::Number,
#                     flux::QN,
#                     is::Index...)
#  return diagITensor(x,flux,IndexSet(is...))
#end
#
#diagITensor(x::Number, is::QNIndexSet) = error("Must specify flux")

function delta(::Type{ElT}, is::QNIndexSet) where {ElT<:Number}
  blocks = nzdiagblocks(QN(), is)
  T = DiagBlockSparseTensor(one(ElT), blocks, is)
  return itensor(T)
end


text:
cd(@__DIR__)
using Weave
weave("README.jmd", out_path=:pwd, doctype="github")


text:
using Printf
#
# functions for getting and setting V blocks required for block respecting QX and SVD
#
# Handles W with only one link index
function getV1(W::ITensor,off::V_offsets)::ITensor
    ils=filterinds(inds(W),tags="Link")
    iss=filterinds(inds(W),tags="Site")
    @assert length(ils)==1
    w1=ils[1]
    v1=redim(w1,dim(w1)-1)
    T=eltype(W)
    V=ITensor(T(0.0),v1,iss...)
    for ilv in eachindval(v1)
        wlv=IndexVal(w1,ilv.second+off.o1)
        s=slice(W,wlv)
        assign!(V,s,ilv)
    end
    return V
end

function getV(W::ITensor,off::V_offsets)::ITensor
    ils=filterinds(inds(W),tags="Link")
    if length(ils)==1
        return getV1(W,off)
    end
    iss=filterinds(inds(W),tags="Site")
    w1=ils[1]
    w2=ils[2]
    v1=redim(w1,dim(w1)-1)
    v2=redim(w2,dim(w2)-1)
    V=ITensor(v1,v2,iss...)
    for ilv in eachindval(v1,v2)
        wlv=(IndexVal(w1,ilv[1].second+off.o1),IndexVal(w2,ilv[2].second+off.o2))
        s=slice(W,wlv...)
        assign!(V,s,ilv...)
    end
    return V
end

# Handles W with only one link index
function setV1(W::ITensor,V::ITensor,ms::matrix_state)::ITensor
    wils=filterinds(inds(W),tags="Link") #should be l=n, l=n-1
    vils=filterinds(inds(V),tags="Link") #should be qx and {l=n,l=n-1} depending on sweep direction
    @assert length(wils)==1
    @assert length(vils)==1
    wil=wils[1]
    vil=vils[1]
    iss=filterinds(inds(W),tags="Site")
    @assert iss==filterinds(inds(V),tags="Site")
    off=V_offsets(ms)
    T=eltype(V)
    @assert(off.o1==off.o2)
    if dim(wil)>dim(vil)+1
        #we need to shrink W
        wil1=Index(dim(vil)+1,tags(wil))
        W1=ITensor(T(0.0),wil1,iss)
        if off.o1==1
            #save first element
            for isv in eachindval(iss)
                W1[wil1=>1,isv...]=W[wil=>1,isv...]
            end
        else
            #save last element
            for isv in eachindval(iss)
                W1[wil1=>dim(wil1),isv...]=W[wil=>dim(wil),isv...]
            end
        end
    else
        W1=W
        wil1=wil
    end

    for ilv in eachindval(vil)
        wlv=IndexVal(wil1,ilv.second+off.o1)
        for isv in eachindval(iss)
            W1[wlv,isv...]=V[ilv,isv...]
        end
    end
    return W1
end

#
#  This function is non trivial for 3 reasons:
#   1) V could have truncted number of rows or columns as a result of rank revealing QX
#      W then has to re-sized accordingly
#   2) If W gets resized we need to preserve the last row or column from the old W.
#   3) V and W should one common link index so we need to find those and pair them together.
#
function setV(W::ITensor,V::ITensor,ms::matrix_state)::ITensor
    wils=filterinds(inds(W),tags="Link") #should be l=n, l=n-1
    if length(wils)==1
        return setV1(W,V,ms) #Handle row/col vectors
    end
    vils=filterinds(inds(V),tags="Link") #should be qx and {l=n,l=n-1} depending on sweep direction
    #@show wils vils
    @assert length(wils)==2
    @assert length(vils)==2
    iss=filterinds(inds(W),tags="Site")
    @assert iss==filterinds(inds(V),tags="Site")
    #
    #  these need to loop in the correct order in order to get the W and V indices to line properly.
    #  one index from each of W & V should have the same tags, so we just need get these
    #  indices paired up so they can loop together.
    #
    # this would be perfect but it only looks at the first tag "Link",
    # we are interested in the second tag l=$n :(
    #icom=ITensors.commontags(tags(wils...),tags(vils...))

    if tags(wils[1])!=tags(vils[1]) && tags(wils[2])!=tags(vils[2])
        vils=vils[2],vils[1] #swap tags the same on index 1 or 2.
    end
    @assert tags(wils[1])==tags(vils[1]) || tags(wils[2])==tags(vils[2])
    if hastags(vils[1],"qx")
        ivqx=vils[1]
        iwqx=wils[1]
        ivl=vils[2]
        iwl=wils[2]
    else
        @assert hastags(vils[2],"qx")
        ivqx=vils[2]
        iwqx=wils[2]
        ivl=vils[1]
        iwl=wils[1]
    end

    off=V_offsets(ms)
    T=eltype(V)
    resize=dim(iwqx)>dim(ivqx)+1
    if resize
        iw1=Index(dim(ivqx)+1,tags(iwqx))
        W1=ITensor(T(0.0),iw1,iwl,iss)
        others=noncommoninds(W,iwqx)
        @assert off.o1==off.o2
        if  off.o1==1 #save row or col 1
            for io in eachindval(others...)
                W1[iw1=>1,io...]=W[iwqx=>1,io...]
            end
        else #off.o1==0 save row or col Dw
            @assert off.o1==0
            for io in eachindval(others...)
                W1[iw1=>dim(iw1),io...]=W[iwqx=>dim(iwqx),io...]
            end
        end

        W=W1
        wils=filterinds(inds(W),tags="Link") #should be l=n, l=n-1
        iwqx=iw1
    end # if resize

    for ilv in eachindval(ivqx,ivl)
        wlv=(IndexVal(iwqx,ilv[1].second+off.o1),IndexVal(iwl,ilv[2].second+off.o2))
        for isv in eachindval(iss)
            W[wlv...,isv...]=V[ilv...,isv...]
        end
    end

    return W
end

#
#  o1   add row to RL       o2  add column to RL
#   0   at bottom, Dw1+1    0   at right, Dw2+1
#   1   at top, 1           1   at left, 1
#
function growRL(RL::ITensor,iWlink::Index,off::V_offsets)::Tuple{ITensor,Index}
    @assert order(RL)==2
    iLlink=filterinds(inds(RL),tags=tags(iWlink))[1] #find the link index of RL
    iLqx=noncommonind(inds(RL),iLlink) #find the qx link of RL
    Dwl=dim(iLlink)
    Dwq=dim(iLqx)
    @assert dim(iWlink)==Dwl+1
    iq=Index(Dwq+1,tags(iLqx))
    T=eltype(RL)
    RLplus=ITensor(T(0.0),iq,iWlink)
    @assert norm(RLplus)==0.0
    for jq in eachindval(iLqx)
        for jl in eachindval(iLlink)
            ip=(IndexVal(iq,jq.second+off.o1),IndexVal(iWlink,jl.second+off.o2))
            RLplus[ip...]=RL[jq,jl]
        end
    end
    #add diagonal 1.0
    if !(off.o1==1 && off.o2==1)
        RLplus[iq=>Dwq+1,iWlink=>Dwl+1]=1.0
    end
    if !(off.o1==0 && off.o2==0)
        RLplus[iq=>1,iWlink=>1]=1.0
    end
    return RLplus,iq
end

#
#  factor LR such that for
#       lr=left  LR=M*RM_prime
#       lr=right LR=RL_primt*M
#  However becuase of how the ITensor index works we don't need to distinguish between left and
#  right matrix multiplication in the code.  BUT we do need to worry about upper and lower RL
#  matrices when they are rectangular.  For an upper triangular matrix we wnat to grab the
#  matrix from the right side of R, since that is where the most meat (numerical weight) is
#  Conversly for the lower tri L we want grab M from left side.  In short we want as few
#  zeros as possible in M in order for the SVD decomp and compression to have maximum effect.
#
function getM(RL::ITensor,ms::matrix_state,eps::Float64)::Tuple{ITensor,ITensor,Index,Bool}
    ils=filterinds(inds(RL),tags="Link")
    iqx=findinds(ils,"qx")[1] #think of this as the row index
    iln=noncommonind(ils,iqx) #think of this as the column index
    Dwq,Dwn=dim(iqx),dim(iln)
    Dwm=Base.min(Dwq,Dwn)
    irm=Index(Dwm,"Link,m") #new common index between Mplus and RL_prime
    imq=Index(Dwq-2,tags(iqx)) #mini version of iqx
    imm=Index(Dwm-2,tags(irm)) #mini version of irm
    M=ITensor(imq,imm)
    # @show inds(M)
    shift=0
    if ms.ul==lower
        shift=Base.max(0,Dwn-Dwq) #for upper rectangular R we want M over at the right
    else #upper
        shift=Base.max(0,Dwq-Dwn)
    end
    #@show shift,ms.ul,Dwn,Dwq
    for j1 in 2:Dwq-1
        for j2 in 2:Dwm-1
            M[imq=>j1-1,imm=>j2-1]=RL[iqx=>j1,iln=>j2+shift]
        end
    end
    #
    # Now we need RL_prime such that RL=M*RL_prime.
    # RL_prime is just the perimeter of RL with 1's on the diagonal
    # Well sort of, if RL is rectangular then htings get a little more involved.
    #
    #@show Dwm dim(iln) Dwn
    non_zero=false
    RL_prime=ITensor(0.0,irm,iln)
    for j1 in 1:dim(irm) #or 1:Dwm
        RL_prime[irm=>j1,iln=>1       ]=RL[iqx=>j1,iln=>1  ] #first col
        RL_prime[irm=>j1,iln=>dim(iln)]=RL[iqx=>j1,iln=>dim(iln)] #last cols
        #check for non-zero elements to right of where I is.
        #@show Dwm iln
        for j2 in Dwm:dim(iln)-1
            #@show j2
            non_zero = non_zero || abs(RL[iqx=>j1,iln=>j2]) >eps
        end
        RL_prime[irm=>j1,iln=>j1+shift]=1.0
    end
    for j2 in 1:dim(iln)
        RL_prime[irm=>1       ,iln=>j2]=RL[iqx=>1  ,iln=>j2]
        RL_prime[irm=>dim(irm),iln=>j2]=RL[iqx=>Dwq,iln=>j2]
    end
    RL_prime[irm=>dim(irm),iln=>dim(iln)]=1.0

    return M,RL_prime,irm,non_zero
end

#                      |1 0 0|
#  given A, spit out G=|0 A 0|
#                      |0 0 1|
#
function grow(A::ITensor,ig1::Index,ig2::Index)
    ils=inds(A)
    @assert length(ils)==order(A)
    #
    # we need to connect the indices of A with ig1,ig2 indices based on matching tags.
    #
    if hastags(ils[1],tags(ig1))
        ia1=ils[1]
        @assert hastags(ils[2],tags(ig2))
        ia2=ils[2]
    elseif hastags(ils[1],tags(ig2))
        ia2=ils[1]
        @assert hastags(ils[2],tags(ig1))
        ia1=ils[2]
    else
        @assert false
    end
    chi1,chi2=dim(ia1),dim(ia2)
    @assert dim(ig1)==chi1+2
    @assert dim(ig2)==chi2+2

    G=ITensor(0.0,ig1,ig2) #would be nice to use delta() but we can't set elements on it.
    G[ig1=>1     ,ig2=>1     ]=1.0;
    G[ig1=>chi1+2,ig2=>chi2+2]=1.0;
    for j1 in 1:chi1
        for j2 in 1:chi2
            G[ig1=>j1+1,ig2=>j2+1]=A[ia1=>j1,ia2=>j2]
        end
    end
    return G
end


text:
include("structs/PlutoViewer.jl")
include("structs/Livestream.jl")

"""
    _draw_image(video::Video, objects::Vector, frame::Int, canvas::Gtk.Canvas,
    img_dims::Vector)

Internal function to create an image that is drawn on a Gtk Canvas.
"""
function _draw_image(
    video::Video,
    objects::Vector,
    frame::Int,
    canvas::Gtk.Canvas,
    img_dims::Vector,
)
    @guarded draw(canvas) do widget
        # Gets a specific frame from graphic; transposed due to returned matrix
        frame_mat = transpose(get_javis_frame(video, objects, frame; layers = video.layers))

        # Gets the correct Canvas context to draw on
        context = getgc(canvas)

        # Uses Cairo to draw on Gtk canvas context
        image(context, CairoImageSurface(frame_mat), 0, 0, img_dims[1], img_dims[2])
    end
end

"""
    _increment(video::Video, widgets::Vector, objects::Vector, dims::Vector,
        canvas::Gtk.Canvas, frames::Int, layers=Vector)

Increments a given value and returns the associated frame.
"""
function _increment(
    video::Video,
    widgets::Vector,
    objects::Vector,
    dims::Vector,
    canvas::Gtk.Canvas,
    frames::Int,
)
    # Get current frame from textbox as an Int value
    curr_frame = parse(Int, get_gtk_property(widgets[2], :text, String))
    if frames > curr_frame
        # `widgets[1]` represents the GtkReactive slider widget
        push!(widgets[1], curr_frame + 1)
        _draw_image(video, objects, curr_frame + 1, canvas, dims)
    else
        # `widgets[2]` represents the GtkReactive textboxwidget
        push!(widgets[2], 1) # Sets the first frame shown to one
        _draw_image(video, objects, 1, canvas, dims)
    end
end

"""
    _decrement(video::Video, widgets::Vector, objects::Vector, dims::Vector,
        canvas::Gtk.Canvas, frames::Int, layers::Vector)

Decrements a given value and returns the associated frame.
"""
function _decrement(
    video::Video,
    widgets::Vector,
    objects::Vector,
    dims::Vector,
    canvas::Gtk.Canvas,
    frames::Int,
)
    # Get current frame from textbox as an Int value
    curr_frame = parse(Int, get_gtk_property(widgets[2], :text, String))
    if curr_frame > 1
        # `widgets[1]` represents the GtkReactive slider widget
        push!(widgets[1], curr_frame - 1)
        _draw_image(video, objects, curr_frame - 1, canvas, dims)
    else
        # `widgets[2]` represents the GtkReactive textboxwidget
        push!(widgets[2], frames) # Sets the first frame shown to one
        _draw_image(video, objects, frames, canvas, dims)
    end
end

"""
     _javis_viewer(video::Video, frames::Int, object_list::Vector, show::Bool)

Internal Javis Viewer built on Gtk that is called for live previewing.
"""
function _javis_viewer(
    video::Video,
    total_frames::Int,
    object_list::Vector,
    show::Bool = true,
)
    #####################################################################
    # VIEWER WINDOW AND CONFIGURATION
    #####################################################################

    # Determine frame size of animation
    frame_dims = [video.width, video.height]

    # Creates a GTK window for drawing; sized based on frame size
    win = GtkWindow("Javis Viewer", frame_dims[1], frame_dims[2])

    # Sets border size of window
    set_gtk_property!(win, :border_width, 20)

    #####################################################################
    # DISPLAY WIDGETS
    #####################################################################

    # Create GtkScale internal widget
    _slide = GtkScale(false, 1:total_frames)

    # Create GtkReactive slider widget
    slide = slider(1:total_frames, value = 1, widget = _slide)

    #=
    #
    # NOTE: We must provide a named GtkScale widget named `_slide` to the
    # GtkReactive `slider` widget so as to perform asynchronous calls
    # via signal_connect. Otherwise, we will be unable to update the
    # widget that is automatically created by the slider object.
    #
    # It should be stated that a `slider` object is essentially a
    # GtkScale widget coupled with a Reactive object.
    #
    =#

    # Create a textbox
    tbox = GtkReactive.textbox(Int; signal = signal(slide))

    # Button for going forward through animation
    forward = GtkButton("==>")

    # Button for going backward through animation
    backward = GtkButton("<==")

    #=
    TODO: Enable widgets of window to dynamically resize based on user changing the size of a window.
    I think I can use the `configure-event` signal in GTK3 documentation
    (link: https://developer.gnome.org/gtk3/stable/GtkWidget.html#GtkWidget-configure-event).
    From there, I can then make a `signal_connect` set-up where I update `set_gtk_property!()`
    of the windows accordingly using `:width_request` and `height_request`.
    =#

    #####################################################################
    # VIEWER CANVAS AND GRID CONFIGURATION
    #####################################################################

    # Gtk Canvas object upon which to draw image; sized via frame size
    canvas = Gtk.Canvas(frame_dims[1], frame_dims[2])

    # Grid to allocate widgets
    grid = Gtk.Grid()

    # Allocate the widgets in a 3x3 grid
    grid[1:3, 1] = canvas
    grid[1:3, 2] = slide
    grid[1, 3] = backward
    grid[2, 3] = tbox
    grid[3, 3] = forward

    # Center all widgets vertically in grid
    set_gtk_property!(grid, :valign, 3)

    # Center all widgets horizontally in grid
    set_gtk_property!(grid, :halign, 3)

    # Adds grid to previously defined window
    push!(win, grid)

    #####################################################################
    # DISPLAY FIRST FRAME
    #####################################################################

    _draw_image(video, object_list, 1, canvas, frame_dims)

    #####################################################################
    # SIGNAL CONNECTION FUNCTIONS
    #####################################################################

    # When the slider is changed, update currently viewed frame
    signal_connect(_slide, "value-changed") do widget
        # Collects GtkScale as an adjustable bounded value object
        bound_slide = Gtk.GAccessor.adjustment(_slide)

        # Get frame number from bounded value object as Int
        slide_val = Gtk.get_gtk_property(bound_slide, "value", Int)

        _draw_image(video, object_list, slide_val, canvas, frame_dims)
    end

    # When the `Enter` key is pressed, update the frame
    signal_connect(win, "key-press-event") do widget, event
        if event.keyval == 65293
            # Get current frame from textbox as an Int value
            curr_frame = parse(Int, get_gtk_property(tbox, :text, String))
            curr_frame = clamp(curr_frame, 1, total_frames)
            _draw_image(video, object_list, curr_frame, canvas, frame_dims)
        end
    end

    # When the `forward` button is clicked, increment current frame number
    # If at final frame, wrap viewer to first frame
    signal_connect(forward, "clicked") do widget
        _increment(video, [slide, tbox], object_list, frame_dims, canvas, total_frames)
    end

    # When the `Right Arrow` key is pressed, increment current frame number
    # If at final frame, wrap viewer to first frame
    signal_connect(win, "key-press-event") do widget, event
        if event.keyval == 65363
            _increment(video, [slide, tbox], object_list, frame_dims, canvas, total_frames)
        end
    end

    # When the `backward` button is clicked, decrement the current frame number
    # If at first frame, wrap viewer to last frame
    signal_connect(backward, "clicked") do widget
        _decrement(video, [slide, tbox], object_list, frame_dims, canvas, total_frames)
    end

    # When the `Left Arrow` key is pressed, decrement current frame number
    # If at first frame, wrap viewer to last frame
    signal_connect(win, "key-press-event") do widget, event
        if event.keyval == 65361
            _decrement(video, [slide, tbox], object_list, frame_dims, canvas, total_frames)
        end
    end

    #####################################################################

    if show
        # Display image viewer
        Gtk.showall(win)
    else
        return win, frame_dims, slide, tbox, canvas, object_list, total_frames, video
    end
end

"""
    _jupyter_viewer(video::Video, frames::Int, actions::Vector)
Creates an interactive viewer in a Jupyter Notebook.
"""
function _jupyter_viewer(
    video::Video,
    frames::Int,
    objects::Vector{AbstractObject},
    framerate::Int,
)
    t = Interact.textbox(1:frames, value = 1, typ = Int)
    f = Interact.slider(1:frames, label = "Frame", value = t)
    obs = Interact.Observables.throttle(1 / framerate, f)
    output = @map get_javis_frame(video, objects, &obs; layers = video.layers)
    wdg = Widget(["f" => f, "t" => t], output = output)
    @layout! wdg vbox(hbox(:f, :t), output)
end

function Base.show(io::IO, ::MIME"image/png", v::PlutoViewer)
    write(io, read(v.filename))
end

"""
    _pluto_viewer(video::Video, frames::Int, actions::Vector)
Creates an interactive viewer in a Pluto Notebook by storing all the frames in-memory
```
# In separate Pluto notebook cells
using PlutoUI

anim = render(
    video;
    pathname = "loading.gif",liveview=true
);

@bind x Slider(1:1:frames)

anim[x]
```
"""
function _pluto_viewer(video::Video, frames::Int, objects::Vector;)
    arr = collect(
        get_javis_frame(video, objects, frame; layers = video.layers) for frame in 1:frames
    )
    return arr
end

"""
    setup_stream(livestreamto=:local; protocol="udp", address="0.0.0.0", port=14015, twitch_key="")

Sets up the livestream configuration.
**NOTE:** Twitch not fully implemented, do not use.
"""
function setup_stream(
    livestreamto::Symbol = :local;
    protocol::String = "udp",
    address::String = "0.0.0.0",
    port::Int = 14015,
    twitch_key::String = "",
)
    StreamConfig(livestreamto, protocol, address, port, twitch_key)
end

"""
    cancel_stream()

Sends a `SIGKILL` signal to the livestreaming process. Though used internally, it can be used stop streaming.
However this method is not guaranted to end the stream on the client side.
"""
function cancel_stream()
    #todo explore better ways of searching and killing processes

    # kill the ffmpeg process
    # ps aux | grep ffmpeg | grep stream_loop | awk '{print $2}' | xargs kill -9
    try
        println("Checking for existing stream....")
        run(
            pipeline(
                `ps aux`,
                pipeline(`grep ffmpeg`, pipeline(`grep stream_loop`, `awk '{print $2}'`)),
            ),
        )
    catch
        return @warn "Not Streaming Anything Currently"
    end

    run(
        pipeline(
            `ps aux`,
            pipeline(
                `grep ffmpeg`,
                pipeline(`grep stream_loop`, pipeline(`awk '{print $2}'`, `xargs kill -9`)),
            ),
        ),
    )
    return "Livestream Cancelled!"
end

"""
    _livestream(streamconfig, framerate, width, height, pathname)

Internal method for livestreaming
"""
function _livestream(
    streamconfig::StreamConfig,
    framerate::Int,
    width::Int,
    height::Int,
    pathname::String,
)
    cancel_stream()

    livestreamto = streamconfig.livestreamto
    twitch_key = streamconfig.twitch_key

    if livestreamto == :twitch && isempty(twitch_key)
        return error("Please enter your twitch stream key")
    end

    command = [
        "-stream_loop", # loop the stream -1 times i.e. indefinitely
        "-1",
        "-r", # frames per second
        "$framerate",
        "-an",  # Tells FFMPEG not to expect any audio
        "-loglevel", # show only ffmpeg errors
        "error",
        "-re", # read input at native frame rate
        "-i", # input file
        "$pathname",
    ]

    if livestreamto == :twitch
        if isempty(twitch_key)
            error("Please enter your twitch api key")
        end

        #
        twitch_cmd = [
            "-f",
            "flv", # force the file to flv format
            "rtmp://live.twitch.tv/app/$twitch_key", # stream to the twitch platform using rtmp protocol
        ]
        push!(command, twitch_cmd...)
        @info "Livestreaming to Twitch!"
    elseif livestreamto == :local
        protocol = streamconfig.protocol
        address = streamconfig.address
        port = streamconfig.port
        local_command = ["-f", "mpegts", "$protocol://$address:$port"] # use an mpeg-ts format, and stream to the given address/port using the protocol
        push!(command, local_command...)
        @info "Livestream Started at $protocol://$address:$port"
    end

    # schedule the streaming process and allow it to run asynchronously
    schedule(@task begin
        ffmpeg_exe(`$command`)
    end)
end

_livestream(
    streamconfig::Nothing,
    framerate::Int,
    width::Int,
    height::Int,
    pathname::String,
) = return