text:
using ParticleFilters
using Distributions
using StaticArrays
using LinearAlgebra
using Random

# 2D Double Integrator
# state is [x, y, xdot, ydot];

const dt = 0.1
const A = [1.0 0.0 dt  0.0;
           0.0 1.0 0.0 dt ;
           0.0 0.0 1.0 0.0;
           0.0 0.0 0.0 1.0]

const B = [0.5*dt^2 0.0     ;
           0.0      0.5*dt^2;
           dt       0.0     ;
           0.0      dt      ]

const W = Matrix(0.001*Diagonal{Float64}(I, 4)) # Process noise covariance
const V = Matrix(Diagonal{Float64}(I, 2)) # Measurement noise covariance

f(x, u, rng) = A*x + B*u + rand(rng, MvNormal(W))
g(x0, u, x, y) = pdf(MvNormal(x[1:2], V), y)
model = ParticleFilterModel{Vector{Float64}}(f, g)

h(x, rng) = rand(rng, MvNormal(x[1:2], V))

@testset "example" begin
    N = 1000
    filter = SIRParticleFilter(model, N)
    Random.seed!(1)
    rng = Random.GLOBAL_RNG
    b = ParticleCollection([4.0*rand(4).-2.0 for i in 1:N])
    x = [0.0, 1.0, 1.0, 0.0]
    for i in 1:100
        print(".")
        m = mean(b)
        u = [-m[1], -m[2]] # try to orbit the origin
        x = f(x, u, rng)
        y = h(x, rng)
        b = update(filter, b, u, y)

        # scatter([p[1] for p in particles(b)], [p[2] for p in particles(b)], color=:black, markersize=0.1, label="")
        # scatter!([s[1]], [s[2]], color=:blue, xlim=(-5,5), ylim=(-5,5), title=t, label="")
    end
    # write("particles.gif", film)
end


text:
# randperm
n = 100
r = [1:n;]
randperm!(r)
@test isperm(r)
@test r != [1:n;]

# randchoice
n = 10
r = [1:n;]
x = zeros(Int, 1000)
for i=1:length(x)
    x[i] = randchoice(r)
    @test 1 <= x[i] <= n
end
@test sort(union(x)) == r

a = randchoice(MersenneTwister(17),r)
@test 1 <= a <= n

r = (i for i in 1:20)
a = randchoice(r)
@test 1<= a <= 20

#getRNG
@test getRNG(-1) === Base.Random.GLOBAL_RNG
@test getRNG(0) === Base.Random.GLOBAL_RNG
@test getRNG(17) == MersenneTwister(17)

# randchoice!

s = randchoice(1:10, 3)
@test length(s) == 3
for  e in s
    @test 1 <= e <= 10
end

s = randchoice!([1:10;], 3)
@test length(s) == 3
for  e in s
    @test 1 <= e <= 10
end

s = randchoice!([1:10;], 6, exclude=[1,2])
@test length(s) == 6
for  e in s
    @test 3 <= e <= 10
end

ext: jl

max_stars_repo_name: JuliaTagBot/LittleScienceTools

max_issues_repo_name: JuliaTagBot/LittleScienceTools

max_forks_repo_name: JuliaTagBot/LittleScienceTools


text:
# This file contains function to enable progress meter log

function progressbarmeter(n::Integer; desc::String = "Computing DEA model", progress::Bool = true)
    if progress == true
        Progress(n; desc = desc, enabled = progress)
    else
        Progress(n; enabled = false)
    end
end


text:
# Autogenerated wrapper script for Wine_jll for x86_64-linux-gnu
export wine64

using Zlib_jll
using libpng_jll
using JpegTurbo_jll
using GnuTLS_jll
JLLWrappers.@generate_wrapper_header("Wine")
JLLWrappers.@declare_executable_product(wine64)
function __init__()
    JLLWrappers.@generate_init_header(Zlib_jll, libpng_jll, JpegTurbo_jll, GnuTLS_jll)
    JLLWrappers.@init_executable_product(
        wine64,
        "bin/wine64",
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
metadata_packages = [
    "DataFrames",
    "DecisionTree",
    "HDF5",
    "JLD"]

Pkg.init()
Pkg.update()

for package=metadata_packages
    Pkg.add(package)
end

Pkg.resolve()

max_stars_repo_name: carlsonp/pachyderm

max_issues_repo_name: carlsonp/pachyderm

max_forks_repo_name: carlsonp/pachyderm


text:
@testset "971.flip-binary-tree-to-match-preorder-traversal.jl" begin
    @test flip_match_voyage(TreeNode(1, TreeNode(2), nothing), [2, 1]) == [-1]
    @test flip_match_voyage(TreeNode(1, TreeNode(2), TreeNode(3)), [1, 3, 2]) == [1]
    @test flip_match_voyage(TreeNode(1, TreeNode(2), TreeNode(3)), [1, 2, 3]) == Int[]
end


text:
mutable struct Flatten <: DiffableFunction
    in_shape
    grad_field :: GradField
    Flatten(in_shape, grad_field) = new(in_shape, grad_field)
end

function forward(::Flatten, x)
    return vcat(x...)
end

function backward(f::Flatten, gy)
    reshape(gy, f.in_shape)
end

"""
    flatten(x::Variable)
The function corresponding to `vcat(x...)`.

# Example
julia> x = Variable([1 2; 3 4; 5 6])
name: nothing
values: [1 2; 3 4; 5 6]
creator: User-Defined(nothing)

julia> JITrench.flatten(x)
name: nothing
values: [1, 3, 5, 2, 4, 6]
creator: JITrench.Flatten
"""
flatten(x::Variable) = Flatten(size(x.values), GradField())(x)


text:
using GPUifyLoops: @launch, @loop, @unroll
using Oceananigans.Utils: @loop_xy, @loop_xz, launch_config

#####
##### Halo filling for value and gradient boundary conditions
#####

function fill_bottom_halo!(c, bc::Union{VBC, GBC}, arch, grid, args...)
    @launch device(arch) config=launch_config(grid, :xy) _fill_bottom_halo!(c, bc, grid, args...)
    return nothing
end

function fill_top_halo!(c, bc::Union{VBC, GBC}, arch, grid, args...)
    @launch device(arch) config=launch_config(grid, :xy) _fill_top_halo!(c, bc, grid, args...)
    return nothing
end

function fill_south_halo!(c, bc::Union{VBC, GBC}, arch, grid, args...)
    @launch device(arch) config=launch_config(grid, :xz) _fill_south_halo!(c, bc, grid, args...)
    return nothing
end

function fill_north_halo!(c, bc::Union{VBC, GBC}, arch, grid, args...)
    @launch device(arch) config=launch_config(grid, :xz) _fill_north_halo!(c, bc, grid, args...)
    return nothing
end

@inline linearly_extrapolate(c₀, ∇c, Δ) = c₀ + ∇c * Δ

@inline  left_gradient(bc::GBC, c¹, Δ, i, j, args...) = getbc(bc, i, j, args...)
@inline right_gradient(bc::GBC, cᴺ, Δ, i, j, args...) = getbc(bc, i, j, args...)

@inline  left_gradient(bc::VBC, c¹, Δ, i, j, args...) = ( c¹ - getbc(bc, i, j, args...) ) / (Δ/2)
@inline right_gradient(bc::VBC, cᴺ, Δ, i, j, args...) = ( getbc(bc, i, j, args...) - cᴺ ) / (Δ/2)

function _fill_bottom_halo!(c, bc::Union{VBC, GBC}, grid, args...)
    @loop_xy i j grid begin
        @inbounds ∇c = left_gradient(bc, c[i, j, 1], grid.Δz, i, j, grid, args...)
        @unroll for k in (1 - grid.Hz):0
            Δ = (k - 1) * grid.Δz  # separation between bottom grid cell and halo is negative
            @inbounds c[i, j, k] = linearly_extrapolate(c[i, j, 1], ∇c, Δ)
        end
    end
    return nothing
end

function _fill_top_halo!(c, bc::Union{VBC, GBC}, grid, args...)
    @loop_xy i j grid begin
        @inbounds ∇c = right_gradient(bc, c[i, j, grid.Nz], grid.Δz, i, j, grid, args...)
        @unroll for k in (grid.Nz + 1) : (grid.Nz + grid.Hz)
            Δ = (k - grid.Nz) * grid.Δz
            @inbounds c[i, j, k] = linearly_extrapolate(c[i, j, grid.Nz], ∇c, Δ)
        end
    end
    return nothing
end

function _fill_south_halo!(c, bc::Union{VBC, GBC}, grid, args...)
    @loop_xz i k grid begin
        @inbounds ∇c = left_gradient(bc, c[i, 1, k], grid.Δy, i, k, grid, args...)
        @unroll for j in (1 - grid.Hy):0
            Δ = (j - 1) * grid.Δy  # separation between southern-most grid cell and halo is negative
            @inbounds c[i, j, k] = linearly_extrapolate(c[i, 1, k], ∇c, Δ)
        end
    end
    return nothing
end

function _fill_north_halo!(c, bc::Union{VBC, GBC}, grid, args...)
    @loop_xz i k grid begin
        @inbounds ∇c = right_gradient(bc, c[i, grid.Ny, k], grid.Δy, i, k, grid, args...)
        @unroll for j in (grid.Ny + 1) : (grid.Ny + grid.Hy)
            Δ = (j - grid.Ny) * grid.Δy
            @inbounds c[i, j, k] = linearly_extrapolate(c[i, grid.Ny, k], ∇c, Δ)
        end
    end
    return nothing
end


text:
module ConstantCellValues

using Gridap

export ConstantCellValue
export ConstantCellNumber
export ConstantCellArray
export ConstantCellVector
export ConstantCellMatrix
export ConstantCellMap

import Gridap: apply
import Gridap: reindex
import Gridap: evaluate
import Base: size
import Base: getindex
import Base: broadcast
import Base: ==, ≈

struct ConstantCellValue{T} <: IndexCellValue{T,1}
  value::T
  length::Int
end

size(self::ConstantCellValue) = (self.length,)

function getindex(c::ConstantCellValue,i::Integer)
  c.value
end

const ConstantCellNumber = ConstantCellValue{<:NumberLike}

const ConstantCellArray{T,N} = ConstantCellValue{Array{T,N}}

const ConstantCellVector{T} = ConstantCellArray{T,1}

const ConstantCellMatrix{T} = ConstantCellArray{T,2}

const ConstantCellMap{S,M,T,N} = ConstantCellValue{<:Map{S,M,T,N}}

function ConstantCellArray(v::AbstractArray{T,N},l::Integer) where {T,N}
  ConstantCellArray{T,N}(v,l)
end

function ConstantCellVector(v::AbstractVector{T},l::Integer) where T
  ConstantCellVector{T}(v,l)
end

function ConstantCellMatrix(v::AbstractMatrix{T},l::Integer) where T
  ConstantCellMatrix{T}(v,l)
end

function ConstantCellMap(v::Map,l::Integer)
  ConstantCellValue(v,l)
end

function ConstantCellNumber(v::NumberLike,l::Integer)
  ConstantCellValue(v,l)
end

function (==)(a::ConstantCellNumber,b::ConstantCellNumber)
  _eq_kernel(==,a,b)
end

function (≈)(a::ConstantCellNumber,b::ConstantCellNumber)
  _eq_kernel(≈,a,b)
end

function (==)(a::ConstantCellArray,b::ConstantCellArray)
   _eq_kernel(==,a,b)
end

function (≈)(a::ConstantCellArray,b::ConstantCellArray)
  _eq_kernel(≈,a,b)
end

function _eq_kernel(op,a,b)
  !(op(a.value,b.value)) && return false
  length(a) != length(b) && return false
  return true
end

function apply(k::NumberKernel,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(v...)
  val = compute_value(k,vals...)
  ConstantCellNumber(val,l)
end

function apply(k::ArrayKernel,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(v...)
  val = compute_value(k,vals...)
  ConstantCellArray(val,l)
end

function apply(k::ArrayKernel,m::ConstantCellMap)
  w = apply(k,m.value)
  ConstantCellMap(w,m.length)
end

function apply(k::ArrayKernel,m::ConstantCellMap,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(m,v...)
  w = apply(k,m.value,vals...)
  ConstantCellMap(w,l)
end

function evaluate(cm::ConstantCellMap{S,M},ca::ConstantCellArray{<:S,M}) where {S,M}
  @assert length(cm) == length(ca)
  m = cm.value
  a = ca.value
  r = evaluate(m,a)
  ConstantCellArray(r,length(cm))
end

function _compute_l(v...)
  @assert length(v) > 0
  v1, = v
  l1 = length(v1)
  @assert all([ length(vi) == l1 for vi in v ])
  l1
end

function reindex(values::ConstantCellValue, indices::CellValue{<:IndexLike})
  ConstantCellValue(values.value,length(indices))
end

function reindex(values::ConstantCellValue, indices::IndexCellValue{<:IndexLike})
  ConstantCellValue(values.value,length(indices))
end

broadcast(f,cvs::ConstantCellValue) = ConstantCellValue(f(cvs.value),cvs.length)

end # module ConstantCellValues


text:
using Documenter
using SyncBarriers

makedocs(
    sitename = "SyncBarriers",
    format = Documenter.HTML(),
    modules = [SyncBarriers]
)

deploydocs(
    repo = "github.com/JuliaConcurrent/SyncBarriers.jl",
    push_preview = true,
)


text:
# =========================================================================================
"""
    rfenergy{T, R <: BEMResult{T}}(
        bem       ::R
    )

Computes the local or nonlocal reaction field energy W* as
```math
W^* = ∫φ^* ρ \\quad dΩ
```
where ``φ^*`` is the reaction field and ``ρ`` is the corresponding charge distribution.

# Unit
``\\frac{kJ}{mol}``

# Return type
`T`
"""
function rfenergy(bem::R) where {T, R <: BEMResult{T}}
    qposs = [charge.pos for charge in bem.model.charges]
    qvals = [charge.val for charge in bem.model.charges]

    # reaction field energy per point charge
    wstar = zeros(T, length(bem.model.charges))

    # W* = -[W ⋅ u](ξ)
    Rjasanow.laplacecoll!(DoubleLayer, wstar, bem.model.elements, qposs, bem.u)
    rmul!(wstar, -1)

    # W* += [V ⋅ q](ξ)
    Rjasanow.laplacecoll!(SingleLayer, wstar, bem.model.elements, qposs, bem.q)

    # Apply ρ, integrate over Ω and apply remaining prefactors (in order)
    # ▶ 4π        for Vtilde, W
    # (in potprefactor:)
    # ▶ 4π⋅ε0     for u and q
    # ▶ 1.602e-19 for elemental charge e; [e] = C
    # ▶ 1e10      for the conversion Å → m; [ε0] = F/m
    #
    # ▶ 1.602e-19 for elemental charge e; [e] = C
    # ▶ 6.022e23  for Avogadro constant Nₐ; [Nₐ] = 1/mol
    # ▶ 1e-3      for the conversion 1/J → 1/kJ
    # ▶ 0.5       since we have counted all interactions twice
    wstar ⋅ qvals / T(4π) * potprefactor(T) * T(ec * 6.022140857e10 / 2)
end

# =========================================================================================
"""
    φΩ{T, R <: BEMResult{T}}(
        Ξ         ::Vector{Vector{T}},
        bem       ::R
    )

Computes the local or nonlocal interior electrostatic potential ``φ_Ω`` for the given set
of observation points `Ξ`.

!!! warning
    This function does not verify whether all points in `Ξ` are located in ``Ω``!

# Unit
``V = \\frac{C}{F}``

# Return type
`Vector{T}`
"""
function φΩ(
        Ξ         ::Vector{Vector{T}},
        bem       ::R
    ) where {T, R <: BEMResult{T}}
    # result vector
    φ = zeros(T, length(Ξ))

    # φ  = -[W ⋅ u](ξ)
    Rjasanow.laplacecoll!(DoubleLayer, φ, bem.model.elements, Ξ, bem.u)
    rmul!(φ, -1)

    # φ += [Vtilde ⋅ q](ξ)
    Rjasanow.laplacecoll!(SingleLayer, φ, bem.model.elements, Ξ, bem.q)

    # φ *= 1/4π
    # (W and Vtilde were premultiplied by 4π! 4π⋅ε0 from u and q still to be applied)
    rmul!(φ, T(1 / 4π))

    # φ += 1/εΩ ⋅ φ*mol(ξ)
    # (φ*mol was premultiplied by 4π⋅ε0⋅εΩ; 4π⋅ε0 remain to be applied)
    axpy!(1/bem.model.params.εΩ, φmol(Ξ, bem.model.charges), φ)

    # Apply remaining prefactors:
    # ▶ 4π⋅ε0     for u, q, and umol
    # ▶ 1.602e-19 for elemental charge e; [e] = C
    # ▶ 1e10      for the conversion Å → m; [ε0] = F/m
    rmul!(φ, potprefactor(T))

    φ
end

# =========================================================================================
"""
    φΣ{T, B <: BEMResult{T}}(
        Ξ         ::Vector{Vector{T}},
        bem       ::B
    )

Computes the local or nonlocal exterior electrostatic potential ``φ_Σ`` for the given set
of observation points `Ξ`.

!!! warning
    This function does not verify whether all points in `Ξ` are located in ``Σ``!

# Unit
``V = \\frac{C}{F}``

# Return type
`Vector{T}`
"""
function φΣ(
        Ξ         ::Vector{Vector{T}},
        bem       ::LocalBEMResult{T}
    ) where T
    # result vector
    φ = zeros(T, length(Ξ))
    buf = Array{T}(undef, length(bem.model.elements))

    # φ  = -εΩ/εΣ ⋅ [Vtilde ⋅ (q + qmol)](ξ)
    copyto!(buf, bem.q)
    axpy!(1, bem.qmol, buf)
    Rjasanow.laplacecoll!(SingleLayer, φ, bem.model.elements, Ξ, buf)
    rmul!(φ, -bem.model.params.εΩ/bem.model.params.εΣ)

    # φ += [W ⋅ (u + umol)](ξ)
    copyto!(buf, bem.u)
    axpy!(1, bem.umol, buf)
    Rjasanow.laplacecoll!(DoubleLayer, φ, bem.model.elements, Ξ, buf)

    # Apply remaining prefactors:
    # ▶ 4π        for Vtilde, W
    # ▶ 4π⋅ε0     for u, q, umol, and qmol
    # ▶ 1.602e-19 for elemental charge e; [e] = C
    # ▶ 1e10      for the conversion Å → m; [ε0] = F/m
    rmul!(φ, potprefactor(T) / T(4π))

    φ
end

function φΣ(
        Ξ         ::Vector{Vector{T}},
        bem       ::NonlocalBEMResult{T}
    ) where T
    # result vector
    φ = zeros(T, length(Ξ))

    # convenience aliases
    εΩ  = bem.model.params.εΩ
    εΣ  = bem.model.params.εΣ
    ε∞  = bem.model.params.ε∞
    yuk = yukawa(bem.model.params)
    elements = bem.model.elements

    buf = Array{T}(undef, length(elements))

    # φ  = -V[εΩ/ε∞ ⋅ (q + qmol)](ξ)
    copyto!(buf, bem.q)
    axpy!(1, bem.qmol, buf)
    Rjasanow.laplacecoll!(SingleLayer, φ, elements, Ξ, buf)
    rmul!(φ, -εΩ/ε∞)

    # φ += (Vʸ-V)[εΩ(1/εΣ - 1/ε∞) ⋅ (q + qmol)](ξ)
    rmul!(buf, εΩ * (1/εΣ - 1/ε∞))
    Radon.regularyukawacoll!(SingleLayer, φ, elements, Ξ, yuk, buf)

    # φ += K[u + umol](ξ)
    copyto!(buf, bem.u)
    axpy!(1, bem.umol, buf)
    Rjasanow.laplacecoll!(DoubleLayer, φ, elements, Ξ, buf)

    # φ += (Kʸ-K)[u + (1-εΩ/εΣ) ⋅ umol - ε∞/εΣ ⋅ w](ξ)
    copyto!(buf, bem.u)
    axpy!(1-εΩ/εΣ, bem.umol, buf)
    axpy!(-ε∞/εΣ, bem.w, buf)
    Radon.regularyukawacoll!(DoubleLayer, φ, elements, Ξ, yuk, buf)

    # Apply remaining prefactors:
    # ▶ 4π        for V, K, (Vʸ-V), and (Kʸ-K)
    # ▶ 4π⋅ε0     for u, q, w, umol, and qmol
    # ▶ 1.602e-19 for elemental charge e; [e] = C
    # ▶ 1e10      for the conversion Å → m; [ε0] = F/m
    rmul!(φ, potprefactor(T) / T(4π))

    φ
end


text:
#== # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Description
#
#   Pre-defined formats.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ==#

export unicode, ascii_dots, ascii_rounded, borderless, compact, markdown, mysql, simple, unicode_rounded, matrix

const unicode = TextFormat()

const ascii_dots = TextFormat(
    up_right_corner     = '.',
    up_left_corner      = '.',
    bottom_left_corner  = ':',
    bottom_right_corner = ':',
    up_intersection     = '.',
    left_intersection   = ':',
    right_intersection  = ':',
    middle_intersection = ':',
    bottom_intersection = ':',
    column              = ':',
    row                 = '.'
)

const ascii_rounded = TextFormat(
    up_right_corner     = '.',
    up_left_corner      = '.',
    bottom_left_corner  = '\'',
    bottom_right_corner = ''',
    up_intersection     = '.',
    left_intersection   = ':',
    right_intersection  = ':',
    middle_intersection = '+',
    bottom_intersection = ''',
    column              = '|',
    row                 = '-'
)

const borderless = TextFormat(
    up_right_corner     = ' ',
    up_left_corner      = ' ',
    bottom_left_corner  = ' ',
    bottom_right_corner = ' ',
    up_intersection     = ' ',
    left_intersection   = ' ',
    right_intersection  = ' ',
    middle_intersection = ' ',
    bottom_intersection = ' ',
    column              = ' ',
    row                 = ' ',
    hlines              = [:header]
)

const compact = TextFormat(
    up_right_corner     = ' ',
    up_left_corner      = ' ',
    bottom_left_corner  = ' ',
    bottom_right_corner = ' ',
    up_intersection     = ' ',
    left_intersection   = ' ',
    right_intersection  = ' ',
    middle_intersection = ' ',
    bottom_intersection  = ' ',
    column              = ' ',
    row                 = '-'
   )

const markdown = TextFormat(
    left_intersection   = '|',
    right_intersection  = '|',
    middle_intersection = '|',
    column              = '|',
    row                 = '-',
    hlines              = [:header]
)

const mysql = TextFormat(
    up_right_corner     = '+',
    up_left_corner      = '+',
    bottom_left_corner  = '+',
    bottom_right_corner = '+',
    up_intersection     = '+',
    left_intersection   = '+',
    right_intersection  = '+',
    middle_intersection = '+',
    bottom_intersection = '+',
    column              = '|',
    row                 = '-'
   )

const simple = TextFormat(
    up_right_corner     = '=',
    up_left_corner      = '=',
    bottom_left_corner  = '=',
    bottom_right_corner = '=',
    up_intersection     = ' ',
    left_intersection   = '=',
    right_intersection  = '=',
    middle_intersection = ' ',
    bottom_intersection  = ' ',
    column              = ' ',
    row                 = '='
)

const unicode_rounded = TextFormat(
    up_right_corner     = '╮',
    up_left_corner      = '╭',
    bottom_left_corner  = '╰',
    bottom_right_corner = '╯',
    up_intersection     = '┬',
    left_intersection   = '├',
    right_intersection  = '┤',
    middle_intersection = '┼',
    bottom_intersection = '┴',
    column              = '│',
    row                 = '─'
)

const matrix = TextFormat(
    left_intersection   = '│',
    right_intersection  = '│',
    row                 = ' ',
    vlines              = [:begin,:end],
    hlines              = [:begin,:end]
)


text:
function prepare_input(x::AbstractMatrix)
    x = MMI.matrix(x) .|> Float32
    return numpy.array(x)
end

function prepare_input(x::AbstractVector)
    x = x .|> Float32 |> numpy.array
    return numpy.array(x)
end

function mlj_to_kwargs(model)
    return Dict{Symbol, Any}(
        name => getfield(model, name)
        for name in fieldnames(typeof(model))
    )
end

include("./cuml/classification.jl")
include("./cuml/clustering.jl")
include("./cuml/dimensionality_reduction.jl")
include("./cuml/regression.jl")
include("./cuml/time_series.jl")

const CUML_MODELS = Union{CUML_CLASSIFICATION,
                            CUML_CLUSTERING,
                            CUML_DIMENSIONALITY_REDUCTION,
                            CUML_REGRESSION,
                            CUML_TIME_SERIES}
include("./cuml/mlj_serialization.jl")

MMI.clean!(model::CUML_MODELS) = ""

# MLJ Package Metadata
MMI.package_name(::Type{<:CUML_MODELS}) = "RAPIDS"
MMI.package_uuid(::Type{<:CUML_MODELS}) = "2764e59e-7dd7-4b2d-a28d-ce06411bac13"
MMI.package_url(::Type{<:CUML_MODELS}) = "https://github.com/tylerjthomas9/RAPIDS.jl"
MMI.is_pure_julia(::Type{<:CUML_MODELS}) = false


text:
# This file was generated, do not modify it. # hide
train = setdiff!(train |> collect, idx_missing_outcome)
test = setdiff!(test |> collect, idx_missing_outcome)
datac = datac[.!missing_outcome, :];


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder

# Collection of sources required to build BKMaxflow
sources = [
    "http://vision.csd.uwo.ca/code/maxflow-v3.01.zip" =>
    "b30891ba393a5fbc9d7d0ec808f502bdfca801f289a5cb79b673ac35c413b06e",

    "https://github.com/Gnimuc/bkmaxflow-c.git" =>
    "3a789aa7ff167e2286a9c8fdeaec184031813019",

]

# Bash recipe for building across all platforms
script = raw"""
cd $WORKSPACE/srcdir
mv bkmaxflow-c/* .
cat > maxflow.patch << 'END'
--- maxflow.cpp
+++ maxflow.cpp
@@ -681,4 +681,4 @@

-#include "instances.inc"
+// #include "instances.inc"
END

patch --ignore-whitespace < maxflow.patch

mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=/opt/$target/$target.toolchain ..
make -j${nproc}
make install

"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line
platforms = BinaryBuilder.supported_platforms()

# The products that we will ensure are always built
products(prefix) = [
    LibraryProduct(prefix, "libbkmaxflow", :libbkmaxflow)
]

# Dependencies that must be installed before this package can be built
dependencies = [

]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, "BKMaxflow", v"3.0.1", sources, script, platforms, products, dependencies)

max_stars_repo_name: Gnimuc/BKMaxflowBuilder

max_issues_repo_name: Gnimuc/BKMaxflowBuilder

max_forks_repo_name: Gnimuc/BKMaxflowBuilder


text:
# ------------------------------------------------------------------------------
# Computations involving triangles
# -------------------------------------------------------------------------------

using SymPy

"""
    equipoints(A, B)

Find the two points the can form an equilateral triangle with `A` and `B`
"""
function equipoints(A, B)
    x, y = @vars x y
    pt = Point(x, y)
    dist1 = squaredist(pt, A)
    dist2 = squaredist(pt, B)
    dist3 = squaredist(A, B)
    sol = solve([dist1-dist3, dist2-dist3], [x,y])
    [Point(simplify(s[1]), simplify(s[2])) for s in sol]
end

"""
    outer_equitri(A, B, C)

Find the outer equilateral triangle of `Triangle(A, B, C)` which is incident to `Edge(A, B)`.
"""
function outer_equitri(A, B, C)
    ptAB = equipoints(A, B)
    dist = map(pt->squaredist(pt, C), ptAB)
    d = simplify(dist[1]-dist[2])
    if simplify(d>=0)
        return Triangle(A, B, ptAB[1])
    else
        return Triangle(A, B, ptAB[2])
    end
end

"""
    outer_equitriangles(📐)

Find all three outer equilateral triangles of the triangle `📐`.
"""
function outer_equitriangles(📐)
    pts = vertices(📐)
    triangles = map(i->outer_equitri(circshift(pts, i)...), 0:2)
    triangles
end

"""
    isequilateral(📐)

Check if `📐` is equilateral, i.e., if the three edges of `📐` are of the same length.
"""
function isequilateral(📐)
    dist = [squaredist(e.src, e.dst) for e in edges(📐)]
    if (dist[1] == dist[2]) && (dist[1] == dist[3])
        return true
    else
        return false
    end
end

"""
    median(A::Point, B::Point, C::Point)

Find the edge from the midpoint of `A` and `B` to `C`.
"""
function median(A::Point, B::Point, C::Point)
    Edge(midpoint(A, B), C)
end


text:
"""
    RingBuffer(size, buf)

A `Channel`-like data structure that rotates through
`size` buffers. `put!`s work by mutating one of the buffers:

```
put!(ringbuffer) do buf
    rand!(buf)
end
```

The result can then be `take!`n:

```
res = take!(ringbuffer)
```

!!! warning "Invalidation"

    Only one result is valid at a time! On the next `take!`, the previous
    result will be reused as a buffer and be mutated by a `put!`
"""
mutable struct RingBuffer{T}
    buffers::Channel{T}
    results::Channel{T}
    current::T
end

function RingBuffer(bufs::Vector{T}) where T
    size = length(bufs) - 1
    buffers = Channel{T}(size + 1)
    results = Channel{T}(size)
    foreach(bufs[2:end]) do buf
        put!(buffers, buf)
    end

    return RingBuffer{T}(buffers, results, bufs[1])
end

function RingBuffer(size, buffer::T) where T
    buffers = [buffer]
    for i ∈ 1:size
        push!(buffers, deepcopy(buffer))
    end
    return RingBuffer(buffers)
end

function Base.take!(ringbuffer::RingBuffer)
    put!(ringbuffer.buffers, ringbuffer.current)
    ringbuffer.current = take!(ringbuffer.results)
    return ringbuffer.current
end

"""
    put!(f!, ringbuffer::RingBuffer)

Apply f! to a buffer in `ringbuffer` and put into the results
channel.

```julia
x = rand(10, 10)
ringbuffer = RingBuffer(1, x)
put!(ringbuffer) do buf
    @test x == buf
    copy!(buf, rand(10, 10))
end
x_ = take!(ringbuffer)
@test !(x ≈ x_)

```
"""
function Base.put!(f!, ringbuffer::RingBuffer)
    buf = take!(ringbuffer.buffers)
    buf_ = f!(buf)
    #@assert buf_ === buf
    put!(ringbuffer.results, buf_)
end

function Base.close(ringbuffer::RingBuffer)
    close(ringbuffer.results)
    close(ringbuffer.buffers)
end


text:
module MiscUtils

import ProgressLogging

"""
    @progress_if cond [options...] expr

If `cond` evaluates to `true`, run `@progress(options..., expr)`; otherwise
just run `expr`.

`options` and `expr` are passed to
[`ProgressLogging.@progress`](https://junolab.org/ProgressLogging.jl/dev/#ProgressLogging.@progress-Tuple)
"""
macro progress_if(cond, args...)
    quote
        if $cond
            $ProgressLogging.@progress($(args...))
        else
            $(args[end])
        end
    end |> esc
end

end  # module


text:
struct TestPQs <: SequentialPQs
  x
  TestPQs(; x=-1) = new(x)
end

qs_score(qs::TestPQs, data::Array{T, 2}, pools::Dict{Symbol, Vector{Int}}) where T <: Real = collect(1:size(data, 2))

function qs_score(qs::TestPQs,
         data::Array{T, 2},
         pools::Dict{Symbol, Vector{Int}},
         subspaces::Vector{Vector{Int}}) where T <: Real
   return [collect(1:size(data, 2)) for i in eachindex(subspaces)]
end


text:
module SigmaRidgeRegression

using BlockDiagonals
using Distributions

using Expectations
using FillArrays #not used yet
using FiniteDifferences
using LinearAlgebra
using MLJBase
import MLJModelInterface
const MMI = MLJModelInterface
import MLJ
import MLJTuning
using MutableNamedTuples

using Random
using Requires
using Roots
using Setfield
using StatsBase
using Tables
using UnPack
using WoodburyMatrices

import Base.\
import Base: reduce, rand
import LinearAlgebra: ldiv!
import StatsBase: fit!, fit, coef, islinear, leverage, modelmatrix, response, predict
import WoodburyMatrices: _ldiv!

#---------- piracy ---------------------------------------------------------
MMI.nrows(X::Tables.MatrixTable) = size(MMI.matrix(X), 1)
MMI.selectrows(X::Tables.MatrixTable, ::Colon) = X
MMI.selectrows(X::Tables.MatrixTable, r::Integer) =
    MMI.selectrows(X::Tables.MatrixTable, r:r)
function MMI.selectrows(X::Tables.MatrixTable, r)
    new_matrix = MMI.matrix(X)[r, :]
    _names = getfield(X, :names)
    MMI.table(new_matrix; names = _names)
end

#----------------------------------------------------------------------------

include("nnls.jl")
include("utils.jl")
include("groupedfeatures.jl")
include("blockridge.jl")
include("end_to_end.jl")
include("covariance_design.jl")
include("simulations.jl")
include("theoretical_risk_curves.jl")
include("mmi.jl")
include("mmi_sigmaridge.jl")
include("grouplasso.jl")
include("mmi_metadata.jl")
include("datasets/CLLData/CLLData.jl")

function __init__()
    @require RCall="6f49c342-dc21-5d91-9882-a32aef131414" include("seagull.jl")
end

export GroupedFeatures,
    ngroups,
    group_idx,
    group_summary,
    group_expand,
    random_betas,
    CholeskyRidgePredictor,
    WoodburyRidgePredictor,
    BasicGroupRidgeWorkspace,
    MomentTunerSetup,
    get_αs_squared,
    get_λs,
    loo_error,
    mse_ridge,
    σ_squared_max,
    sigma_squared_path,
    CovarianceDesign,
    nfeatures,
    get_Σ,
    spectrum,
    simulate_rotated_design,
    AR1Design,
    set_groups,
    DiagonalCovarianceDesign,
    IdentityCovarianceDesign,
    UniformScalingCovarianceDesign,
    ExponentialOrderStatsCovarianceDesign,
    BlockCovarianceDesign,
    simulate,
    GroupRidgeSimulationSettings,
    RandomLinearResponseModel,
    optimal_risk,
    optimal_single_λ_risk,
    optimal_ignore_second_group_risk,
    SingleGroupRidgeRegressor,
    MultiGroupRidgeRegressor,
    LooRidgeRegressor,
    TunedRidgeRegressor,
    SigmaRidgeRegressor,
    GroupLassoRegressor,
    DefaultTuning,
    LooSigmaRidgeRegressor

end # module


text:
using Sass
using Test

filename = joinpath(Sass.examplefolder, "test.sass")

@testset "outputstyle" begin
    css = Sass.compile_file(filename; output_style = Sass.nested)
    @test css == "body {\n  font: 100% Helvetica, sans-serif;\n  color: #333; }\n"
    css = Sass.compile_file(filename; output_style = Sass.expanded)
    @test css == "body {\n  font: 100% Helvetica, sans-serif;\n  color: #333;\n}\n"
    css = Sass.compile_file(filename; output_style = Sass.compact)
    @test css == "body { font: 100% Helvetica, sans-serif; color: #333; }\n"
    css = Sass.compile_file(filename; output_style = Sass.compressed)
    @test css == "body{font:100% Helvetica,sans-serif;color:#333}\n"
end


text:
module GeoFormatTypes

# Use the README as the module docs
@doc let
    path = joinpath(dirname(@__DIR__), "README.md")
    include_dependency(path)
    read(path, String)
end GeoFormatTypes

export GeoFormat

export CoordinateReferenceSystemFormat, EPSG, ProjString, ProjJSON, CoordSys, GML

export GeometryFormat, GeoJSON, KML

export MixedFormat, GML

export AbstractWellKnownText, WellKnownText, WellKnownText2, ESRIWellKnownText, WellKnownBinary

const PROJ_PREFIX = "+proj="
const EPSG_PREFIX = "EPSG:"
# TODO more verification that types are wrapping the right format.

"""
    FormatMode

Traits to indicate the format type, such as `Geom`, `CRS` or `MixedFormat`.
"""
abstract type FormatMode end

"""
    Geom <: FormatMode

    Geom()

Trait specifying that a format object, like [`WellKnownText`](@ref),
contains geometry data.
"""
struct Geom <: FormatMode end

"""
    CRS <: FormatMode

    CRS()

Trait specifying that a format object, like [`WellKnownText`](@ref),
contains only coordinate reference system data.
"""
struct CRS <: FormatMode end

"""
   MixedFormatMode <: FormatMode

Abstract subtype of [`FormatMode`](@ref) where both
geometry and coordinate reference system data are or may be present.
"""
abstract type MixedFormatMode <: FormatMode end
"""
    Extended <: MixedFormatMode <: FormatMode

    Extended()

Trait specifying that a mixed format object, like [`WellKnownText`](@ref),
contains both geometry and coordinate reference system.
"""
struct Extended <: MixedFormatMode end

"""
    Unknown <: MixedFormatMode <: FormatMode

    Unknown()

Trait specifying that for a mixed format object, like [`WellKnownText`](@ref),
it is unknown whether it stores geometry or coordinate reference system data, or both.
"""
struct Unknown <: MixedFormatMode end

"""
    val(f::GeoFormat)

Get the contained value of a GeoFormat type.
"""
function val end

"""
    GeoFormat

Abstract supertype for geospatial data formats
"""
abstract type GeoFormat end

# Convert from the same type does nothing.
Base.convert(::Type{T1}, source::T2) where {T1<:GeoFormat,T2<:T1} = source
# Convert uses the `mode` trait to distinguish crs form geometry conversion
Base.convert(target::Type{T1}, source::T2; kwargs...) where {T1<:GeoFormat,T2<:GeoFormat} = begin
    sourcemode = mode(source)
    targetmode = mode(target)
    convertmode = if targetmode isa Geom
        if sourcemode isa Union{MixedFormatMode,Geom}
            Geom() # Geom is the default if both formats are mixed
        else
            throw(ArgumentError("cannot convert $(typeof(source)) to $target"))
        end
    elseif targetmode isa CRS
        if sourcemode isa Union{MixedFormatMode,CRS}
            CRS()
        else
            throw(ArgumentError("cannot convert $(typeof(source)) to $target"))
        end
    else # targetmode isa MixedFormatMode
        # MixedFormatMode to MixedFormatMode defaults to Geom
        if sourcemode isa Union{MixedFormatMode,Geom}
            Geom()
        else
            CRS()
        end
    end
    convert(target, convertmode, source; kwargs...)
end

"""
    CoordinateReferenceSystemFormat <: GeoFormat

Formats representing coordinate reference systems
"""
abstract type CoordinateReferenceSystemFormat <: GeoFormat end

"""
    GeometryFormat <: GeoFormat

Formats representing geometries. These wrappers simply mark string
formats that may optionally be converted to Geoetry objects at a later point.
"""
abstract type GeometryFormat <: GeoFormat end

"""
    MixedFormat <: GeoFormat

Formats that may hold either or both coordinate reference systems and geometries.
"""
abstract type MixedFormat{X} <: GeoFormat end

val(x::GeoFormat) = x.val

mode(format::GeoFormat) = mode(typeof(format))
mode(::Type{<:GeometryFormat}) = Geom()
mode(::Type{<:CoordinateReferenceSystemFormat}) = CRS()
mode(::Type{<:MixedFormat}) = Unknown()
mode(::Type{<:MixedFormat{M}}) where {M} = M()

# Most GeoFormat types wrap String or have a constructor for string inputs
Base.convert(::Type{String}, input::GeoFormat) = val(input)
Base.convert(::Type{T}, input::AbstractString) where {T<:GeoFormat} = T(convert(String, (input)))

"""
    ProjString <: CoordinateReferenceSystemFormat

    ProjString(x::String)

Wrapper for Proj strings. String input must start with "$PROJ_PREFIX".
"""
struct ProjString <: CoordinateReferenceSystemFormat
    val::String
    ProjString(input::String) = begin
        startswith(input, PROJ_PREFIX) ||
            throw(ArgumentError("Not a Proj string: $input does not start with $PROJ_PREFIX"))
        new(input)
    end
end

"""
    ProjJSON <: CoordinateReferenceSystemFormat

    ProjJSON(x::Dict{String,<:Any})
    ProjJSON(x::String)

Wrapper for [PROJJSON](https://proj.org/specifications/projjson.html).
"""
struct ProjJSON <: CoordinateReferenceSystemFormat
    val::Union{String,Dict{String,<:Any}}
    ProjJSON(input::Dict{String,<:Any}) = begin
        haskey(input, "type") ||
            throw(ArgumentError("Not a ProjJSON: $input does not have the required key 'type'"))
        new(input)
    end
    ProjJSON(input::String) = begin
        occursin("type", input) ||
            throw(ArgumentError("Not a ProjJSON: $input does not have the required key 'type'"))
        new(input)
    end
end

"""
    CoordSys <: CoordinateReferenceSystemFormat

    CoordSys(val)

Wrapper for a Mapinfo CoordSys string.
"""
struct CoordSys <: CoordinateReferenceSystemFormat
    val::String
end

"""
    AbstractWellKnownText <: MixedFormat

Well known text has a number of versions and standards, and can hold
either coordinate reference systems or geometric data in string format.
"""
abstract type AbstractWellKnownText{X} <: MixedFormat{X} end

"""
    WellKnownText <: AbstractWellKnownText

    WellKnownText(val)
    WellKnownText(mode, val)

Weapper for Well-Known-Text v1, following the OGC standard.
These may hold CRS or geometry data.

These may hold CRS or geometry data. The default mode is `Mixed()`,
and conversions to either type will be attempted where possible.
A specific type can be specified if it is known, e.g.:

```julia
geom = WellKnownText(Geom(), geom_string)
```
"""
struct WellKnownText{X} <: AbstractWellKnownText{X}
    mode::X
    val::String
end
WellKnownText(val) = WellKnownText(Unknown(), val)

"""
    WellKnownText2 <: AbstractWellKnownText

    WellKnownText2(val)
    WellKnownText2(mode, val)

Weapper for Well-Known-Text v2 objects, following the new OGC standard.

These may hold CRS or geometry data. The default mode is `Unknown()`,
and conversions to either type will be attempted where possible.
A specific type can be specified if it is known, e.g.:

```julia
crs = WellKnownText2(CRS(), crs_string)
```
"""
struct WellKnownText2{X} <: AbstractWellKnownText{X}
    mode::X
    val::String
end
WellKnownText2(val) = WellKnownText2(Unknown(), val)

"""
    ESRIWellKnownText <: AbstractWellKnownText

    ESRIWellKnownText(x::String)
    ESRIWellKnownText(::CRS, x::String)
    ESRIWellKnownText(::Geom, x::String)

Wrapper for Well-Known-Text strings, following the ESRI standard.

These may hold CRS or geometry data. The default mode is `Unknown`,
and conversions to either type will be attempted where possible.
A specific type can be specified if it is known, e.g:

```julia
crs = ESRIWellKnownText(CRS(), crs_string)
```
"""
struct ESRIWellKnownText{X} <: AbstractWellKnownText{X}
    mode::X
    val::String
end
ESRIWellKnownText(val) = ESRIWellKnownText(Unknown(), val)

"""
    WellKnownBinary <: MixedFormat

Wrapper for Well-Known-Binary objects.

These may hold CRS or geometry data. The default mode is `Unknown`,
and conversions to either type will be attempted where possible.
A specific type can be specified if it is known, e.g:

```julia
crs = WellKnownBinary(CRS(), crs_blob)
```
"""
struct WellKnownBinary{X,T} <: MixedFormat{X}
    mode::X
    val::T
end
WellKnownBinary(val) = WellKnownBinary(Unknown(), val)

Base.convert(::Type{String}, input::WellKnownBinary) =
    error("`convert` to `String` is not defined for `WellKnownBinary`")

"""
    EPSG <: CoordinateReferenceSystemFormat

    EPSG(input)

EPSG code representing a coordinate reference system from the
EPSG spatial reference system registry.

String input must start with "$EPSG_PREFIX". `EPSG` can be converted to an `Int` or `String`
using `convert`, or another `CoordinateReferenceSystemFormat` when ArchGDAL.jl is loaded.
"""
struct EPSG <: CoordinateReferenceSystemFormat
    val::Int
end
function EPSG(input::AbstractString)
    startswith(input, EPSG_PREFIX) || throw(ArgumentError("String $input does no start with $EPSG_PREFIX"))
    code = parse(Int, input[findlast(EPSG_PREFIX, input).stop+1:end])
    EPSG(code)
end

Base.convert(::Type{Int}, input::EPSG) = val(input)
Base.convert(::Type{String}, input::EPSG) = string(EPSG_PREFIX, val(input))
Base.convert(::Type{EPSG}, input::Int) = EPSG(input)

"""
    KML <: GeometryFormat

Wrapper object for "Keyhole Markup Language" (KML) strings.

See: https://www.ogc.org/standards/kml/

Can be converted to a `String`. Conversion to crs will convert from EPSG(4326),
which is the default for KML.
"""
struct KML <: GeometryFormat
    val::String
end
# We know KML always has a crs of EPSG(4326)
Base.convert(::Type{T}, ::KML) where {T<:CoordinateReferenceSystemFormat} = convert(T, EPSG(4326))

"""
    GML <: MixedFormat

Wrapper for Geography Markup Language string.

These contain geometry data, but may also have embedded crs information.
`GML` can be converted to either a `GeometryFormat` or `CoordinateReferenceSystemFormat`.
"""
struct GML{X} <: MixedFormat{X}
    mode::X
    val::String
end
GML(val) = GML(Unknown(), val)

"""
    GeoJSON <: GeometryFormat

Wrapper for a GeoJSON `String` or `Dict`.

Conversion between `Dict` and `String` values is not yet handles.
"""
struct GeoJSON{T} <: GeometryFormat
    val::T
end

end # module


text:
#== # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Description
#
#    SatelliteToolbox orbit propagator API: epoch
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ==#

export epoch

"""
    epoch(orbp)

Return the epoch of the propagator `orbp` [JD].

"""
epoch(orbp::OrbitPropagatorJ2)      = orbp.j2d.epoch
epoch(orbp::OrbitPropagatorJ4)      = orbp.j4d.epoch
epoch(orbp::OrbitPropagatorSGP4)    = orbp.sgp4d.epoch
epoch(orbp::OrbitPropagatorTwoBody) = orbp.tbd.epoch


text:
include("euler/euler.jl")
using .BigIntegers: BigInteger
using .Numbers: digits_sum

compute(n::Int)::Int = digits_sum(prod(map(BigInteger, 1:n)))

max_stars_repo_name: Dynortice/Project-Euler

max_issues_repo_name: Dynortice/Project-Euler

max_forks_repo_name: Dynortice/Project-Euler


text:
#load and process funds as desired
function prepfunds(returns::DataFrame,
    ms::Union{Nothing, MeasureSpec}=nothing,
    ;refreshfunds::Bool = PARAM[:refreshfunds],
    refreshcomom::Bool = PARAM[:refreshcomom],
    workingpath::String = PARAM[:workingpath],
    loadpath::NString = nothing,
    hfrname::String = PARAM[:hfrname],
    mfname::String = PARAM[:mfname],
    returntype::String = PARAM[:crspdatalabel],
    plname::String = PARAM[:plname],
    weeklycrsptype::String = string(PARAM[:crspweekly]),
    dtformat = PARAM[:wrdsdateformat])

  local hf::DataFrame
  local mf::DataFrame
  local comom::DataFrame

  if refreshfunds

    hf = prephf(returns::DataFrame, ms::MeasureSpec)
    hf |> CSV.write("$workingpath\\$(returntype)_$(hfrname).csv")

    mf = prepmf(returns::DataFrame, ms::MeasureSpec)
    mf |> CSV.write("$workingpath\\$(returntype)_$(mfname).csv")
  else
    hf = CSV.File("$workingpath\\$(returntype)_$(hfrname).csv") |> DataFrame
    mf = CSV.File("$workingpath\\$(returntype)_$(mfname).csv") |> DataFrame
  end
  if refreshcomom
    if PARAM[:crspfrequencysuffix] ≡ "m"
      comom = generatemonthlycomomentum()
      #comom = generatecomomentumbb()
    elseif PARAM[:crspfrequencysuffix] ≡ "w"
      comom = generateweeklycomomentum()
    else
      @assert false
    end
    comom |> CSV.write("$workingpath\\$(returntype)_$(plname).csv")
  else
    if isfile("$workingpath\\$(returntype)_$(plname).csv")
      comom = CSV.File("$workingpath\\$(returntype)_$(plname).csv") |> DataFrame
    else
      comom = DataFrame()
    end
  end

  return (;hf, mf, comom)
end

#=function betaweightsslice(df, extrabetaweights::Vector{Symbol}=PARAM[:fundextrabetaweights],
  Fbetacontrols=PARAM[:fundbetacontrols],
  betacorridorradius=PARAM[:fundbetacorridorradius])

  throw("THIS NEEDS TO BE COMPLETED")
  #this will hold the weights by fundid
  #the below version must work with missing data
  weights = DataFrame(fundid=unique(df.fundid))
  function betaw(t,Frhs)
    completeindices = sum()
    X = reduce(hcat, (F->t[F]).(Frhs)

    (svd(Xclean))\(t.netret))[1]
  end
  local Fweights = Vector{Symbol}()
  #now execute the regressions
  for Fbetaweight ∈ Fbetaweights
    Fweight = Symbol(:w_, Fbetaweight)
    Fabsweight = Symbol(:absw_, Fbetaweight)
    Fabscorweight = Symbol(:abscorw_, Fbetaweight)
    newweightfields = [Fweight,Fabsweight,Fabscorweight]

    #form a view with no missing values
    focalfields = [:date; :fundid; :some1s; Fbetaweight; :netret; Fbetacontrols]
    Frhs = [Fbetaweight; Fbetacontrols; :some1s;]
    cleandf = view(df, completecases(df, [Fbetaweight; Fbetacontrols; :netret]), focalfields)

    #run the regression for each fundid
    #we are primarily interested in the absolute vlaue of the weights
    #the corrodor version creates a buffer to guard against measurement error in the controls
    weight = combine(groupby(cleandf, :fundid), (:) |> AsTable =>
      ((t)->betaw(t,Frhs)) => Fweight)
    weight[!, Fabsweight] = abs.(weight[!, Fweight])
    weight[!, Fabscorweight] = (x->ifelse(x<betacorridorradius,0.0, x)).(weight[!, Fabsweight])
    weights = innerjoin(weights, weight[:, [:fundid; newweightfields]], on=[:fundid])
    append!(Fweights, newweightfields)
  end

end=#

function betaweights(df, returns, ms::MeasureSpec;
  extrabetaportfolios::Vector{Vector{Symbol}}=PARAM[:fundextrabetaportfolios],
  Fbetacontrols=PARAM[:fundbetacontrols],
  betacorridorradius=PARAM[:fundbetacorridorradius],
  datefrequency::Symbol = throw("datefrequency is required"),
  testlabel=throw("testlabel is required"),
  Frfr::Symbol = PARAM[:fundrfr],
  )

  @assert issorted(df, [:fundid, :date])

  #just get the parts we want to merge from the returns dataframe
      #crashed here in the returns file
  #throwing the error here from funds-mf: Merge key(s) in df1 are not unique
  df = mergereturns(df, returns,
    Fgroup = :fundid,
    Frets = [ms.Fξs; extrabetaportfolios...; Fbetacontrols;Frfr],
    validateexactfrequency=true;
    datefrequency, testlabel)

  #=WARNING- the below code uses all available data, but will use different fund-date
  #universes for different beta controls. I leave it disabled because its also very slow
  #also it shouldn't mkae a big difference
  for Fret ∈ [Fbetaweights;Fbetacontrols]
    if :indexdate ∈ propertynames(df)
      select!(df, Not([:indexdate]))
    end
    df = mergereturns(df, returns,
      Fgroup = :fundid,
      Frets = [Fret;],
      validateexactfrequency=true;
      datefrequency, testlabel)
  end=#

  #this is the intercept
  df.some1s = ones(nrow(df))
  df.netret = df.ret .- df[!, Frfr]

  #this will hold the weights by fundid
  weights = DataFrame(fundid=unique(df.fundid))

  #runs the regression
  #form the LHS matrix then
  #betaw(t,Fbetaweight) = (svd([t[Fbetaweight] t.some1s])\(t.ret))[1]
  #extracts the relevant coefficeints of interest for the betas
  function betaw(t,Frhs,Fportfolioweights)
    β = (svd(reduce(hcat, (F->t[F]).(Frhs)))\(t.netret))
    return Dict(Fw => β[i] for (i,Fw) ∈ enumerate(Fportfolioweights)) |> NamedTuple
  end

  local Fweights = Vector{Symbol}()
  Fbetaportfolios::Vector{Vector{Symbol}} =   [ms.Fξs]
  append!(Fbetaportfolios, extrabetaportfolios)
  #now execute the regressions
  for Fbetaportfolio ∈ Fbetaportfolios
    Fportfolioweights = Fbetaportfolio .|> s->Symbol(:w_, s)
    Fportfolioabsweights = Fbetaportfolio .|> s->Symbol(:absw_, s)
    Fportfolioabscorweights = Fbetaportfolio .|> s->Symbol(:abscorw_, s)
    newweightfields = [Fportfolioweights; Fportfolioabsweights; Fportfolioabscorweights]

    #form a view with no missing values
    focalfields = [:date; :fundid; :some1s; Fbetaportfolio; :netret; Fbetacontrols]
    Frhs = [Fbetaportfolio; Fbetacontrols; :some1s;]
    cleandf = view(df, completecases(df, [Fbetaportfolio; Fbetacontrols; :netret]), focalfields)

    #run the regression for each fundid
    #we are primarily interested in the absolute vlaue of the weights
    #the corrodor version creates a buffer to guard against measurement error in the controls
    weight = combine(groupby(cleandf, :fundid), (:) |> AsTable =>
      ((t)->betaw(t,Frhs,Fportfolioweights)) => AsTable)

    transform!(weight,
      Fportfolioweights .=>
        (v-> abs.(v)) .=> Fportfolioabsweights,
      Fportfolioweights .=>
        (v->(x->ifelse(x<betacorridorradius,0.0, x)).(abs.(v))) .=> Fportfolioabscorweights,
      )

    weights = innerjoin(weights, weight[:, [:fundid; newweightfields]], on=[:fundid])
    append!(Fweights, newweightfields)
  end

  #merge in the new weight fields
  df = leftjoin(df, weights, on=:fundid, validate=(false, true))
  if !issorted(df, [:fundid, :date])
    sort!(df, [:fundid, :date])
  end
  select!(df, Not([:some1s]))

  for Fweight ∈ Fweights
    #=winsorizewithin!(df, Fsource=Fweight, prop=PARAM[:fundwinsorizebeta], Fgroup=:date,
      twosided=true, Fnew=Symbol(:temp, Fweight))=#
    df[!, Symbol(:temp, Fweight)] = winsorizequantile(df[!,Fweight],
      PARAM[:fundwinsorizebeta]; twosided=true)
  end

  if PARAM[:testfundbeta]
    testmonth = PARAM[:fundbetatestmonth]
    testview::SubDataFrame = view(df, (month.(df.date) .== testmonth.month) .&
      (year.(df.date) .== testmonth.year), :)
    testview[1:(min(20_000, nrow(testview))),:] |> CSV.write(
      "$(PARAM[:testpath])\\$(PARAM[:crspdatalabel])_$(Tuple(testmonth))_$testlabel.csv")

    testfundid = PARAM[:fundbetatestfundid][testlabel |> Symbol]
    testview = view(df, df.fundid .== testfundid, :)
    testview[1:(min(20_000, nrow(testview))),:] |> CSV.write(
      "$(PARAM[:testpath])\\$(PARAM[:crspdatalabel])_$(testfundid)_$testlabel.csv")
  end

  #@info propertynames(df)
  for Fweight ∈ Fweights
    select!(df, Not([Fweight]))

    rename!(df, Dict(Symbol(:temp, Fweight)=>Fweight))
  end

  return Fweights, df
end

hfrbetaweights(hfr, returns, ms::MeasureSpec) = betaweights(
    hfr, returns, ms, testlabel="hfr", datefrequency=:month)

function prephf(returns::AbstractDataFrame, ms::MeasureSpec;
  hfrnamealive::String = PARAM[:hfrnamealive],
  hfrnamedead::String = PARAM[:hfrnamedead],
  hfrpath::String = PARAM[:hfrpath],
  minfundmonths::Int = PARAM[:fundminfundmonths],
  dropfinalmonths::Int = PARAM[:hfrdropfinalmonths],
  lpfilterperiods::Vector{<:Union{Day, Nothing}} = PARAM[:analysislpfilterperiods],
  incsvstream::Function = IN_CSV_STREAM,
  csvextension::String = CSV_EXTENSION)::DataFrame

  hfralive::DataFrame = incsvstream("$hfrpath\\$hfrnamealive.$csvextension") |> CSV.File |> DataFrame
  hfralive.alive = trues(size(hfralive,1))
  hfrdead::DataFrame = incsvstream("$hfrpath\\$hfrnamedead.$csvextension") |> CSV.File |> DataFrame
  hfrdead.alive = falses(size(hfrdead,1))

  hfr = vcat(hfralive, hfrdead)

  normalizename(s::String) = replace(s, "_"=>"") |> lowercase
  rename!(hfr, normalizename.(names(hfr)))

  #need to parse the dates
  for datefield ∈ [:date, :inception, :fundassetsasof, :firmassetsasof]
    hfr[!, datefield] = (d->parsehfr(Date, d)).(hfr[!, datefield])
  end

  sort!(hfr, [:fundid, :date,])
  hfr.firstdomdate = hfr.date .|> firstdayofmonth
  @assert sum(nonunique(hfr, [:fundid, :firstdomdate]))==0

  #classify the strategies- first read in a spreadsheet of the classifications
  stratcodes = CSV.File("$hfrpath\\classifications.csv") |>  DataFrame
  rename!(stratcodes, normalizename.(names(stratcodes)))
  stratcodes=stratcodes[:, Not([:mainstrategy, :substrategy])]

  #conform index classifications
  hfr.hfri = hfr.inhfri .=== "Yes"
  hfr.hfrx = hfr.inhfrx .=== "Yes"

  #then join the classifications with the funds
  hfr = innerjoin(hfr, stratcodes,on=[:strategycode, :substrategycode])
  if !issorted(hfr, [:fundid, :date])
    sort!(hfr, [:fundid, :date,])
  end
  @assert size(hfr,1) == size(hfrdead,1) + size(hfralive,1) #sanity check

  #keep only the relevant dates- is this the right approach? should I take the max range?
  #keep only dates within the return range
  #also drop the final month(s) due to incomplete data
  mindate=minimum(returns.date)|>firstdayofmonth
  maxdate = min(maximum(returns.date) |> lastdayofmonth,
    maximum(hfr.date) - Month(dropfinalmonths) |> lastdayofmonth)
  hfr = hfr[(mindate .< hfr.date) .& (hfr.date .< maxdate), :]
  hfr.ret = hfr.performance ./ 100
  select!(hfr, Not(:performance))

  #now compute AUMs by strategy bunches
  hfr = hfr[hfr.fundassetsdenomin .== "USD",:]
  hfr.assets ./= 1_000
  rename!(hfr, :assets=>:aum) #what about nav?

  #acquire the index of strategy returns
  hfr = hfr[completecases(hfr, [:aum, :ret]), :]
  transform!(groupby(hfr, :fundid), nrow => :fundmonths)
  hfr = hfr[hfr.fundmonths .≥ minfundmonths, :]

  println("Attempting to weight hfr fund aum by returns...")
  Fhfweights, hfr = hfrbetaweights(hfr, returns, ms)
  println("weighting successful.")

  #sums values in an array if the conditional is true and nothing is missing
  dotifs(v1,v2 = 1.0, v3 = 1.0) = sum((v1 .* v2 .* v3) .|> (x)->coalesce(x, 0.0))

  #now group the assets
  shfs = groupby(hfr, :date)
  hf = combine(shfs ,
    :aum => dotifs => :aum,
    [:aum, :equityonly] => dotifs => :A_aumequityonly,
    [:aum, :equity] => dotifs => :A_aumequity,
    [:aum, :quant] => dotifs => :A_aumquant,
    [:aum, :hfri] => dotifs => :A_aumhfri,
    [:aum, :hfrx] => dotifs => :A_aumhfrx,
  )
  sort!(hf, :date)

  #group the regression weights
  for Fhfweight ∈ Fhfweights
    hfweightdf = combine(shfs,
      [:aum, Fhfweight] => dotifs => Symbol(:A_,Fhfweight),
      [:aum, Fhfweight, :equity] => dotifs => Symbol(:A_aumequityX,Fhfweight),
      [:aum, Fhfweight, :equityonly] => dotifs => Symbol(:A_aumequityonlyX,Fhfweight),
    )
    hf = innerjoin(hf, hfweightdf, on=:date, validate=(true,true))
  end
  sort!(hf, :date)

  #create the log growwth columns
  logmaybe(::Missing) = missing
  logmaybe(x) = x ≤ 0 ? missing : log(x)
  for acol ∈ findcols(hf, :asset)
    @assert occursin(r"^A", string(acol))
    hf[!, Symbol(:lG_,acol)] = [missing; logmaybe.(hf[2:end, acol] ./ hf[1:(end-1), acol])]
    #hf[!, Symbol(:G_,acol)] = [missing; hf[2:end, acol] ./ hf[1:(end-1), acol] .- 1]
  end
  #average all cols via the LP filter
  hf = lpfilter(hf, focalcols=setdiff(propertynames(hf), [:date]); lpfilterperiods)

  #create normalized versions NOTE- I don't think this is the right approach- disabled
  #create normalized versions
  #=focalcols = setdiff(propertynames(hf), [:date, :aum])
  for f ∈ focalcols
    transform!(hf, [f, :aum] => ((x,y)->x ./ y) =>Symbol(:N, f))
  end

  for per ∈ lpfilterperiods
    (per === nothing) && continue

    #the suffix is how we will identify the lp columns during analysis
    hf = lpfilter(hf, lpfilterperiods=[per], focalcols=[focalcols; :aum])
    lpcols = focalcols .|> (f)->Symbol(f,lpsuffix(per))
    aumcol = Symbol(:aum, lpsuffix(per))
    lpcols = (s->Symbol(s, lpsuffix(per))).(focalcols)
    for lpcol ∈ lpcols
      transform!(hf, [lpcol, aumcol] => ((x,y)->x ./ y) =>Symbol(:N, lpcol),)
    end
  end=#

  return hf
end

parsehfr(::Type{Date}, s::String, hfrdateformat::DateFormat=dateformat"yyyymmdd"
  ) = Dates.Date(s, hfrdateformat)
parsehfr(::Type{<:Any}, ::Missing) = missing
parsehfr(::Type{Date}, i::Int, hfrdateformat::DateFormat=dateformat"yyyymmdd") = parsehfr(
  Date, "$i", hfrdateformat)

############################MF

mfbetaweights(mf, returns, ms::MeasureSpec;
  datefrequency=:month) = betaweights(
    mf, returns, ms;  testlabel="mf", datefrequency)

function prepmf(returns::AbstractDataFrame, ms::MeasureSpec;
  mfdatefrequency::Symbol=PARAM[:mfdatefrequency],
  mfname::String = PARAM[:mfname],
  mfnamedesc::String = PARAM[:mfnamedesc],
  mfpath::String = PARAM[:mfpath],
  minfundmonths::Int = PARAM[:fundminfundmonths],
  minfunds::Int = PARAM[:fundminfunds],
  lpfilterperiods::Vector{<:Union{Day, Nothing}} = PARAM[:analysislpfilterperiods],
  incsvstream::Function = IN_CSV_STREAM,
  csvextension::String = CSV_EXTENSION,
  dtformat::DateFormat = PARAM[:wrdsdateformat])

  #scrubs the names and dates
  function initialclean(df::DataFrame; #=startdate::Date = PARAM[:mfstartdate]=#)
    rename!(df, (cleanname).(names(df)))
    df.date = (i::Int->Date("$i", dtformat)).(df.caldt)
    #df.year = (d->year(d)).(df.date)
    transform!(df, :crspfundno => ByRow((s)->Symbol(:mf, s)) => :fundid)
    select!(df, Not([:crspfundno, :caldt]))

    return df
  end

  #start by working with the description file
  mfdesc::DataFrame = CSV.File("$mfpath\\$mfnamedesc.csv") |> DataFrame
  mfdesc = initialclean(mfdesc)
  println(describe(mfdesc))
  rename!(mfdesc, :crspobjcd =>:obj)

  #tehse are the classificaiton codes for selection
  mfdesc = mfdesc[mfdesc.obj .!== missing, :]
  mfdesc = mfdesc[length.(mfdesc.obj) .≥ 1,:]
  (!issorted(mfdesc, [:fundid, :date])) && sort!(mfdesc, [:fundid, :date])
  select!(mfdesc, Not([:date]))
  mfdesc = combine(last, groupby(mfdesc, :fundid), renamecols=false)

  transform!(mfdesc,
    :obj => ByRow(obj->obj[1] == 'E' || obj[1]=='M' || obj[1]=='O') => :equity,
    :obj => ByRow(obj->obj[1] == 'E') => :equityonly,
    :obj => ByRow((obj->(length(obj) > 1) && (obj[1] == 'E') && (obj[2] == 'D'))) => :domesticeo,
    )

  mfdesc.lipperclassname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperclassname)
  mfdesc.lipperobjname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperobjname)
  #=mfdesc.equity = (mfdesc.obj[1] .== 'E') .| (mfdesc.obj[1] .== 'M') .| (mfdesc.obj[1] .== 'O')
  mfdesc.equityonly = mfdesc.obj[1] .== 'E'
  mfdesc.domesticeo = .(mfdesc.obj)=#

  #mfdesc = combine(groupby(mfdesc, [:fundid, :year]), propertynames(mfdesc) .=>
  #  (v)->v[end], renamecols=false)
  #dups = mfdesc[nonunique(mfdesc[:, [:fundid, :year]]),:]
  #dups.id = 1:nrow(dups)
  ##println(innerjoin(mfdesc, dups[:, [:fundid, :year, :id]], on=[:fundid, :year]))
  #@assert sum(nonunique(mfdesc[:, [:fundid, :year]])) == 0

  #=mfdesc.lipperclassname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperclassname)
  mfdesc.lipperobjname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperobjname)

  #note- the classifications are by hand- the below is ONLY for support of the by-hand classifications
  unique(mfdesc[:, [:lipperclassname, :lipperobjname]]) |> CSV.write("mfclassificationsraw.csv")

  #only interested in the most recent, non-missing row for each fund
  mfdesc = mfdesc[(mfdesc.lipperclassname .!== missing) .| (mfdesc.lipperobjname .!== missing), :]
  mfdesc = combine(last, groupby(mfdesc, :fundid), renamecols=false)
  @assert sum(nonunique(mfdesc[:, [:fundid]])) == 0

  #get the classifications from a seperate file
  stratcodes = CSV.File("$mfpath\\mfclassifications.csv") |>  DataFrame
  select!(stratcodes, [:lipperclassname, :lipperobjname, :equity,:equityonly, :domesticeo])
  oldnrows = nrow(mfdesc)
  mfdesc = innerjoin(mfdesc, stratcodes, on=[:lipperclassname, :lipperobjname], matchmissing=:equal)=#

  #@assert nrow(mfdesc) == oldnrows
  @assert sum(nonunique(mfdesc[:, [:fundid]])) == 0

  #clean the data
  mf::DataFrame = incsvstream("$mfpath\\$mfname.$csvextension") |> CSV.File |> DataFrame
  mf = initialclean(mf)

  sort!(mf, [:fundid, :date])
  @assert sum(nonunique(mf[!, [:fundid, :date]])) == 0

  #parse relevant fields to a number- letter codes generally indicate missing
  for (Fold, Fnew) ∈ [:mret=>:ret, :mtna=>:aum, :mnav=>:nav]
    mf[!, Fnew] = (s->parsemf(Float64, s)).(mf[!, Fold])
  end

  #invalid values also generally represent missing
  mf.ret .= mf.ret .|> (x)-> (x ≡ missing ) || (x < -1.0) ? missing : x
  mf.aum .= mf.aum .|> (x)-> (x ≡ missing ) || (x < 0.0) ? missing : x
  mf.aum ./= 1000

  mf = dropmissing!(mf, :ret)

  #implmeent the reversal check of PST2015
  lagwithin2!(mf, [:aum], :fundid, date=:date)
  leadwithin2!(mf, [:aum], :fundid, date=:date)
  mf.Gaum = (mf.aum .- mf.Laum) ./ mf.Laum
  mf.rev = (mf.Naum .- mf.aum) ./ (mf.aum .- mf.Laum)
  transform!(groupby(mf, :fundid), [:Gaum, :rev, :Laum, :aum] => ByRow(
    (Gaum, rev, Laum, aum)->ifelse(
      (Gaum !== missing) && (rev !== missing) &&
      (abs(Gaum) > 0.5) && (-1.25 ≤ rev ≤ -0.75) && (Laum > 0.01), missing, aum))
    => :adjaum)

  @info "Dropped $(sum(mf.aum .!== mf.adjaum)) month-fund rows due to rapid  aum reversals"
  select!(mf, Not([:Gaum, :rev, :Laum, :aum, :Naum]))
  rename!(mf, :adjaum => :aum)

  mf = patchmfmonthlyreturngaps(mf)

  #no point in keeping a row if the assets field is missing
  mf = dropmissing!(mf, :aum)
  transform!(groupby(mf, :fundid), nrow => :fundmonths)
  mf = mf[mf.fundmonths .≥ minfundmonths, :]

  #trim down to the desired range with a buffer
  startdate = minimum(returns.date) - Year(1)
  enddate =  maximum(returns.date) + Year(1)
  mf = mf[(mf.date .≥ startdate) .&  (mf.date .≤ enddate), :]

  #if we want to patch in daily or weekly return data, this is where we do that
  select!(mf, [:fundid, :date, :aum, :ret])
  mf = patchmfreturns(mf, mfdatefrequency)

  #patch in the description info
  mf = leftjoin(mf, mfdesc[:, [:fundid, :equity, :equityonly, :domesticeo,
    :obj, :lipperclassname, :lipperobjname]], on=[:fundid], validate=(false, true))
  if !issorted(mf, [:fundid, :date])
    sort!(mf, [:fundid, :date])
  end

  transform!(groupby(mf, :date), nrow => :numfunds)
  mf = mf[mf.numfunds .≥  minfunds,:]

  #merge in the beta-based fund weights
  println("Attempting to weight mf fund aum by returns...")
  Fmfweights, mf = mfbetaweights(mf, returns, ms; datefrequency=mfdatefrequency)
  println("...weighting successful.")

  #sums values in an array if the conditional is true and nothing is missing
  zeromissings(x) =  coalesce(x, 0.0)
  dotifs(v1,v2 = 1.0, v3 = 1.0) = sum((v1 .* v2 .* v3) .|> zeromissings)
  desccount(aum,beta = 1.0, cond = 1.0) = sum((aum .* cond .* (beta .!==missing)) .|> zeromissings .|> (x)->x>0.0)
  descmin(beta,cond=true) = minimum((beta .* cond) |> skipmissing)
  descmax(beta,cond=true) = maximum((beta .* cond) |> skipmissing)
  descstd(beta,cond=true) = std((beta .* cond) |> skipmissing)
  descmedianabs(beta,cond=true) = median((beta .* cond) .|> abs |> skipmissing)
  descmeanabs(beta,cond=true) = mean((beta .* cond) .|> abs |> skipmissing)

  combine(groupby(view(mf,
    mf.date .== maximum(mf.date[(mf.date .≥ Date(2019,12,1)) .& (mf.date .≤ Date(2019,12,31))]),:),
      [:lipperclassname, :lipperobjname]),
      Fmfweights .=> ((v)->mean(skipmissing(abs.(v)))) .=> (s->Symbol(:meanabs_,s)).(Fmfweights),
        :aum => ((v)->sum(skipmissing(v))) => :aum,
      zip([:aum for i ∈ 1:length(Fmfweights)], Fmfweights) .|> collect .=> dotifs .=> (
        s->Symbol(:A_,s)).(Fmfweights),
      :equity => last => :equity,
      :equityonly => last => :equityonly,
      :domesticeo => last => :domesticeo) |> CSV.write(
        "$(PARAM[:testpath])\\mf_asset aggregates 20191227.csv")

  #now group the assets
  smfs = groupby(mf, :date)
  funds = combine(smfs,
    :aum => dotifs => :aum,
    [:aum, :equity] => dotifs => :aumequity,
    [:aum, :equityonly] => dotifs => :aumequityonly,
    [:aum, :domesticeo] => dotifs => :aumdomesticeo,
  )
  if !issorted(funds, :date)
    sort!(funds, :date)
  end

  #group the regression weights
  for Fmfweight ∈ Fmfweights
    oldnames = propertynames(funds)
    mfweightdf = combine(smfs,
      [:aum, Fmfweight] => dotifs => Symbol(:A_, Fmfweight),
      [:aum, Fmfweight, :equity] => dotifs => Symbol(:A_aumequityX,Fmfweight),
      [:aum, Fmfweight, :equityonly] => dotifs => Symbol(:A_aumequityonlyX,Fmfweight),
      [:aum, Fmfweight, :domesticeo] => dotifs => Symbol(:A_aumdomesticeoX,Fmfweight),

      [:aum, Fmfweight] => desccount => Symbol(:descn_, Fmfweight),
      [Fmfweight] => descmedianabs => Symbol(:descmedabs_, Fmfweight),
      [Fmfweight] => descmeanabs => Symbol(:descmeanabs_, Fmfweight),
      [Fmfweight] => descstd => Symbol(:descstd_, Fmfweight),
      [Fmfweight] => descmin => Symbol(:descmin_, Fmfweight),
      [Fmfweight] => descmax => Symbol(:descmax_, Fmfweight),

      [:aum, Fmfweight, :equity] => desccount => Symbol(:descn_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmedianabs => Symbol(:descmedabs_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmeanabs => Symbol(:descmeanabs_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descstd => Symbol(:descstd_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmin => Symbol(:descmin_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmax => Symbol(:descmax_aumequityX,Fmfweight),

      [:aum, Fmfweight, :equityonly] => desccount => Symbol(:descn_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmedianabs => Symbol(:descmedabs_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmeanabs => Symbol(:descmeanabs_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descstd => Symbol(:descstd_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmin => Symbol(:descmin_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmax => Symbol(:descmax_aumequityonlyX,Fmfweight),

      [:aum, Fmfweight, :domesticeo] => desccount => Symbol(:descn_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmedianabs => Symbol(:descmedabs_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmeanabs => Symbol(:descmeanabs_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descstd => Symbol(:descstd_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmin => Symbol(:descmin_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmax => Symbol(:descmax_aumdomesticeoX,Fmfweight),

    )
    funds = innerjoin(funds, mfweightdf, on=:date, validate=(true,true))
  end
  sort!(funds, :date)

  logmaybe(::Missing) = missing
  logmaybe(x) = x ≤ 0 ? missing : log(x)
  lpfiltercols = findcols(funds, :asset)
  for acol ∈ findcols(funds, :asset)
    @assert occursin(r"^A", string(acol))

    push!(lpfiltercols, Symbol(:lG_,acol))
    funds[!, lpfiltercols[end]] = [missing; logmaybe.(funds[2:end, acol] ./ funds[1:(end-1), acol])]
    #funds[!, Symbol(:G_,acol)] = [missing; funds[2:end, acol] ./ funds[1:(end-1), acol] .- 1]
  end

  funds = lpfilter(funds, focalcols=lpfiltercols; lpfilterperiods)

  return funds
end

function patchmfmonthlyreturngaps(mf::AbstractDataFrame,
    maxpatchlength=PARAM[:mftimetostale])

  #sanity checks to make sure everything is as expected
  @assert issorted(mf, [:fundid, :date])
  @assert sum(completecases(mf, [:ret])) == nrow(mf)

  #temporary helper fields
  mf.ret1 = mf.ret .+ 1
  mf.maxvaliddate = (mf.date .+ maxpatchlength) .|> lastdayofmonth

  function patchwithret(dates, maxdate, aum, ret)

    sum(aum .≡ missing) == length(aum) && return aum
    T = searchsortedfirst(dates, maximum(dates[aum .!== missing]))

    maxvaliddate = Date(1900,1,1)
    for t ∈ 1:T

      #if the aum is not missing, then record the max valid date
      if aum[t] !== missing
        maxvaliddate = maxdate[t]
        continue
      end

      if dates[t] ≤ maxvaliddate
        aum[t] = (1+ret[t]) * aum[t-1]
      end
    end

    return aum
  end

  #correct the dates
  mf.aumcorrected = mf.aum |> deepcopy
  transform!(groupby(mf, :fundid),
    [:date, :maxvaliddate, :aumcorrected, :ret] => patchwithret => :aumcorrected)
  mf.corrected = (mf.aum .≡ missing ) .& (mf.aumcorrected .!== missing)

  #check
  uncorrectedcomplete = view(mf, completecases(mf, :aum), :)
  @assert (uncorrectedcomplete.aum .== uncorrectedcomplete.aumcorrected) |> all

  testfunds = mf[mf.corrected, :fundid] |> unique |> (v)->v[1:3]
  mf[
    (mf.fundid .== testfunds[1] ) .| (mf.fundid .== testfunds[2] ) .| (mf.fundid .== testfunds[3])
    , :] |> CSV.write("$(PARAM[:testpath])\\mf_monthlyaumpatch.csv")

  mf.aum = mf.aumcorrected |> deepcopy
  select!(mf, Not([:aumcorrected, :ret1, :maxvaliddate]))
  return mf
end

function patchmfreturns(mf::DataFrame, mfdatefrequency::Symbol;
    mfdailyname::String = PARAM[:mfdailyname],
    mfpath::String = PARAM[:mfpath],
    incsvstream::Function = IN_CSV_STREAM,
    csvextension::String = CSV_EXTENSION,
    dtformat = PARAM[:wrdsdateformat],
    interpolate::Bool = PARAM[:mfdailyinterpolate],
  )

  if mfdatefrequency ≡ :month
    dropmissing!(mf, [:ret])
    winsorizewithin!(mf, Fsource=:ret, prop=PARAM[:mfqwinsorize], Fgroup=:date,
      twosided=true, Fnew=:WXret)
    select!(mf, Not([:ret]))
    rename!(mf, :WXret=>:ret)
    return mf #nothing to patch in this case
  end

  #we may be able to drop this assertion if it is checked later
  @assert mfdatefrequency ∈ [:day, :week]
  oldmonthlyaum = mf[!, [:date, :fundid, :aum]]
  rename!(oldmonthlyaum, :aum=>:aumoldmonthly)

  #load and clean daily data
  mfd::DataFrame = incsvstream(
    "$mfpath\\$mfdailyname.$csvextension") |> CSV.File |> DataFrame
  rename!(mfd, (cleanname).(names(mfd)))
  mfd.date = (i::Int->Date("$i", dtformat)).(mfd.caldt)
  mfd.yearmonth = mfd.date .|> (d->year(d) + month(d)/100)
  #df.year = (d->year(d)).(df.date)
  transform!(mfd, :crspfundno => ByRow((s)->Symbol(:mf, s)) => :fundid)
  mfd.ret = mfd.dret .|> (r-> parsemf(Float64, r))
  dropmissing!(mfd, [:fundid, :date, :ret])

  #quick and easy performance enhancement based on the time period covered
  mfd = mfd[
    (mfd.date .≥ firstdayofmonth(minimum(mf.date))) .&
    (mfd.date .≤ lastdayofmonth(maximum(mf.date))),:]

  #prep mf
  @assert issorted(mf, [:fundid, :date])

  #we want the beginning of period AUM, not the end of period
  mfN = deepcopy(mf)
  mfN.Nyearmonth = mf.date .+ Month(1) .|> (d->year(d) + month(d)/100)
  mfN = select(mfN, [:fundid, :Nyearmonth, :aum])

  #we want to use the eom aum onthe last day of the month,
  #and then use returns to interpolate at other parts of the month
  mf.yearmonth = mf.date .|> (d->year(d) + month(d)/100)
  mf = select!(mf, [:fundid, :yearmonth, :aum])
  rename!(mf, :aum=>:aumeom)
  mfN = leftjoin(mfN, mf, on=[:fundid, :Nyearmonth => :yearmonth])

  #NOTE: the below restriction ensures that only the AUM is shifted as desired
  #WARNING: read above before adding columns to below

  select!(mfd, [:fundid, :date, :yearmonth, :ret])
  mf = innerjoin(mfN, mfd, on=[:fundid, :Nyearmonth => :yearmonth], validate=(true, false))

  #validate df and make sure everything is as expected
  @assert nonunique(mf, [:fundid, :date]) |> !all

  mf = sort(mf, [:fundid, :date])
  @assert issorted(mf, [:fundid, :Nyearmonth])
  dropmissing!(mf, [:ret, :aum])

  #winsorze for data quality
  winsorizewithin!(mf, Fsource=:ret, prop=PARAM[:mfqwinsorize], Fgroup=:Nyearmonth,
    twosided=true, Fnew=:WXret)
  select!(mf, Not([:ret]))
  rename!(mf, :WXret=>:ret)
  println(describe(mf))

  mf.aumbom = mf.aum |> deepcopy
  mfbyfund = groupby(mf, [:fundid, :Nyearmonth])

  #the interpolation routine backs out net deposits, then discounts
  #them back to the beginning of period. The discounted deposits are then added
  #pro-rata to the base beginning of period amount and discoutned back up to the
  #focal day. In equations,
  # Let CUMR(t) be the cumulative return for the month on day t. Then
  # totaldepbom = (AUM(T)-AUM(0)*(1+CUMR(T))/(1+CUMR(T))
  # AUM(t)=(AUM(0)+totaldepbom*t/T)*CUMR(t)
  function interpolatereturns(
      ret::AbstractVector,  aum::AbstractVector, aumeom::AbstractVector) where T

    cumret = cumprod(ret .+ 1.0) .- 1.0
    netgains = aumeom[end] - (1.0 + cumret[end]) * aum[1]
    netgainsbom = coalesce(netgains / (1.0+cumret[end]), 0.0)
    N::Int = length(ret)

    interpolated = [(t/N*netgainsbom+aum[1])*(1+cumr) for (t,cumr) ∈ enumerate(cumret)]

    #maybe disable the assertion if it slows things down
    @assert (interpolated[end] + 1.0 ≈ coalesce(aumeom[end], aum[1]*(1+cumret[end])) + 1.0) "
      aumeom[end]=$(aumeom[end])
      but interpolated[end]=$(interpolated[end])"
    return interpolated
  end

  @assert issorted(mf, [:fundid, :date])
  if interpolate

    #this will be correct for all but the last day of the month
    transform!(mfbyfund, [:ret, :aum, :aumeom] => interpolatereturns => :aum)
  else
    transform!(mfbyfund,
      [:ret, :aum] => ((ret,aum)->cumprod(ret .+ 1.0) .* aum) => :aum)
    #now patch in the eom aum at the end of the month
    transform!(mfbyfund,
      [:aum, :aumeom] => ((aum, aumeom)->aumeom[end] ≡ missing ?
        aum : [aum[1:(end-1)]; aumeom[end]]) => :aum)
  end

  if mfdatefrequency ≡ :week
    mf.weekid = assignweekid(mf.date)
    mfw = combine(groupby(mf, [:fundid, :weekid], sort=true),
      :ret => (r -> prod(1.0 .+ r) - 1.0) => :ret,
      :date => last => :date,
      :aum => last => :aum,
      )

    #we need to conform the weekly dates
    conformeddates = combine(groupby(mfw[!, [:weekid, :date]], :weekid),
      :date => maximum => :date)
    select!(mfw, Not([:date]))
    oldrows = nrow(mfw)
    mfw = innerjoin(mfw, conformeddates, on=:weekid, validate=(false,true))
    @assert oldrows == nrow(mfw)
    if !issorted(mfw, [:fundid, :weekid])
      sort!(mfw, [:fundid, :weekid])
    end

    #writes the output to a CSV for inspection
    if PARAM[:testfunds]
      testfundid = mfw[findmax(mfw.aum)[2], :fundid]
      testmfw = mfw[mfw.fundid .== testfundid,:]
      rename!(testmfw, Dict(:ret=>:retw, :aum=>:aumw, :date=>:datew))
      testmf = innerjoin(testmfw, mf, on=[:fundid, :weekid])

      testmf.ym = testmf.date .|> (d->year(d) + month(d)/100)
      oldmonthlyaum.ym = oldmonthlyaum.date .|> (d->year(d) + month(d)/100)
      rename!(oldmonthlyaum, :date=>:dateoldaum)
      testmf = leftjoin(testmf, oldmonthlyaum, on=[:fundid, :ym])
      sort!(testmf, [:fundid, :date])
      testmf |> CSV.write(
        "$(PARAM[:testpath])\\$(PARAM[:crsptypesuffix])dailyforweeklymf_$(mfdatefrequency).csv")

    end
    mf = mfw
  else
    @assert mfdatefrequency ≡ :day
  end

  #throw("got through it- check now!")

  return mf

end

parsemf(::Type{Date}, s::String, mfdateformat::DateFormat=dateformat"yyyymmdd"
  ) = Dates.Date(s, mfdateformat)
parsemf(::Type{<:Any}, ::Missing) = missing
parsemf(::Type{Date}, i::Int, mfdateformat::DateFormat=dateformat"yyyymmdd") = parsemf(
  Date, "$i", mfdateformat)

#parse to a number if possible
parsemf(T::Type{<:Real}, s::String) = something(tryparse(T, s), missing)
parsemf(::Type{T}, x::T) where T = x

max_stars_repo_name: clintonTE/CCA

max_issues_repo_name: clintonTE/CCA

max_forks_repo_name: clintonTE/CCA


text:
module Getters
    export connection_params, channel_to_add, channel

    function connection_params(broker_config)
        adapter_field::String = "adapter"
        login_field::String = "login"
        password_field::String = "password"
        mechanism_field::String = "mechanism"
        host_field::String = "host"
        port_field::String = "port"
        vhost_field::String = "virtual_host"

        default_adapter::String = "rabbit"
        default_login::String = "guest"
        default_password::String = "guest"
        default_mechanism::String = "AMQPLAIN"
        default_host::String = "localhost"
        default_vhost::String = "/"
        default_port::Int64 = 5672

        adapter::String = get(broker_config, adapter_field, default_adapter)
        login::String = get(broker_config, login_field, default_login)
        password::String = get(broker_config, password_field, default_password)
        mechanism::String = get(broker_config, mechanism_field, default_mechanism)
        host::String = get(broker_config, host_field, default_host)
        virtual_host::String = get(broker_config, vhost_field, default_vhost)
        port::Int64 = get(broker_config, port_field, default_port)

        return adapter, mechanism, login, password, host, port, virtual_host
    end

    function channel_to_add(channel, exchangers, queues)
        result = nothing

        if channel !== nothing
            result = Dict{String, Any}(
                "CHANNEL" => channel,
                "EXCHANGERS" => exchangers,
                "QUEUES" => queues
            )
        end

        return result
    end

    function module_name(some_module)
        return last(fullname(some_module))
    end

    function channel(connection, name, to_create, Adapter)
        return Adapter.get_channel(connection, name, to_create)
    end
end


text:
# This file is a part of DataDeps.jl. License is MIT.

if Sys.isunix() && Sys.KERNEL != :FreeBSD
    function unpack_cmd(file,directory,extension,secondary_extension)
        if ((extension == ".gz" || extension == ".Z") && secondary_extension == ".tar") || extension == ".tgz"
            return (`tar xzf $file --directory=$directory`)
        elseif (extension == ".bz2" && secondary_extension == ".tar") || extension == ".tbz"
            return (`tar xjf $file --directory=$directory`)
        elseif extension == ".xz" && secondary_extension == ".tar"
            return pipeline(`unxz -c $file `, `tar xv --directory=$directory`)
        elseif extension == ".tar"
            return (`tar xf $file --directory=$directory`)
        elseif extension == ".zip"
            return (`unzip -x $file -d $directory`)
        elseif extension == ".gz"
            target = joinpath(directory, file)
            return pipeline(`gzip -k -d $target`)
        end
        throw(ArguementError("I don't know how to unpack $file"))
    end
end

if Sys.KERNEL == :FreeBSD
    # The `tar` on FreeBSD can auto-detect the archive format via libarchive.
    # The supported formats can be found in libarchive-formats(5).
    # For NetBSD and OpenBSD, libarchive is not available.
    # For macOS, it is. But the previous unpack function works fine already.
    function unpack_cmd(file, dir, ext, secondary_ext)
        tar_args = ["--no-same-owner", "--no-same-permissions"]
        return pipeline(
            `/bin/mkdir -p $dir`,
            `/usr/bin/tar -xf $file -C $dir $tar_args`)
    end
end

if Sys.iswindows()
    const exe7z = joinpath(Sys.BINDIR, "7z.exe")

    function unpack_cmd(file,directory,extension,secondary_extension)
        if ((extension == ".Z" || extension == ".gz" || extension == ".xz" || extension == ".bz2") &&
                secondary_extension == ".tar") || extension == ".tgz" || extension == ".tbz"
            return pipeline(`$exe7z x $file -y -so`, `$exe7z x -si -y -ttar -o$directory`)
        elseif (extension == ".zip" || extension== ".gz" || extension == ".7z" || extension == ".tar" ||
                (extension == ".exe" && secondary_extension == ".7z"))
            return (`$exe7z x $file -y -o$directory`)
        end
        throw(ArguementError("I don't know how to unpack $file"))
    end
end

"""
    unpack(f; keep_originals=false)

Extracts the content of an archive in the current directory;
deleting the original archive, unless the `keep_originals` flag is set.
"""
function unpack(f; keep_originals=false)
    run(unpack_cmd(f, pwd(), last(splitext(f)), last(splitext(first(splitext(f))))))
    rm("pax_global_header"; force=true)# Non-compliant tarball extractors dump out this file. It is meaningly (google it's filename for more)
    !keep_originals && rm(f)
end


text:
#=
Generate Symbolic Jacobian with ModelingToolkit. The testing equations are from Matlab tutorial:
- Jacobian and gradient: https://www.mathworks.com/help/symbolic/jacobian.html
- Hessian: https://www.mathworks.com/help/symbolic/hessian.html
=#

using ModelingToolkit

@variables x, y, z

# Jacobian: Vector{1, N} → Matrix{N,M} where M is Number of variables
jac = ModelingToolkit.jacobian([x*y*z, y^2, x + z], [x, y, z])

# Gradient: Equation → Vector
grad = ModelingToolkit.gradient(2*x + 3*y + 4*z, [x, y, z])

# Hessian
hess = ModelingToolkit.hessian(x*y+2*z*x, [x,y,z])

# Hessian: Jacobian of Gradient
hess_ = ModelingToolkit.jacobian(ModelingToolkit.gradient(x*y+2*z*x, [x,y,z]), [x,y,z])


text:
#
# This file is a part of Code module. Contains Code functions related
# to Organism. For example eating, moving, working with memory,...
#
# @author DeadbraiN
#
import Helper
import Config

export lookAt
export eatLeft
export eatRight
export eatUp
export eatDown
export stepLeft
export stepRight
export stepUp
export stepDown
export toMem
export fromMem
export idLeft
export idRight
export idUp
export idDown
export energyLeft
export energyRight
export energyUp
export energyDown
export myColor
export myX
export myY
export codeSizeLeft
export codeSizeRight
export codeSizeUp
export codeSizeDown
#
# Divider for "for" operator
#
const FOR_DIVIDER = f16(655.0)
#
# @cmd
# @line
# Returns energy amount in specified point in a world
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr|Expr(:nothing)} New expression or Expr(:nothing) if skipped
#
function lookAt(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.getEnergy(o, $(@randVar()), $(@randVar())))
end
#
# @cmd
# @line
# Is called when organism bites on the left. Format: var_xx = eatLeft(var_xx):Float16
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr}
#
function eatLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.eatLeft(c, o, $(@randVar())))
end
#
# @cmd
# @line
# Is called when organism bites on the right. Format: var_xx = eatRight(cfg.orgEatSize):Float16
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr}
#
function eatRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.eatRight(c, o, $(@randVar())))
end
#
# @cmd
# @line
# Is called when organism bites up. Format: var_xx = eatUp(cfg.orgEatSize):Float16
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr}
#
function eatUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.eatUp(c, o, $(@randVar())))
end
#
# @cmd
# @line
# Is called when organism bites down. Format: var_xx = eatDown(cfg.orgEatSize):Float16
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr}
#
function eatDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.eatDown(c, o, $(@randVar())))
end
#
# @cmd
# @line
# Is called when organism makes step left
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr}
#
function stepLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.stepLeft(o))
end
#
# @cmd
# @line
# Is called when organism makes step right
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function stepRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.stepRight(o))
end
#
# @cmd
# @line
# Is called when organism make step up
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function stepUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.stepUp(o))
end
#
# @cmd
# @line
# Is called when organism make step down
# @param cfg Global configuration type
# @param org Organism we have to mutate
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function stepDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.stepDown(o))
end
#
# @cmd
# @line
# Saves custom value to organism's private memory
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function toMem(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :(o.mem[$(@randVar())] = $(@randVar()))
end
#
# @cmd
# @line
# Extracts custom value from organism's private memory
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function fromMem(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  local var::Symbol = @randVar()
  #
  # We don't need to create separate block here, because
  # this is one code line.
  #
  :($(@randVar()) = haskey(o.mem, $(var)) ? o.mem[$(var)] : $(var))
end
# #
# # @cmd
# # @line
# # Obtains unique id of organism from the left. UInt id will
# # be converted to Int type. If there is no organism on the left,
# # it will obtain 0.
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function idLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = Creature.idLeft(o))
# end
# #
# # @cmd
# # @line
# # Obtains unique id of organism from the right. UInt id will
# # be converted to Int type. If there is no organism on the right,
# # it will obtain 0.
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function idRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = Creature.idRight(o))
# end
# #
# # @cmd
# # @line
# # Obtains unique id of organism from the above. UInt id will
# # be converted to Int type. If there is no organism on the above,
# # it will obtain 0.
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function idUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = Creature.idUp(o))
# end
# #
# # @cmd
# # @line
# # Obtains unique id of organism from the right. UInt id will
# # be converted to Int type. If there is no organism on the right,
# # it will obtain 0.
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function idDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = Creature.idDown(o))
# end
#
# @cmd
# @line
# Obtains amount of energy of organism from the left. If there is no organism
# on the left, then 0 will be obtained
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function energyLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.energyLeft(o))
end
#
# @cmd
# @line
# Obtains amount of energy of organism from the right. If there is no organism
# on the right, then 0 will be obtained
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function energyRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.energyRight(o))
end
#
# @cmd
# @line
# Obtains amount of energy of organism from the above. If there is no organism
# on the above, then 0 will be obtained
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function energyUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.energyUp(o))
end
#
# @cmd
# @line
# Obtains amount of energy of organism from the below. If there is no organism
# on the below, then 0 will be obtained
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function energyDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = Creature.energyDown(o))
end
#
# @cmd
# @line
# Obtains it's own color.
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function myColor(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = f16(o.color))
end
#
# @cmd
# @line
# Obtains it's own X coordinate.
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function myX(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = f16(o.pos.x))
end
#
# @cmd
# @line
# Obtains it's own Y coordinate.
# @param cfg Global configuration type
# @param org Organism we are working with
# @param pos Position in code
# @return {Expr|Expr(:nothing)}
#
function myY(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
  :($(@randVar()) = f16(o.pos.y))
end
# #
# # @cmd
# # @line
# # Obtains code size of organism on the left
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function codeSizeLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.codeSizeLeft(o)))
# end
# #
# # @cmd
# # @line
# # Obtains code size of organism on the right
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function codeSizeRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.codeSizeRight(o)))
# end
# #
# # @cmd
# # @line
# # Obtains code size of organism on the up
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function codeSizeUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.codeSizeUp(o)))
# end
# #
# # @cmd
# # @line
# # Obtains code size of organism on the down
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function codeSizeDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.codeSizeDown(o)))
# end
# #
# # @cmd
# # @line
# # Obtains clone energy percent of organism on the left
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function cloneEnergyPercentLeft(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.cloneEnergyPercentLeft(o)))
# end
# #
# # @cmd
# # @line
# # Obtains clone energy percent of organism on the right
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function cloneEnergyPercentRight(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.cloneEnergyPercentRight(o)))
# end
# #
# # @cmd
# # @line
# # Obtains clone energy percent of organism on the above
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function cloneEnergyPercentUp(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.cloneEnergyPercentUp(o)))
# end
# #
# # @cmd
# # @line
# # Obtains clone energy percent of organism on the bottom
# # @param cfg Global configuration type
# # @param org Organism we are working with
# # @param pos Position in code
# # @return {Expr|Expr(:nothing)}
# #
# function cloneEnergyPercentDown(cfg::Config.ConfigData, org::Creature.Organism, pos::Helper.CodePos)
#   :($(@randVar()) = f16(Creature.cloneEnergyPercentDown(o)))
# end

max_stars_repo_name: tmptrash/jevo

max_issues_repo_name: tmptrash/jevo

max_forks_repo_name: tmptrash/jevo


text:
using Test
using ModelingToolkit, OrdinaryDiffEq

include("../examples/rc_model.jl")

sys = structural_simplify(rc_model)
@test !isempty(ModelingToolkit.defaults(sys))
u0 = [
      capacitor.v => 0.0
      capacitor.p.i => 0.0
      resistor.v => 0.0
     ]
prob = ODEProblem(sys, u0, (0, 10.0))
sol = solve(prob, Rodas4())

@test sol[resistor.p.i] == sol[capacitor.p.i]
@test sol[resistor.n.i] == -sol[capacitor.p.i]
@test sol[capacitor.n.i] == -sol[capacitor.p.i]
@test iszero(sol[ground.g.i])
@test iszero(sol[ground.g.v])
@test sol[resistor.v] == sol[source.p.v] - sol[capacitor.p.v]

prob = ODAEProblem(sys, u0, (0, 10.0))
sol = solve(prob, Tsit5())

@test sol[resistor.p.i] == sol[capacitor.p.i]
@test sol[resistor.n.i] == -sol[capacitor.p.i]
@test sol[capacitor.n.i] == -sol[capacitor.p.i]
@test iszero(sol[ground.g.i])
@test iszero(sol[ground.g.v])
@test sol[resistor.v] == sol[source.p.v] - sol[capacitor.p.v]
#using Plots
#plot(sol)

include("../examples/serial_inductor.jl")
sys = structural_simplify(ll_model)
u0 = [
      inductor1.i => 0.0
      inductor2.i => 0.0
      inductor2.v => 0.0
     ]
prob = ODEProblem(sys, u0, (0, 10.0))
sol = solve(prob, Rodas4())

prob = ODAEProblem(sys, u0, (0, 10.0))
sol = solve(prob, Tsit5())


text:
@testset "Nodes initialization algorithm entry  dimension 1 output dimension 1" begin
    # Comprobamos que las hipótesis de selección son correctas
    M = 1
    @test RampFunction(M) == 1
    @test RampFunction(-M) == 0
    # Bien definido para tamaño n = 2 y salida de dimensión 1
    f_regression(x)=(x<=1) ? exp(-x) : log(x)
    data_set_size =  5
    entry_dimension = 1
    output_dimension = 1
    # Número de neuronas
    n = data_set_size  # Debe de ser mayor que 1 para que no de error
    X_train= map(
        x-> (x-0.5)*10, # reescalamos al intervalo [-5,5]
        rand(Float64, data_set_size)
    )

    Y_train = map(f_regression, X_train)
    h = nn_from_data(X_train, Y_train, n, M)

    # veamos que el tamaño de la salida es la adecuada
    @test size(h.W1) == (n,2)
    @test size(h.W2) == (1,n)

    # Si ha sido bien construida:
    # Evaluar la red neuronal en los datos con los que se construyó
    # debería de resultar el valor de Y_train respectivo
    evaluar(x)=forward_propagation(h,
     RampFunction,x)

    for (x,y) in zip(X_train,Y_train)
        @test evaluar([x]) ≈ [y]
    end

end


text:
"""
This struct contains data for the capacity market product.
    name: Capacity.
    derating: The derating factor for capacity market participation based on technology type.
    accepted_perc: The percentage of capacity cleared in capacity markets for each scenario.
    capacity_bid: Capacity market bid placed by the project.
"""
mutable struct Capacity <: Product
    name::Symbol
    derating::Float64
    accepted_perc::Dict{String, Vector{Float64}}
    capacity_bid::Float64
end

# Derating factors, accepted percentage and capacity bids only returned when product is of type Capacity
get_derating(prod::Product) = nothing
get_derating(prod::Capacity) = prod.derating
get_accepted_perc(prod::Product) = nothing
get_accepted_perc(prod::Capacity) = prod.accepted_perc
get_capacity_bid(prod::Product) = nothing
get_capacity_bid(prod::Capacity) = prod.capacity_bid

# Derating factors only set when product is of type Capacity
function set_derating!(prod::T, derating_factor) where T <: Product
    return
end

function set_derating!(prod::Capacity, derating_factor)
    prod.derating = derating_factor
    return
end

# Capacity bids only set when product is of type Capacity
function set_capacity_bid!(prod::T, capacity_bid) where T <: Product
    return
end

function set_capacity_bid!(prod::Capacity, capacity_bid)
    prod.capacity_bid = capacity_bid
    return
end

# Capacity accepted percentage only set when product is of type Capacity
function set_accepted_perc!(product::T,
                            scenario_name::String,
                            capacity_accepted_perc::Array{Float64, 1}) where T <: Product
    return
end

function set_accepted_perc!(product::Capacity,
                            scenario_name::String,
                            capacity_accepted_perc::Array{Float64, 1})
    product.accepted_perc[scenario_name] = capacity_accepted_perc
    return
end


text:
module ZeroMeanFESpacesTests

using Test
using Gridap

model = CartesianDiscreteModel(partition=(2,2))

order = 2

fespace = FESpace(
  reffe = :PLagrangian,
  conformity = :L2,
  valuetype = Float64,
  order = order,
  model = model)

quadorder = order
fespace = ZeroMeanFESpace(fespace,quadorder)

@test fespace.vol ≈ 1.0

ufun(x) = x[1] + x[2] + 2.0

trian = Triangulation(model)
quad = CellQuadrature(trian,degree=order)

bh = FEBasis(fespace)
uh = zero(fespace)

cellmat = integrate(inner(bh,bh),trian,quad)
cellvec = integrate(inner(bh,uh),trian,quad)

nfree = 23

ndiri = 0

test_fe_space(fespace, nfree, ndiri, cellmat, cellvec, ufun)

V = TrialFESpace(fespace)

x = rand(Float64,num_free_dofs(V))
uh = FEFunction(V,x)

i = integrate(uh,trian,quad)

@test sum(i) + 1.0 ≈ 1.0

end # module


text:
"""
    SimpleSDMPredictor(::Type{EarthEnv}, ::Type{LandCover}, layer::Integer=1; full::Bool=false, kwargs...)

Download and prepare the EarthEnv consensus landcover data, and returns them as
an array of `SimpleSDMPredictor`s. Layers are called by their number, from 1 to
19. The list of available layers is given in a table below. The raw data come
from https://www.earthenv.org/landcover.

THe `full` keyword indicates whether the *DISCover* information must be
included. Quoting from the reference website:

> Although DISCover is based on older remote sensing imagery (1992-1993), it
> contains some complementary information which is useful for capturing
> sub-pixel land cover heterogeneity (please see the associated article for
> details). Therefore, it is recommended to use the full version of the
> consensus land cover dataset for most applications. However, the reduced
> version may provide an alternative for applications in regions with large land
> cover change in the past two decades.

It is recommended to *keep* the content of the `path` folder, as it will
eliminate the need to download and/or extract the tiff files. For example,
calling `landcover(1:12)` will download and extract everything, and future calls
will be much faster. Please keep in mind that the layers can be quite large, so
keeping the models stored is particularly important.

| Variable |             Description              |
| -------- | ------------------------------------ |
| 1        | Evergreen/Deciduous Needleleaf Trees |
| 2        | Evergreen Broadleaf Trees            |
| 3        | Deciduous Broadleaf Trees            |
| 4        | Mixed/Other Trees                    |
| 5        | Shrubs                               |
| 6        | Herbaceous Vegetation                |
| 7        | Cultivated and Managed Vegetation    |
| 8        | Regularly Flooded Vegetation         |
| 9        | Urban/Built-up                       |
| 10       | Snow/Ice                             |
| 11       | Barren                               |
| 12       | Open Water                           |

These data are released under a CC-BY-NC license to Tuanmu & Jetz.
"""
function SimpleSDMPredictor(::Type{EarthEnv}, ::Type{LandCover}, layer::Integer=1; full::Bool=false, kwargs...)
    file = _get_raster(EarthEnv, LandCover, layer, full)
    return geotiff(SimpleSDMPredictor, file; kwargs...)
end

function SimpleSDMPredictor(::Type{EarthEnv}, ::Type{LandCover}, layers::AbstractArray; kwargs...)
    @assert eltype(layers) <: Integer
    return [SimpleSDMPredictor(EarthEnv, LandCover, l; kwargs...) for l in layers]
end