text:
context("Distance Fields") do

context("SignedDistanceField") do
    s1 = SignedDistanceField(HyperRectangle(Vec(1,2),Vec(3,4)), [1 2;3 4])
    @fact typeof(s1) --> GeometryTypes.SignedDistanceField{2,Int,Int}
    s1 = SignedDistanceField(HyperRectangle(Vec(1,2),Vec(3,4)), [1. 2.;3. 4.])
    @fact typeof(s1) --> GeometryTypes.SignedDistanceField{2,Int,Float64}

    # functional
    s2 = SignedDistanceField(HyperRectangle(Vec(0,0,0.),Vec(1,1,1.))) do v
        sqrt(sum(v.*v)) - 1 # sphere
    end
    @fact size(s2) --> (11, 11, 11)
    # functional
    s2 = SignedDistanceField(HyperRectangle(Vec(-1,-1.),Vec(2,3.))) do v
        sqrt(sum(v.*v)) - 1 # circle
    end
    @fact size(s2) --> (21, 31)
    @fact size(s2, 1) --> 21
    @fact size(s2, 2) --> 31

    @fact HyperRectangle(s2) --> HyperRectangle(Vec(-1,-1.),Vec(2,3.))
end

context("getindex") do
    sdf = SignedDistanceField(HyperRectangle(Vec(-1,-1.),Vec(2,2.))) do v
        sqrt(sum(v.*v)) - 1 # circle
    end

    @fact sdf[1,1] --> sdf[21*21] # by circle symmetry
    @fact_throws BoundsError sdf[21*21+1]
end

end


text:
#################################################################################
# Questions
#
# - why do exported functions (e.g. ids, var, con, ref) need pacakge qualification?
# - why do non-qualified exported functions (e.g. ids, var) still throw warnings?
#
#
# Comments
#
# - Ideally the net_injection variables would be bounded.  This can be done using an adhoc data model extention
#

#################################################################################
# Model Definitions

""
function build_nip_model(data::Dict{String,Any}, model_constructor; multinetwork=true, kwargs...)
    return PM.build_generic_model(data, model_constructor, post_nip; multinetwork=multinetwork, kwargs...)
end

""
function post_nip(pm::PM.GenericPowerModel)
    for (n, network) in PM.nws(pm)
        @assert !PM.ismulticonductor(pm, nw=n)
        PM.variable_voltage(pm, nw=n)
        variable_net_injection(pm, nw=n)
        PM.variable_branch_flow(pm, nw=n)#, bounded=false)
        PM.variable_dcline_flow(pm, nw=n)

        PM.constraint_voltage(pm, nw=n)

        for i in PM.ids(pm, :ref_buses, nw=n)
            PM.constraint_theta_ref(pm, i, nw=n)
        end

        for i in PM.ids(pm, :bus, nw=n)
            constraint_kcl_ni(pm, i, nw=n)
        end

        for i in PM.ids(pm, :branch, nw=n)
            PM.constraint_ohms_yt_from(pm, i, nw=n)
            PM.constraint_ohms_yt_to(pm, i, nw=n)

            PM.constraint_voltage_angle_difference(pm, i, nw=n)

            PM.constraint_thermal_limit_from(pm, i, nw=n)
            PM.constraint_thermal_limit_to(pm, i, nw=n)
        end

        for i in PM.ids(pm, :dcline)
            PM.constraint_dcline(pm, i, nw=n)
        end
    end
end

""
function build_nip_expr_model(data::Dict{String,Any}, model_constructor; multinetwork=true, kwargs...)
    return PM.build_generic_model(data, model_constructor, post_nip_expr; multinetwork=multinetwork, kwargs...)
end

""
function post_nip_expr(pm::PM.GenericPowerModel)
    for (n, network) in PM.nws(pm)
        @assert !PM.ismulticonductor(pm, nw=n)
        PM.variable_voltage(pm, nw=n)
        PM.variable_branch_flow(pm, nw=n)#, bounded=false)
        PM.variable_dcline_flow(pm, nw=n)

        PM.constraint_voltage(pm, nw=n)

        for i in PM.ids(pm, :ref_buses, nw=n)
            PM.constraint_theta_ref(pm, i, nw=n)
        end

        for i in PM.ids(pm, :bus, nw=n)
            constraint_kcl_ni_expr(pm, i, nw=n)
        end

        for i in PM.ids(pm, :branch, nw=n)
            PM.constraint_ohms_yt_from(pm, i, nw=n)
            PM.constraint_ohms_yt_to(pm, i, nw=n)

            PM.constraint_voltage_angle_difference(pm, i, nw=n)

            PM.constraint_thermal_limit_from(pm, i, nw=n)
            PM.constraint_thermal_limit_to(pm, i, nw=n)
        end

        for i in PM.ids(pm, :dcline)
            PM.constraint_dcline(pm, i, nw=n)
        end
    end
end

#################################################################################
# Model Extention Functions

"generates variables for both `active` and `reactive` net injection"
function variable_net_injection(pm::PM.GenericPowerModel; kwargs...)
    variable_active_net_injection(pm; kwargs...)
    variable_reactive_net_injection(pm; kwargs...)
end

""
function variable_active_net_injection(pm::PM.GenericPowerModel; nw::Int=pm.cnw, cnd::Int=pm.ccnd)
    PM.var(pm, nw, cnd)[:pni] = @variable(pm.model,
        [i in PM.ids(pm, nw, :bus)], base_name="$(nw)_$(cnd)_pni",
        start = 0.0
    )
end

""
function variable_reactive_net_injection(pm::PM.GenericPowerModel; nw::Int=pm.cnw, cnd::Int=pm.ccnd)
    PM.var(pm, nw, cnd)[:qni] = @variable(pm.model,
        [i in PM.ids(pm, nw, :bus)], base_name="$(nw)_$(cnd)_qni",
        start = 0.0
    )
end

""
function constraint_kcl_ni(pm::PM.GenericPowerModel, i::Int; nw::Int=pm.cnw, cnd::Int=pm.ccnd)
    if !haskey(PM.con(pm, nw, cnd), :kcl_p)
        PM.con(pm, nw, cnd)[:kcl_p] = Dict{Int,ConstraintRef}()
    end
    if !haskey(PM.con(pm, nw, cnd), :kcl_q)
        PM.con(pm, nw, cnd)[:kcl_q] = Dict{Int,ConstraintRef}()
    end

    bus = PM.ref(pm, nw, :bus, i)
    bus_arcs = PM.ref(pm, nw, :bus_arcs, i)
    bus_arcs_dc = PM.ref(pm, nw, :bus_arcs_dc, i)

    constraint_kcl_ni(pm, nw, cnd, i, bus_arcs, bus_arcs_dc)
end

""
function constraint_kcl_ni(pm::PM.GenericPowerModel, n::Int, c::Int, i::Int, bus_arcs, bus_arcs_dc)
    p = PM.var(pm, n, c, :p)
    q = PM.var(pm, n, c, :q)
    pni = PM.var(pm, n, c, :pni, i)
    qni = PM.var(pm, n, c, :qni, i)
    p_dc = PM.var(pm, n, c, :p_dc)
    q_dc = PM.var(pm, n, c, :q_dc)

    PM.con(pm, n, c, :kcl_p)[i] = @constraint(pm.model, sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == pni)
    PM.con(pm, n, c, :kcl_q)[i] = @constraint(pm.model, sum(q[a] for a in bus_arcs) + sum(q_dc[a_dc] for a_dc in bus_arcs_dc) == qni)
end

""
function constraint_kcl_ni_expr(pm::PM.GenericPowerModel, i::Int; nw::Int=pm.cnw, cnd::Int=pm.ccnd)
    if !haskey(PM.con(pm, nw, cnd), :kcl_p)
        PM.con(pm, nw, cnd)[:kcl_p] = Dict{Int,ConstraintRef}()
    end
    if !haskey(PM.con(pm, nw, cnd), :kcl_q)
        PM.con(pm, nw, cnd)[:kcl_q] = Dict{Int,ConstraintRef}()
    end

    bus = PM.ref(pm, nw, :bus, i)
    bus_arcs = PM.ref(pm, nw, :bus_arcs, i)
    bus_arcs_dc = PM.ref(pm, nw, :bus_arcs_dc, i)

    pni_expr = PM.ref(pm, nw, :bus, i, "pni")
    qni_expr = PM.ref(pm, nw, :bus, i, "qni")

    constraint_kcl_ni_expr(pm, nw, cnd, i, bus_arcs, bus_arcs_dc, pni_expr, qni_expr)
end

""
function constraint_kcl_ni_expr(pm::PM.GenericPowerModel, n::Int, c::Int, i::Int, bus_arcs, bus_arcs_dc, pni_expr, qni_expr)
    p = PM.var(pm, n, c, :p)
    q = PM.var(pm, n, c, :q)
    p_dc = PM.var(pm, n, c, :p_dc)
    q_dc = PM.var(pm, n, c, :q_dc)

    PM.con(pm, n, c, :kcl_p)[i] = @constraint(pm.model, sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == pni_expr)
    PM.con(pm, n, c, :kcl_q)[i] = @constraint(pm.model, sum(q[a] for a in bus_arcs) + sum(q_dc[a_dc] for a_dc in bus_arcs_dc) == qni_expr)
end

"active power only models ignore reactive power variables"
function variable_reactive_net_injection(pm::PM.GenericPowerModel{T}; kwargs...) where T <: PM.AbstractDCPForm
end

"active power only models ignore reactive power flows"
function constraint_kcl_ni(pm::PM.GenericPowerModel{T}, n::Int, c::Int, i::Int, bus_arcs, bus_arcs_dc) where T <: PM.AbstractDCPForm
    p = PM.var(pm, n, c, :p)
    pni = PM.var(pm, n, c, :pni, i)
    p_dc = PM.var(pm, n, c, :p_dc)

    PM.con(pm, n, c, :kcl_p)[i] = @constraint(pm.model, sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == pni)
end

""
function constraint_kcl_ni_expr(pm::PM.GenericPowerModel{T}, n::Int, c::Int, i::Int, bus_arcs, bus_arcs_dc, pni_expr, qni_expr) where T <: PM.AbstractDCPForm
    p = PM.var(pm, n, c, :p)
    p_dc = PM.var(pm, n, c, :p_dc)

    PM.con(pm, n, c, :kcl_p)[i] = @constraint(pm.model, sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == pni_expr)
end

ext: jl


text:
module Runner

using Pkg
using GitHub
using Base64

using ..Utils
using ..Reporter
using ..Config

using ..ContinuousBenchmarks: BENCHMARK_CONFIG_FILE, get_benchmark_files

const report_repo = Config.get_config("github.report_repo")

function trigger(push_event)
    commit = push_event.payload["head_commit"]
    commit_id =  commit["id"]
    commit_msg = commit["message"]
    reg = r"^BM-\d{12}.*"
    @debug "Begin to run benchmark $commit_msg..."
    if match(reg, commit_msg) != nothing
        run_bm(commit_msg)
    end
end

function run_bm(name::String)
    bot_auth = GitHub.authenticate(Config.get_config("github.token"))
    params = Dict("ref" => name) # on the PR branch
    bm_file_content = GitHub.file(report_repo, "jobs/$(name).toml"; params=params, auth=bot_auth)
    content = String(base64decode(replace(bm_file_content.content, "\n" =>"")))
    bm_info = Pkg.TOML.parse(content)
    report_path = nothing
    try
        report_path = local_benchmark(name, bm_info["benchmark"]["branches"])
    catch ex
        Reporter.send_error(name, bm_info, ex)
        return
    end
    Reporter.send(name, bm_info, report_path)
end

function local_benchmark(name, branch_names)
    @assert length(branch_names) > 1
    branch_names = [branch_names...]
    project_dir = Config.get_config("target.project_dir", ".")
    gitbranches(project_dir, branch_names)
    result_path = result_dir(name)
    shas = gitbranchshas(project_dir, branch_names)

    for (branch, sha) in zip(branch_names, shas)
        onbranch(project_dir, branch) do
            save_path = joinpath(result_path, snip7(sha))
            isdir(save_path) || mkdir(save_path)
            for bm_file in get_benchmark_files()
                run_benchmarks([bm_file], save_path=save_path)
            end
        end
    end
    report_path = joinpath(result_path, "report.md")
    Reporter.write_report!(name, report_path, branch_names, shas)
    return report_path
end

function run_benchmarks(files; ignore_error=true, save_path="")
    @info("Benchmarking started.")
    for file in files
        try
            run_benchmark(file, save_path=save_path)
        catch err
            @error("Error occurs while running the benchmark: $file.")
            if :msg in fieldnames(typeof(err))
                @error(err.msg)
            end
            !ignore_error && rethrow(err)
        end
    end
    @info("Benchmarking completed.")
end

function run_benchmark(bm_path; save_path="")
    @info("Benchmarking `$bm_path` ... ")
    data = Dict(
        :config_file => BENCHMARK_CONFIG_FILE[],
        :bm_file => bm_path,
        :save_path => save_path,
    )
    julia_path = joinpath(Sys.BINDIR, Base.julia_exename())
    project_dir = Config.get_config("target.project_dir", ".")
    code = code_bm_run(data)
    job = `$julia_path --project=$(project_dir) -e $code`
    @info(job);
    run(job)
    @info("`$bm_path` ✓")
    return
end

function run_bm_on_travis(name, branch_names, cid)
    report_path = local_benchmark(name, branch_names)
    Reporter.send_from_travis(name, cid, report_path)
end

end


text:
module PrettyPlots

using Colors
using DataFrames
using Plots
#import Plots.gr
#import Plots.pyplot
import Plots.xlims!, Plots.ylims!, Plots.plot
gr(); # default backend
#pyplot()
using VehicleModels # need for parameters
using NLOptControl  # need for resultsDir!()

include("PrettyUtils.jl")
include("NLOptControl_plots.jl")
include("MPCPlots.jl")
include("VehicleModels_plots.jl")

export
    # PrettyUtils.jl
    minDF,
    maxDF,
    plotSettings,
    _pretty_defaults,
    currentSettings,

    # NLOptControl.jl plots
    statePlot,
    controlPlot,
    costatesPlot,
    costatesPlots,
    allPlots,
    adjust_axis,

    # MPC plots
    mpcPlot,
    tPlot,
    optPlot,

  # MAVs.jl & VehicleModel.jl plots
    obstaclePlot,
    trackPlot,
    mainSim,
    posterP,
    posPlot,
    vtPlot,
    pSimGR,
    axLimsPlot,

    # Plots.jl exported functions
    xlims!,
    ylims!,
    plot
    #Plots.gui(),
    #Plots.pyplot(),
    #Plots.gr(),
    #Plots.pgfplots()

end # module


text:
using Test, QMC
using Statistics

@testset "Small Vector of size $n" for n in 1:10
    p = rand(1:10, n)
    mean_ = p' * collect(1:size(p,1)) / sum(p)
    var_ = (p' * (collect(1:size(p,1)) .^ 2) / sum(p)) - mean_^2

    @testset "ProbabilityVector" begin
        pvec = ProbabilityVector(p)
        X = [rand(pvec) for _ in 1:1_000_000]

        @test isapprox(mean_, mean(X), atol=0.05)
        @test isapprox(var_, var(X), atol=0.05)

        # X = [QMC.rand_bisect(pvec) for _ in 1:1_000_000]
        # @test isapprox(mean_, mean(X), atol=0.05)
        # @test isapprox(var_, var(X), atol=0.05)
    end

    @testset "ProbabilityHeap" begin
        pvec = ProbabilityHeap(p)
        X = [rand(pvec) for _ in 1:1_000_000]

        @test isapprox(mean_, mean(X), atol=0.05)
        @test isapprox(var_, var(X), atol=0.05)
    end
end


text:
import ModiaMath

include("../../../src/language/ModiaLogging.jl")
include("../../../src/language/Instantiation.jl")

include("../../../src/symbolic/BLTandPantelides/BLTandPantelidesUtilities.jl")
include("../../../src/symbolic/BLTandPantelides/BLTandPantelides.jl")


text:
# This script generates and draws examples for the circle detection

using PyPlot
using Printf
using DelimitedFiles
using RAFF

"""

    gen_circle(np::Int, p::Int; std::Float64=0.1,
               θSol::Vector{Float64}=10.0*randn(Float64, 3),
               outTimes::Float64=5.0,
               interval::Vector{Float64}=rand(np)*2.0*π)

Generate perturbed points in a circle given by `θSol`. Construct a
test file for `RAFF`.

"""
function gen_circle(np::Int, p::Int; std::Float64=0.1,
                    θSol::Vector{Float64}=10.0*randn(Float64, 3),
                    outTimes::Float64=5.0, interval::Vector{Float64}=rand(np)*2.0*π)

    ρ = (α, ρ) -> [ρ * cos(α) + θSol[1], ρ * sin(α) + θSol[2]]
    f = (x) -> (x[1] - θSol[1])^2 + (x[2] - θSol[2])^2 - θSol[3]^2

    data = Array{Float64, 2}(undef, np, 4)

    # Points selected to be outliers
    v = RAFF.get_unique_random_points(np, np - p)

    for (i, α) in enumerate(interval)

        pt = ρ(α, θSol[3] + std * randn())

        data[i, 3:4] = 0.0 #f(pt)

        if i in v

            pt = ρ(α, θSol[3] * (1.0 + 2 * rand()) * outTimes * std * sign(randn()))

            data[i, 4] = 1.0

        end

        data[i, 1:2] = pt

    end

    open("/tmp/output.txt", "w") do fp

        # Dimension of the domain of the function to fit
        @printf(fp, "%d\n", 2)

        for k = 1:np

            @printf(fp, "%20.15f %20.15f %20.15f %1d\n",
                    data[k, 1], data[k, 2], data[k, 3], Int(k in v))

        end

    end

    return data, v

end

function gen_ncircle(np::Int, p::Int; std::Float64=0.1,
                    θSol::Vector{Float64}=10.0*randn(Float64, 3),
                    outTimes::Float64=5.0, interval::Vector{Float64}=rand(p)*2.0*π)

    ρ = (α, ρ) -> [ρ * cos(α) + θSol[1], ρ * sin(α) + θSol[2]]
    f = (x) -> (x[1] - θSol[1])^2 + (x[2] - θSol[2])^2 - θSol[3]^2

    data = Array{Float64, 2}(undef, np, 4)

    for (i, α) in enumerate(interval)

        pt = ρ(α, θSol[3] + std * randn())

        data[i, 1:2]  = pt
        data[i, 3:4] .= 0.0 #f(pt)

    end

    # Just random noise

    v = Vector{Int}(undef, np - p)

    for i = p + 1:np

        data[i, 1] = θSol[1] - 2.0 * θSol[3] + rand() * 4.0 * θSol[3]
        data[i, 2] = θSol[2] - 2.0 * θSol[3] + rand() * 4.0 * θSol[3]
        data[i, 3] = 0.0
        data[i, 4] = 1.0

        v[i - p] = i

    end

    open("/tmp/output.txt", "w") do fp

        # Dimension of the domain of the function to fit
        @printf(fp, "%d\n", 2)

        for k = 1:np

            @printf(fp, "%20.15f %20.15f %20.15f %1d\n",
                    data[k, 1], data[k, 2], data[k, 3], Int(k in v))

        end

    end

    return data, v

end

"""

Draw the points generated by the previous function.

"""
function draw_circle(data, outliers)

    np, = size(data)

    c = zeros(np)

    c[outliers] .= 1.0

    PyPlot.scatter(data[:, 1], data[:, 2], c=c, marker="o", s=50.0, linewidths=0.2,
                   cmap=PyPlot.cm["Paired"], alpha=0.9)

    PyPlot.axis("scaled")

    PyPlot.xticks([])

    PyPlot.yticks([])

    PyPlot.savefig("/tmp/circle.png", dpi=72)

end

"""

Draw the points and the solutions obtained. Save the picture in a file.

"""
function draw_circle_sol(M; model_str="circle", raff_output=nothing, other_sols...)

    x  = M[:, 1]
    y  = M[:, 2]
    co = M[:, 4]

    t = 0:0.1:2.1 * π

    ptx = (α, ρ, d) -> ρ * cos(α) + d[1]
    pty = (α, ρ, d) -> ρ * sin(α) + d[2]

    # Plot data

    true_outliers = findall(co .!= 0.0)

    PyPlot.scatter(x[co .== 0.0], y[co .== 0.0], color=PyPlot.cm."Pastel1"(2.0/9.0),
                   marker="o", s=50.0, linewidths=0.2)

    PyPlot.scatter(x[co .!= 0.0], y[co .!= 0.0], color=PyPlot.cm."Pastel1"(2.0/9.0),
                   marker="^", s=25.0, linewidths=0.2, label="Outliers")

    if raff_output != nothing

        n, model, modelstr = RAFF.model_list[model_str]

        fSol = raff_output.solution

        modl1x = (α) -> ptx(α, fSol[3], fSol[1:2])
        modl1y = (α) -> pty(α, fSol[3], fSol[1:2])

        PyPlot.plot(modl1x.(t), modl1y.(t), color=PyPlot.cm."Set1"(2.0/9.0))

        # Draw outliers found by RAFF

        true_positives = intersect(true_outliers, raff_output.outliers)
        false_positives = setdiff(raff_output.outliers, true_positives)

        if length(false_positives) > 0

            PyPlot.scatter(x[false_positives], y[false_positives],
                           color=PyPlot.cm."Pastel1"(0.0/9.0), marker="o",
                           linewidths=0.2, edgecolors="k", s=50.0, label="False positives")

        end

        if length(true_positives) > 0

            PyPlot.scatter(x[true_positives], y[true_positives],
                           color=PyPlot.cm."Pastel1"(0.0/9.0), marker="^",
                           s=50.0, linewidths=0.2, edgecolors="k", label="Identified outliers")

        end

    end

    PyPlot.legend(loc="best")

    PyPlot.axis("scaled")

    PyPlot.xticks([])

    PyPlot.yticks([])

    PyPlot.savefig("/tmp/circle.png", dpi=150, bbox_inches="tight")

end

function draw_circle_sol(tSol, fSol, lsSol)

    datafile = "/tmp/output.txt"

    fp = open(datafile, "r")

    N = parse(Int, readline(fp))

    M = readdlm(fp)

    close(fp)

    x  = M[:, 1]
    y  = M[:, 2]
    ρ  = M[:, 3]
    co = M[:, 4]

    t = [0:0.1:2.1 * π;]

    ptx = (α, ρ, d) -> ρ * cos(α) + d[1]
    pty = (α, ρ, d) -> ρ * sin(α) + d[2]

    # True solution

    pptx = (α) -> ptx(α, tSol[3], tSol[1:2])
    ppty = (α) -> pty(α, tSol[3], tSol[1:2])

    PyPlot.plot(pptx.(t), ppty.(t), "b--", label="True solution")

    # RAFF solution

    pptx = (α) -> ptx(α, fSol[3], fSol[1:2])
    ppty = (α) -> pty(α, fSol[3], fSol[1:2])

    PyPlot.plot(pptx.(t), ppty.(t), "g-", label="RAFF")

    # # LS solution

    # pptx = (α) -> ptx(α, lsSol[3], lsSol[1:2])
    # ppty = (α) -> pty(α, lsSol[3], lsSol[1:2])

    # PyPlot.plot(pptx.(t), ppty.(t), "r-", label="Least squares")

    PyPlot.scatter(x[co .== 0.0], y[co .== 0.0], color=PyPlot.cm."Pastel1"(2.0/9.0),
                   marker="o", s=50.0, linewidths=0.2)

    PyPlot.scatter(x[co .!= 0.0], y[co .!= 0.0], color=PyPlot.cm."Pastel1"(1.0/9.0),
                   marker=".", s=25.0, linewidths=0.2, label="Outliers")

    PyPlot.legend(loc=4)

    PyPlot.axis("scaled")

    PyPlot.xticks([])

    PyPlot.yticks([])

    PyPlot.savefig("/tmp/circle.png", dpi=150, bbox_inches="tight")

end


text:
abstract type Pet end

struct Dog <: Pet
    name::AbstractString
end

struct Cat <: Pet
    name::AbstractString
end

encounter(a, b) = "$(name(a)) meets $(name(b)) and $(meets(a, b))."

# name
name(p::Pet) = p.name

# fallbacks
meets(a::Pet, b::Pet) = "is cautious"
meets(a::Pet, b) = "runs away"
meets(a, b) = "nothing happens"

# specific types
meets(a::Dog, b::Dog) = "sniffs"
meets(a::Dog, b::Cat) = "chases"
meets(a::Cat, b::Dog) = "hisses"
meets(a::Cat, b::Cat) = "slinks"


text:
# Important, for broadcast joins we cannot assume c and d have same number of columns:
# c could have more columns than d
rowcmp(::Tuple, i, ::Tuple{}, j) = 0

function rowcmp(tc::Tuple, i, td::Tuple, j)
    c, d = tc[1], td[1]
    let k = rowcmp(c, i, d, j)
        (k == 0) ? rowcmp(tail(tc), i, tail(td), j) : k
    end
end

function rowcmp(c::Columns, i, d::Columns, j)
    tc = Tuple(fieldarrays(c))
    td = Tuple(fieldarrays(d))
    return rowcmp(tc, i, td, j)
end

@inline function rowcmp(c::AbstractVector, i, d::AbstractVector, j)
    cmp(c[i], d[j])
end

struct GroupJoinPerm{LP<:GroupPerm, RP<:GroupPerm}
    left::LP
    right::RP
end

GroupJoinPerm(lkeys::AbstractVector, rkeys::AbstractVector, lperm=sortperm(lkeys), rperm=sortperm(rkeys)) =
    GroupJoinPerm(GroupPerm(lkeys, lperm), GroupPerm(rkeys, rperm))

function _pick(s, a, b)
    if a === nothing && b === nothing
        return nothing
    elseif a === nothing
        return (1:0, b[1]), (1, a, b)
    elseif b === nothing
        return (a[1], 1:0), (-1, a, b)
    else
        lp = sortperm(s.left)
        rp = sortperm(s.right)
        cmp = rowcmp(parent(s.left), lp[first(a[1])], parent(s.right), rp[first(b[1])])
        if cmp < 0
            return (a[1], 1:0), (-1, a, b)
        elseif cmp == 0
            return (a[1], b[1]), (0, a, b)
        else
            return (1:0, b[1]), (1, a, b)
        end
    end
end

function Base.iterate(s::GroupJoinPerm)
    l = iterate(s.left)
    r = iterate(s.right)
    _pick(s, l, r)
end

function Base.iterate(s::GroupJoinPerm, (select, l, r))
    (select <= 0) && (l = iterate(s.left, l[2]))
    (select >= 0) && (r = iterate(s.right, r[2]))
    _pick(s, l, r)
end

Base.IteratorSize(::Type{<:GroupJoinPerm}) = Base.SizeUnknown()
Base.IteratorEltype(::Type{<:GroupJoinPerm}) = Base.HasEltype()
Base.eltype(::Type{<:GroupJoinPerm}) = Tuple{UnitRange{Int}, UnitRange{Int}}

# Missing
nullrow(t::Type{<:Tuple}, ::Type{Missing}) = Tuple(map(x->missing, fieldtypes(t)))
nullrow(t::Type{<:NamedTuple}, ::Type{Missing}) = t(Tuple(map(x->missing, fieldtypes(t))))

# DataValue
nullrow(::Type{T}, ::Type{DataValue}) where {T <: Tuple} = Tuple(fieldtype(T, i)() for i = 1:fieldcount(T))
function nullrow(::Type{NamedTuple{names, T}}, ::Type{DataValue}) where {names, T}
    NamedTuple{names, T}(Tuple(fieldtype(T, i)() for i = 1:fieldcount(T)))
end

nullrow(T, M) = missing_instance(M)

nullrowtype(::Type{T}, ::Type{S}) where {T<:Tup, S} = map_params(t -> type2missingtype(t, S), T)
nullrowtype(::Type{T}, ::Type{S}) where {T, S} = type2missingtype(T, S)

nullablerows(s::Columns{C}, ::Type{S}) where {C, S} = Columns{nullrowtype(C, S)}(fieldarrays(s))
nullablerows(s::AbstractVector, ::Type{S}) where {S} = vec_missing(s, S)

function init_left_right(ldata::AbstractVector{L}, rdata::AbstractVector{R}) where {L, R}
    (left = similar(arrayof(L), 0), right = similar(arrayof(R), 0))
end

_reduce(f, iter, ::Nothing) = reduce(f, iter)
_reduce(f, iter, init_group) = reduce(f, iter, init = init_group())
_reduce(::Nothing, iter, ::Nothing) = collect_columns(iter)
_reduce(::Nothing, iter, init_group) = collect_columns(iter)

# In a plain non group join with f === concat_tup, we avoid creating large structs and prefer to do things in place
# In every step of the iteration, instead of just iterating we push the values to init
# Even in the general case, type of keys is known, so we push to I while iterating data
function _join!(I, init, ::Val{typ}, ::Val{grp}, f, iter::GroupJoinPerm, ldata::AbstractVector{L}, rdata::AbstractVector{R};
    missingtype=Missing, init_group=nothing, accumulate=nothing) where {typ, grp, L, R}

    lkey, rkey = parent(iter.left), parent(iter.right)
    lperm, rperm = sortperm(iter.left), sortperm(iter.right)
    lnullable = grp === false && typ === :outer
    rnullable = grp === false && typ !== :inner

    filter_func = if typ === :anti
        ((lidxs, ridxs),) -> !isempty(lidxs) && isempty(ridxs)
    elseif typ === :inner
        ((lidxs, ridxs),) -> !isempty(lidxs) && !isempty(ridxs)
    elseif typ === :left
        ((lidxs, _),) -> !isempty(lidxs)
    elseif typ === :outer
        _ -> true
    end

    function iterate_value_push_key((lidxs, ridxs))
        key = isempty(lidxs) ? rkey[rperm[ridxs[1]]] : lkey[lperm[lidxs[1]]]
        liter = lnullable && isempty(lidxs) ? (nullrow(L, missingtype),) : (ldata[lperm[i]] for i in lidxs)
        riter = rnullable && isempty(ridxs) ? (nullrow(R, missingtype),) : (rdata[rperm[i]] for i in ridxs)
        if init === nothing
            if grp
                push!(I, key)
                joint_iter = (f(l::L, r::R) for (l, r) in product(liter, riter))
                return _reduce(accumulate, joint_iter, init_group)
            else
                return ((push!(I, key); f(l::L, r::R)) for (l, r) in product(liter, riter))
            end
        else
            Base.foreach(product(liter, riter)) do (l, r)
                push!(I, key)
                push!(init.left, l::L)
                push!(init.right, r::R)
            end
            return
        end
    end
    filtered_iter = Iterators.filter(filter_func, iter)
    if init !== nothing
        Base.foreach(iterate_value_push_key, filtered_iter)
        left, right = init
        data = Columns(concat_tup(columns(left), columns(right)))
        return data
    else
        data_iter = (iterate_value_push_key(idxs) for idxs in filtered_iter)
        data = grp ? collect_columns(data_iter) : collect_columns_flattened(data_iter)
        return data
    end
end

"""
    join(left, right; kw...)
    join(f, left, right; kw...)

Join tables `left` and `right`.

If a function `f(leftrow, rightrow)` is provided, the returned table will have a single
output column.  See the Examples below.

If the same key occurs multiple times in either table, each `left` row will get matched
with each `right` row, resulting in `n_occurrences_left * n_occurrences_right` output rows.

# Options (keyword arguments)

- `how = :inner`
    - Join method to use. Described below.
- `lkey = pkeys(left)`
    - Fields from `left` to match on (see [`pkeys`](@ref)).
- `rkey = pkeys(right)`
    - Fields from `right` to match on.
- `lselect = Not(lkey)`
    - Output columns from `left` (see [`Not`](@ref))
- `rselect = Not(rkey)`
    - Output columns from `right`.
- `missingtype = Missing`
    - Type of missing values that can be created through `:left` and `:outer` joins.
    - Other supported option is `DataValue`.

## Join methods (`how = :inner`)

- `:inner` -- rows with matching keys in both tables
- `:left` -- all rows from `left`, plus matched rows from `right` (missing values can occur)
- `:outer` -- all rows from both tables (missing values can occur)
- `:anti` -- rows in `left` WITHOUT matching keys in `right`

# Examples

    a = table((x = 1:10,   y = rand(10)), pkey = :x)
    b = table((x = 1:2:20, z = rand(10)), pkey = :x)

    join(a, b; how = :inner)
    join(a, b; how = :left)
    join(a, b; how = :outer)
    join(a, b; how = :anti)

    join((l, r) -> l.y + r.z, a, b)
"""
function Base.join(f, left::Dataset, right::Dataset;
                   how=:inner, group=false,
                   lkey=pkeynames(left), rkey=pkeynames(right),
                   lselect=isa(left, NDSparse) ?
                       valuenames(left) : excludecols(left, lkey),
                   rselect=isa(right, NDSparse) ?
                       valuenames(right) : excludecols(right, rkey),
                   name = nothing,
                   cache=true,
                   missingtype=Missing,
                   init_group=nothing,
                   accumulate=nothing)

    if !(how in [:inner, :left, :outer, :anti])
        error("Invalid how: supported join types are :inner, :left, :outer, and :anti")
    end
    lkey = lowerselection(left, lkey)
    rkey = lowerselection(right, rkey)

    if !isa(lkey, Tuple)
        lkey = (lkey,)
    end

    if !isa(rkey, Tuple)
        rkey = (rkey,)
    end

    lperm, lkey = sortpermby(left, lkey; cache=cache, return_keys=true)
    rperm, rkey = sortpermby(right, rkey; cache=cache, return_keys=true)

    lselect = lowerselection(left, lselect)
    rselect = lowerselection(right, rselect)
    if f === concat_tup
        if !isa(lselect, Tuple)
            lselect = (lselect,)
        end

        if !isa(rselect, Tuple)
            rselect = (rselect,)
        end
    end

    ldata = rows(left, lselect)
    rdata = rows(right, rselect)

    if !group
        (how == :outer) && (ldata = nullablerows(ldata, missingtype))
        (how == :inner) || (rdata = nullablerows(rdata, missingtype))
    end

    ldata = replace_placeholder(left, ldata)
    rdata = replace_placeholder(right, rdata)

    KT = map_params(promote_type, eltype(lkey), eltype(rkey))
    lkey = Columns{KT}(Tuple(fieldarrays(lkey)))
    rkey = Columns{KT}(Tuple(fieldarrays(rkey)))
    join_iter = GroupJoinPerm(GroupPerm(lkey, lperm), GroupPerm(rkey, rperm))
    init = !group && f === concat_tup ? init_left_right(ldata, rdata) : nothing
    typ, grp = Val{how}(), Val{group}()
    I = similar(arrayof(KT), 0)
    data = _join!(I, init, typ, grp, f, join_iter, ldata, rdata;
        missingtype=missingtype, init_group=init_group, accumulate=accumulate)
    if group && left isa IndexedTable && !(data isa Columns)
        data = Columns(groups=data)
    end
    convert(collectiontype(left), I, data, presorted=true, copy=false)
end

function Base.join(left::Dataset, right::Dataset; how=:inner, kwargs...)
    f = how === :anti ? (x,y) -> x : concat_tup
    join(f, left, right; how=how, kwargs...)
end

"""
    groupjoin(left, right; kw...)
    groupjoin(f, left, right; kw...)

Join `left` and `right` creating groups of values with matching keys.

For keyword argument options, see [`join`](@ref).

# Examples

    l = table([1,1,1,2], [1,2,2,1], [1,2,3,4], names=[:a,:b,:c], pkey=(:a, :b))
    r = table([0,1,1,2], [1,2,2,1], [1,2,3,4], names=[:a,:b,:d], pkey=(:a, :b))

    groupjoin(l, r)
    groupjoin(l, r; how = :left)
    groupjoin(l, r; how = :outer)
    groupjoin(l, r; how = :anti)
"""
function groupjoin(left::Dataset, right::Dataset; how=:inner, kwargs...)
    f = how === :anti ? (x,y) -> x : concat_tup
    groupjoin(f, left, right; how=how, kwargs...)
end

function groupjoin(f, left::Dataset, right::Dataset; how=:inner, kwargs...)
    join(f, left, right; group=true, how=how, kwargs...)
end

for (fn, how) in [:naturaljoin =>     (:inner, false, concat_tup),
                  :leftjoin =>        (:left,  false, concat_tup),
                  :outerjoin =>       (:outer, false, concat_tup),
                  :antijoin =>        (:anti,  false, (x, y) -> x),
                  :naturalgroupjoin =>(:inner, true, concat_tup),
                  :leftgroupjoin =>   (:left,  true, concat_tup),
                  :outergroupjoin =>  (:outer, true, concat_tup)]

    how, group, f = how

    @eval function $fn(f, left::Dataset, right::Dataset; kwargs...)
        join(f, left, right; group=$group, how=$(Expr(:quote, how)), kwargs...)
    end

    @eval function $fn(left::Dataset, right::Dataset; kwargs...)
        $fn($f, left, right; kwargs...)
    end
end

## Joins

const innerjoin = naturaljoin

map(f, x::NDSparse{T,D}, y::NDSparse{S,D}) where {T,S,D} = naturaljoin(f, x, y)

# asof join

"""
    asofjoin(left::NDSparse, right::NDSparse)

Join rows from `left` with the "most recent" value from `right`.

# Example

    using Dates
    akey1 = ["A", "A", "B", "B"]
    akey2 = [Date(2017,11,11), Date(2017,11,12), Date(2017,11,11), Date(2017,11,12)]
    avals = collect(1:4)

    bkey1 = ["A", "A", "B", "B"]
    bkey2 = [Date(2017,11,12), Date(2017,11,13), Date(2017,11,10), Date(2017,11,13)]
    bvals = collect(5:8)

    a = ndsparse((akey1, akey2), avals)
    b = ndsparse((bkey1, bkey2), bvals)

    asofjoin(a, b)
"""
function asofjoin(left::NDSparse, right::NDSparse)
    flush!(left); flush!(right)
    lI, rI = left.index, right.index
    lD, rD = left.data, right.data
    ll, rr = length(lI), length(rI)

    data = similar(lD)

    i = j = 1

    while i <= ll && j <= rr
        c = rowcmp(lI, i, rI, j)
        if c < 0
            @inbounds data[i] = lD[i]
            i += 1
        elseif row_asof(lI, i, rI, j)  # all equal except last col left>=right
            j += 1
            while j <= rr && row_asof(lI, i, rI, j)
                j += 1
            end
            j -= 1
            @inbounds data[i] = rD[j]
            i += 1
        else
            j += 1
        end
    end
    data[i:ll] = lD[i:ll]

    NDSparse(copy(lI), data, presorted=true)
end

# merge - union join

function count_overlap(I::Columns{D}, J::Columns{D}) where D
    lI, lJ = length(I), length(J)
    i = j = 1
    overlap = 0
    while i <= lI && j <= lJ
        c = rowcmp(I, i, J, j)
        if c == 0
            overlap += 1
            i += 1
            j += 1
        elseif c < 0
            i += 1
        else
            j += 1
        end
    end
    return overlap
end

function promoted_similar(x::Columns, y::Columns, n)
    Columns(map((a,b)->promoted_similar(a, b, n), columns(x), columns(y)))
end

function promoted_similar(x::AbstractArray, y::AbstractArray, n)
    similar(x, promote_type(eltype(x),eltype(y)), n)
end

# assign y into x out-of-place
merge(x::NDSparse{T,D}, y::NDSparse{S,D}; agg = IndexedTables.right) where {T,S,D<:Tuple} = (flush!(x);flush!(y); _merge(x, y, agg))
# merge without flush!
function _merge(x::NDSparse{T,D}, y::NDSparse{S,D}, agg) where {T,S,D}
    I, J = x.index, y.index
    lI, lJ = length(I), length(J)
    #if isless(I[end], J[1])
    #    return NDSparse(vcat(x.index, y.index), vcat(x.data, y.data), presorted=true)
    #elseif isless(J[end], I[1])
    #    return NDSparse(vcat(y.index, x.index), vcat(y.data, x.data), presorted=true)
    #end
    if agg === nothing
        n = lI + lJ
    else
        n = lI + lJ - count_overlap(I, J)
    end

    K = promoted_similar(I, J, n)
    data = promoted_similar(x.data, y.data, n)
    _merge!(K, data, x, y, agg)
end

function _merge!(K, data, x::NDSparse, y::NDSparse, agg)
    I, J = x.index, y.index
    lI, lJ = length(I), length(J)
    n = length(K)
    i = j = k = 1
    @inbounds while k <= n
        if i <= lI && j <= lJ
            c = rowcmp(I, i, J, j)
            if c > 0
                copyrow!(K, k, J, j)
                copyrow!(data, k, y.data, j)
                j += 1
            elseif c < 0
                copyrow!(K, k, I, i)
                copyrow!(data, k, x.data, i)
                i += 1
            else
                copyrow!(K, k, I, i)
                data[k] = x.data[i]
                if isa(agg, Nothing)
                    k += 1
                    copyrow!(K, k, I, i)
                    copyrow!(data, k, y.data, j) # repeat the data
                else
                    data[k] = agg(x.data[i], y.data[j])
                end
                i += 1
                j += 1
            end
        elseif i <= lI
            # TODO: copy remaining data columnwise
            copyrow!(K, k, I, i)
            copyrow!(data, k, x.data, i)
            i += 1
        elseif j <= lJ
            copyrow!(K, k, J, j)
            copyrow!(data, k, y.data, j)
            j += 1
        else
            break
        end
        k += 1
    end
    NDSparse(K, data, presorted=true)
end

"""
    merge(a::IndexedTable, b::IndexedTable; pkey)

Merge rows of `a` with rows of `b` and remain ordered by the primary key(s).  `a` and `b` must
have the same column names.

    merge(a::NDSparse, a::NDSparse; agg)

Merge rows of `a` with rows of `b`.  To keep unique keys, the value from `b` takes priority.
A provided function `agg` will aggregate values from `a` and `b` that have the same key(s).

# Example:

    a = table((x = 1:5, y = rand(5)); pkey = :x)
    b = table((x = 6:10, y = rand(5)); pkey = :x)
    merge(a, b)

    a = ndsparse([1,3,5], [1,2,3])
    b = ndsparse([2,3,4], [4,5,6])
    merge(a, b)
    merge(a, b; agg = (x,y) -> x)
"""
function Base.merge(a::Dataset, b) end

function Base.merge(a::IndexedTable, b::IndexedTable;
                    pkey = pkeynames(a) == pkeynames(b) ? a.pkey : [])

    if colnames(a) != colnames(b)
        if Set(collect(colnames(a))) == Set(collect(colnames(b)))
            b = ColDict(b, copy=false)[(colnames(a)...,)]
        else
            throw(ArgumentError("the tables don't have the same column names. Use `select` first."))
        end
    end
    table(map(opt_vcat, columns(a), columns(b)), pkey=pkey, copy=false)
end

opt_vcat(a, b) = vcat(a, b)
opt_vcat(a::PooledArray{<:Any, <:Integer, 1},
         b::PooledArray{<:Any, <:Integer,1}) = vcat(a, b)
opt_vcat(a::AbstractArray{<:Any, 1}, b::PooledArray{<:Any, <:Integer, 1}) = vcat(is_approx_uniqs_less_than(a, length(b.pool)) ? PooledArray(a) : a, b)
opt_vcat(a::PooledArray{<:Any, <:Integer, 1}, b::AbstractArray{<:Any, 1}) = vcat(a, is_approx_uniqs_less_than(b, length(a.pool)) ? PooledArray(b) : b)
function is_approx_uniqs_less_than(itr, maxuniq)
    hset = Set{UInt64}()
    for item in itr
        (length(push!(hset, hash(item))) >= maxuniq) && (return false)
    end
    true
end

function merge(x::NDSparse, xs::NDSparse...; agg = nothing)
    as = [x, xs...]
    filter!(a->length(a)>0, as)
    length(as) == 0 && return x
    length(as) == 1 && return as[1]
    for a in as; flush!(a); end
    sort!(as, by=y->first(y.index))
    if all(i->isless(as[i-1].index[end], as[i].index[1]), 2:length(as))
        # non-overlapping
        return NDSparse(vcat(map(a->a.index, as)...),
                            vcat(map(a->a.data,  as)...),
                            presorted=true)
    end
    error("this case of `merge` is not yet implemented")
end

# merge in place
function merge!(x::NDSparse{T,D}, y::NDSparse{S,D}; agg = IndexedTables.right) where {T,S,D<:Tuple}
    flush!(x)
    flush!(y)
    _merge!(x, y, agg)
end
# merge! without flush!
function _merge!(dst::NDSparse, src::NDSparse, f)
    if length(dst.index)==0 || isless(dst.index[end], src.index[1])
        append!(dst.index, src.index)
        append!(dst.data, src.data)
    else
        # merge to a new copy
        new = _merge(dst, src, f)
        ln = length(new)
        # resize and copy data into dst
        resize!(dst.index, ln)
        copyto!(dst.index, new.index)
        resize!(dst.data, ln)
        copyto!(dst.data, new.data)
    end
    return dst
end

# broadcast join - repeat data along a dimension missing from one array

function find_corresponding(Ap, Bp)
    matches = zeros(Int, length(Ap))
    J = BitSet(1:length(Bp))
    for i = 1:length(Ap)
        for j in J
            if Ap[i] == Bp[j]
                matches[i] = j
                delete!(J, j)
                break
            end
        end
    end
    isempty(J) || error("unmatched source indices: $(collect(J))")
    tuple(matches...)
end

function match_indices(A::NDSparse, B::NDSparse)
    if isa(columns(A.index), NamedTuple) && isa(columns(B.index), NamedTuple)
        Ap = colnames(A.index)
        Bp = colnames(B.index)
    else
        Ap = typeof(A).parameters[2].parameters
        Bp = typeof(B).parameters[2].parameters
    end
    find_corresponding(Ap, Bp)
end

# broadcast over trailing dimensions, i.e. C's dimensions are a prefix
# of B's. this is an easy case since it's just an inner join plus
# sometimes repeating values from the right argument.
function _broadcast_trailing(f, B::NDSparse, C::NDSparse, B_common)
    lI, rI = B.index, C.index
    lD, rD = B.data, C.data
    ll, rr = length(lI), length(rI)
    iter = GroupJoinPerm(GroupPerm(B_common, Base.OneTo(ll)), GroupPerm(rI, Base.OneTo(rr)))
    filt = Iterators.filter(((_, ridxs),) -> !isempty(ridxs), iter)
    I = similar(lI, 0)
    function step((lidxs, ridxs),)
        @inbounds Ck = rD[first(ridxs)]
        @inbounds ((pushrow!(I, lI, i); f(lD[i], Ck)) for i in lidxs)
    end
    vals = collect_columns_flattened(step(idxs) for idxs in filt)
    NDSparse(I, vals, copy=false, presorted=true)
end

function _bcast_loop(f::Function, B::NDSparse, C::NDSparse, B_common, B_perm)
    m, n = length(B_perm), length(C)
    iperm = zeros(Int, m)
    idxperm = Int32[]
    C_perm = Base.OneTo(n)
    iter = GroupJoinPerm(GroupPerm(B_common, B_perm), GroupPerm(C.index, C_perm))
    filt = Iterators.filter(((_, ridxs),) -> !isempty(ridxs), iter)
    function step((bidxs, cidxs))
        @inbounds Ck = C.data[first(cidxs)]
        @inbounds ((pj = B_perm[j]; push!(idxperm, pj); iperm[pj] = length(idxperm); f(B.data[pj], Ck)) for j in bidxs)
    end
    vals = collect_columns_flattened(step(idxs) for idxs in filt)
    B.index[idxperm], filter!(i->i!=0, iperm), vals
end

# broadcast C over B. assumes ndims(B) >= ndims(C)
function _broadcast(f::Function, B::NDSparse, C::NDSparse; dimmap=nothing)
    flush!(B); flush!(C)
    if dimmap === nothing
        C_inds = match_indices(B, C)
    else
        C_inds = dimmap
    end
    C_dims = ntuple(identity, ndims(C))
    if C_inds[1:ndims(C)] == C_dims
        return _broadcast_trailing(f, B, C, rows(B.index, C_dims))
    end
    common = filter(i->C_inds[i] > 0, 1:ndims(B))
    C_common = C_inds[common]
    B_common_cols = Columns(getsubfields(columns(B.index), common))
    B_perm = sortperm(B_common_cols)
    if C_common == C_dims
        idx, iperm, vals = _bcast_loop(f, B, C, B_common_cols, B_perm)
        A = NDSparse(idx, vals, copy=false, presorted=true)
        if !issorted(A.index)
            copyto!(A.data, A.data[iperm])
            Base.permute!!(refs(A.index), iperm)
        end
    else
        # TODO
        #C_perm = sortperm(Columns(columns(C.index)[[C_common...]]))
        error("dimensions of one argument to `broadcast` must be a subset of the dimensions of the other")
    end
    return A
end

"""
    broadcast(f, A::NDSparse, B::NDSparse; dimmap::Tuple{Vararg{Int}})
    A .* B

Compute an inner join of `A` and `B` using function `f`, where the dimensions
of `B` are a subset of the dimensions of `A`. Values from `B` are repeated over
the extra dimensions.

`dimmap` optionally specifies how dimensions of `A` correspond to dimensions
of `B`. It is a tuple where `dimmap[i]==j` means the `i`th dimension of `A`
matches the `j`th dimension of `B`. Extra dimensions that do not match any
dimensions of `j` should have `dimmap[i]==0`.

If `dimmap` is not specified, it is determined automatically using index column
names and types.

# Example

    a = ndsparse(([1,1,2,2], [1,2,1,2]), [1,2,3,4])
    b = ndsparse([1,2], [1/1, 1/2])
    broadcast(*, a, b)

`dimmap` maps dimensions that should be broadcasted:

    broadcast(*, a, b, dimmap=(0,1))
"""
function broadcast(f::Function, A::NDSparse, B::NDSparse; dimmap=nothing)
    if ndims(B) > ndims(A)
        _broadcast((x,y)->f(y,x), B, A, dimmap=dimmap)
    else
        _broadcast(f, A, B, dimmap=dimmap)
    end
end

broadcast(f::Function, x::NDSparse) = NDSparse(x.index, broadcast(f, x.data), presorted=true)
broadcast(f::Function, x::NDSparse, y) = NDSparse(x.index, broadcast(f, x.data, y), presorted=true)
broadcast(f::Function, y, x::NDSparse) = NDSparse(x.index, broadcast(f, y, x.data), presorted=true)

Broadcast.broadcasted(f::Function, A::NDSparse) = broadcast(f, A)
Broadcast.broadcasted(f::Function, A::NDSparse, B::NDSparse) = broadcast(f, A, B)
Broadcast.broadcasted(f::Function, A, B::NDSparse) = broadcast(f, A, B)
Broadcast.broadcasted(f::Function, A::NDSparse, B) = broadcast(f, A, B)


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    BallFolding(ball)

A method for creating folds from a spatial object that
are centers of balls.
"""
struct BallFolding{B} <: FoldingMethod
  ball::B
end

function folds(object, method::BallFolding)
  # retrieve parameters
  ball = method.ball

  # create search method
  searcher = NeighborhoodSearch(object, ball)
  n = nelms(object)

  function pair(i)
    # source and target indices
    coords = coordinates(object, i)
    inside = search(coords, searcher)
    sinds  = setdiff(1:n, inside)
    tinds  = [i]

    sinds, tinds
  end

  (pair(i) for i in 1:n)
end


text:
function copy_multidimensional_arrays(n_rep=5, N=[5000 7000 9000])

    loop_time = zeros(size(N))
    vec_time = zeros(size(N))

    for i = 1:length(N); n = N[i]

        A_ = rand(n, n, 3)

        A = A_;
        loop_time[i] = @elapsed begin
            for k = 1:n_rep; A = loop_strategy(A, n); end
        end
        loop_time[i] /= n_rep;

        A = A_;
        vec_time[i] = @elapsed begin
            for k = 1:n_rep; A = vec_strategy(A); end
        end
        vec_time[i] /= n_rep;
    end
    println("Loop time [s]:"); println(loop_time)
    println(" Vec time [s]:"); println(vec_time)

end

function loop_strategy(A, n)
    for j = 1:n, i = 1:n
        A[i,j,1] = A[i,j,2]
        A[i,j,3] = A[i,j,1]
        A[i,j,2] = A[i,j,3]
    end
    return A
end

function vec_strategy(A)
    A[:,:,[1 3 2]] = A[:,:,[2 1 3]];
    return A
end

copy_multidimensional_arrays()

max_stars_repo_name: ricardoaf/juliafirststeps

max_issues_repo_name: ricardoaf/juliafirststeps

max_forks_repo_name: ricardoaf/juliafirststeps


text:
const named_array_plot_types = [BarPlot, Heatmap, Volume]

function convert_arguments(P::PlotFunc, m::NamedArray)
    n = length(axes(m))
    ptype = plottype(P, named_array_plot_types[n])
    args = (names(m, i) for i in 1:n)
    v = convert(Array, m)
    to_plotspec(ptype, convert_arguments(ptype, args..., v))
end

const frequency = Analysis(freqtable)


text:
select count(a.household_id), count(distinct a.household_id), b.operating_company
from wh_supplmental.household_dependent_id_map a
join wh_fsp.partitioned_fsp_wkly_fact b
     on a.dependent_id = b.card_num
     and UPPER(trim(b.Operating_company)) = trim(UPPER(case when a.retailer='FOODLION' then 'FOOD LION' else a.retailer end ))
where a.current_state = 'MATCHED'
  and a.household_type = 'EXPERIAN'
  and a.dependent_type = 'FSP_CARD'
  and TM_DIM_KEY_WEEK>=1909
group by b.operating_company
"""
149243  17012   FREDS
196025906       7033205 RITEAID
330798388       994861  WEGMANS
38008898        221081  KEYFOOD
6020735400      15722602        KROGER
16025680        100830  SUPERVALU
302291297       2137143 BJS
"""

hive -e 'SELECT distinct household_id FROM WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP where household_type = "EXPERIAN" and dependent_type = "FSP_CARD" and  dependent_id is not null' | sed 's/[\t]/,/g' > /mapr/mapr04p/analytics0001/analytic_users/mddak/fsp_ids_retailer2.csv

set hive.cli.print.header=true;
SELECT retailer, count(distinct household_id) as ids FROM WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP where household_type = "EXPERIAN" and dependent_type = "FSP_CARD" and  dependent_id is not null GROUP BY retailer;

hadoop fs -put /mapr/mapr04p/analytics0001/analytic_users/mddak/Experian/IRI_Key_Food_Shipment.txt /externaldata01/prd/experian/xwalk/raw/

set hive.cli.print.header=true; select * from wh_fsp.partitioned_fsp_wkly_fact limit 10;
hive -e 'set hive.cli.print.header=true; select * from MEDIA_LIFT_CAMPAIGN.lift_product WHERE registration_request_id=931 limit 10' | sed 's/[\t]/,/g' > /mapr/mapr04p/analytics0001/analytic_users/mddak/lift_product_example10.csv
set hive.cli.print.header=true; select * from MEDIA_LIFT_CAMPAIGN.lift_product_item_filter_map WHERE registration_request_id=931 limit 10;
set hive.cli.print.header=true; select * from MEDIA_LIFT_CAMPAIGN.POS_WKLY_FACT POS WHERE registration_request_id=931 limit 10;

set hive.cli.print.header=true; select * from Jennieo6.Kantar_FSP_POS_Jennieo6_931 limit 10;

# ---------------------------------
Insert Overwrite Table Jennieo6.Kantar_FSP_Jennieo6_2_931_dk
select
FSP.VENUE_DIM_KEY,
FSP.TM_DIM_KEY_DAY,
FROM_UNIXTIME((unix_timestamp(cast(FSP.Trans_DATE as string),'YYYY-MM-DD'))) as date1,
FSP.Trans_TIME,
FSP.Trans_NUM,
FSP.STORE_NUM,
FSP.CARD_NUM,
FSP.UPC,
FSP.ndc_upc_ind,
FSP.QUANTITY,
FSP.WEIGHT,
FSP.item_list_price,
FSP.NET_PRICE,
FSP.CARD_DISCOUNT,
FSP.ad_discount,
FSP.other_discount,
FSP.ITEM_TYPE,
FSP.TENDER_TYPE,
FSP.ad_event,
FSP.ad_version ,
FSP.TM_DIM_KEY_WEEK,
FSP.OPERATING_COMPANY,
FSP.item_dim_key,
FSP.UPC10,
FSP.UNITS,
FSP.CENTS,
FSP.BASELINE_UNITS,
FSP.BASELINE_CENTS,
FSP.FEATURE,
FSP.DISPLAY,
FSP.TOTL_PRC_REDUC,
FSP.registration_request_id,
FSP.Product_id_Jennieo6_final,
exp.household_id,
0 as HTML5,
0 as Static,
0 as Native_creative,
0 as Video,
0 as Content,
0 as Contextual,
0 as Direct,
0 as Prospecting,
0 as Retargeting,
0 as Behavorial,
0 as Native,
0 as Predictive,
0 as 3rdparty,
0 as PMP,
0 as Buzzfeed,
0 as Hulu,
0 as PopSugarUS,
0 as RachaelRayMag,
0 as Shape,
0 as Blank,
0 as EatingWell,
0 as MeredithCorporation,
0 as RealSimple,
0 as TURNDSP,
0 as CookingLight,
0 as dataxu,
0 as Allrecipes,
0 as Amazon,
0 as MyFitnessPal,
0 as WomensHealth,
0 as Yummly,
0 as YouTube,
0 as gross_imps
from Jennieo6.Kantar_FSP_POS_Jennieo6_PRDMAP_931_final FSP
join WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP exp on
    cast(FSP.CARD_NUM as BIGINT) = cast(exp.dependent_id as BIGINT)
    AND FSP.Operating_company= exp.retailer
where (FSP.card_num is not NULL)
       and (trim(FSP.operating_company) not in ('FREDS'))
       and (FSP.UPC10<>'2265570029')
       and (FSP.UPC10<>'2265570051')
       and (FSP.UPC10<>'2265570140')
       and exp.household_type = 'EXPERIAN'
       and exp.dependent_type = 'FSP_CARD'
       and exp.current_state='MATCHED'
order by household_id, date1, Trans_TIME;

set hive.cli.print.header=true; select date1 from Jennieo6.Kantar_FSP_Jennieo6_2_931_dk limit 10;
hive -e 'set hive.cli.print.header=true; select * from Jennieo6.Kantar_FSP_Jennieo6_2_931_dk limit 10' | sed 's/[\t]/,/g' > /mapr/mapr04p/analytics0001/analytic_users/mddak/lift_product_example10.csv

Insert Overwrite Table jennie.jennie6_daily_breaks_ru
select
'0' as VENUE_DIM_KEY,
'0' as TM_DIM_KEY_DAY,
FROM_UNIXTIME((unix_timestamp(cast(a.date1 as string),"yyyyMMdd"))) as date1,
'na' as Trans_TIME,
'0' as Trans_NUM,
'0' as STORE_NUM,
'0' as CARD_NUM,
'0' as UPC,
'na' as ndc_upc_ind,
'0' as QUANTITY,
'0' as WEIGHT,
'0' as item_list_price,
'0' as NET_PRICE,
'0' as CARD_DISCOUNT,
'0' as ad_discount,
'0' as other_discount,
'na' as ITEM_TYPE,
'0' as TENDER_TYPE,
'na' as ad_event,
'0' as ad_version,
'0' as TM_DIM_KEY_WEEK,
max(b.retailer) as OPERATING_COMPANY,
'0' as item_dim_key,
'na' as UPC10,
'0' as UNITS,
'0' as CENTS,
'0' as BASELINE_UNITS,
'0' as BASELINE_CENTS,
'0' as FEATURE,
'0' as DISPLAY,
'0' as TOTL_PRC_REDUC,
'0' as registration_request_id,
'0' as product_id_Jennieo6,
b.household_id,
sum(a.HTML5) as HTML5,
sum(a.Static) as Static,
sum(a.Native_creative) as Native_creative,
sum(a.Video) as Video,
sum(a.Content) as Content,
sum(a.Contextual) as Contextual,
sum(a.Direct) as Direct,
sum(a.Prospecting) as Prospecting,
sum(a.Retargeting) as Retargeting,
sum(a.Behavorial) as Behavorial,
sum(a.Native) as Native,
sum(a.Predictive) as Predictive,
sum(a.3rdparty) as 3rdparty,
sum(a.PMP) as PMP,
sum(a.Buzzfeed) as Buzzfeed,
sum(a.Hulu) as Hulu,
sum(a.PopSugarUS) as PopSugarUS,
sum(a.RachaelRayMag) as RachaelRayMag,
sum(a.Shape) as Shape,
sum(a.Blank) as Blank,
sum(a.EatingWell) as EatingWell,
sum(a.MeredithCorporation) as MeredithCorporation,
sum(a.RealSimple) as RealSimple,
sum(a.TURNDSP) as TURNDSP,
sum(a.CookingLight) as CookingLight,
sum(a.dataxu) as dataxu,
sum(a.Allrecipes) as Allrecipes,
sum(a.Amazon) as Amazon,
sum(a.MyFitnessPal) as MyFitnessPal,
sum(a.WomensHealth) as WomensHealth,
sum(a.Yummly) as Yummly,
sum(a.YouTube) as YouTube,
sum(a.gross_imps) as gross_imps
from jennie.jennie6_daily_breaks a
inner JOIN WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP b on a.lr_id = b.dependent_id
where b.dependent_type = 'LVRAMP_ID' and b.household_type = 'EXPERIAN' and b.retailer = 'COMSCORE' and b.data_supplier = 'EXPERIAN' and b.current_state = 'MATCHED'
group by household_id, date1
order by household_id, date1;

set hive.cli.print.header=true; select date1 from jennie.jennie6_daily_breaks_ru limit 10;

-- union the pos and exposure file into one table

jennie.jennie6_daily_breaks_ru    Jennieo6.Kantar_FSP_Jennieo6_2_931_dk
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

Insert Overwrite Table jennie.jennie_rf_pos_exposure
select * from (
select
a.VENUE_DIM_KEY,
a.TM_DIM_KEY_DAY,
a.date1,
a.Trans_TIME,
a.Trans_NUM,
a.STORE_NUM,
a.CARD_NUM,
a.UPC,
a.ndc_upc_ind,
a.QUANTITY,
a.WEIGHT,
a.item_list_price,
a.NET_PRICE,
a.CARD_DISCOUNT,
a.ad_discount,
a.other_discount,
a.ITEM_TYPE,
a.TENDER_TYPE,
a.ad_event,
a.ad_version ,
a.TM_DIM_KEY_WEEK,
a.OPERATING_COMPANY,
a.item_dim_key,
a.UPC10,
a.UNITS,
a.CENTS,
a.BASELINE_UNITS,
a.BASELINE_CENTS,
a.FEATURE,
a.DISPLAY,
a.TOTL_PRC_REDUC,
a.registration_request_id,
a.Product_id_Jennieo6,
a.household_id,
a.HTML5,
a.Static,
a.Native_creative,
a.Video,
a.Content,
a.Contextual,
a.Direct,
a.Prospecting,
a.Retargeting,
a.Behavorial,
a.Native,
a.Predictive,
a.3rdparty,
a.PMP,
a.Buzzfeed,
a.Hulu,
a.PopSugarUS,
a.RachaelRayMag,
a.Shape,
a.Blank,
a.EatingWell,
a.MeredithCorporation,
a.RealSimple,
a.TURNDSP,
a.CookingLight,
a.dataxu,
a.Allrecipes,
a.Amazon,
a.MyFitnessPal,
a.WomensHealth,
a.Yummly,
a.YouTube,
a.gross_imps
from Jennieo6.Kantar_FSP_Jennieo6_2_931_dk a
union all
select
b.VENUE_DIM_KEY,
b.TM_DIM_KEY_DAY,
b.date1,
b.Trans_TIME,
b.Trans_NUM,
b.STORE_NUM,
b.CARD_NUM,
b.UPC,
b.ndc_upc_ind,
b.QUANTITY,
b.WEIGHT,
b.item_list_price,
b.NET_PRICE,
b.CARD_DISCOUNT,
b.ad_discount,
b.other_discount,
b.ITEM_TYPE,
b.TENDER_TYPE,
b.ad_event,
b.ad_version ,
b.TM_DIM_KEY_WEEK,
b.OPERATING_COMPANY,
b.item_dim_key,
b.UPC10,
b.UNITS,
b.CENTS,
b.BASELINE_UNITS,
b.BASELINE_CENTS,
b.FEATURE,
b.DISPLAY,
b.TOTL_PRC_REDUC,
b.registration_request_id,
b.Product_id_Jennieo6,
b.household_id,
b.HTML5,
b.Static,
b.Native_creative,
b.Video,
b.Content,
b.Contextual,
b.Direct,
b.Prospecting,
b.Retargeting,
b.Behavorial,
b.Native,
b.Predictive,
b.3rdparty,
b.PMP,
b.Buzzfeed,
b.Hulu,
b.PopSugarUS,
b.RachaelRayMag,
b.Shape,
b.Blank,
b.EatingWell,
b.MeredithCorporation,
b.RealSimple,
b.TURNDSP,
b.CookingLight,
b.dataxu,
b.Allrecipes,
b.Amazon,
b.MyFitnessPal,
b.WomensHealth,
b.Yummly,
b.YouTube,
b.gross_imps
from jennie.jennie6_daily_breaks_ru b
) unioned;

# -------------------------- TEST -------------------------------------------
# select household_id from jennie_rf_pos_exposure where gross_imps > 0;
select avg(a.net_price) from jennie_rf_pos_exposure a where a.household_id in (select b.household_id from jennie_rf_pos_exposure b where b.gross_imps > 0);

# ---------------------------------------------------------------------------

set hive.cli.print.header=true; select * from jennie.jennie_rf_pos_exposure limit 10;

-- prepare table for analysis

Insert Overwrite Table jennie.jennie_rf_pos_exposure2
select
max(a.VENUE_DIM_KEY) as VENUE_DIM_KEY,
max(a.TM_DIM_KEY_DAY) as TM_DIM_KEY_DAY,
a.date1,
max(a.Trans_TIME) as Trans_TIME,
max(a.Trans_NUM) as Trans_NUM,
max(a.STORE_NUM) as STORE_NUM,
max(a.CARD_NUM) as CARD_NUM,
max(a.UPC) as UPC,
max(a.ndc_upc_ind) as ndc_upc_ind,
sum(a.QUANTITY) as QUANTITY,
max(a.WEIGHT) as WEIGHT,
max(a.item_list_price) as item_list_price,
max(a.NET_PRICE) as NET_PRICE,
max(a.CARD_DISCOUNT) as CARD_DISCOUNT,
max(a.ad_discount) as ad_discount,
max(a.other_discount) as other_discount,
max(a.ITEM_TYPE) as ITEM_TYPE,
max(a.TENDER_TYPE) as TENDER_TYPE,
max(a.ad_event) as ad_event,
max(a.ad_version) as ad_version,
max(a.TM_DIM_KEY_WEEK) as TM_DIM_KEY_WEEK,
max(a.OPERATING_COMPANY) as OPERATING_COMPANY,
max(a.item_dim_key) as item_dim_key,
max(a.UPC10) as UPC10,
sum(a.UNITS) as UNITS,
sum(a.CENTS) as CENTS,
sum(a.BASELINE_UNITS) as BASELINE_UNITS,
sum(a.BASELINE_CENTS) as BASELINE_CENTS,
max(a.FEATURE) as FEATURE,
max(a.DISPLAY) as DISPLAY,
sum(a.TOTL_PRC_REDUC) as TOTL_PRC_REDUC,
max(a.registration_request_id) as registration_request_id,
max(a.Product_id_Jennieo6) as product_id_Jennieo6,
a.household_id,
sum(a.HTML5) as HTML5,
sum(a.Static) as Static,
sum(a.Native_creative) as Native_creative,
sum(a.Video) as Video,
sum(a.Content) as Content,
sum(a.Contextual) as Contextual,
sum(a.Direct) as Direct,
sum(a.Prospecting) as Prospecting,
sum(a.Retargeting) as Retargeting,
sum(a.Behavorial) as Behavorial,
sum(a.Native) as Native,
sum(a.Predictive) as Predictive,
sum(a.3rdparty) as 3rdparty,
sum(a.PMP) as PMP,
sum(a.Buzzfeed) as Buzzfeed,
sum(a.Hulu) as Hulu,
sum(a.PopSugarUS) as PopSugarUS,
sum(a.RachaelRayMag) as RachaelRayMag,
sum(a.Shape) as Shape,
sum(a.Blank) as Blank,
sum(a.EatingWell) as EatingWell,
sum(a.MeredithCorporation) as MeredithCorporation,
sum(a.RealSimple) as RealSimple,
sum(a.TURNDSP) as TURNDSP,
sum(a.CookingLight) as CookingLight,
sum(a.dataxu) as dataxu,
sum(a.Allrecipes) as Allrecipes,
sum(a.Amazon) as Amazon,
sum(a.MyFitnessPal) as MyFitnessPal,
sum(a.WomensHealth) as WomensHealth,
sum(a.Yummly) as Yummly,
sum(a.YouTube) as YouTube,
sum(a.gross_imps) as gross_imps
from jennie.jennie_rf_pos_exposure a
group by household_id, date1
order by household_id, date1;

set hive.cli.print.header=true; select * from jennie.jennie_rf_pos_exposure2 limit 10;

===================================================================================
select count(*) from jennie.jennie_rf_pos_exposure where date1 is null;

select quantity, item_type, ndc_upc_ind, '---', html5, static , native_creative , prospecting from  jennie.jennie_rf_pos_exposure where date1 is null;

Jennieo6.Kantar_FSP_Jennieo6_2_931_dk
select count(*) from Jennieo6.Kantar_FSP_Jennieo6_2_931_dk;
select count(*) from Jennieo6.Kantar_FSP_Jennieo6_2_931_dk where date1 is null;

  2988336 row(s)

# -------------------------- TEST -------------------------------------------
# select household_id from jennie_rf_pos_exposure where gross_imps > 0;
select avg(a.net_price) from jennie_rf_pos_exposure a where a.household_id in (select b.household_id from jennie_rf_pos_exposure b where b.gross_imps > 0);

select a.household_id,a.date1, from jennie_rf_pos_exposure a;

# ---------------------------------------------------------------------------

# ---------------------------------- TESTING NEGATIVE UPLIFT -------------------------------------------------

drop table x;
create table x as select a.household_id, a.date1, max(a.NET_PRICE) as NET_PRICE, sum(a.gross_imps) as gross_imps
from jennie.jennie_rf_pos_exposure a
group by household_id, date1
order by household_id, date1;
select avg(net_price) from x where gross_imps > 0;

create table y as select household_id from x where gross_imps > 0;

drop table z;
create table z as select a.household_id, a.date1, max(a.net_price) as net_price, sum(a.gross_imps) as gross_imps from y left join jennie.jennie_rf_pos_exposure a
on y.household_id = a.household_id
group by a.household_id, a.date1;

select max(net_price) from z;

--------------------------

create table t as select household_id, date1, gross_imps from jennie.jennie_rf_pos_exposure where gross_imps > 0;
1375413305      2016-07-27 00:00:00     10
1375413600      2016-07-12 00:00:00     4
1375413600      2016-07-17 00:00:00     1
1375413600      2016-07-26 00:00:00     16
1375413600      2016-07-27 00:00:00     15
1375413600      2016-07-28 00:00:00     16
1375413600      2016-07-29 00:00:00     14
1375413600      2016-07-30 00:00:00     3
1375413600      2016-07-31 00:00:00     8
1375413600      2016-08-01 00:00:00     14
1375413600      2016-08-02 00:00:00     13
select household_id, net_price, date1, gross_imps from jennie.jennie_rf_pos_exposure where household_id = 1375413600;

drop table z;
create table z as select a.household_id, a.date1, max(a.net_price) as net_price, sum(a.gross_imps) as gross_imps from jennie.jennie_rf_pos_exposure a
group by a.household_id, a.date1;

select max(net_price) from z where gross_imps > 0;

    select sum(net_price) from jennie.jennie_rf_pos_exposure where

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

drop table jennie.gfsp;
create table jennie.gfsp as select
FSP.VENUE_DIM_KEY,
FSP.TM_DIM_KEY_DAY,
FSP.Trans_DATE as date1,
FROM_UNIXTIME((unix_timestamp(cast(FSP.Trans_DATE as string),'YYYY-MM-DD'))) as date2,
FSP.Trans_TIME,
FSP.Trans_NUM,
FSP.STORE_NUM,
FSP.CARD_NUM,
FSP.UPC,
FSP.ndc_upc_ind,
FSP.QUANTITY,
FSP.WEIGHT,
FSP.item_list_price,
FSP.NET_PRICE,
FSP.CARD_DISCOUNT,
FSP.ad_discount,
FSP.other_discount,
FSP.ITEM_TYPE,
FSP.TENDER_TYPE,
FSP.ad_event,
FSP.ad_version ,
FSP.TM_DIM_KEY_WEEK,
FSP.OPERATING_COMPANY,
FSP.item_dim_key,
FSP.UPC10,
FSP.UNITS,
FSP.CENTS,
FSP.BASELINE_UNITS,
FSP.BASELINE_CENTS,
FSP.FEATURE,
FSP.DISPLAY,
FSP.TOTL_PRC_REDUC,
FSP.registration_request_id,
FSP.Product_id_Jennieo6_final,
exp.household_id
from Kantar_FSP_POS_Jennieo6_PRDMAP_931_final FSP
join WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP exp on
    cast(FSP.CARD_NUM as BIGINT) = cast(exp.dependent_id as BIGINT)
    AND FSP.Operating_company= exp.retailer
where (FSP.card_num is not NULL)
       and (trim(FSP.operating_company) not in ('FREDS'))
       and (FSP.UPC10<>'2265570029')
       and (FSP.UPC10<>'2265570051')
       and (FSP.UPC10<>'2265570140')
       and exp.household_type = 'EXPERIAN'
       and exp.dependent_type = 'FSP_CARD'
       and exp.current_state='MATCHED'
order by household_id, date1, Trans_TIME;

    set hive.cli.print.header=true; select date1 from Jennie.gfsp limit 10;

drop table jennie.gexp;
create table jennie.gexp as
select
a.date1,
FROM_UNIXTIME((unix_timestamp(cast(a.date1 as string),"yyyyMMdd"))) as date2,
max(b.retailer) as OPERATING_COMPANY,
b.household_id,
sum(a.HTML5) as HTML5,
sum(a.Static) as Static,
sum(a.Native_creative) as Native_creative,
sum(a.Video) as Video,
sum(a.Content) as Content,
sum(a.Contextual) as Contextual,
sum(a.Direct) as Direct,
sum(a.Prospecting) as Prospecting,
sum(a.Retargeting) as Retargeting,
sum(a.Behavorial) as Behavorial,
sum(a.Native) as Native,
sum(a.Predictive) as Predictive,
sum(a.3rdparty) as 3rdparty,
sum(a.PMP) as PMP,
sum(a.Buzzfeed) as Buzzfeed,
sum(a.Hulu) as Hulu,
sum(a.PopSugarUS) as PopSugarUS,
sum(a.RachaelRayMag) as RachaelRayMag,
sum(a.Shape) as Shape,
sum(a.Blank) as Blank,
sum(a.EatingWell) as EatingWell,
sum(a.MeredithCorporation) as MeredithCorporation,
sum(a.RealSimple) as RealSimple,
sum(a.TURNDSP) as TURNDSP,
sum(a.CookingLight) as CookingLight,
sum(a.dataxu) as dataxu,
sum(a.Allrecipes) as Allrecipes,
sum(a.Amazon) as Amazon,
sum(a.MyFitnessPal) as MyFitnessPal,
sum(a.WomensHealth) as WomensHealth,
sum(a.Yummly) as Yummly,
sum(a.YouTube) as YouTube,
sum(a.gross_imps) as gross_imps
from jennie.jennie6_daily_breaks a
inner JOIN WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP b on a.lr_id = b.dependent_id
where b.dependent_type = 'EXPOSURE_HOUSEHOLD_ID' and b.household_type = 'EXPERIAN' and b.retailer = 'COMSCORE'
      and b.data_supplier = 'EXPERIAN' and b.current_state = 'MATCHED'
group by household_id, date1
order by household_id, date1;

select count(dependent_id) from WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP b
where b.dependent_type = 'EXPOSURE_HOUSEHOLD_ID'
      and b.household_type = 'EXPERIAN' and b.retailer = 'COMSCORE'
      and b.data_supplier = 'EXPERIAN' and b.current_state = 'MATCHED';

    select * from gexp, gfsp where gfsp.household_id=gexp.household_id and gfsp.date1=gexp.date1;
group by household_id, date1
order by household_id, date1;

    set hive.cli.print.header=true; select date1 from jennie.gexp limit 10;

-- union the pos and exposure file into one table

jennie.jennie6_daily_breaks_ru    Jennieo6.Kantar_FSP_Jennieo6_2_931_dk

select household_id,date1, gross_imps from gexp limit 10;
select household_id,date1,net_price from gfsp limit 10;

select a.household_id from gexp a where a.gross_imps > 0 and a.household_id in (select b.household_id from gfsp b);

select household_id, date1,net_price from gfsp where household_id=2991413339;
select household_id, date1,gross_imps from gexp where household_id=2991413339;

2991413339
2991413339
2991413339
2991413339
2991413339
2991413339
2991413339
2991413339
2991413339
2991413339
2991748577
2991817061
2991817061
2991817061

# -----------------------------------------------------------------------------------------------------------------
- dependent_type
RETAILER_HOUSEHOLD_ID
FSP_CARD
RETAILER_UNIVERSAL_ID
EXPERIAN
EXPOSURE_HOUSEHOLD_ID

- household_type
    EXPERIAN
- data_supplier
EXPERIAN
- retailer
 BJS
COMSCORE
EXPERIAN
FOODLION
FREDS
KEYFOOD
KROGER
RITEAID
SUPERVALU
WEGMANS

-current_state
DUPLICATE
MATCHED
UNMATCHED

INSERT INTO TABLE FSP_POS_MSTR_STEP1_${hiveconf:WS_CAMPAIGN_ID}

SELECT
    COUNT(DISTINCT TRANS_NUM) AS TRANS_NUM,
    STORE_NUM,
    SUM(QUANTITY) AS QUANTITY,
    SUM(WEIGHT) AS WEIGHT,
    SUM(NET_PRICE) AS NET_PRICE,
    SUM(VOLUMEEQUIVALENCYVALUE*QUANTITY) AS VOLUME,
    SUM(CARD_DISCOUNT) AS CARD_DISCOUNT,
    TM_DIM_KEY_WEEK,
    OPERATING_COMPANY,
    SUM(COALESCE(UNITS,0)) AS UNITS,
    SUM(COALESCE(CENTS,0)) AS CENTS,
    SUM(COALESCE(BASELINE_UNITS,0)) AS BASELINE_UNITS,
    SUM(COALESCE(BASELINE_CENTS,0)) AS BASELINE_CENTS,
    SUM(COALESCE(POS_PRICE,0)) AS POS_PRICE,
    MAX(COALESCE(FEATURE,0)) AS FEATURE,
    MAX(COALESCE(DISPLAY,0)) AS DISPLAY,
    MAX(COALESCE(TOTL_PRC_REDUC,0)) AS TOTL_PRC_REDUC,
    PRODUCT_GRP_ID, EXPERIAN_ID, EXPOSED_FLAG
FROM( SELECT T2.*, PRGP.PRODUCT_ID AS PRODUCT_GRP_ID, PRGP.VOLUMEEQUIVALENCYVALUE AS VOLUMEEQUIVALENCYVALUE
      FROM( SELECT FSP_POS.*, HDM.EXPERIAN_ID, HDM.EXPOSED_FLAG
            FROM( SELECT FSP.*, POS.ITEM_DIM_KEY, POS.UNITS, POS.CENTS, POS.BASELINE_UNITS, POS.BASELINE_CENTS,
                         POS.CENTS/(100*POS.UNITS) AS POS_PRICE, POS.FEATURE, POS.DISPLAY, POS.TOTL_PRC_REDUC, POS.REGISTRATION_REQUEST_ID
                  FROM ( SELECT VENUE_DIM_KEY, TM_DIM_KEY_DAY, TRANS_DATE, TRANS_TIME, TRANS_NUM, STORE_NUM, CARD_NUM, UPC, DERIVED_UPC10,
                                NDC_UPC_IND, QUANTITY, WEIGHT, ITEM_LIST_PRICE, NET_PRICE, CARD_DISCOUNT, AD_DISCOUNT, OTHER_DISCOUNT,
                                ITEM_TYPE, TENDER_TYPE, AD_EVENT, AD_VERSION, TM_DIM_KEY_WEEK, OPERATING_COMPANY
                         FROM WH_FSP.PARTITIONED_FSP_WKLY_FACT
                         WHERE TM_DIM_KEY_WEEK BETWEEN ${hiveconf:WS_PRE_START_WEEK} AND ${hiveconf:WS_POS_END_WEEK}
                               AND  OPERATING_COMPANY NOT IN (${hiveconf:WS_RETAILER_EXCLUDE_LIST})
                               AND  DERIVED_UPC10 IN (SELECT DISTINCT UPC10 FROM POS_DATA_${hiveconf:WS_CAMPAIGN_ID})
                       ) FSP
                         INNER JOIN ( SELECT * FROM POS_DATA_${hiveconf:WS_CAMPAIGN_ID} ) POS
                         ON  FSP.TM_DIM_KEY_WEEK = POS.TIME_DIM_KEY
                         AND FSP.VENUE_DIM_KEY = POS.VENUE_DIM_KEY
                         AND FSP.DERIVED_UPC10 = POS.UPC10
                 ) FSP_POS
                   INNER JOIN ( SELECT RETAILER, DEPENDENT_ID, EXPERIAN_ID, COALESCE(EXPOSED_FLAG,0) AS EXPOSED_FLAG
                                FROM( SELECT RETAILER, cast(DEPENDENT_ID as bigint) as DEPENDENT_ID, HOUSEHOLD_ID AS EXPERIAN_ID
                                      FROM WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP
                                      WHERE DATA_SUPPLIER IN (${hiveconf:WS_DATA_SUPPLIER})
                                      AND   RETAILER NOT IN (${hiveconf:WS_RETAILER_EXCLUDE_LIST})
                                      AND   CURRENT_STATE = 'MATCHED'
                                      AND   HOUSEHOLD_TYPE = 'EXPERIAN'
                                      AND   DEPENDENT_TYPE = 'FSP_CARD'
                                   ) HOU_EXP
                                     LEFT OUTER JOIN( SELECT HOUSEHOLD_ID AS HOUSEHOLD_ID, MAX(EXPOSED_FLAG) AS EXPOSED_FLAG
                                                      FROM( SELECT CAST(${hiveconf:WS_EXPOSURE_HH_COL} AS BIGINT) AS HOUSEHOLD_ID,
                                                                   1 AS EXPOSED_FLAG
                                                            FROM ${hiveconf:WS_INTER_TBL}
                                                          ) MLIFT
                                                      GROUP BY HOUSEHOLD_ID
                                                    ) EXP_FLAG
                                     ON HOU_EXP.EXPERIAN_ID = EXP_FLAG.HOUSEHOLD_ID
                             ) HDM
                    ON FSP_POS.OPERATING_COMPANY = HDM.RETAILER
                    AND FSP_POS.CARD_NUM = HDM.DEPENDENT_ID
         ) T2
    INNER JOIN ( SELECT * FROM PRD_GRP_TBL_${hiveconf:WS_CAMPAIGN_ID}_TEMP )PRGP
    ON T2.ITEM_DIM_KEY = PRGP.ITEM_DIM_KEY
  ) FSP_AGG
GROUP BY STORE_NUM, OPERATING_COMPANY, TM_DIM_KEY_WEEK, PRODUCT_GRP_ID, EXPERIAN_ID, EXPOSED_FLAG

--Code section to handle BJS UPC issue post 1930
UNION ALL

SELECT
COUNT(DISTINCT TRANS_NUM) AS TRANS_NUM,
STORE_NUM,
SUM(QUANTITY) AS QUANTITY,
SUM(WEIGHT) AS WEIGHT,
SUM(NET_PRICE) AS NET_PRICE,
SUM(VOLUMEEQUIVALENCYVALUE*QUANTITY) AS VOLUME,
SUM(CARD_DISCOUNT) AS CARD_DISCOUNT,
TM_DIM_KEY_WEEK,
OPERATING_COMPANY,
SUM(COALESCE(UNITS,0)) AS UNITS,
SUM(COALESCE(CENTS,0)) AS CENTS,
SUM(COALESCE(BASELINE_UNITS,0)) AS BASELINE_UNITS,
SUM(COALESCE(BASELINE_CENTS,0)) AS BASELINE_CENTS,
SUM(COALESCE(POS_PRICE,0)) AS POS_PRICE,
MAX(COALESCE(FEATURE,0)) AS FEATURE,
MAX(COALESCE(DISPLAY,0)) AS DISPLAY,
MAX(COALESCE(TOTL_PRC_REDUC,0)) AS TOTL_PRC_REDUC,
PRODUCT_GRP_ID,
EXPERIAN_ID,
EXPOSED_FLAG
FROM(
SELECT T2.*,
PRGP.PRODUCT_ID AS PRODUCT_GRP_ID,
PRGP.VOLUMEEQUIVALENCYVALUE AS VOLUMEEQUIVALENCYVALUE
FROM(
SELECT FSP_POS.*,
HDM.EXPERIAN_ID,
HDM.EXPOSED_FLAG
FROM(
SELECT
FSP.*,
POS.ITEM_DIM_KEY,
POS.UNITS,
POS.CENTS,
POS.BASELINE_UNITS,
POS.BASELINE_CENTS,
POS.CENTS/(100*POS.UNITS) AS POS_PRICE,
POS.FEATURE,
POS.DISPLAY,
POS.TOTL_PRC_REDUC,
POS.REGISTRATION_REQUEST_ID
FROM (
SELECT VENUE_DIM_KEY,
TM_DIM_KEY_DAY,
TRANS_DATE,
TRANS_TIME,
TRANS_NUM,
STORE_NUM,
CARD_NUM,
UPC,
DERIVED_UPC10,
NDC_UPC_IND,
QUANTITY,
WEIGHT,
ITEM_LIST_PRICE,
NET_PRICE,
CARD_DISCOUNT,
AD_DISCOUNT,
OTHER_DISCOUNT,
ITEM_TYPE,
TENDER_TYPE,
AD_EVENT,
AD_VERSION,
TM_DIM_KEY_WEEK,
OPERATING_COMPANY,
substring(Lpad(cast(upc as string),14,"0"),length(Lpad(cast(upc as string),14,"0"))-10,10) as New_DERIVED_UPC10
FROM WH_FSP.PARTITIONED_FSP_WKLY_FACT
WHERE
TM_DIM_KEY_WEEK BETWEEN 1930 AND ${hiveconf:WS_POS_END_WEEK}
AND  OPERATING_COMPANY IN ('BJS')
AND  substring(Lpad(cast(upc as string),14,"0"),length(Lpad(cast(upc as string),14,"0"))-10,10) IN (SELECT DISTINCT UPC10  FROM POS_DATA_${hiveconf:WS_CAMPAIGN_ID})
) FSP
INNER JOIN (
SELECT * FROM POS_DATA_${hiveconf:WS_CAMPAIGN_ID}
) POS
ON  FSP.TM_DIM_KEY_WEEK = POS.TIME_DIM_KEY
AND FSP.VENUE_DIM_KEY = POS.VENUE_DIM_KEY
AND FSP.New_DERIVED_UPC10 = POS.UPC10
) FSP_POS
INNER JOIN (
SELECT RETAILER, DEPENDENT_ID, EXPERIAN_ID,
COALESCE(EXPOSED_FLAG,0) AS EXPOSED_FLAG
FROM(
SELECT RETAILER, cast(DEPENDENT_ID as bigint) as DEPENDENT_ID, HOUSEHOLD_ID AS EXPERIAN_ID
FROM WH_SUPPLMENTAL.HOUSEHOLD_DEPENDENT_ID_MAP
WHERE DATA_SUPPLIER IN (${hiveconf:WS_DATA_SUPPLIER})
AND   RETAILER NOT IN (${hiveconf:WS_RETAILER_EXCLUDE_LIST})
AND   CURRENT_STATE = 'MATCHED'
AND   HOUSEHOLD_TYPE = 'EXPERIAN'
AND   DEPENDENT_TYPE = 'FSP_CARD'
) HOU_EXP
LEFT OUTER JOIN(
SELECT
HOUSEHOLD_ID AS HOUSEHOLD_ID,
MAX(EXPOSED_FLAG) AS EXPOSED_FLAG
FROM(
SELECT
CAST(${hiveconf:WS_EXPOSURE_HH_COL} AS BIGINT) AS HOUSEHOLD_ID,
1 AS EXPOSED_FLAG
FROM ${hiveconf:WS_INTER_TBL}
) MLIFT
GROUP BY HOUSEHOLD_ID
) EXP_FLAG
ON HOU_EXP.EXPERIAN_ID = EXP_FLAG.HOUSEHOLD_ID
) HDM
ON  FSP_POS.OPERATING_COMPANY = HDM.RETAILER
AND FSP_POS.CARD_NUM = HDM.DEPENDENT_ID
) T2
INNER JOIN (
SELECT * FROM PRD_GRP_TBL_${hiveconf:WS_CAMPAIGN_ID}_TEMP
)PRGP
ON T2.ITEM_DIM_KEY = PRGP.ITEM_DIM_KEY
) FSP_AGG
GROUP BY STORE_NUM, OPERATING_COMPANY, TM_DIM_KEY_WEEK, PRODUCT_GRP_ID, EXPERIAN_ID, EXPOSED_FLAG
;

;
----Time taken: 2469.589

max_stars_repo_name: werbac/jlib

max_issues_repo_name: werbac/jlib

max_forks_repo_name: werbac/jlib


text:
using WellLogs
using Test

@testset "WellLogs.jl" begin
    # Write your own tests here.
end


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule Cairo_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("Cairo")
JLLWrappers.@generate_main_file("Cairo", UUID("83423d85-b0ee-5818-9007-b63ccbeb887a"))
end  # module Cairo_jll


text:
using ConvergencePlots
using Test

@testset "ConvergencePlots.jl" begin
    # Write your tests here.
end


text:
# evaluate right bound of scan_func
function get_right_endpoint(
    theta_init::Vector{Float64}, # initial point of parameters
    scan_func::Function, # h(theta) function for predictions or parameters
    loss_func::Function, # lambda(theta) - labmbda_min - delta_lambda
    method::Val{:CICO_ONE_PASS}; # function works only for method ONE_PASS;

    theta_bounds::Vector{Tuple{Float64,Float64}} = fill(
        (-Inf, Inf), length(theta_init)
        ),
    scan_bound::Float64 = 9.0,
    scan_tol::Float64 = 1e-3,
    loss_tol::Float64 = 1e-3, # we have no ideas how to implement loss_tol
    # better results for LN_NELDERMEAD, :LD_MMA, :LD_SLSQP, :LD_CCSAQ
    # worse results for :LN_COBYLA, :LN_PRAXIS
    # errors for :LN_BOBYQA, :LN_SBPLX, :LN_NEWUOA
    local_alg::Symbol = :LN_NELDERMEAD,
    # options for local fitter :max_iter
    max_iter::Int = 10^5,
    ftol_abs::Float64 = 1e-3,
    kwargs...
    )
    # dim of the theta vector
    n_theta = length(theta_init)

    # checking arguments
    # when using :LN_NELDERMEAD initial parameters should not be zero
    if local_alg == :LN_NELDERMEAD
        zeroParameter = [ isapprox(theta_init[i], 0., atol=1e-2) for i in 1:n_theta]
        if any(zeroParameter)
            @warn "Close-to-zero parameters found when using :LN_NELDERMEAD."
            show(findall(zeroParameter))
        end
    end

    # optimizer
    local_opt = Opt(local_alg, n_theta)
    ftol_abs!(local_opt, scan_tol) #ftol_abs

    # flags to analyze fitting stop
    out_of_bound::Bool = false
    #constraints_func_error = false

    function constraints_func(x, g) # testing grad methods
    #function constraints_func(x) # testing grad methods
        # this part is necessary to understand the difference between
        # "stop out of bounds" and "stop because of function call error"
        try
            loss = loss_func(x)
        catch e
            @warn "Error when call loss_func($x)"
            throw(e)
        end

        if (loss < 0.) && (scan_func(x) > scan_bound)
            out_of_bound = true
            throw(NLopt.ForcedStop())
            # throw(OutOfBoundException())
        #elseif isapprox(loss, 0., atol=loss_tol)
            #@warn "loss_tol reached... but..."
            #return loss
        elseif length(g) > 0
            try ForwardDiff.gradient!(g, loss_func, x)
            catch e
                @show e
            end
            #Calculus.finite_difference!(loss_func,x,g,:central)
            #g .= Zygote.gradient(loss_func,x)[1]
        end

        return loss
    end

    function obj_func(x,g)
        if length(g) > 0
            try ForwardDiff.gradient!(g, scan_func, x)
            catch e
                @show e
            end
            #Calculus.finite_difference!(scan_func,x,g,:central)
            #g .= Zygote.gradient(scan_func,x)[1]
        end

        scan_func(x)
    end

    # constrain optimizer
    opt = Opt(:LN_AUGLAG, n_theta)
    ftol_abs!(opt, scan_tol)

    max_objective!(
        opt,
        obj_func
        )

    local_optimizer!(opt, local_opt)
    maxeval!(opt, max_iter)

    # inequality constraints
    inequality_constraint!(
        opt,
        constraints_func,
        loss_tol
    )

    # version 1: internal :LN_AUGLAG box constrains
    opt.lower_bounds = [tb[1] for tb in theta_bounds]
    opt.upper_bounds = [tb[2] for tb in theta_bounds]

    # version 2: creating lower and upper bounds as general constraints
    #=
    [ inequality_constraint!(
        opt,
        (x, g) -> x[i] - theta_bounds[i][2],
        0.
    ) for i in 1:n_theta ]
    [ inequality_constraint!(
        opt,
        (x, g) -> theta_bounds[i][1] - x[i],
        0.
    ) for i in 1:n_theta ]
    =#

    # start optimization
    (optf, optx, ret) = optimize(opt, theta_init)

    if ret == :FORCED_STOP && !out_of_bound
        pp = ProfilePoint[]
        res = (nothing, pp, :LOSS_ERROR_STOP)
    elseif ret == :MAXEVAL_REACHED
        pp = ProfilePoint[]
        res = (nothing, pp, :MAX_ITER_STOP)
    elseif (ret == :FORCED_STOP || ret == :FAILURE) && out_of_bound # successfull result
        pp = ProfilePoint[]
        res = (nothing, pp, :SCAN_BOUND_REACHED)
    elseif ret == :FTOL_REACHED # successfull result
        loss = loss_func(optx)
        pp = [ ProfilePoint(optf, loss, optx, ret, nothing) ]
        res = (optf, pp, :BORDER_FOUND_BY_SCAN_TOL)
    else
        # this part is reached in case of :FAILURE return code
        # we have seen it in case of gradient methods
        pp = ProfilePoint[]
        res = (nothing, pp, :UNKNOWN_STOP)
    end

    return res
end # get_right_endpoint

function get_right_endpoint(
    theta_init::Vector{Float64}, # initial point of parameters
    theta_num::Int, # number of parameter to scan
    loss_func::Function, # lambda(theta) - labmbda_min - delta_lambda
    method::Val{:CICO_ONE_PASS}; # function works only for method ONE_PASS;

    theta_bounds::Vector{Tuple{Float64,Float64}} = fill(
        (-Inf, Inf), length(theta_init)
        ),
    scan_bound::Float64 = 9.0,
    scan_tol::Float64 = 1e-3,
    loss_tol::Float64 = 1e-3,
    local_alg::Symbol = :LN_NELDERMEAD,
    kwargs... # options for local fitter
    )
    # checking arguments
    if theta_num > length(theta_init)
        throw(DomainError(theta_num, "theta_num exceed theta dimention"))
    end

    scan_func(theta::Vector) = theta[theta_num]

    get_right_endpoint(
        theta_init,
        scan_func,
        loss_func,
        method;

        theta_bounds = theta_bounds,
        scan_bound = scan_bound,
        scan_tol = scan_tol,
        loss_tol = loss_tol,
        local_alg = local_alg,
        kwargs... # options for local fitter
    )
end

# struct OutOfBoundException <: Exception end


text:
export CoOccurrence

"""

    CoOccurrence(
        data::DataAccessor,
        i_ref::Int
    )

Recommend items which are most frequently co-occurred with a reference item `i_ref`.
"""
struct CoOccurrence <: Recommender
    data::DataAccessor
    i_ref::Int
    scores::AbstractVector

    function CoOccurrence(data::DataAccessor, i_ref::Int)
        n_item = size(data.R, 2)
        new(data, i_ref, vector(n_item))
    end
end

isbuilt(recommender::CoOccurrence) = isfilled(recommender.scores)

function build!(recommender::CoOccurrence)
    n_item = size(recommender.data.R, 2)

    v_ref = recommender.data.R[:, recommender.i_ref]
    c = count(!isalmostzero, v_ref)

    for i in 1:n_item
        v = recommender.data.R[:, i]
        cc = length(v_ref[(v_ref .> almost_zero) .& (v .> almost_zero)])
        recommender.scores[i] = cc / c * 100.0
    end
end

function ranking(recommender::CoOccurrence, u::Int, i::Int)
    check_build_status(recommender)
    recommender.scores[i]
end


text:
#
# This file is part of the SlowActors.jl Julia package,
# MIT license, part of https://github.com/JuliaActors
#

function _act(mbx::Mailbox)
    task_local_storage("_ACT", mbx.A.act)
    magic = hash(rand(Int))
    while true
        msg = lock(mbx.lock) do
            isempty(mbx.box) ?
                magic : popfirst!(mbx.box)
        end
        msg == magic && break
        msg isa Actors.Exit && break
        onmessage(mbx.A.act, msg)
    end
end

function startActor(mbx::Mailbox)
    mbx.A.t = Task() do
        _act(mbx)
    end
    mbx.A.t.sticky = mbx.A.sticky
    schedule(mbx.A.t)
end

"""
    spawn(bhv::Func; sticky=false)

Create a new slow actor with a behavior `bhv`. If `sticky=true`
it is started on the same thread as the sender of a message.

**Note:** a slow actor is actually started as a task if a
message is sent to it.
"""
function spawn(bhv::Func; sticky=false)
    lk = Link(
        Mailbox(Deque{Any}(), ReentrantLock(),
            SlowActor(nothing, sticky, _ACT())),
        myid(), :local
    )
    lk.chn.A.act.bhv = bhv
    lk.chn.A.act.self = lk
end


text:
##### Generic methods #####

"""
    size(d::MatrixDistribution)

Return the size of each sample from distribution `d`.
"""
Base.size(d::MatrixDistribution)

size(d::MatrixDistribution, i) = size(d)[i]

"""
    length(d::MatrixDistribution)

The length (*i.e* number of elements) of each sample from the distribution `d`.
"""
Base.length(d::MatrixDistribution)

"""
    rank(d::MatrixDistribution)

The rank of each sample from the distribution `d`.
"""
LinearAlgebra.rank(d::MatrixDistribution)

"""
    vec(d::MatrixDistribution)

If known, returns a `MultivariateDistribution` instance representing the
distribution of vec(X), where X is a random matrix with distribution `d`.
"""
Base.vec(d::MatrixDistribution)

"""
    inv(d::MatrixDistribution)

If known, returns a `MatrixDistribution` instance representing the
distribution of inv(X), where X is a random matrix with distribution `d`.
"""
Base.inv(d::MatrixDistribution)

"""
    mean(d::MatrixDistribution)

Return the mean matrix of `d`.
"""
mean(d::MatrixDistribution)

"""
    var(d::MatrixDistribution)

Compute the matrix of element-wise variances for distribution `d`.
"""
var(d::MatrixDistribution) = ((n, p) = size(d); [var(d, i, j) for i in 1:n, j in 1:p])

"""
    cov(d::MatrixDistribution)

Compute the covariance matrix for `vec(X)`, where `X` is a random matrix with distribution `d`.
"""
function cov(d::MatrixDistribution, ::Val{true}=Val(true))
    M = length(d)
    V = zeros(partype(d), M, M)
    iter = CartesianIndices(size(d))
    for el1 = 1:M
        for el2 = 1:el1
            i, j = Tuple(iter[el1])
            k, l = Tuple(iter[el2])
            V[el1, el2] = cov(d, i, j, k, l)
        end
    end
    return V + tril(V, -1)'
end

"""
    cov(d::MatrixDistribution, flattened = Val(false))

Compute the 4-dimensional array whose `(i, j, k, l)` element is `cov(X[i,j], X[k, l])`.
"""
function cov(d::MatrixDistribution, ::Val{false})
    n, p = size(d)
    [cov(d, i, j, k, l) for i in 1:n, j in 1:p, k in 1:n, l in 1:p]
end

"""
    _rand!(::AbstractRNG, ::MatrixDistribution, A::AbstractMatrix)

Sample the matrix distribution and store the result in `A`.
Must be implemented by matrix-variate distributions.
"""
_rand!(::AbstractRNG, ::MatrixDistribution, A::AbstractMatrix)

## sampling

# multivariate with pre-allocated 3D array
function _rand!(rng::AbstractRNG, s::Sampleable{Matrixvariate},
                m::AbstractArray{<:Real, 3})
    @boundscheck (size(m, 1), size(m, 2)) == (size(s, 1), size(s, 2)) ||
        throw(DimensionMismatch("Output size inconsistent with matrix size."))
    smp = sampler(s)
    for i in Base.OneTo(size(m,3))
        _rand!(rng, smp, view(m,:,:,i))
    end
    return m
end

# multiple matrix-variates with pre-allocated array of maybe pre-allocated matrices
rand!(rng::AbstractRNG, s::Sampleable{Matrixvariate},
      X::AbstractArray{<:AbstractMatrix}) =
          @inbounds rand!(rng, s, X,
                          !all([isassigned(X,i) for i in eachindex(X)]) ||
                          (sz = size(s); !all(size(x) == sz for x in X)))

function rand!(rng::AbstractRNG, s::Sampleable{Matrixvariate},
               X::AbstractArray{M}, allocate::Bool) where M <: AbstractMatrix
    smp = sampler(s)
    if allocate
        for i in eachindex(X)
            X[i] = _rand!(rng, smp, M(undef, size(s)))
        end
    else
        for x in X
            rand!(rng, smp, x)
        end
    end
    return X
end

# multiple matrix-variates, must allocate array of arrays
rand(rng::AbstractRNG, s::Sampleable{Matrixvariate}, dims::Dims) =
    rand!(rng, s, Array{Matrix{eltype(s)}}(undef, dims), true)
rand(rng::AbstractRNG, s::Sampleable{Matrixvariate,Continuous}, dims::Dims) =
    rand!(rng, s, Array{Matrix{float(eltype(s))}}(undef, dims), true)

# single matrix-variate, must allocate one matrix
rand(rng::AbstractRNG, s::Sampleable{Matrixvariate}) =
    _rand!(rng, s, Matrix{eltype(s)}(undef, size(s)))
rand(rng::AbstractRNG, s::Sampleable{Matrixvariate,Continuous}) =
    _rand!(rng, s, Matrix{float(eltype(s))}(undef, size(s)))

# single matrix-variate with pre-allocated matrix
function rand!(rng::AbstractRNG, s::Sampleable{Matrixvariate},
               A::AbstractMatrix{<:Real})
    @boundscheck size(A) == size(s) ||
        throw(DimensionMismatch("Output size inconsistent with matrix size."))
    return _rand!(rng, s, A)
end

# pdf & logpdf

_logpdf(d::MatrixDistribution, X::AbstractMatrix) = logkernel(d, X) + d.logc0

_pdf(d::MatrixDistribution, x::AbstractMatrix{T}) where {T<:Real} = exp(_logpdf(d, x))

"""
    logpdf(d::MatrixDistribution, AbstractMatrix)

Compute the logarithm of the probability density at the input matrix `x`.
"""
function logpdf(d::MatrixDistribution, x::AbstractMatrix{T}) where T<:Real
    size(x) == size(d) ||
        throw(DimensionMismatch("Inconsistent array dimensions."))
    _logpdf(d, x)
end

"""
    pdf(d::MatrixDistribution, x::AbstractArray)

Compute the probability density at the input matrix `x`.
"""
function pdf(d::MatrixDistribution, x::AbstractMatrix{T}) where T<:Real
    size(x) == size(d) ||
        throw(DimensionMismatch("Inconsistent array dimensions."))
    _pdf(d, x)
end

function _logpdf!(r::AbstractArray, d::MatrixDistribution, X::AbstractArray{M}) where M<:Matrix
    for i = 1:length(X)
        r[i] = logpdf(d, X[i])
    end
    return r
end

function _pdf!(r::AbstractArray, d::MatrixDistribution, X::AbstractArray{M}) where M<:Matrix
    for i = 1:length(X)
        r[i] = pdf(d, X[i])
    end
    return r
end

function logpdf!(r::AbstractArray, d::MatrixDistribution, X::AbstractArray{M}) where M<:Matrix
    length(X) == length(r) ||
        throw(DimensionMismatch("Inconsistent array dimensions."))
    _logpdf!(r, d, X)
end

function pdf!(r::AbstractArray, d::MatrixDistribution, X::AbstractArray{M}) where M<:Matrix
    length(X) == length(r) ||
        throw(DimensionMismatch("Inconsistent array dimensions."))
    _pdf!(r, d, X)
end

function logpdf(d::MatrixDistribution, X::AbstractArray{<:AbstractMatrix{<:Real}})
    map(Base.Fix1(logpdf, d), X)
end

function pdf(d::MatrixDistribution, X::AbstractArray{<:AbstractMatrix{<:Real}})
    map(Base.Fix1(pdf, d), X)
end

"""
    _logpdf(d::MatrixDistribution, x::AbstractArray)

Evaluate logarithm of pdf value for a given sample `x`. This function need not perform dimension checking.
"""
_logpdf(d::MatrixDistribution, x::AbstractArray)

"""
    loglikelihood(d::MatrixDistribution, x::AbstractArray)

The log-likelihood of distribution `d` with respect to all samples contained in array `x`.

Here, `x` can be a matrix of size `size(d)`, a three-dimensional array with `size(d, 1)`
rows and `size(d, 2)` columns, or an array of matrices of size `size(d)`.
"""
loglikelihood(d::MatrixDistribution, X::AbstractMatrix{<:Real}) = logpdf(d, X)
function loglikelihood(d::MatrixDistribution, X::AbstractArray{<:Real,3})
    (size(X, 1), size(X, 2)) == size(d) || throw(DimensionMismatch("Inconsistent array dimensions."))
    return sum(i -> _logpdf(d, view(X, :, :, i)), axes(X, 3))
end
function loglikelihood(d::MatrixDistribution, X::AbstractArray{<:AbstractMatrix{<:Real}})
    return sum(x -> logpdf(d, x), X)
end

#  for testing
is_univariate(d::MatrixDistribution) = size(d) == (1, 1)
check_univariate(d::MatrixDistribution) = is_univariate(d) || throw(ArgumentError("not 1 x 1"))

##### Specific distributions #####

for fname in ["wishart.jl", "inversewishart.jl", "matrixnormal.jl",
              "matrixreshaped.jl", "matrixtdist.jl", "matrixbeta.jl",
              "matrixfdist.jl", "lkj.jl"]
    include(joinpath("matrix", fname))
end


text:
inpt = "3113322113"

output = ""
for x = 1:50
	output = ""
	current = inpt[1]
	count = 0
	for i in inpt
		if i == current
			count += 1
		else
			output *= "$count$current"
			count = 1
			current = i
		end
	end
	output *= "$count$current"
	inpt = output
end
println(length(output))

max_stars_repo_name: jstrube/adventofcode

max_issues_repo_name: jstrube/adventofcode

max_forks_repo_name: jstrube/adventofcode


text:
using Test, LinearMaps, LinearAlgebra

@testset "wrapped maps" begin
    A = rand(10, 20)
    B = rand(ComplexF64, 10, 20)
    SA = A'A + I
    SB = B'B + I
    L = @inferred LinearMap{Float64}(A)
    MA = @inferred LinearMap(SA)
    MB = @inferred LinearMap(SB)
    @test size(L) == size(A)
    @test @inferred !issymmetric(L)
    @test @inferred issymmetric(MA)
    @test @inferred !issymmetric(MB)
    @test @inferred isposdef(MA)
    @test @inferred isposdef(MB)
end


text:
export add!, add, +, -, merge

## Polynomial addition
function add!(p::Polynomial, p1::T) where T<:AbstractPolynomial
    return add!(p, convert(Polynomial, p1))
end

function add!(p::Polynomial, p1::Number)
    return add!(p, convert(Polynomial, p1))
end

function add!(p::Polynomial, p1::Polynomial)
    for (cur_expo, λ) in p1
        λ != 0 || continue
        add_to_dict!(p.poly, cur_expo, λ)
    end
    p.degree.explvar = max(p.degree.explvar, p1.degree.explvar)
    p.degree.conjvar = max(p.degree.conjvar, p1.degree.conjvar)
end

function add(p1::Polynomial, p2::Polynomial)
    p = Polynomial()
    add!(p, p1)
    add!(p, p2)
    return p
end

function +(p::T) where T<:AbstractPolynomial
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    return convert(Polynomial, p)
end

function +(p1::T, p2::U) where T<:AbstractPolynomial where U<:AbstractPolynomial
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    return add(convert(Polynomial, p1), convert(Polynomial, p2))
end
function +(p1::Number, p2::T) where T<:AbstractPolynomial
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    return add(convert(Polynomial, p1), convert(Polynomial, p2))
end
function +(p1::T, p2::Number) where T<:AbstractPolynomial
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    return add(convert(Polynomial, p1), convert(Polynomial, p2))
end

### Soustraction
function -(p::T) where T<:AbstractPolynomial
    return -1 * convert(Polynomial, p)
end

function -(p1::T, p2::U) where T<:AbstractPolynomial where U<:AbstractPolynomial
    return p1 + (-p2)
end
function -(p1::Number, p2::T) where T<:AbstractPolynomial
    return p1 + (-p2)
end
function -(p1::T, p2::Number) where T<:AbstractPolynomial
    return p1 + (-p2)
end

## Point addition
"""
  add_coord!(pt::Point, var::Variable, val::Number)

  Sparsely add `val` to the `var` coordinate of `pt`.
"""
function add_coord!(pt::Point, var::Variable, val::Number)
  return add_to_dict!(pt.coords, var, val, isdense=pt.isdense)
end

function add!(pt1::Point, pt2::Point)
    for (var, val) in pt2.coords
        add_coord!(pt1, var, val)
    end
end

function add(pt1::Point, pt2::Point)
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    pt_coord = deepcopy(pt1.coords)
    pt = Point(pt_coord)
    add!(pt, pt2)
    return pt
end

function +(pt1::Point, pt2::Point)
    debug(LOGGER, "Inefficient implementation\n$(string(stacktrace()))")
    add(pt1, pt2)
end

function -(pt1::Point, pt2::Point)
    return pt1 + (-1)*pt2
end

function merge(x::Point ...)
    pt_res = Point()
    for pt in x
        add!(pt_res, pt)
    end
    return pt_res
end


text:
@testset "kernelsum" begin
    rng = MersenneTwister(123456)
    x = rand(rng)*2
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)

    k1 = LinearKernel()
    k2 = SqExponentialKernel()
    k3 = RationalQuadraticKernel()
    X = rand(rng, 2,2)

    w = [2.0,0.5]
    k = KernelSum([k1,k2],w)
    ks1 = 2.0*k1
    ks2 = 0.5*k2
    @test length(k) == 2
    @test k(v1, v2) == (2.0 * k1 + 0.5 * k2)(v1, v2)
    @test (k + k3)(v1,v2) ≈ (k3 + k)(v1, v2)
    @test (k1 + k2)(v1, v2) == KernelSum([k1, k2])(v1, v2)
    @test (k + ks1)(v1, v2) ≈ (ks1 + k)(v1, v2)
    @test (k + k)(v1, v2) == KernelSum([k1, k2, k1, k2], vcat(w, w))(v1, v2)

    @testset "kernelmatrix" begin
        rng = MersenneTwister(123456)

        Nx = 5
        Ny = 4
        D = 3

        w1 = rand(rng) + 1e-3
        w2 = rand(rng) + 1e-3
        k1 = w1 * SqExponentialKernel()
        k2 = w2 * LinearKernel()
        k = k1 + k2

        @testset "$(typeof(x))" for (x, y) in [
            (ColVecs(randn(rng, D, Nx)), ColVecs(randn(rng, D, Ny))),
            (RowVecs(randn(rng, Nx, D)), RowVecs(randn(rng, Ny, D))),
        ]
            @test kernelmatrix(k, x, y) ≈ kernelmatrix(k1, x, y) + kernelmatrix(k2, x, y)

            @test kernelmatrix(k, x) ≈ kernelmatrix(k1, x) + kernelmatrix(k2, x)

            @test kerneldiagmatrix(k, x) ≈ kerneldiagmatrix(k1, x) + kerneldiagmatrix(k2, x)

            tmp = Matrix{Float64}(undef, length(x), length(y))
            @test kernelmatrix!(tmp, k, x, y) ≈ kernelmatrix(k, x, y)

            tmp_square = Matrix{Float64}(undef, length(x), length(x))
            @test kernelmatrix!(tmp_square, k, x) ≈ kernelmatrix(k, x)

            tmp_diag = Vector{Float64}(undef, length(x))
            @test kerneldiagmatrix!(tmp_diag, k, x) ≈ kerneldiagmatrix(k, x)
        end
    end
    test_ADs(x->KernelSum([SqExponentialKernel(),LinearKernel(c= x[1])], x[2:3]), rand(3), ADs = [:ForwardDiff, :ReverseDiff])
    @test_broken "Zygote failing because of mutating array"
end


text:
module Words

export getwords, hamming, getwordedges

using HTTP, STMO

"""Get all English words"""
function getwords()
    io = HTTP.get("https://github.com/dwyl/english-words/raw/master/words_alpha.txt")
    words = split(String(io.body), "\n") .|> rstrip .|> String
    return words
end

"""Get all English words of length `wordsize`"""
function getwords(wordsize::Integer)
    return filter(w -> length(w) == wordsize, getwords())
end

"""
    getwordedges(words; cutoff=5)

Returns all the weigthed edges between a list of `words` where the Hamming
distance is less than `cutoff`.
"""
function getwordedges(words; cutoff=5)
    edges = Tuple{Int,String,String}[]
    for (i, w1) in enumerate(words[1:end-1])
        for w2 in words[(i+1):end]
            d = hamming(w1, w2)
            d ≤ cutoff && push!(edges, (d, w1, w2))
        end
    end
    return edges
end

end #Words


text:
# Multiscale HINT network
# Author: Gabrio Rizzuti, grizzuti3@gatech.edu
# Date: October 2020

using InvertibleNetworks, LinearAlgebra, Test, Random
Random.seed!(11)

# Define network
nx = 64
ny = 64
n_in = 2
n_hidden = 4
batchsize = 2
L = 2
K = 2

# Multi-scale and single scale network
H = NetworkMultiScaleHINT(n_in, n_hidden, L, K; split_scales=false, k1=3, k2=1, p1=1, p2=0)

print("\nInvertibility test multiscale HINT network\n")

# Test layers
test_size = 10
X = randn(Float32, nx, ny, n_in, test_size)

# Forward-backward
Z, logdet = H.forward(X)
X_ = H.backward(0f0.*Z, Z)[2]
@test isapprox(X, X_; rtol=1f-3)

# Forward-inverse
Z, logdet = H.forward(X)
X_ = H.inverse(Z)
@test isapprox(X, X_; rtol=1f-3)

# Loss
function loss(H, X)
    Z, logdet = H.forward(X)
    f = -log_likelihood(Z) - logdet
    ΔZ = -∇log_likelihood(Z)
    ΔX = H.backward(ΔZ, Z)[1]
    return f, ΔX
end

test_size = 10
X = randn(Float32, nx, ny, n_in, test_size)
X0 = randn(Float32, nx, ny, n_in, test_size)
dX = X - X0

f0, ΔX = loss(H, X0)
h = 0.1f0
maxiter = 6
err1 = zeros(Float32, maxiter)
err2 = zeros(Float32, maxiter)

print("\nGradient test multiscale HINT net: input\n")
for j=1:maxiter
    f = loss(H, X0 + h*dX)[1]
    err1[j] = abs(f - f0)
    err2[j] = abs(f - f0 - h*dot(dX, ΔX))
    print(err1[j], "; ", err2[j], "\n")
    global h = h/2f0
end

@test isapprox(err1[end] / (err1[1]/2^(maxiter-1)), 1f0; atol=1f1)
@test isapprox(err2[end] / (err2[1]/4^(maxiter-1)), 1f0; atol=1f1)

###################################################################################################
# Jacobian-related tests

# Gradient test

# Initialization
H = NetworkMultiScaleHINT(n_in, n_hidden, L, K; k1=3, k2=1, p1=1, p2=0); H.forward(randn(Float32, nx, ny, n_in, batchsize))
θ = deepcopy(get_params(H))
H0 = NetworkMultiScaleHINT(n_in, n_hidden, L, K; k1=3, k2=1, p1=1, p2=0); H0.forward(randn(Float32, nx, ny, n_in, batchsize))
θ0 = deepcopy(get_params(H0))
X = randn(Float32, nx, ny, n_in, batchsize)

# Perturbation (normalized)
dθ = θ-θ0; dθ .*= norm.(θ0)./(norm.(dθ).+1f-10)
dX = randn(Float32, nx, ny, n_in, batchsize); dX *= norm(X)/norm(dX)

# Jacobian eval
dZ, Z, _, _ = H.jacobian(dX, dθ, X)

# Test
print("\nJacobian test\n")
h = 0.1f0
maxiter = 5
err5 = zeros(Float32, maxiter)
err6 = zeros(Float32, maxiter)
for j=1:maxiter
    set_params!(H, θ+h*dθ)
    Z_, _ = H.forward(X+h*dX)
    err5[j] = norm(Z_ - Z)
    err6[j] = norm(Z_ - Z - h*dZ)
    print(err5[j], "; ", err6[j], "\n")
    global h = h/2f0
end

@test isapprox(err5[end] / (err5[1]/2^(maxiter-1)), 1f0; atol=1f1)
@test isapprox(err6[end] / (err6[1]/4^(maxiter-1)), 1f0; atol=1f1)

# Adjoint test

set_params!(H, θ)
dZ, Z, _, _ = H.jacobian(dX, dθ, X)
dZ_ = randn(Float32, size(dZ))
dX_, dθ_, _ = H.adjointJacobian(dZ_, Z)
a = dot(dZ, dZ_)
b = dot(dX, dX_)+dot(dθ, dθ_)
@test isapprox(a, b; rtol=1f-1) ####### need to check low accuracy here


text:
module iJR904

    import BSON
    import ..Chemostat_Nanchen2006
    const ChN = Chemostat_Nanchen2006
    import ..BegData
    const Bd = BegData
    import ..NanchenData
    const Nd = NanchenData
    import Chemostat
    const ChU = Chemostat.Utils
    import MAT

    using ProjAssistant
    @gen_sub_proj

    include("const.jl")
    include("load_data.jl")
    include("beg_enz_cost.jl")
    include("load_model.jl")
    include("ME_MODES.jl")

    function __init__()
        @create_proj_dirs
    end

end