text:
using MultipleViewGeometry, Test
using MultipleViewGeometry.ModuleCostFunction
using MultipleViewGeometry.ModuleTypes
using MultipleViewGeometry.ModuleConstraints
using StaticArrays, Random, LinearAlgebra

# Fix random seed.
Random.seed!(1234)

# Test for cost functions.

# Test cost function on Fundamental matrix estimation.

𝒳 = [Point3D(x,y,rand(50:100)) for x = -100:5:100 for y = -100:5:100]
𝒳 = 𝒳[1:50:end]

# Specify the coordinate systems of the world, the camera frame and the picture
# plane.
world_basis = (Vec(1.0, 0.0, 0.0), Vec(0.0, 1.0, 0.0), Vec(0.0, 0.0, 1.0))
camera_basis = (Point(0.0, 0.0, 0.0), Vec(-1.0, 0.0, 0.0), Vec(0.0, -1.0, 0.0), Vec(0.0, 0.0, 1.0))
picture_basis = (Point(0.0, 0.0), Vec(-1.0, 0.0), Vec(0.0, -1.0))

# The focal length for both cameras is one.
f = 1
image_width = 640 / 10
image_height = 480 / 10

camera₁ = Pinhole(image_width, image_height, f, camera_basis..., picture_basis...)
camera₂ = Pinhole(image_width, image_height, f, camera_basis..., picture_basis...)

# Rotate and translate camera one.
𝐑₁ = Matrix{Float64}(I,3,3)
𝐭₁ = [-50.0, -2.0, 0.0]
relocate!(camera₁, 𝐑₁, 𝐭₁)

# Rotate and translate camera two.
𝐑₂ = Matrix{Float64}(I,3,3)
𝐭₂ = [50.0, 2.0, 0.0]
relocate!(camera₂, 𝐑₂, 𝐭₂)

𝐑₁′, 𝐭₁′ = ascertain_pose(camera₁, world_basis... )
𝐊₁′ = obtain_intrinsics(camera₁, CartesianSystem())
𝐑₂′, 𝐭₂′ = ascertain_pose(camera₂, world_basis... )
𝐊₂′ = obtain_intrinsics(camera₂, CartesianSystem())

# Camera projection matrices.
𝐏₁ = construct(ProjectionMatrix(),𝐊₁′,𝐑₁′,𝐭₁′)
𝐏₂ = construct(ProjectionMatrix(),𝐊₂′,𝐑₂′,𝐭₂′)

# Set of corresponding points.
ℳ = project(camera₁,𝐏₁,𝒳)
ℳʹ = project(camera₂,𝐏₂,𝒳)

𝐅 = construct(FundamentalMatrix(),𝐊₁′,𝐑₁′,-𝐭₁′,𝐊₂′,𝐑₂′,-𝐭₂′)

# Verify that the algorithm returns the correct answer when the
# constraint is already satisfied.
𝒪,𝒪ʹ = satisfy(FundamentalMatrix(), EpipolarConstraint(), 𝐅, (ℳ, ℳʹ))

# Verify that the original corresponding points satisfy the epipolar constraint.
N = length(ℳ)
for n = 1:N
    𝐦 = hom(ℳ[n])
    𝐦ʹ = hom(ℳʹ[n])
    @test  isapprox(𝐦'*𝐅*𝐦ʹ, 0.0; atol = 1e-12)
end

# Verify that the 'corrected' points satisfy the epipolar constraint.
N = length(ℳ)
for n = 1:N
    𝐦 = hom(𝒪[n])
    𝐦ʹ = hom(𝒪ʹ[n])
    @test  isapprox(𝐦'*𝐅*𝐦ʹ, 0.0; atol = 1e-12)
end

# Perturb the original corresponding points slightly so that they no-longer
# satisfy the epipolar constraint.
N = length(ℳ)
σ = 1e-7
for n = 1:N
    ℳ[n] = ℳ[n] + SVector{2}(σ * rand(2,1))
    ℳʹ[n] = ℳʹ[n] + SVector{2}(σ * rand(2,1))
    𝐦 = hom(ℳ[n])
    𝐦ʹ = hom(ℳʹ[n])
    @test abs(𝐦'*𝐅*𝐦ʹ) > 1e-12
end

# Verify that the algorithm returns the correct answer when applied
# to sets of correspondences that do not satisfy the epipolar constraint.
𝒪 ,𝒪ʹ = satisfy(FundamentalMatrix(), EpipolarConstraint(), 𝐅, (ℳ, ℳʹ))

# Verify that the 'corrected' points satisfy the epipolar constraint.
N = length(ℳ)
for n = 1:N
    𝐦 = hom(𝒪[n])
    𝐦ʹ = hom(𝒪ʹ[n])
    @test  isapprox(𝐦'*𝐅*𝐦ʹ, 0.0; atol = 1e-12)
end


text:
module _Test

using Base.Test
# using Lint.lintpkg

include("Meta/meta.jl")
include("DataStructure/datastructure.jl")
include("Algorithm/algorithm.jl")
include("Math/math.jl")
include("Finance/finance.jl")

function test()
    _Meta.test_all()
    _DataStructure.test_all()
    _Algorithm.test_all()
    _Math.test_all()
    _Finance.test_all()

    println("")

    # @test isempty(lintpkg("Brainstorm", returnMsgs=true))
    # lintpkg("Brainstorm", returnMsgs=true)
end

end

ext: jl


text:
"""
        mcdm(df, w, fns, method)

    Perform selected method for a given decision matrix, weight vector, and function list.

# Arguments:
 - `df::DataFrame`: n × m matrix of decision matrix in type of DataFrame.
 - `weights::Array{Float64, 1}`: m-vector of weights for criteria.
 - `fs::Array{Function, 1}`: m-vector of functions that are either maximize or minimize for each single criterion.
 - `method::MCDMMethod`: Preferred MCDMMethod.

# Description
The method is one of the subtypes of MCDMMethod type. See examples.

# Output
- `::MCDMResult`: An object derived from subtypes of MCDMResult type.

# Examples

```julia-repl

julia> subtypes(MCDMMethod)
18-element Vector{Any}:
 ArasMethod
 CocosoMethod
 CodasMethod
 CoprasMethod
 CriticMethod
 EdasMethod
 ElectreMethod
 GreyMethod
 MabacMethod
 MaircaMethod
 MarcosMethod
 MooraMethod
 PrometheeMethod
 SawMethod
 TopsisMethod
 VikorMethod
 WPMMethod
 WaspasMethod

julia> # mcdm() for Topsis:
julia> # mcdm(df, w, fns, TopsisMethod())

julia> # mcdm() for Saw:
julia> # mcdm(df, w, fns, SawMethod())

julia> # mcdm() with optional parameters:
julia> # mcdm(df, w, fns, GreyMethod(0.6))
```

"""
function mcdm(df::DataFrame,
    w::Array{Float64, 1},
    fns::Array{Function, 1},
    method::T1)::MCDMResult where {T1 <: MCDMMethod}

    if method isa TopsisMethod
        topsis(df, w, fns)
    elseif method isa ElectreMethod
        electre(df, w, fns)
    elseif method isa ArasMethod
        aras(df, w, fns)
    elseif method isa CocosoMethod
        cocoso(df, w, fns, lambda = method.lambda)
    elseif method isa CodasMethod
        codas(df, w, fns, tau = method.tau)
    elseif method isa CoprasMethod
        copras(df, w, fns)
    elseif method isa CriticMethod
        critic(df, fns)
    elseif method isa EdasMethod
        edas(df, w, fns)
    elseif method isa GreyMethod
        grey(df, w, fns, zeta = method.zeta)
    elseif method isa MabacMethod
        mabac(df, w, fns)
    elseif method isa MaircaMethod
        mairca(df, w, fns)
    elseif method isa MooraMethod
        moora(df, w, fns)
    elseif method isa PrometheeMethod
        promethee(df, w, fns, method.pref, method.qs, method.ps)
    elseif method isa SawMethod
        saw(df, w, fns)
    elseif method isa VikorMethod
        vikor(df, w, fns, v = method.v)
    elseif method isa WaspasMethod
        waspas(df, w, fns, lambda = method.lambda)
    elseif method isa WPMMethod
        wpm(df, w, fns)
    elseif method isa MarcosMethod
        marcos(df, w, fns)
    else
        error("Method is not defined")
    end
end

"""
    mcdm(setting, method)

    Perform selected method for a given decision matrix, weight vector, and function list.

# Arguments:
 - `setting::MCDMSetting`: MCDMSetting object that holds the decision matrix, weight vector, and functions.
 - `method::MCDMMethod`: Preferred MCDMMethod.

# Description
The method is one of the subtypes of MCDMMethod type. See examples.

# Output
- `::MCDMResult`: An object derived from subtypes of MCDMResult type.

# Examples

```julia-repl
julia> # mcdm() for Topsis:
julia> # mcdm(setting, TopsisMethod())

julia> # mcdm() for Saw:
julia> # mcdm(setting, SawMethod())

julia> # mcdm() with optional parameters:
julia> # mcdm(setting, GreyMethod(0.6))
```

"""
function mcdm(setting::MCDMSetting,
    method::T1)::MCDMResult where {T1 <: MCDMMethod}

    mcdm(
        setting.df,
        setting.weights,
        setting.fns,
        method
    )
end

max_stars_repo_name: drvinceknight/JMcDM

max_issues_repo_name: drvinceknight/JMcDM

max_forks_repo_name: drvinceknight/JMcDM


text:
"""
    SplitOctonion{T <: Real} = Hyperbolic{Quaternion{T}}

A split-octonion is a hyperbolic Cayley-Dickson construct with Quaternion quaternions.
"""
const SplitOctonion{T <: Real} = Hyperbolic{Quaternion{T}}

function SplitOctonion(z::Quaternion{T}) where T <: Real
    SplitOctonion{T}(z, zero(Quaternion{T}))
end

function SplitOctonion(z::Binion{T}) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(z, zero(Binion{T})),
        zero(Quaternion{T})
    )
end

function SplitOctonion(a::T, b::T, c::T, d::T, f::T, g::T, h::T, j::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, d)
        ),
        Quaternion{T}(
            Binion{T}(f, g),
            Binion{T}(h, j)
        )
    )
end

function SplitOctonion(a::T, b::T, c::T, d::T, f::T, g::T, h::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, d)
        ),
        Quaternion{T}(
            Binion{T}(f, g),
            Binion{T}(h, zero(T))
        )
    )
end

function SplitOctonion(a::T, b::T, c::T, d::T, f::T, g::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, d)
        ),
        Quaternion{T}(
            Binion{T}(f, g),
            zero(Binion{T})
        )
    )
end

function SplitOctonion(a::T, b::T, c::T, d::T, f::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, d)
        ),
        Quaternion{T}(
            Binion{T}(f, zero(T)),
            zero(Binion{T})
        )
    )
end

function SplitOctonion(a::T, b::T, c::T, d::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, d)
        ),
        zero(Quaternion{T})
    )
end

function SplitOctonion(a::T, b::T, c::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            Binion{T}(c, zero(T))
        ),
        zero(Quaternion{T})
    )
end

function SplitOctonion(a::T, b::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, b),
            zero(Binion{T})
        ),
        zero(Quaternion{T})
    )
end

function SplitOctonion(a::T) where T <: Real
    SplitOctonion{T}(
        Quaternion{T}(
            Binion{T}(a, zero(T)),
            zero(Binion{T})
        ),
        zero(Quaternion{T})
    )
end

function SplitOctonion(a::Real, b::Real)
    SplitOctonion(promote(a, b)...)
end

function SplitOctonion(a::Real, b::Real, c::Real)
    SplitOctonion(promote(a, b, c)...)
end

function SplitOctonion(a::Real, b::Real, c::Real, d::Real)
    SplitOctonion(promote(a, b, c, d)...)
end

function SplitOctonion(a::Real, b::Real, c::Real, d::Real, f::Real)
    SplitOctonion(promote(a, b, c, d, f)...)
end

function SplitOctonion(a::Real, b::Real, c::Real, d::Real, f::Real, g::Real)
    SplitOctonion(promote(a, b, c, d, f, g)...)
end

function SplitOctonion(a::Real, b::Real, c::Real, d::Real, f::Real, g::Real, h::Real)
    SplitOctonion(promote(a, b, c, d, f, g, h)...)
end

function SplitOctonion(a::Real, b::Real, c::Real, d::Real, f::Real, g::Real, h::Real, j::Real)
    SplitOctonion(promote(a, b, c, d, f, g, h, j)...)
end

"""
    unreal(z::SplitOctonion)

The unreal part of a split-octonion is a 7-vector.
"""
function unreal(z::SplitOctonion)
    vcat(unreal(z.l), array(z.r))
end

function show(io::IO, z::SplitOctonion)
    print(io, "[1: ")
    print(io, z.l.l.l)
    print(io, ", i: ")
    print(io, z.l.l.r)
    print(io, ", j: ")
    print(io, z.l.r.l)
    print(io, ", ij: ")
    print(io, z.l.r.r)
    print(io, ", s: ")
    print(io, z.r.l.l)
    print(io, ", is: ")
    print(io, z.r.l.r)
    print(io, ", js: ")
    print(io, z.r.r.l)
    print(io, ", (ij)s: ")
    print(io, z.r.r.r)
    print(io, "]")
end

function random(::Type{SplitOctonion{T}}) where T <: Real
    SplitOctonion{T}(random(Quaternion{T}), random(Quaternion{T}))
end


text:
using LinearAlgebra
using Plots

chebp(n) = cos.(pi*collect(0:n-1)/(n-1))
function cheb(m,l)
    x=chebp(m)
    c=ones(m)
    c[1] = c[end] = 2.
    d = [i==j ? 0 : (2/l)*c[i]/c[j]*(-1)^(i+j)/(x[i]-x[j]) for i = 1:m, j = 1:m]
    d-Diagonal(vec(sum(d,dims=2)))
end
function meshgrid(vx::AbstractVector{T}, vy::AbstractVector{T}) where {T}
    m, n = length(vy), length(vx)
    vx = reshape(vx, 1, n)
    vy = reshape(vy, m, 1)
    (repeat(vx, m, 1), repeat(vy, 1, n))
end

mu = 5.
Lz, Lx = 1., 24.
Nz, Nx = 30, 60
N = Nz * Nx
z, x = chebp(Nz), chebp(Nx)
z = (1 .+ z)/2
x = (Lx/2) * x
dz, dx = cheb(Nz,Lz), cheb(Nx,Lx)
zz, xx = meshgrid(z,x)
Z, X = zz[:], xx[:]
Ez = Matrix{Float64}(I,Nz,Nz)
Ex = Matrix{Float64}(I,Nx,Nx)
E = Matrix{Float64}(I,N,N)
Dz, Dx = kron(dz,Ex), kron(Ez,dx)
Dz2, Dx2 = kron(dz^2,Ex), kron(Ez,dx^2)
z0 = findall(in(0.),Z)
z1 = findall(in(1.),Z)
xb = findall(in(Lx/2),abs.(X))
border = union(z0,z1,xb)
psi = 4Z .* tanh.(X)
phi = mu*(1 .- Z)
seed = [psi;phi]
fz = 1 .- Z.^3
fzp = -3Z.^2
k, kmax, epsilon = 0, 10, 1e-6
done = true
while done && k < kmax
    # computing b
    b1 = -phi.^2 .* psi + fz.*(Z.*psi - Dx2*psi - fzp.*(Dz*psi) - fz.*(Dz2*psi))
    b2 = 2phi .* psi.^2 - Dx2*phi - fz.*(Dz2*phi)
    b1[z1] = fzp[z1].*(Dz[z1,:]*psi) + Dx2[z1,:]*psi - psi[z1]
    b1[z0] = psi[z0]
    b1[xb] = Dx[xb,:]*psi
    b2[z1] = phi[z1]
    b2[z0] = phi[z0] .- mu
    b2[xb] = Dx[xb,:]*phi
    b = -[b1;b2]
    # computing A
    A11 = Diagonal(Z.*fz - phi.^2) - fz.*Dx2 - fz.^2 .* Dz2 - fz.*fzp.*Dz
    A12 = Diagonal(-2phi.*psi)
    A21 = Diagonal(4psi.*phi)
    A22 = Diagonal(2psi.^2) - Dx2 - fz.*Dz2
    A11[z0,:] = E[z0,:]
    A11[z1,:] = fzp[z1].*Dz[z1,:] + Dx2[z1,:] - E[z1,:]
    A11[xb,:] = Dx[xb,:]
    A12[border,:] .= 0.
    A21[border,:] .= 0.
    A22[z0,:] = E[z0,:]
    A22[z1,:] = E[z1,:]
    A22[xb,:] = Dx[xb,:]
    A = [A11 A12;A21 A22]
    sol = A \ b
    global seed += sol
    reseed = reshape(seed,:,2)
    global psi = reseed[:,1]
    global phi = reseed[:,2]
    global k += 1
    rmssol = sqrt(sum(abs2.(sol))/(2N))
    if rmssol < epsilon
        global done = false
    end
    println(rmssol)
end
plot(z,x,reshape(psi,Nx,Nz),st=:surface)

max_stars_repo_name: zhongshanxu/graduate

max_issues_repo_name: zhongshanxu/MASTER

max_forks_repo_name: zhongshanxu/MASTER


text:
#=
The training script for the 100-20 node MLP for the MA2 model
=#
using MXNet,PyPlot
include("dataprep.jl") # creates X,Y,XT,YT, the training and testing inputs and output

# how to set up data providers using data in memory
batchsize = 2048 # can adjust this later, but must be defined now for next line
trainprovider = mx.ArrayDataProvider(:data => X, batch_size=batchsize, shuffle=false, :label => Y)
evalprovider = mx.ArrayDataProvider(:data => XT, batch_size=batchsize, shuffle=false, :label => YT)

# set up 2 layer MLP with 2 outputs
data = mx.Variable(:data)
label = mx.Variable(:label)
net  = @mx.chain    mx.FullyConnected(data = data, num_hidden=100) =>
                    mx.Activation(act_type=:tanh) =>
                    mx.FullyConnected(num_hidden=20) =>
                    mx.Activation(act_type=:tanh) =>
                    mx.FullyConnected(num_hidden=2)

# squared error loss is appropriate for regression, don't change
cost = mx.LinearRegressionOutput(data = net, label=label)

# final model definition, don't change, except if using gpu
model = mx.FeedForward(cost, context=mx.cpu())

# set up the optimizer: select one, explore parameters, if desired
#optimizer = mx.SGD(lr=0.01, momentum=0.9, weight_decay=0.00001)
optimizer = mx.ADAM()

# train, reporting loss for training and evaluation sets
# initial training with small batch size, to get to a good neighborhood
batchsize = 128
mx.fit(model, optimizer, initializer=mx.NormalInitializer(0.0,0.1), eval_metric=mx.MSE(), trainprovider, eval_data=evalprovider, n_epoch = 100)
# more training with larger sample
batchsize = 2048
mx.fit(model, optimizer, eval_metric=mx.MSE(), trainprovider, eval_data=evalprovider, n_epoch = 10)

# obtain predictions
plotprovider = mx.ArrayDataProvider(:data => XT, :label => YT)
fit = mx.predict(model, plotprovider)
fit = fit'

# compute RMSE using the testing data
trainsize = 900000
testsize = 100000
thetas = thetas[trainsize+1:trainsize+testsize,:]
preprocess = preprocess[trainsize+1:trainsize+testsize,:]
fit = preprocess + sErrors.*fit
error = thetas - fit
# compute RMSE
bias = mean(error,1)
mse = mean(error.^2,1)
rmse = sqrt(mse)
@printf("    bias      rmse       mse\n")
for i=1:size(bias,2)
    @printf("%8.5f  %8.5f  %8.5f\n", bias[i], rmse[i], mse[i])
end

# get the first layer parameters for influence analysis
beta = copy(model.arg_params[:fullyconnected0_weight])
z = maximum(abs(beta),2);
cax3 = matshow(z', interpolation="nearest")
colorbar(cax3)
xlabels = [string(i) for i=1:11]
xticks(0:10, xlabels)

figure()
cax1 = scatter(thetas[:,1],fit[:,1])
xlabel("theta1")
ylabel("fitted theta1")

figure()
cax2 = scatter(thetas[:,2], fit[:,2])
xlabel("theta2")
ylabel("fitted theta2")


text:
default_value_of(::Type{String}) = ""
default_value_of(::Type{Bool}) = false
default_value_of(::Type{<:Number}) = 0
default_value_of(::Type{DbId}) = DbId()

#### read section ###########################

function filter_attributes(a::Vector{<:Any}, struct_type::Type{<:Any})
    # Secure that all elements of a are of type Build_Types
    @assert all(typeof.(a).<: Build_Types)

    attr_rel_names = [x.type.label.name for x in a]
    names_of_fields = string.(fieldnames(struct_type))
    result_names = intersect(attr_rel_names, names_of_fields)
    attribs = filter(x->in(x.type.label.name, result_names), a)

    return attribs
end

function filter_attributes(concept::ConceptMap)
    return filter(x->typeof(x.type) <: Build_Types, collect(values(concept.data)))
end

function _get_value(::AbstractCoreTransaction, inp::AbstractAttribute, ::Type{<:Any}, ::AbstractThing)
    return inp.value
end

function _get_value(::AbstractCoreTransaction,
    inp::Attribute{VALUE_TYPE.DATETIME, Int64},
    ::Type{<:Any},
    ::AbstractThing )

    return unix2datetime(inp.value / 1000)
end

function _get_value(transaction::AbstractCoreTransaction,
    inp_relation::AbstractRelation,
    inp_struct_type::Type{<:Any},
    thing::AbstractThing)

    result = nothing
    # get the fieldtype of the relation struct
    field_type = _resulting_fieldtype(inp_struct_type, Symbol(inp_relation.type.label.name))
    # get all players for the given relation
    rel_player = filter(x->string(field_type) == x.type.label.name, get_players(transaction, inp_relation))
    # process the right struct_type for a given iid to a struct
    if length(rel_player) == 1
        read_result = read_concept_from_struct(transaction, field_type, typeof(rel_player[1].type), rel_player[1].iid)
        result = !isempty(read_result) ? read_result[1] : nothing
    else
        throw("get_value of relations for iid: $(thing.iid) went wrong")
    end
    return result
end

function values_of_attributes(transaction::AbstractCoreTransaction ,
    attributes::Vector,
    inp_type_struct::Type{<:Any},
    thing::AbstractThing)

    attribute_values = Dict{String,Any}()
    for attribute in attributes
        attribute_values[attribute.type.label.name] = _get_value(transaction, attribute, inp_type_struct, thing)
    end
    return attribute_values
end

function define_conversation(concept_or_attribute, struct_type::Type{<:Any})
    dict_convers = Dict{Symbol,Any}()
    attr_filter = filter_attributes(concept_or_attribute, struct_type)
    for attr in attr_filter
        dict_convers[Symbol(attr.type.label.name)] = attr.type.label.name
    end
    return dict_convers
end

function concept_to_struct(transaction::AbstractCoreTransaction,
    concept_or_attributes,
    inp_struct_type::Type{<:Any},
    id_field::FieldName_Type,
    thing::AbstractThing)

    filtered_attributes = filter_attributes(concept_or_attributes, inp_struct_type)

    attributes = values_of_attributes(transaction, filtered_attributes, inp_struct_type, thing)
    conversation_attributes = define_conversation(concept_or_attributes, inp_struct_type)
    # add iid to the attributes
    conversation_attributes[id_field.fieldname] = DbId(thing.iid)

    values = Vector{Any}()
    for field in fieldnames(inp_struct_type)
        if haskey(conversation_attributes, field) && haskey(attributes, conversation_attributes[field])
            push!(values, attributes[conversation_attributes[field]])
        else
            push!(values, default_value_of(fieldtype(inp_struct_type, field)))
        end
    end

    result = inp_struct_type(values...)
    # set the id right
    setfield!(result, id_field.fieldname, DbId(thing.iid))

    return result
end

function read_concept_from_struct(transaction::AbstractCoreTransaction,
    inp_struct_type::Type{<:Any},
    type::Type{<:AbstractThingType},
    iid::String = "")

    if iid == ""
        thing_type = get(ConceptManager(transaction), type, string(inp_struct_type))
        instances_of_type = get_instances(as_remote(thing_type, transaction))
    else
        instances_of_type = [get(ConceptManager(transaction), iid)]
    end

    # detect the id field inside the given struct type
    tuple_id_field = tuple_field_id(inp_struct_type)

    thing_attributs = Dict()
    for instance in instances_of_type
        # get attributes
        thing_attributs[instance] = Build_Types[]
        thing_attributs[instance] = vcat(thing_attributs[instance], get_has(transaction, instance))
        # get relations
        thing_attributs[instance] = vcat(thing_attributs[instance], get_relations(transaction, instance))
    end

    result = inp_struct_type[]
    for (instance,attributes) in thing_attributs
        push!(result, concept_to_struct(transaction,attributes, inp_struct_type, tuple_id_field, instance))
    end
    return result
end

####### write section contains new and update ##############
function write_struct_new(transaction::AbstractCoreTransaction,
    save_struct,
    type::Type{<:AbstractThingType})

    # get type of the struct and create an object in the database
    type = _get_type(transaction, save_struct, type)
    obj = _create_obj_from_type(transaction, type, save_struct)

    # take all fields of the struct and set attributes to the object
    thing = _set_attributes_of_entity!(transaction, obj, save_struct)

    return thing
end

function write_struct_update(transaction::AbstractCoreTransaction,
    write_struct,
    db_iid::DbId)

    # get the thing from the database
    obj = get(ConceptManager(transaction), db_iid.iid)

    # take all fields of the struct and set attributes to the object
    obj = _set_attributes_of_entity!(transaction, obj, write_struct)

    return obj
end

function write_struct(transaction::AbstractCoreTransaction,
    write_struct,
    type::Type{<:AbstractThingType} = EntityType)

    db_iid = _get_struct_iid(write_struct)
    thing = nothing

    if isempty(db_iid.iid)
        thing = write_struct_new(transaction, write_struct, type)
        # write back the iid to the struct
        db_iid.iid = thing.iid
    else
        thing = write_struct_update(transaction, write_struct, db_iid)
    end

    return thing
end

####### delete functionality #####################
function delete!(transaction::AbstractCoreTransaction, inp_struct::Any)
    struct_iid = _get_struct_iid(inp_struct)
    isempty(struct_iid.iid) && throw("Please provide the iid inside the struct type $(typeof(inp_struct))")
    thing = get(ConceptManager(transaction), struct_iid.iid)
    try
        delete(as_remote(thing,transaction))
    catch ex
        @info ex
    end
end

# Utility function
function _get_type(transaction::AbstractCoreTransaction, obj::Any, type::Type{<:AbstractThingType})
    thing_type = get(ConceptManager(transaction), type, string(typeof(obj)))
    return thing_type
end

function _get_type(transaction::AbstractCoreTransaction, obj::Type{<:Any}, type::Type{<:AbstractThingType})
    thing_type = get(ConceptManager(transaction), type, string(obj))
    return thing_type
end

function _create_obj_from_type(transaction::AbstractCoreTransaction, type::AbstractThingType, save_struct )
    obj = nothing
    if typeof(type) <: EntityType
        obj = create(as_remote(type, transaction))
    elseif typeof(type)<:AbstractAttributeType
        obj = put(as_remote(type, transaction), value(save_struct))
    else
        throw("Type requested not supported yet")
    end

    return obj
end

function _create_or_load_attribute(transaction::AbstractCoreTransaction, attribute_name::Symbol, value)
    result = nothing
    attr_type =  get(ConceptManager(transaction), AttributeType, string(attribute_name))
    attr = get(as_remote(attr_type, transaction), value)
    if attr === nothing
        result = put(as_remote(attr_type, transaction), value)
    else
        result = attr
    end
    return result
end

function _create_or_load_relation(transaction::AbstractCoreTransaction,
    thing::AbstractThing,
    owner_struct,
    relation_name::AbstractString,
    relation_struct)

    # make the relation type
    rel_type = RelationType(Label("", relation_name), false)
    # get the roles associated with the relation
    roles = get_relates(as_remote(rel_type, transaction))
    # filter the roles for the relation struct name
    role_for_relation_struct = filter(x-> lowercase(x.label.name) ==
                                        lowercase(string(typeof(relation_struct))), roles)

    role_for_owner_struct = filter(x-> lowercase(x.label.name) ==
                                        lowercase(string(typeof(owner_struct))), roles)

    # getting the thingtype of the relation_struct for writing it to the database
    relation_struct_type = _get_thingtype_for_role(transaction,
                                role_for_relation_struct[1],
                                relation_struct)

    # write relation struct to the databse
    rel_thing = write_struct(transaction, relation_struct, relation_struct_type)

    # get the relations according the roles selected. For now there should be only one relation
    # for a struct which will be stored.
    # TODO: build the ability to store more then one struct aka Vector of structs
    rels = Relation[]
    try
        rels = get_relations(transaction, thing, roles)
    catch ex
        occursin("Concept does not exist", ex.error_message) && throw("Please check your relation
        name $relation_name in your struct $owner_struct against the definition in the schema")
    end

    if !isempty(rels)
        @assert length(rels) == 1 "For now only one struct per relation $relation_name
                                    in $(string(owner_struct)) is supported"

        player_rel = get_players(transaction, rels[1], role_for_relation_struct)
        if length(player_rel) == 1
            if player_rel[1].iid != rel_thing.iid
                remove_player(transaction, rels[1], role_for_relation_struct[1], player_rel[1])
                add_player(transaction, rels[1], role_for_relation_struct[1], rel_thing)
            end
        else
            throw("something went wrong in updating the relation $relation_name
            for struct $(string(owner_struct))")
        end
    else
        # build new relations
        relation = create(as_remote(rel_type, transaction))
        # adding the relation struct to the give relation
        add_player(transaction, relation, role_for_relation_struct[1], rel_thing)
        # adding the owner struct to the give relation
        add_player(transaction, relation, role_for_owner_struct[1], thing)
        #TODO: Workout relates and get it work if more than one item is meant.
    end

    return nothing
end

function _set_attributes_of_entity!(transaction::AbstractCoreTransaction,
    obj::AbstractThing,
    save_struct)

    for field in fieldnames(typeof(save_struct))
        value = getproperty(save_struct,Symbol(field))

        if (typeof(value) <: Direct_Storables) && value !== nothing
            attr_type = get(ConceptManager(transaction), AttributeType, string(field))
            attr_old = get_has(transaction, obj, attr_type)
            attr = _create_or_load_attribute(transaction, field, value)

            if !(isempty(attr_old))
                if attr.value != attr_old[1].value
                    unset_has(transaction, obj, attr_old[1])
                end
            end

            set_has(transaction, obj, attr)
        elseif (typeof(value) <: Any) && typeof(value) != DbId && value !== nothing
            _create_or_load_relation(transaction, obj, save_struct, string(field), value)
        end
    end
    return obj
end

function value(save_struct::Any)
    return values(save_struct)
end

## The needed function should return the type of a Union type structure
## which is not Nothing but Nothing has to be there.
function _get_union_type(x::Union)
    typeof(x.b) == Union && throw("The Union type $x contains too many types")
    Nothing <: x || throw("One Nothing in your Union definition $x has to be there")
    (x.a == Nothing) ? x.b : x.a
end

function _resulting_fieldtype(inp_struct_type::Type{<:Any}, field::Symbol)
    field_type = fieldtype(inp_struct_type, field)
    type_of_field_type = typeof(field_type)

    inside_types = nothing
    if type_of_field_type == Union
        inside_types = _get_union_type(field_type)
    else
        inside_types = fieldtype(inp_struct_type, field)
    end
    return inside_types
end

function _get_thingtype_for_role(transaction::AbstractCoreTransaction,
    role::RoleType,
    inp_struct::Any)

   players = get_players(transaction, Label(role.label.scope, role.label.name))
   filter!(x->lowercase(x.label.name) == lowercase(string(typeof(inp_struct))), players)
   length(players) != 1 && throw("something went wrong detecting the ThingType from the role $role")
   return typeof(players[1])
end

max_stars_repo_name: Humans-of-Julia/TypeDBORM

max_issues_repo_name: Humans-of-Julia/TypeDBORM

max_forks_repo_name: Humans-of-Julia/TypeDBORM


text:
using StringDistances, Unicode, Test, Random

@testset "Distances" begin
	@testset "Hamming" begin
		@test Hamming()("martha", "marhta") ≈  2
		@test Hamming()("es an ", " vs an") ≈ 6
		@test Hamming()([1, 2, 3], [1,2, 4]) ≈ 1
		@inferred Hamming()("", "")
		@test ismissing(Hamming()("", missing))
	end

	@testset "Jaro" begin
		@test Jaro()("martha", "marhta") ≈  0.05555555555555547
		@test Jaro()("es an ", " vs an") ≈ 0.2777777777777777
		@test Jaro()(" vs an", "es an ") ≈ 0.2777777777777777
		@test Jaro()([1, 2, 3], [1,2, 4]) ≈ 0.2222222222222222
		@test Jaro()(graphemes("alborgów"), graphemes("amoniak")) == Jaro()("alborgów", "amoniak")
		@test Jaro()(" vs an", "es an ") ≈ 0.2777777777777777
		@test result_type(Jaro(), "hello", "world") == typeof(float(1))
		@inferred Jaro()("", "")
		@test ismissing(Jaro()("", missing))
	end

	@testset "Levenshtein" begin
		@test Levenshtein()("", "") == 0
		@test Levenshtein()("abc", "") == 3
		@test Levenshtein()("", "abc") == 3
		@test Levenshtein()("bc", "abc") == 1
		@test Levenshtein()("kitten", "sitting") == 3
		@test Levenshtein()("saturday", "sunday") == 3
		@test Levenshtein()("hi, my name is", "my name is") == 4
		@test Levenshtein()("a cat", "an act") == 3
		@test Levenshtein()("alborgów", "amoniak") == 6
		prefix = "my_prefix"
		@test Levenshtein()(prefix * "alborgów", prefix * "amoniak") == Levenshtein()("alborgów", "amoniak")
		@test Levenshtein()([1, 2, 3], [1, 2, 4]) == 1
		@test Levenshtein()(graphemes("alborgów"), graphemes("amoniak")) == Levenshtein()("alborgów", "amoniak")
		@test Levenshtein()("", "abc") == 3
		@test result_type(Levenshtein(), "hello", "world") == Int
		@inferred Levenshtein()("", "")
		@test ismissing(Levenshtein()("", missing))
	end

	@testset "OptimalStringAlignement" begin
		@test OptimalStringAlignement()("", "") == 0
		@test OptimalStringAlignement()("abc", "") == 3
		@test OptimalStringAlignement()("bc", "abc") == 1
		@test OptimalStringAlignement()("fuor", "four") == 1
		@test OptimalStringAlignement()("abcd", "acb") == 2
		@test OptimalStringAlignement()("cape sand recycling ", "edith ann graham") == 17
		@test OptimalStringAlignement()("jellyifhs", "jellyfish") == 2
		@test OptimalStringAlignement()("ifhs", "fish") == 2
		@test OptimalStringAlignement()("a cat", "an act") == 2
		@test OptimalStringAlignement()("a cat", "an abct") == 4
		@test OptimalStringAlignement()("a cat", "a tc") == 3
		@test OptimalStringAlignement()("abcdef", "abcxyf") == 2
		@test OptimalStringAlignement()("abcdef", "abcxyf"; max_dist = 2) == 2
		prefix = "my_prefix"
		@test OptimalStringAlignement()(prefix * "alborgów", prefix * "amoniak") == OptimalStringAlignement()("alborgów", "amoniak")
		@test OptimalStringAlignement()([1, 2, 3], [1,2, 4]) == 1
		@test OptimalStringAlignement()(graphemes("alborgów"), graphemes("amoniak")) == OptimalStringAlignement()("alborgów", "amoniak")
		@test OptimalStringAlignement()("bc", "abc") == 1
		@test result_type(OptimalStringAlignement(), "hello", "world") == Int
		@inferred OptimalStringAlignement()("", "")
		@test ismissing(OptimalStringAlignement()("", missing))
	end

	@testset "DamerauLevenshtein" begin
		@test DamerauLevenshtein()("", "") == 0
		@test DamerauLevenshtein()("CA", "ABC") == 2
		@test DamerauLevenshtein()("ABCDEF", "ABDCEF") == 1
		@test DamerauLevenshtein()("ABCDEF", "BACDFE") == 2
		@test DamerauLevenshtein()("ABCDEF", "ABCDE") == 1
		@test DamerauLevenshtein()("a cat", "an act") == 2
		@test DamerauLevenshtein()("a cat", "an abct") == 3
		@test DamerauLevenshtein()("a cat", "a tc") == 2
		prefix = "my_prefix"
		@test DamerauLevenshtein()(prefix * "alborgów", prefix * "amoniak") == DamerauLevenshtein()("alborgów", "amoniak")
		@test result_type(DamerauLevenshtein(), "hello", "world") == Int
		@inferred DamerauLevenshtein()("", "")
		@test ismissing(DamerauLevenshtein()("", missing))
	end

	@testset "RatcliffObershelp" begin
		@test RatcliffObershelp()("dixon", "dicksonx") ≈ 1 - 0.6153846153846154
		@test RatcliffObershelp()("alexandre", "aleksander") ≈ 1 - 0.7368421052631579
		@test RatcliffObershelp()("pennsylvania",  "pencilvaneya") ≈ 1 - 0.6666666666666
		@test RatcliffObershelp()("",  "pencilvaneya") ≈ 1.0
		@test RatcliffObershelp()("NEW YORK METS", "NEW YORK MEATS") ≈ 1 -  0.962962962963
		@test RatcliffObershelp()("Yankees",  "New York Yankees") ≈ 0.3913043478260869
		@test RatcliffObershelp()("New York Mets",  "New York Yankees") ≈ 0.24137931034482762
		@test RatcliffObershelp()([1, 2, 3], [1,2, 4]) ≈ 1/3
		@test RatcliffObershelp()(graphemes("alborgów"), graphemes("amoniak")) == RatcliffObershelp()("alborgów", "amoniak")
		@test RatcliffObershelp()("pennsylvania",  "pencilvaneya") ≈ 1 - 0.6666666666666
		@test result_type(RatcliffObershelp(), "hello", "world") == typeof(float(1))
		@inferred RatcliffObershelp()("", "")
		@test ismissing(RatcliffObershelp()("", missing))
	end

	@testset "QGram" begin
		@test QGram(1)("abc", "abc") == 0
		@test QGram(1)("", "abc") == 3
		@test QGram(1)("abc", "cba") == 0
		@test QGram(1)("abc", "ccc") == 4
		@test QGram(4)("aü☃", "aüaüafs") == 4
		@test QGram(2)(SubString("aü☃", 1, 4), SubString("aüaüafs", 1, 4)) == 2
		@test QGram(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ QGram(2)("alborgów", "amoniak")
		@test QGram(1)("abc", "cba") == 0
		@test result_type(QGram(1), "hello", "world") == Int
		@test ismissing(QGram(1)("", missing))
		@inferred QGram(1)("", "")
	end

	@testset "Cosine" begin
		@test isnan(Cosine(2)("", "abc"))
		@test Cosine(2)("abc", "ccc") ≈ 1 atol = 1e-4
		@test Cosine(2)("leia", "leela") ≈ 0.7113249 atol = 1e-4
		@test Cosine(2)([1, 2, 3], [1, 2, 4]) ≈ 0.5
		@test Cosine(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ Cosine(2)("alborgów", "amoniak")
		@test Cosine(2)("leia", "leela") ≈ 0.7113249 atol = 1e-4
		@test result_type(Cosine(2), "hello", "world") == typeof(float(1))
		@inferred Cosine(2)("", "")
		@test ismissing(Cosine(2)("", missing))
	end

	@testset "Jaccard" begin
		@test Jaccard(1)("", "abc") ≈ 1.0
		@test Jaccard(1)("abc", "ccc") ≈ 2/3 atol = 1e-4
		@test Jaccard(2)("leia", "leela") ≈ 0.83333 atol = 1e-4
		@test Jaccard(2)([1, 2, 3], [1, 2, 4]) ≈ 2/3 atol = 1e-4
		@test Jaccard(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ Jaccard(2)("alborgów", "amoniak")
		@test Jaccard(2)("leia", "leela") ≈ 0.83333 atol = 1e-4
		@test result_type(Jaccard(1), "hello", "world") == typeof(float(1))
		@inferred Jaccard(1)("", "")
		@test ismissing(Jaccard(1)("", missing))
	end

	@testset "SorensenDice" begin
		@test SorensenDice(1)("night", "nacht") ≈ 0.4 atol = 1e-4
		@test SorensenDice(2)("night", "nacht") ≈ 0.75 atol = 1e-4
		@test SorensenDice(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ SorensenDice(2)("alborgów", "amoniak")
		@test SorensenDice(2)("night", "nacht") ≈ 0.75 atol = 1e-4
		@test result_type(SorensenDice(1), "hello", "world") == typeof(float(1))
		@inferred SorensenDice(1)("", "")
		@test ismissing(SorensenDice(1)("", missing))
	end

	@testset "Overlap" begin
		@test Overlap(1)("night", "nacht") ≈ 0.4 atol = 1e-4
		@test Overlap(1)("context", "contact") ≈ .2 atol = 1e-4
		@test Overlap(1)("context", "contact") ≈ .2 atol = 1e-4
		@test result_type(Overlap(1), "hello", "world") == typeof(float(1))
		@inferred Overlap(1)("", "")
		@test ismissing(Overlap(1)("", missing))
	end

	@testset "MorisitaOverlap" begin
		# overlap for 'n', 'h', and 't' and 5 q-grams per string:
		@test MorisitaOverlap(1)("night", "nacht") == 0.4 # 1.0-((2*3)/(5*5/5 + 5*5/5))

		# overlap for 'o', 'n', 2-overlap for 'c' and 't' and 7 unique q-grams in total so multiplicity vectors
		# ms1 = [1, 1, 1, 2, 1, 1, 0]
		# ms2 = [2, 1, 1, 2, 0, 0, 1]
		# sum(ms1 .* ms2) = 8, sum(ms1 .^ 2) = 9, sum(ms2 .^ 2) = 11, sum(ms1) = 7, sum(ms2) = 7
		@test MorisitaOverlap(1)("context", "contact") ≈ .2 atol = 1e-4 # 1.0-((2*8)/(9*7/7 + 11*7/7)) = 16/20
		@test MorisitaOverlap(1)("context", "contact") ≈ .2 atol = 1e-4

		# Multiplicity vectors for 2-grams "co", "on", "nt", "te", "ex", "xt", "ta", "ac", "ct"
		# ms1 = [1, 1, 1, 1, 1, 1, 0, 0, 0]
		# ms2 = [1, 1, 1, 0, 0, 0, 1, 1, 1]
		# sum(ms1 .* ms2) = 3, sum(ms1 .^ 2) = 6, sum(ms2 .^ 2) = 6, sum(ms1) = 6, sum(ms2) = 6
		@test MorisitaOverlap(2)("context", "contact") == 0.5 # 1.0-((2*3)/(6*6/6 + 6*6/6))

		@test result_type(MorisitaOverlap(1), "hello", "world") == typeof(float(1))
		@inferred MorisitaOverlap(1)("", "")
		@test ismissing(MorisitaOverlap(1)("", missing))
	end

	@testset "NMD" begin
		# m(s1) = [1, 1, 1, 1, 1, 0, 0], m(s2) = [1, 0, 0, 1, 1, 1, 1]
		@test NMD(1)("night", "nacht") == 0.4 # (7-5)/5

		# ms1 = [1, 1, 1, 2, 1, 1, 0]
		# ms2 = [2, 1, 1, 2, 0, 0, 1]
		@test NMD(1)("context", "contact") ≈ 0.2857 atol = 1e-4 # ((2+1+1+2+1+1+1)-7)/(7)
		@test NMD(1)("context", "contact") ≈ 0.2857 atol = 1e-4

		# ms1 = [1, 1, 1, 1, 1, 1, 0, 0, 0]
		# ms2 = [1, 1, 1, 0, 0, 0, 1, 1, 1]
		@test NMD(2)("context", "contact") == 0.5 # ((1+1+1+1+1+1+1+1+1)-6)/6

		@test result_type(NMD(1), "hello", "world") == typeof(float(1))
		@inferred NMD(1)("", "")
		@test ismissing(NMD(1)("", missing))
	end

	@testset "QGramDict and QGramSortedVector counts qgrams" begin
		# To get something we can more easily compare to:
		stringify(p::Pair{<:AbstractString, <:Integer}) = (string(first(p)), last(p))
		stringify(p::Pair{V, <:Integer}) where {S<:AbstractString,V<:AbstractVector{S}} = (map(string, first(p)), last(p))
		sortedcounts(qc) = sort(collect(qc.counts), by = first)
		totuples(qc) = map(stringify, sortedcounts(qc))

		s1, s2   = "arnearne", "arnebeda"

		qd1, qd2 = QGramDict(s1, 2), QGramDict(s2, 2)
		@test totuples(qd1) == [("ar", 2), ("ea", 1), ("ne", 2), ("rn", 2)]
		@test totuples(qd2) == [("ar", 1), ("be", 1), ("da", 1), ("eb", 1), ("ed", 1), ("ne", 1), ("rn", 1)]

		qc1, qc2 = QGramSortedVector(s1, 2), QGramSortedVector(s2, 2)
		@test totuples(qc1) == [("ar", 2), ("ea", 1), ("ne", 2), ("rn", 2)]
		@test totuples(qc2) == [("ar", 1), ("be", 1), ("da", 1), ("eb", 1), ("ed", 1), ("ne", 1), ("rn", 1)]

		s3 = "rgówów"
		qd3a = QGramDict(s3, 2)
		@test totuples(qd3a) == [("gó", 1), ("rg", 1), ("wó", 1), ("ów", 2)]

		qd3b = QGramDict(graphemes(s3), 2)
		@test totuples(qd3b) == [(["g", "ó"], 1), (["r", "g"], 1), (["w", "ó"], 1), (["ó", "w"], 2)]

		qc3a = QGramSortedVector(s3, 2)
		@test totuples(qc3a) == [("gó", 1), ("rg", 1), ("wó", 1), ("ów", 2)]

		qd3b = QGramDict(graphemes(s3), 2)
		@test totuples(qd3b) == [(["g", "ó"], 1), (["r", "g"], 1), (["w", "ó"], 1), (["ó", "w"], 2)]
	end

	function partlyoverlappingstrings(sizerange, chars = nothing)
		l = rand(sizerange)
		str1 = isnothing(chars) ? randstring(l) : randstring(chars, l)
		ci1 = thisind(str1, rand(1:l))
		ci2 = thisind(str1, rand(ci1:l))
		copied = join(str1[ci1:ci2])
		prefix = isnothing(chars) ? randstring(ci1-1) : randstring(chars, ci1-1)
		slen = l - length(copied) - length(prefix)
		suffix = isnothing(chars) ? randstring(slen) : randstring(chars, slen)
		return str1, (prefix * copied * suffix)
	end

	@testset "Precalculation on unicode strings" begin
		Chars = vcat(map(collect, ["δσμΣèìòâôîêûÊÂÛ", 'a':'z', '0':'9'])...)
		for _ in 1:100
			qlen = rand(2:5)
			str1, str2 = partlyoverlappingstrings(6:100, Chars)
			dist = Jaccard(qlen)

			qd1 = QGramDict(str1, qlen)
			qd2 = QGramDict(str2, qlen)
			@test dist(str1, str2) == dist(qd1, qd2)

			qd1b = QGramDict(graphemes(str1), qlen)
			qd2b = QGramDict(graphemes(str2), qlen)
			@test dist(str1, str2) == dist(qd1b, qd2b)

			qc1 = QGramSortedVector(str1, qlen)
			qc2 = QGramSortedVector(str2, qlen)
			@test dist(str1, str2) == dist(qc1, qc2)

			qc1b = QGramSortedVector(graphemes(str1), qlen)
			qc2b = QGramSortedVector(graphemes(str2), qlen)
			@test dist(str1, str2) == dist(qc1b, qc2b)
		end
	end

	@testset "QGram distance on short strings" begin
		@test isnan(Overlap(2)( "1",  "2"))
		@test isnan(Jaccard(3)("s1", "s2"))
		@test isnan(Cosine(5)( "s1", "s2"))

		@test !isnan(Overlap(2)( "s1",  "s2"))
		@test !isnan(Jaccard(3)("st1", "st2"))
		@test !isnan(Cosine(5)( "stri1", "stri2"))

		@test !isnan(Jaccard(3)("st1", "str2"))
		@test !isnan(Jaccard(3)("str1", "st2"))
	end

	@testset "Differential testing of String, QGramDict, and QGramSortedVector" begin
		for D in [QGram, Cosine, Jaccard, SorensenDice, Overlap, MorisitaOverlap, NMD]
			for _ in 1:100
				qlen = rand(2:9)
				dist = D(qlen)
				str1, str2 = partlyoverlappingstrings(10:10000)

				# QGramDict gets same result as for standard string
				qd1 = QGramDict(str1, qlen)
				qd2 = QGramDict(str2, qlen)
				expected = dist(str1, str2)
				@test expected == dist(qd1, qd2)

				# QGramSortedVector gets same result as for standard string
				qc1 = QGramSortedVector(str1, qlen)
				qc2 = QGramSortedVector(str2, qlen)
				@test expected == dist(qc1, qc2)
			end
		end
	end

	strings = [
	("martha", "marhta"),
	("dwayne", "duane") ,
	("dixon", "dicksonx"),
	("william", "williams"),
	("", "foo"),
	("a", "a"),
	("abc", "xyz"),
	("abc", "ccc"),
	("kitten", "sitting"),
	("saturday", "sunday"),
	("hi, my name is", "my name is"),
	("alborgów", "amoniak"),
	("cape sand recycling ", "edith ann graham"),
	( "jellyifhs", "jellyfish"),
	("ifhs", "fish"),
	("leia", "leela"),
	]

	solutions = ((Levenshtein(), [2  2  4  1  3  0  3  2  3  3  4  6 17  3  3  2]),
			(OptimalStringAlignement(), [1  2  4  1  3  0  3  2  3  3  4  6 17  2  2  2]),
			(Jaro(), [0.05555556 0.17777778 0.23333333 0.04166667 1.00000000 0.00000000 1.00000000 0.44444444 0.25396825 0.2805556 0.2285714 0.48809524 0.3916667 0.07407407 0.16666667 0.21666667]),
			(QGram(1), [0   3   3   1 3  0   6   4   5   4   4  11  14   0   0   3]),
			(QGram(2), [  6   7   7   1 2 0   4   4   7   8   4  13  32   8   6   5]),
			(Jaccard(1), [0.0 0.4285714 0.3750000 0.1666667       1.0 0.0 1.0000000 0.6666667 0.5714286 0.3750000 0.2000000 0.8333333 0.5000000 0.0 0.0 0.2500000]),
			(Jaccard(2),  [ 0.7500000 0.8750000 0.7777778 0.1428571       1.0     NaN 1.0000000 1.0000000 0.7777778 0.8000000 0.3076923 1.0000000 0.9696970 0.6666667 1.0000000 0.8333333]),
			(Cosine(2), [0.6000000 0.7763932 0.6220355 0.0741799  NaN  NaN 1.0000000 1.0000000 0.6348516 0.6619383 0.1679497 1.0000000 0.9407651 0.5000000 1.0000000 0.7113249]))
	# Test with R package StringDist
	for x in solutions
		dist, solution = x
		for i in eachindex(solution)
			if isnan(dist(strings[i]...))
				@test isnan(solution[i])
			else
				@test dist(strings[i]...) ≈ solution[i] atol = 1e-4
			end
		end
	end
	# test  RatcliffObershelp
	solution = [83, 73, 62, 93, 0, 100, 0, 33, 62, 71, 83, 27, 33, 78, 50, 67]
	for i in eachindex(strings)
		@test round(Int, (1 - RatcliffObershelp()(strings[i]...)) * 100) ≈ solution[i] atol = 1e-4
	end

	# test max_dist
	for i in eachindex(strings)
		d = Levenshtein()(strings[i]...)
		@test Levenshtein()(strings[i]...; max_dist = d) == d
		d = OptimalStringAlignement()(strings[i]...)
		@test OptimalStringAlignement()(strings[i]...; max_dist = d) == d
	end
end

#= R test
library(stringdist)
strings = matrix(data = c(
"martha", "marhta",
"dwayne", "duane",
"dixon", "dicksonx",
"william", "williams",
"", "foo",
"a", "a",
"abc", "xyz",
"abc", "ccc",
"kitten", "sitting",
"saturday", "sunday",
"hi, my name is", "my name is",
"alborgów", "amoniak",
"cape sand recycling ", "edith ann graham",
 "jellyifhs", "jellyfish",
"ifhs", "fish",
"leia", "leela"),
nrow = 2
)
stringdist(strings[1,], strings[2,], method = "jw", p = 0)
stringdist(strings[1,], strings[2,], method = "jw", p = 0.1)
stringdist(strings[1,], strings[2,], method = "qgram", q = 1)

=#

#= Fuzzywuzzy usesRatcliffObershelp  if python-Levenshtein not installed, fuzzywuzzy uses RatcliffObershelp)
from fuzzywuzzy import fuzz
strings = [
("martha", "marhta"),
("dwayne", "duane") ,
("dixon", "dicksonx"),
("william", "williams"),
("", "foo"),
("a", "a"),
("abc", "xyz"),
("abc", "ccc"),
("kitten", "sitting"),
("saturday", "sunday"),
("hi, my name is", "my name is"),
("alborgów", "amoniak"),
("cape sand recycling ", "edith ann graham"),
( "jellyifhs", "jellyfish"),
("ifhs", "fish"),
("leia", "leela"),
]
for x in strings:
   print(fuzz.ratio(x[0], x[1]))
=#


text:
# Julia wrapper for header: /usr/include/libavutil/file.h
# Automatically generated using Clang.jl wrap_c, version 0.0.0

export
    av_file_map,
    av_file_unmap,
    av_tempfile

function av_file_map(filename, bufptr, size, log_offset::Integer, log_ctx)
    ccall((:av_file_map, libavutil), Cint, (Cstring, Ptr{Ptr{UInt8}}, Ptr{Csize_t}, Cint, Ptr{Cvoid}), filename, bufptr, size, log_offset, log_ctx)
end

function av_file_unmap(bufptr, size::Csize_t)
    ccall((:av_file_unmap, libavutil), Cvoid, (Ptr{UInt8}, Csize_t), bufptr, size)
end

function av_tempfile(prefix, filename, log_offset::Integer, log_ctx)
    ccall((:av_tempfile, libavutil), Cint, (Cstring, Ptr{Cstring}, Cint, Ptr{Cvoid}), prefix, filename, log_offset, log_ctx)
end


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder, Pkg

name = "GMAT"
version = v"2020.0.0"

# Collection of sources required to complete build
sources = [
    ArchiveSource("https://downloads.sourceforge.net/project/gmat/GMAT/GMAT-R2020a/GMAT-src-R2020a.zip", "943f403ac04d958b313b1d99d64fd09e3fa8e4c65363809d5bb88dd8c66e43e4"),
    DirectorySource("./bundled"),
]

# Bash recipe for building across all platforms
script = raw"""
cd $WORKSPACE/srcdir
apk add --upgrade cmake --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main
cd GMAT-R2020a/
cp -r $WORKSPACE/srcdir/patches/cmake .
dos2unix plugins/EstimationPlugin/src/base/measurement/Ionosphere/Ionosphere.hpp
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0001-Remove-hard-coded-CSPICE-paths.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0002-Remove-MSVC-flags.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0003-Fix-non-portable-cast.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0004-Use-std-chrono-on-all-platforms.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0005-Use-Linux-typedefs-for-cross-compile.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0006-Use-standard-CMake-boost-module.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0007-Remove-explicit-Ws2_32-linking.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0008-Use-standard-install-locations.patch"
atomic_patch -p1 "${WORKSPACE}/srcdir/patches/0009-FreeBSD-compat.patch"
mkdir builddir
cd builddir/
cmake -DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} -DCMAKE_BUILD_TYPE=Release \
  -DGMAT_INCLUDE_GUI=OFF \
  -DCSPICE_LIB=${libdir}/libcspice.${dlext} -DCSPICE_DIR=${prefix} -DCSPICE_INCLUDE_DIR=${includedir} \
  -DF2C_DIR=${includedir} \
  ..
make -j${nproc}
make install
"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line
platforms = expand_cxxstring_abis(supported_platforms())

# The products that we will ensure are always built
products = [
    LibraryProduct("libGmatFunction", :libgmatfunction, "plugins"),
    LibraryProduct("libGmatUtil", :libgmatutil),
    LibraryProduct("libScriptTools", :libgmatscripttools, "plugins"),
    LibraryProduct("libCInterface", :libgmatcinterface),
    LibraryProduct("libGmatBase", :libgmatbase),
    # FIXME: Enable once Python_jll is available for Windows
    # LibraryProduct("libPythonInterface", :libgmatpythoninterface, "plugins"),
    LibraryProduct("libGmatEstimation", :libgmatestimation, "plugins"),
    LibraryProduct("libDataInterface", :libgmatdatainterface, "plugins"),
    LibraryProduct("libYukonOptimizer", :libgmatyukonoptimizer, "plugins"),
    LibraryProduct("libThrustFile", :libgmatthrustfile, "plugins"),
    LibraryProduct("libEphemPropagator", :libgmatephempropagator, "plugins"),
    LibraryProduct("libExtraPropagators", :libgmatextrapropagators, "plugins"),
    LibraryProduct("libNewParameters", :libgmatnewparameters, "plugins"),
    LibraryProduct("libFormation", :libgmatformation, "plugins"),
    LibraryProduct("libPolyhedronGravity", :libgmatpolyhedrongravity, "plugins"),
    LibraryProduct("libStation", :libgmatstation, "plugins"),
    LibraryProduct("libSaveCommand", :libgmatsavecommand, "plugins"),
    LibraryProduct("libEventLocator", :libgmateventlocator, "plugins"),
    LibraryProduct("libProductionPropagators", :libgmatproductionpropagators, "plugins"),
    LibraryProduct("libEKF", :libgmatekf, "plugins"),
    ExecutableProduct("GmatConsole", :gmatconsole)
]

# Dependencies that must be installed before this package can be built
dependencies = [
    Dependency(PackageSpec(name="CSPICE_jll", uuid="07f52509-e9d9-513c-a20d-3b911885bf96"))
    Dependency(PackageSpec(name="Python_jll", uuid="93d3a430-8e7c-50da-8e8d-3dfcfb3baf05"))
    Dependency(PackageSpec(name="Xerces_jll", uuid="637d83c4-b86a-5d90-b82d-5cf0573a8cfc"))
    BuildDependency(PackageSpec(name="boost_jll", uuid="28df3c45-c428-5900-9ff8-a3135698ca75"))
]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; preferred_gcc_version = v"7.1.0")

max_stars_repo_name: sharanry/Yggdrasil

max_issues_repo_name: sharanry/Yggdrasil

max_forks_repo_name: sharanry/Yggdrasil


text:
module DispatchedTuples

import Base

export AbstractDispatchedTuple, dispatch

export DispatchedTuple
export DispatchedSet

struct NoDefaults end

"""
    AbstractDispatchedTuple{T <: Tuple, D}

An abstract dispatch tuple type, for sub-typing
dispatched tuples.
"""
abstract type AbstractDispatchedTuple{T <: Tuple, D} end

unwrap_pair(tup) = eltype(tup) <: Pair ?
    map(x->(x.first, x.second), tup) : tup

function match_expr(dt, TT, T)
    expr = quote tuple() end
    found_match = false
    for (i,k) in enumerate(fieldnames(TT))
        if first_eltype(fieldtype(TT, i)) == T
            found_match = true
            push!(expr.args[2].args, :(dt.tup[$i][2]))
        end
    end
    return expr, found_match
end

function match_expr_val(dt, TT, T)
    match_count = 0
    expr = quote end
    for (i,k) in enumerate(fieldnames(TT))
        if first_eltype(fieldtype(TT, i)) == T
            match_count += 1
            expr = :(dt.tup[$i][2])
        end
    end
    return expr, match_count
end

#####
##### DispatchedTuple
#####

"""
    DispatchedTuple(tup[, default_value])

A dispatch-able tuple.

`DispatchedTuple` takes a `Tuple` of `Pair`s, where the
`first` field of the `Pair` (the "key") is **an instance
of the type you want to dispatch on**. The `second` field
of the `Pair` is the quantity (the "value", which can be
anything) returned by `dispatch`.

If a `DispatchedTuple` has non-unique keys, then all values
are returned in the returned `Tuple`.

The second (optional) argument to `DispatchedTuple` is a
default value, which is returned for any unrecognized keys.
If the default value is not given, then `dispatch` on an
unregistered key will return an empty tuple.

For convenience, `DispatchedTuple` can alternatively take a
`Tuple` of two-element `Tuple`s.

## Example

```julia
using DispatchedTuples
struct Foo end;
struct Bar end;
struct Baz end;

dtup = DispatchedTuple((
   Pair(Foo(), 1),
   Pair(Foo(), 2),
   Pair(Bar(), 3),
));

dispatch(dtup, Foo()) # returns (1, 2)
dispatch(dtup, Bar()) # returns (3,)
dispatch(dtup, Baz()) # returns ()
```
"""
struct DispatchedTuple{T,D} <: AbstractDispatchedTuple{T, D}
    tup::T
    default::D
    function DispatchedTuple(tup_in::T, default=NoDefaults()) where {T<:Tuple}
        tup = unwrap_pair(tup_in)
        return new{typeof(tup), typeof(default)}(tup, default)
    end
end

# Accept vararg Pairs:
DispatchedTuple(p...; default=NoDefaults()) = DispatchedTuple(Tuple(p), default)

"""
    dispatch(::DispatchedTuple, type_instance)

Dispatch on the [`DispatchedTuple`](@ref), based
on the instance of the input type `type_instance`.
"""
@generated function dispatch(dt::DispatchedTuple{TT, NoDefaults}, ::T) where {TT, T}
    expr, found_match = match_expr(dt, TT, T)
    expr
end

@generated function dispatch(dt::DispatchedTuple{TT,D}, ::T) where {TT, D, T}
    expr, found_match = match_expr(dt, TT, T)
    if !found_match
        push!(expr.args[2].args, :(dt.default))
    end
    expr
end

#####
##### DispatchedSet
#####

"""
    DispatchedSet(tup[, default_value])

Similar to [`DispatchedTuple`](@ref), except:
 - keys must be unique.
 - returns the value, and not a tuple of values.
 - throws an error in `dispatch` if the key is not unique (without a default value).
"""
struct DispatchedSet{T,D} <: AbstractDispatchedTuple{T, D}
    tup::T
    default::D
    function DispatchedSet(tup_in::T, default=NoDefaults()) where {T<:Tuple}
        tup = unwrap_pair(tup_in)
        return new{typeof(tup), typeof(default)}(tup, default)
    end
end

# Accept vararg Pairs:
DispatchedSet(p...; default=NoDefaults()) = DispatchedSet(Tuple(p), default)

"""
    dispatch(::DispatchedSet, type_instance)

Dispatch on the [`DispatchedSet`](@ref), based
on the instance of the input type `type_instance`.
"""
@generated function dispatch(dt::DispatchedSet{TT, NoDefaults}, ::T) where {TT, T}
    expr, match_count = match_expr_val(dt, TT, T)
    if match_count == 0
        return :(throw(error("No method dispatch defined for type $T")))
    elseif match_count > 1
        return :(throw(error("DispatchedSet has non-unique keys for type $T")))
    else
        return expr
    end
end

@generated function dispatch(dt::DispatchedSet{TT,D}, ::T) where {TT, D, T}
    expr, match_count = match_expr_val(dt, TT, T)
    if match_count == 0
        return :(dt.default)
    elseif match_count > 1
        return :(throw(error("DispatchedSet has non-unique keys for type $T")))
    else
        return expr
    end
end

# Nested dispatch calls:
dispatch(dt::AbstractDispatchedTuple, a, b...) = dispatch(dispatch(dt, a), b...)

# Interface / extending:
Base.isempty(dt::AbstractDispatchedTuple) = Base.isempty(dt.tup)
Base.length(dt::AbstractDispatchedTuple) = Base.length(dt.tup)
Base.map(f, dt::AbstractDispatchedTuple) = Base.map(f, dt.tup)
Base.keys(dt::AbstractDispatchedTuple) = map(x->x[1], dt)
Base.values(dt::AbstractDispatchedTuple) = map(x->x[2], dt)
Base.getindex(dt::AbstractDispatchedTuple, e) = dispatch(dt, e)
Base.iterate(dt::AbstractDispatchedTuple, state = 1) = Base.iterate(dt.tup, state)

# Note: this is not GPU-friendly
show_default(io::IO, dt::AbstractDispatchedTuple, any) = println(io, "  default => $(dt.default)")

show_default(io::IO, dt::DispatchedTuple, ::NoDefaults) = println(io, "  default => ()")
show_default(io::IO, dt::DispatchedSet, ::NoDefaults) = println(io, "  default => error")

function Base.show(io::IO, dt::AbstractDispatchedTuple)
    show(io, typeof(dt))
    print(io, " with $(length(dt)) entries:")
    println(io)
    foreach(dt) do tup
        print(io, "  ")
        show(io, Pair(tup...))
        println(io)
    end
    show_default(io, dt, dt.default)
end

# Helper
first_eltype(::Type{Tuple{T, V}}) where {T,V} = T

end # module


text:
using HPAT
using HDF5
using Base.Test
using CompilerTools
using ParallelAccelerator

function create_dataset_small(dataset_name)
    wcs_item_sk = [2,3,5,7,9,11,1,1,9]
    wcs_user_sk = [3,3,3,3,8,2,2,3,3]

    i_item_sk = [1,2,2,2,2,6,7,8,9]
    i_category_id = [3,3,3,3,8,2,2,3,3]
    i_category = [2,3,4,4,8,8,7,7,7]

    c_customer_sk = [1,2,2,2,8,6,7,8,9]
    c_current_cdemo_sk = [3,3,3,3,8,2,2,3,3]

    cd_demo_sk = [1,12,2,2,2,3,3,8,9]
    cd_gender = [3,3,3,3,8,9,9,9,9]
    cd_education_status = [2,7,7,4,8,8,7,7,7]

    file_name = dataset_name
    h5write(file_name ,"/wcs_item_sk", wcs_item_sk)
    h5write(file_name ,"/wcs_user_sk", wcs_user_sk)

    h5write(file_name ,"/i_item_sk", i_item_sk)
    h5write(file_name ,"/i_category_id",i_category_id)
    h5write(file_name ,"/i_category", i_category)

    h5write(file_name ,"/c_customer_sk", c_customer_sk)
    h5write(file_name ,"/c_current_cdemo_sk", c_current_cdemo_sk)

    h5write(file_name ,"/cd_demo_sk", cd_demo_sk)
    h5write(file_name ,"/cd_gender", cd_gender)
    h5write(file_name ,"/cd_education_status", cd_education_status)
end

@acc hpat function q05(category, education, gender, file_name)
    web_clickstreams = DataSource(DataTable{:wcs_item_sk=Int64, :wcs_user_sk=Int64}, HDF5, file_name)
    item = DataSource(DataTable{:i_item_sk=Int64,:i_category_id=Int64,:i_category=Int64}, HDF5, file_name)
    customer = DataSource(DataTable{:c_customer_sk=Int64,:c_current_cdemo_sk=Int64}, HDF5, file_name)
    customer_demographics = DataSource(DataTable{:cd_demo_sk=Int64,:cd_gender=Int64,:cd_education_status=Int64}, HDF5, file_name)
    # Used for Not NULL
    web_clickstreams = web_clickstreams[:wcs_item_sk>typemin(Int32)]

    user_items = join(web_clickstreams, item, :wcs_item_sk==:i_item_sk, :user_items_sk)

    user_clicks_in_cat = aggregate(user_items, :wcs_user_sk, :clicks_in_category = sum(:i_category==category),
                                                         :clicks_in_1 = sum(:i_category_id==1),
                                                         :clicks_in_2 = sum(:i_category_id==2),
                                                         :clicks_in_3 = sum(:i_category_id==3),
                                                         :clicks_in_4 = sum(:i_category_id==4),
                                                         :clicks_in_5 = sum(:i_category_id==5),
                                                         :clicks_in_6 = sum(:i_category_id==6),
                                                         :clicks_in_7 = sum(:i_category_id==7))
    customer_clicks = join(user_clicks_in_cat, customer, :wcs_user_sk==:c_customer_sk, :customer_clicks_sk)
    customer_demo_clicks = join(customer_clicks, customer_demographics, :c_current_cdemo_sk==:cd_demo_sk , :customer_demo_clicks_sk)
    return customer_demo_clicks[:customer_clicks_sk], customer_demo_clicks[:customer_demo_clicks_sk] , customer_demo_clicks[:clicks_in_2], customer_demo_clicks[:clicks_in_3]
end

println("     Testing Query 05[small]")
create_dataset_small("test_q05.hdf5")
return_arr= q05(3, 7, 9, "test_q05.hdf5")

@test return_arr[1] == [2,2,2,8,2,2,2,8,8]
@test return_arr[2] == [3,3,3,3,3,3,3,3,8]
@test return_arr[3] == [0,0,0,0,0,0,0,0,0]
@test return_arr[4] == [1,1,1,1,1,1,1,1,1]

rm("test_q05.hdf5")