text:
# This file is a part of TypeDBClient.  License is MIT: https://github.com/Humans-of-Julia/TypeDBClient.jl/blob/main/LICENSE
using Dates: length
using Base: func_for_method_checked
# This file is a part of TypeDBClient.  License is MIT: https://github.com/Humans-of-Julia/TypeDBClient.jl/blob/main/LICENSE

using Behavior
using TypeDBClient
using Dates

g = TypeDBClient

# Utility functions
function _attribute_subtypes(context, attribute_label_name, value_type::g.EnumType, contain::Bool; debug::Bool= false)
    types_inp = [db[1] for db in context.datatable]
    result_bools = Bool[]

    attr = g.get(ConceptManager(context[:transaction]), AttributeType, attribute_label_name)
    sub_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))

    # Print out subtypes if needed
    if debug
        @info "Subtypes:"
        println.(sub_types)
        @info "should be:"
        println.(types_inp)
    end

    res = filter(x->g.proto_value_type(x) == value_type ||
                    g.proto_value_type(x) == VALUE_TYPE.OBJECT, sub_types)

    for i in 1:length(types_inp)
        inp_type = g.get(ConceptManager(context[:transaction]), AttributeType, types_inp[i])
        in_types = contain ? in(inp_type, res) : !in(inp_type, res)
        push!(result_bools, in_types)
    end
    return result_bools
end

# Scenario: Attribute types can be created
@then("attribute(name) is null: false") do context
    context[:attr_name] = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    @expect context[:attr_name] !== nothing
end

@then("put attribute type: is-open, with value type: boolean") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "is-open", VALUE_TYPE.BOOLEAN)
end

@then("attribute(is-open) get value type: boolean") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    @expect attr.label.name == "is-open"
end

# Scenario: Attribute types can be created with value class long
@then("attribute(age) get value type: long") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    comparison = typeof(attr) == AttributeType{VALUE_TYPE.LONG}
    @expect comparison
end

# Scenario: Attribute types can be created with value class double
@when("put attribute type: rating, with value type: double") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "rating", VALUE_TYPE.DOUBLE)
end

@then("attribute(rating) get value type: double") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    comparison = typeof(attr) == AttributeType{VALUE_TYPE.DOUBLE}
    @expect comparison
end

# Scenario: Attribute types can be created with value class string
@then("attribute(name) get value type: string") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    comparison = typeof(attr) == AttributeType{VALUE_TYPE.STRING}
    @expect comparison
end

# Scenario: Attribute types with value type string and regular expression can be created
@then("attribute(email) as(string) get regex: \\S+@\\S+\\.\\S+") do context
    attr_email = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    regex = g.get_regex(g.as_remote(attr_email, context[:transaction]))
    @expect regex == "\\S+@\\S+\\.\\S+"
end

# Scenario: Attribute types can be created with value class datetime
@then("put attribute type: timestamp, with value type: datetime") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "timestamp", VALUE_TYPE.DATETIME)
end

@then("attribute(timestamp) get value type: datetime") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    comparison = typeof(attr) == AttributeType{VALUE_TYPE.DATETIME}
    @expect comparison
end

# Scenario: Attribute types can be deleted
@then("attribute(age) is null: false") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    @expect attr !== nothing
end

@when("delete attribute type: age") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    g.delete(g.as_remote(attr, context[:transaction]))
end

@then("attribute(age) is null: true") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    @expect attr === nothing
end

@then("attribute(attribute) get subtypes do not contain:") do context
    types_inp = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "attribute")
    sub_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for i in 1:length(types_inp)
        inp_type = attr = g.get(ConceptManager(context[:transaction]), AttributeType, types_inp[i])
        @expect !in(inp_type, sub_types)
    end
end

@when("delete attribute type: name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    g.delete(g.as_remote(attr, context[:transaction]))
end

@then("attribute(name) is null: true") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    @expect attr === nothing
end

# Scenario: Attribute types that have instances cannot be deleted
@then("delete attribute type: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.delete(g.as_remote(attr, context[:transaction]))
    catch ex
        @expect ex !== nothing
    end
end

# Scenario: Attribute types can change labels
@then("attribute(name) get label: name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    @expect attr.label.name == "name"
end

@when("attribute(name) set label: username") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    g.set_label(g.as_remote(attr_name, context[:transaction]), "username")
end

@then("attribute(username) is null: false") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    @expect attr_name !== nothing
end

@then("attribute(username) get label: username") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    @expect attr_name.label.name == "username"
end

@then("attribute(username) set label: email") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    g.set_label(g.as_remote(attr_name, context[:transaction]), "email")
end

@then("attribute(username) is null: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    @expect attr_name === nothing
end

@then("attribute(email) is null: false") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    @expect attr_name !== nothing
end

@then("attribute(email) get label: email") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    @expect attr_name.label.name == "email"
end

# Scenario: Attribute types can be set to abstract
@then("attribute(name) set abstract: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    g.set_abstract(g.as_remote(attr_name, context[:transaction]))
end

@then("attribute(name) is abstract: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    res_abstract = g.is_abstract(g.as_remote(attr_name, context[:transaction]))
    @expect res_abstract
end

@then("attribute(name) as(string) put: alice; throws exception") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.put(g.as_remote(attr_name, context[:transaction]), "alice")
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(email) is abstract: false") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    res_abstract = g.is_abstract(g.as_remote(attr_name, context[:transaction]))
    @expect res_abstract === false
end

@when("attribute(email) set abstract: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    res_abstract = g.set_abstract(g.as_remote(attr_name, context[:transaction]))
end

@then("attribute(email) is abstract: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    res_abstract = g.is_abstract(g.as_remote(attr_name, context[:transaction]))
    @expect res_abstract
end

@then("attribute(email) as(string) put: alice@email.com; throws exception") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    try
        g.put(g.as_remote(attr_name, context[:transaction]), "alice@email.com")
    catch ex
        @expect ex !== nothing
    end
end

# Scenario: Attribute types can be subtypes of other attribute types
@when("put attribute type: first-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "first-name", VALUE_TYPE.STRING)
end

@when("put attribute type: last-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "last-name", VALUE_TYPE.STRING)
end

@when("put attribute type: real-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "real-name", VALUE_TYPE.STRING)
end

@when("attribute(real-name) set abstract: true") do context
    attr_name = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    res_abstract = g.set_abstract(g.as_remote(attr_name, context[:transaction]))
end

@when("attribute(first-name) set supertype: real-name") do context
    attr_first_name = g.get(ConceptManager(context[:transaction]), AttributeType, "first-name")
    attr_real_name = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    g.set_supertype(g.as_remote(attr_first_name, context[:transaction]),
                    attr_real_name)
end

@when("attribute(last-name) set supertype: real-name") do context
    attr_first_name = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    attr_real_name = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    g.set_supertype(g.as_remote(attr_first_name, context[:transaction]),
                    attr_real_name)
end

@when("attribute(real-name) set supertype: name") do context
    attr_first_name = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    attr_real_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    g.set_supertype(g.as_remote(attr_first_name, context[:transaction]),
                    attr_real_name)
end

@when("attribute(username) set supertype: name") do context
    attr_first_name = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    attr_real_name = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    g.set_supertype(g.as_remote(attr_first_name, context[:transaction]),
                    attr_real_name)
end

@then("attribute(first-name) get supertype: real-name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "first-name")
    super_type = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect super_type.label.name == "real-name"
end

@then("attribute(last-name) get supertype: real-name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    super_type = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect super_type.label.name == "real-name"
end

@then("attribute(real-name) get supertype: name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    super_type = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect super_type.label.name == "name"
end

@then("attribute(username) get supertype: name") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    super_type = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect super_type.label.name == "name"
end

@then("attribute(first-name) get supertypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "first-name")
    res_types = g.get_supertypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(last-name) get supertypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    res_types = g.get_supertypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(real-name) get supertypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    res_types = g.get_supertypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(username) get supertypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    res_types = g.get_supertypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(first-name) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "first-name")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(last-name) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(real-name) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(username) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "username")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(name) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(attribute) get subtypes contain:") do context
    super_types = [db[1] for db in context.datatable]
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "attribute")
    res_types = g.get_subtypes(g.as_remote(attr, context[:transaction]))
    for attr in res_types
        @expect in(attr.label.name, super_types)
    end
end

@then("attribute(is-open) set supertype: is-open; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    try
        g.set_supertype(g.as_remote(attr, trans), attr)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(age) set supertype: age; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    try
        g.set_supertype(g.as_remote(attr, trans), attr)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(rating) set supertype: rating; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    try
        g.set_supertype(g.as_remote(attr, trans), attr)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(name) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(timestamp) set supertype: timestamp; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "time-stamp")
    try
        g.set_supertype(g.as_remote(attr, trans), attr)
    catch ex
        @expect ex !== nothing
    end
end

# Scenario: Attribute types cannot subtype non abstract attribute types
@then("attribute(first-name) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "first-name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(last-name) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(is-open) set supertype: age; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(is-open) set supertype: rating; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(is-open) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(is-open) set supertype: timestamp; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(age) set supertype: is-open; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(age) set supertype: rating; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(age) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(age) set supertype: timestamp; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(rating) set supertype: is-open; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "is.open")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(rating) set supertype: age; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(rating) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(rating) set supertype: timestamp; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(name) set supertype: is-open; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(name) set supertype: age; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(name) set supertype: rating; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(name) set supertype: timestamp; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(timestamp) set supertype: is-open; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "is-open")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(timestamp) set supertype: age; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(timestamp) set supertype: rating; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

@then("attribute(timestamp) set supertype: name; throws exception") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    try
        g.set_supertype(g.as_remote(attr, trans), attr_super)
    catch ex
        @expect ex !== nothing
    end
end

# Scenario: Attribute types can get the root type
@then("attribute(is-open) get supertype: attribute") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect attr_super.label.name == "attribute"
end

@then("attribute(age) get supertype: attribute") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "age")
    attr_super = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect attr_super.label.name == "attribute"
end

@then("attribute(rating) get supertype: attribute") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "rating")
    attr_super = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect attr_super.label.name == "attribute"
end

@then("attribute(timestamp) get supertype: attribute") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_super = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect attr_super.label.name == "attribute"
end

@then("attribute(name) get supertype: attribute") do context
    attr = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_super = g.get_supertype(g.as_remote(attr, context[:transaction]))
    @expect attr_super.label.name == "attribute"
end

# Scenario: Attribute type root can get attribute types of a specific value class
@then("attribute(attribute) as(boolean) get subtypes contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.BOOLEAN, true, debug = true)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(boolean) get subtypes do not contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.BOOLEAN, false)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(long) get subtypes contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.LONG, true)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(long) get subtypes do not contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.LONG, false)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(double) get subtypes contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.DOUBLE, true)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(double) get subtypes do not contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.DOUBLE, false)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(string) get subtypes contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.STRING, true)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(string) get subtypes do not contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.STRING, false)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(datetime) get subtypes contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.DATETIME, true)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

@then("attribute(attribute) as(datetime) get subtypes do not contain:") do context
    res = _attribute_subtypes(context, "attribute", VALUE_TYPE.DATETIME, false)
    @expect length(context.datatable) == length(res)
    @expect all(res)
end

# Scenario: Attribute types can have keys
@when("put attribute type: country-code, with value type: string") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "country-code", VALUE_TYPE.STRING)
end

@when("put attribute type: country-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]), AttributeType, "country-name", VALUE_TYPE.STRING)
end

@when("attribute(country-name) set owns key type: country-code") do context
    attr_country = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-code")
    g.set_owns(g.as_remote(attr_country, context[:transaction]),
                attr_country_name,
                true)
end

@then("attribute(country-name) get owns key types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    types_owns = g.get_owns(g.as_remote(attr_country_name, context[:transaction]),
                                nothing,
                                true)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

# Scenario: Attribute types can unset keys
@when("put attribute type: country-code-1, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "country-code-1",
            VALUE_TYPE.STRING)
end

@when("put attribute type: country-code-2, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "country-code-2",
            VALUE_TYPE.STRING)
end

@when("attribute(country-name) set owns key type: country-code-1") do context
    attr_country = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-code-1")
    g.set_owns(g.as_remote(attr_country, context[:transaction]),
                attr_country_name,
                true)
end

@when("attribute(country-name) set owns key type: country-code-2") do context
    attr_country = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-code-2")
    g.set_owns(g.as_remote(attr_country, context[:transaction]),
                attr_country_name,
                true)
end

@when("attribute(country-name) unset owns key type: country-code-1") do context
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_country_code = g.get(ConceptManager(context[:transaction]), AttributeType, "country-code-1")
    g.unset_owns(g.as_remote(attr_country_name, context[:transaction]),
                    attr_country_code)
end

@then("attribute(country-name) get owns key types do not contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    types_owns = g.get_owns(g.as_remote(attr_country_name, context[:transaction]),
                                nothing,
                                true)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect !in(key_type, types_owns)
    end
end

@when("attribute(country-name) unset owns key type: country-code-2") do context
    attr_country_name = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_country_code = g.get(ConceptManager(context[:transaction]), AttributeType, "country-code-2")
    g.unset_owns(g.as_remote(attr_country_name, context[:transaction]),
                    attr_country_code)
end

# Scenario: Attribute types can have attributes
@when("put attribute type: utc-zone-code, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "utc-zone-code",
            VALUE_TYPE.STRING)
end

@when("put attribute type: utc-zone-hour, with value type: double") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "utc-zone-hour",
            VALUE_TYPE.DOUBLE)
end

@when("attribute(timestamp) set owns attribute type: utc-zone-code") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_is_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "utc-zone-code")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]),
                attr_is_owned,
                false)
end

@when("attribute(timestamp) set owns attribute type: utc-zone-hour") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_is_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "utc-zone-hour")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]),
                attr_is_owned,
                false)
end

@then("attribute(timestamp) get owns attribute types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                false)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

# Scenario: Attribute types can unset attributes
@when("attribute(timestamp) unset owns attribute type: utc-zone-hour") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "utc-zone-hour")
    g.unset_owns(g.as_remote(attr_owns, context[:transaction]),
                    attr_owned)
end

@then("attribute(timestamp) get owns attribute types do not contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                true)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect !in(key_type, types_owns)
    end
end

@when("attribute(timestamp) unset owns attribute type: utc-zone-code") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "timestamp")
    attr_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "utc-zone-code")
    g.unset_owns(g.as_remote(attr_owns, context[:transaction]),
                    attr_owned)
end

# Scenario: Attribute types can have keys and attributes
@when("put attribute type: country-abbreviation, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "country-abbreviation",
            VALUE_TYPE.STRING)
end

@when("attribute(country-name) set owns attribute type: country-abbreviation") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    attr_is_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "country-abbreviation")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]),
                attr_is_owned,
                false)
end

@then("attribute(country-name) get owns attribute types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "country-name")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                false)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

# Scenario: Attribute types can inherit keys and attributes
@when("put attribute type: hash, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "hash",
            VALUE_TYPE.STRING)
end

@when("put attribute type: abbreviation, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "abbreviation",
            VALUE_TYPE.STRING)
end

@when("attribute(name) set owns key type: hash") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_is_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "hash")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]),
                attr_is_owned,
                true)
end

@when("attribute(name) set owns attribute type: abbreviation") do context
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "name")
    attr_is_owned = g.get(ConceptManager(context[:transaction]), AttributeType, "abbreviation")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]),
                attr_is_owned,
                false)
end

@then("attribute(real-name) get owns key types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                true)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

@then("attribute(real-name) get owns attribute types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "real-name")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                false)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

@then("attribute(last-name) get owns key types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                true)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

@then("attribute(last-name) get owns attribute types contain:") do context
    key_types = [db[1] for db in context.datatable]
    attr_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "last-name")
    types_owns = g.get_owns(g.as_remote(attr_owns, context[:transaction]),
                                nothing,
                                false)
    for i in 1:length(key_types)
        key_type = g.get(ConceptManager(context[:transaction]),
                          AttributeType,
                          key_types[i])
        @expect in(key_type, types_owns)
    end
end

# Scenario: Attribute types can be owned as attributes
@when("put attribute type: boy-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "boy-name",
            VALUE_TYPE.STRING)
end

@when("attribute(boy-name) set supertype: name") do context
    type = g.get(ConceptManager(context[:transaction]),
                    AttributeType,
                    "boy-name")
    super_type = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "name")
    g.set_supertype(g.as_remote(type, context[:transaction]),
                    super_type)
end

@when("put attribute type: girl-name, with value type: string") do context
    g.put(ConceptManager(context[:transaction]),
            AttributeType,
            "girl-name",
            VALUE_TYPE.STRING)
end

@when("attribute(girl-name) set supertype: name") do context
    type = g.get(ConceptManager(context[:transaction]),
                    AttributeType,
                    "girl-name")
    super_type = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "name")
    g.set_supertype(g.as_remote(type, context[:transaction]),
                    super_type)
end

@when("entity(person) set abstract: true") do context
    entity_to_abstract = g.get(ConceptManager(context[:transaction]),
                                                EntityType,
                                                "person")
    g.set_abstract(g.as_remote(entity_to_abstract, context[:transaction]))
end

@when("put entity type: boy") do context
    g.put(ConceptManager(context[:transaction]),
            EntityType,
            "boy")
end

@when("entity(boy) set supertype: person") do context
    type = g.get(ConceptManager(context[:transaction]),
                    EntityType,
                    "boy")
    super_type = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        "person")
    g.set_supertype(g.as_remote(type, context[:transaction]),
                    super_type)
end

@when("entity(boy) set owns attribute type: boy-name as name") do context
    entity_owns = g.get(ConceptManager(context[:transaction]), EntityType, "boy")
    attr_set_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "boy-name")
    attr_overriden = g.get(ConceptManager(context[:transaction]), AttributeType, "name")

    g.set_owns(g.as_remote(entity_owns, context[:transaction]),
                attr_set_owns,
                false,
                attr_overriden)
end

@when("put entity type: girl") do context
    g.put(ConceptManager(context[:transaction]),
            EntityType,
            "girl")
end

@when("entity(girl) set supertype: person") do context
    type = g.get(ConceptManager(context[:transaction]),
                    EntityType,
                    "girl")
    super_type = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        "person")
    g.set_supertype(g.as_remote(type, context[:transaction]),
                    super_type)
end

@when("entity(girl) set owns attribute type: girl-name as name") do context
    entity_owns = g.get(ConceptManager(context[:transaction]), EntityType, "girl")
    attr_set_owns = g.get(ConceptManager(context[:transaction]), AttributeType, "girl-name")
    attr_overriden = g.get(ConceptManager(context[:transaction]), AttributeType, "name")

    g.set_owns(g.as_remote(entity_owns, context[:transaction]),
                attr_set_owns,
                false,
                attr_overriden)
end

@then("attribute(age) get attribute owners contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "age")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]))

    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        contain_owners[i])
        @expect in(owner, owners)
    end
end

@then("attribute(name) get attribute owners contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "name")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]))
    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        ThingType,
                        contain_owners[i])
        @expect in(owner, owners)
    end
end

@then("attribute(name) get attribute owners do not contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "name")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]))
    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        contain_owners[i])
        @expect !in(owner, owners)
    end
end

# Scenario: Attribute types can be owned as keys
@when("put entity type: company") do context
    g.put(ConceptManager(context[:transaction]), EntityType, "company")
end

@when("entity(company) set owns attribute type: email") do context
    attr_owns = g.get(ConceptManager(context[:transaction]),
                            EntityType,
                            "company")
    attr_set_owns = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "email")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]), attr_set_owns, false)
end

@when("entity(person) set owns key type: email") do context
    attr_owns = g.get(ConceptManager(context[:transaction]),
                            EntityType,
                            "person")
    attr_set_owns = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "email")
    g.set_owns(g.as_remote(attr_owns, context[:transaction]), attr_set_owns, true)
end

@then("attribute(email) get attribute owners contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "email")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]))

    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        contain_owners[i])
        @expect in(owner, owners)
    end
end

@then("attribute(email) get key owners contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "email")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]), true)

    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        contain_owners[i])
        @expect in(owner, owners)
    end
end

@then("attribute(email) get key owners do not contain:") do context
    contain_owners = [db[1] for db in context.datatable]
    attr_owner = g.get(ConceptManager(context[:transaction]),
                        AttributeType,
                        "email")
    owners = g.get_owners(g.as_remote(attr_owner, context[:transaction]), true)

    for i in length(contain_owners)
        owner = g.get(ConceptManager(context[:transaction]),
                        EntityType,
                        contain_owners[i])
        @expect !in(owner, owners)
    end
end

# Scenario: Attribute types with value type string can unset their regular expression
@then("attribute(email) as(string) unset regex") do context
    attr_owns_regex = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    g.set_regex(g.as_remote(attr_owns_regex, context[:transaction]), nothing)
end

@then("attribute(email) as(string) does not have any regex") do context
    attr_owns_regex = g.get(ConceptManager(context[:transaction]), AttributeType, "email")
    result_regex = g.get_regex(g.as_remote(attr_owns_regex, context[:transaction]))
    @expect result_regex === nothing
end


text:
struct MAParams <: AbstractParams
    q::Vector{<:Integer}
end

MAParams(q::Integer) = MAParams(collect(1:q))

function fit(params::MAParams, z::AbstractVector{T}; n::Integer=20) where T
    q = params.q
    Q = maximum(q)
    if q == collect(1:Q)
        μ = mean(z)
        z = z .- μ
        θ, σ2 = innovations(z, Q, n=n)
        ma = MAModel{Q}(μ, θ, σ2)
    else
        ma = fit(ARMAParams(false, eltype(q)[], q), z, n=n)
    end
    return ma
end

const MAModel{q,T} = ARMAXModel{0,q,0,T}
MAModel{q}(μ::T,θ::AbstractVector{T},σ2::T) where {q,T} = ARMAXModel{0,q,0,T}(μ, T[], θ, T[], σ2)

isstationary(model::MAModel) = true

isinvertible(model::MAModel) = all(norm.(roots(Polynomial([1;.-model.θ]))) .> 1)

function innovations!(θ::AbstractMatrix{T}, σ2::AbstractVector{T}, γ::AbstractVector{T}, q::Integer, n::Integer) where {T}
    θ[n,n] = γ[1 + n] / σ2[1]
    for k in 2:n
        tmp = zero(T)
        for j in 1:k
            tmp += θ[k - 1,(k - 1) - (j - 1) + 1] * θ[n,n - (j - 1)] * σ2[1 + (j - 1)]
        end
        θ[n,n - (k - 1)] =  n - (k - 1) > q ? zero(T) : (-one(T) / σ2[1 + (k - 1)]) * (γ[1 + n - (k - 1)] - tmp)
    end
    σ2[n + 1] = γ[1]
    for j in 1:n
        σ2[n + 1] -= θ[n,n - (j - 1)]^2 * σ2[1 + (j - 1)]
    end
end

function innovations(z::AbstractVector, q::Integer; n::Integer=20)
    n >= q || throw(ArgumentError("number of iterations should be higher than the number of coefficients q"))
    γ = map(i -> autocovariance(z, i), 0:n)
    T = typeof(zero(eltype(z)) / 1)
    σ2 = Vector{T}(undef, n + 1)
    σ2[1] = γ[1]
    θ = zeros(T, n, n)
    for k in 1:n
        innovations!(θ, σ2, γ, q, k)
    end
    return θ[n,1:q], σ2[n + 1]
end

ext: jl


text:
using Plots
using StatsPlots
using CSV

epochs = 10

function plot_train_val(batch_size, samples,fname_train, fname_val)
    batch_size = 2
    samples = 2975
    batches_per_epoch = samples / batch_size

    data_train_loss = CSV.read(fname_train)
    data_val_loss = CSV.read(fname_val)

    @df data_train_loss plot(:global_step / batches_per_epoch, :value,
        label="Training")
    @df data_val_loss plot!(:global_step / batches_per_epoch, :value,
        label="Validation")

    xlabel!("Epoch")
    ylabel!("Value")

    xticks!(0:1:epochs)
end

plot_train_val(2,2975,"data/baseline/train_loss_lr0-01.csv","data/baseline/val_loss_lr0-01.csv")

plot_train_val(2,2975,"data/augment/train_loss_hflip.csv", "data/augment/val_loss_hflip.csv")

plot_train_val(2,2975,"data/threshold/train_loss.csv", "data/threshold/val_loss.csv")
plot_train_val(2,2975,"data/threshold/train_iou.csv", "data/threshold/val_iou.csv")

plot_train_val(2,2975,"data/reduced-dims/train_loss.csv", "data/reduced-dims/val_loss.csv")
plot_train_val(2,2975,"data/reduced-dims/train_iou.csv", "data/reduced-dims/val_iou.csv")

plot_train_val(2,2975,"data/dropout/train_loss.csv", "data/dropout/val_loss.csv")
plot_train_val(2,2975,"data/dropout/train_iou.csv", "data/dropout/val_iou.csv")

plot_train_val(2,2975,"data/iou-loss/train_loss.csv", "data/iou-loss/val_loss.csv")
plot_train_val(2,2975,"data/iou-loss/train_iou.csv", "data/iou-loss/val_iou.csv")

plot_train_val(2,2975,"data/alpha-edges/train_loss.csv", "data/alpha-edges/val_loss.csv")
plot_train_val(2,2975,"data/alpha-edges/train_iou.csv", "data/alpha-edges/val_iou.csv")

plot_train_val(2,2975,"data/bilinear-false/train_loss.csv", "data/bilinear-false/val_loss.csv")
plot_train_val(2,2975,"data/bilinear-false/train_iou.csv", "data/bilinear-false/val_iou.csv")

plot_train_val(2,2975,"data/optimized-lr/0-001/train_loss.csv", "data/optimized-lr/0-001/val_loss.csv")
plot_train_val(2,2975,"data/optimized-lr/0-001/train_iou.csv", "data/optimized-lr/0-001/val_iou.csv")
plot_train_val(2,2975,"data/optimized-lr/0-01/train_loss.csv", "data/optimized-lr/0-01/val_loss.csv")
plot_train_val(2,2975,"data/optimized-lr/0-01/train_iou.csv", "data/optimized-lr/0-01/val_iou.csv")

max_stars_repo_name: kurt-stolle/tue-cityscapes-segmentation

max_issues_repo_name: kurt-stolle/tue-cityscapes-segmentation

max_forks_repo_name: kurt-stolle/tue-cityscapes-segmentation


text:
# Date and DateTime

# R -> Julia

rcopy(::Type{Date}, s::Ptr{RealSxp}) = rcopy(Date, s[1])
rcopy(::Type{DateTime}, s::Ptr{RealSxp}) = rcopy(DateTime, s[1])

rcopy(::Type{Date}, x::Float64) = Date(Dates.UTInstant(Dates.Day((isnan(x) ? 0 : x) + 719163)))
rcopy(::Type{DateTime}, x::Float64) =
    DateTime(Dates.UTInstant(Dates.Millisecond(((isnan(x) ? 0 : x) + 62135683200) * 1000)))

# implicit conversion `rcopy(d)`.
function rcopytype(::Type{RClass{:Date}}, s::Ptr{RealSxp})
    if anyna(s)
        length(s) == 1 ? Missing : Array{Union{Date, Missing}}
    else
        length(s) == 1 ? Date : Array{Date}
    end
end
function rcopytype(::Type{RClass{:POSIXct}}, s::Ptr{RealSxp})
    if anyna(s)
        length(s) == 1 ? Missing : Array{Union{DateTime, Missing}}
    else
        length(s) == 1 ? DateTime : Array{DateTime}
    end
end

# implicit Array conversion `rcopy(Array, d)`.
function eltype(::Type{RClass{:Date}}, s::Ptr{RealSxp})
    if anyna(s)
        Union{Date, Missing}
    else
        Date
    end
end
function eltype(::Type{RClass{:POSIXct}}, s::Ptr{RealSxp})
    if anyna(s)
        Union{DateTime, Missing}
    else
        DateTime
    end
end

# Julia -> R

function sexp(::Type{RClass{:Date}}, d::Date)
    res = protect(sexp(RClass{:numeric}, Float64(Dates.value(d)) - 719163))
    setclass!(res, sexp("Date"))
    unprotect(1)
    res
end
function sexp(::Type{RClass{:Date}}, a::AbstractArray{Date})
    res = protect(sexp(RClass{:numeric}, map((x) -> Float64(Dates.value(x)) - 719163, a)))
    setclass!(res, sexp("Date"))
    unprotect(1)
    res
end
function sexp(::Type{RClass{:Date}}, a::AbstractArray{Union{Date, Missing}})
    rv = protect(allocArray(RealSxp, size(a)...))
    try
        for (i, x) in enumerate(a)
            if ismissing(x)
                rv[i] = Const.NaReal
            else
                rv[i] = Float64(Dates.value(x)) - 719163
            end
        end
        setclass!(rv, sexp("Date"))
    finally
        unprotect(1)
    end
    rv
end
function sexp(::Type{RClass{:POSIXct}}, d::DateTime)
    res = protect(sexp(RClass{:numeric}, Float64(Dates.value(d) / 1000) - 62135683200))
    setclass!(res, sexp(["POSIXct", "POSIXt"]))
    setattrib!(res, "tzone", "UTC")
    unprotect(1)
    res
end
function sexp(::Type{RClass{:POSIXct}}, a::AbstractArray{DateTime})
    res = protect(sexp(RClass{:numeric}, map((x) -> Float64(Dates.value(x) / 1000) - 62135683200, a)))
    setclass!(res, sexp(["POSIXct", "POSIXt"]))
    setattrib!(res, "tzone", "UTC")
    unprotect(1)
    res
end
function sexp(::Type{RClass{:POSIXct}}, a::AbstractArray{Union{DateTime, Missing}})
    rv = protect(allocArray(RealSxp, size(a)...))
    try
        for (i, x) in enumerate(a)
            if ismissing(x)
                rv[i] = Const.NaReal
            else
                rv[i] = Float64(Dates.value(x) / 1000) - 62135683200
            end
        end
        setclass!(rv, sexp(["POSIXct", "POSIXt"]))
        setattrib!(rv, "tzone", "UTC")
    finally
        unprotect(1)
    end
    rv
end


text:
using LinearAlgebra
using Plots

# sphere to capsule

# capsule points
ca = [0.0; 0.0; 0.0]
cb = [0.0; 0.0; 1.0]

# sphere point
s = [0.0; 1.0; -1.0]

dab = cb - ca
das = s - ca

len = dot(dab, dab)

t = dot(das, dab) ./ dot(dab, dab)
t_clamp = min(max(t, 0.0), 1.0)

# closest point
if t <= 0.0
    p = ca
elseif t >= 1.0
    p = cb
else
    p = ca + t * dab
end

###
# https://stackoverflow.com/questions/44824512/how-to-find-the-closest-point-on-a-right-rectangular-prism-3d-rectangle/44824522#44824522
using LinearAlgebra
p = [0.0; 1.0; 1.0]
origin = [0.0; 0.0; 0.0]
r = 0.5
v1 = [r; 0.0; 0.0]
v2 = [0.0; r; 0.0]
v3 = [0.0; 0.0; r]

tx = dot(p - origin, v1) / dot(v1, v1)
ty = dot(p - origin, v2) / dot(v2, v2)
tz = dot(p - origin, v3) / dot(v3, v3)

tx_clamp = min(max(tx, 0.0), 1.0)
ty_clamp = min(max(ty, 0.0), 1.0)
tz_clamp = min(max(tz, 0.0), 1.0)

cp = tx_clamp * v1 + ty_clamp * v2 + tz_clamp * v3 + origin


text:
## Matrix operations on SymMatrix

## * we support Array{Sym} using Julia's generic matrix functions
## * Array{Sym} allows calling of SymPy methods via the dot-call syntax (A.det())
## * we support SymPy's immutable matrices via the SymMatrix type. These use dot call style.

## for mapping sympy.Matrix -> Array{Sym} this is used
function Base.convert(::Type{Array{Sym}}, x::PyCall.PyObject)
    if PyCall.hasproperty(x, :__class__)
        nm = Symbol(x.__class__.__name__)
        _convert(Val(nm), x)
    else
        ## error?
        x
    end
end

function _convert(::Val{:MutableDenseMatrix}, x)
    sh = x.shape
    reshape(x.tolist(), sh)
end

function _convert(::Type{Val{T}}, x) where {T}
    x
end

function Base.convert(::Type{Array{T}}, M::SymMatrix) where {T <: SymbolicObject}
    sh = M.shape
    reshape(M.tolist(), sh)
end

## This allows abstract arrays of Sym Objects to slip through sympy.meth() calls
PyCall.PyObject(A::AbstractArray{Sym,2}) =
    PyCall.pycall(sympy.Matrix, PyCall.PyObject, [PyCall.PyObject.(A[i,:]) for i in 1:size(A)[1]])

PyCall.PyObject(V::AbstractArray{Sym,1}) =
    PyCall.pycall(sympy.Matrix, PyCall.PyObject,[[PyCall.PyObject(v)] for v in V])

# call SymMatrix method on Matrix{Sym}
## Eg. A.norm() where A = [x 1; 1 x], say
function Base.getproperty(A::AbstractArray{T}, k::Symbol) where {T <: SymbolicObject}
    if k in fieldnames(typeof(A))
        return getfield(A,k)
    else
        M1 = getproperty(PyCall.PyObject(A), k)
        M1
    end
end

##################################################
##
## special case generic methods that fail on Array{Sym}:

## inv, det may fail as lu need no pivoting specified in general
function Base.inv(A::Matrix{T}) where {T <: SymbolicObject}
    A.inv()
end

function Base.exp(A::Matrix{T}) where {T <: Sym}
    A.exp()
end

function LinearAlgebra.det(A::Matrix{T}) where {T <: Sym}
    A.det()
end

function LinearAlgebra.norm(A::AbstractArray{T,N}) where {T <: SymbolicObject,N}
    A.norm()
end

function LinearAlgebra.eigvals(a::Matrix{Sym})
    Sym[k for k in keys(a.eigenvals())]
end

function LinearAlgebra.eigvecs(a::Matrix{Sym})
    ds =  a.eigenvects()
    hcat((hcat(di[3]...) for di in ds)...)
end

# solve Ax=b for x, avoiding generic `lu`, which can be very slow for bigger n values
# fix suggested by @olof3 in issue 355
function LinearAlgebra.:\(A::AbstractArray{Sym,2}, b::AbstractArray{S,1}) where {S}

    m,n  = size(A)
    x =  Sym["x$i" for  i in 1:n]
    out = solve(A*x-b, x)
    isempty(out) && throw(SingularException(0)) # Could also return out here?
    ret = Vector{Sym}(undef, n)
    for (i,xᵢ)  in enumerate(x)
        ret[i] =  get(out,  xᵢ, xᵢ)
    end

    return ret

end

function LinearAlgebra.:\(A::AbstractArray{T,2}, B::AbstractArray{S,2}) where {T <: Sym, S}
    hcat([A \ bⱼ for bⱼ in eachcol(B)]...)
end

## Issue #359 so that A  +  λI is of type Sym
Base.:+(A::AbstractMatrix{T}, J::UniformScaling) where {T <: SymbolicObject} = (n=LinearAlgebra.checksquare(A); A .+ J.λ*I(n))
Base.:+(A::AbstractMatrix, J::UniformScaling{T}) where {T <: SymbolicObject} = (n=LinearAlgebra.checksquare(A); A .+ J.λ*I(n))
Base.:+(A::AbstractMatrix{T}, J::UniformScaling{T}) where {T <: SymbolicObject} = (n=LinearAlgebra.checksquare(A); A .+ J.λ*I(n))

Base.:-(J::UniformScaling, A::AbstractMatrix{T}) where {T <: SymbolicObject} = (-A) + J
Base.:-(J::UniformScaling{T}, A::AbstractMatrix) where {T <: SymbolicObject} = (-A) + J
Base.:-(J::UniformScaling{T}, A::AbstractMatrix{T}) where {T <: SymbolicObject} = (-A) + J

# Issue 397 so that A' infers correctly
Base.adjoint(x::Sym)::Sym = x.adjoint()

##################################################
##
## Support for ImmutableMatrix via SymMatrix

#### basic matrix operations must be delegated

## literal powers are tricky without this
Base.inv(x::SymMatrix) = x.inv()
*(x::SymMatrix, y::SymbolicObject) = x.multiply(y)
*(x::SymbolicObject, y::SymMatrix) = y.multiply(x)
*(x::SymMatrix, y::Number) = x.multiply(y)
*(x::Number, y::SymMatrix) = y.multiply(x)
*(x::SymMatrix, y::SymMatrix) = x.multiply(y)
/(x::SymMatrix, y::Number) = x.multiply(1/Sym(y))
/(x::SymMatrix, y::SymbolicObject) = x.multiply(1/y)
+(x::SymMatrix, y::SymMatrix) = x.add(y)
-(x::SymMatrix, y::SymMatrix) = x + y.multiply(-1)
^(x::SymMatrix, y::Union{Int, SymbolicObject}) = pycall(sympy.Pow, SymMatrix, x, y)

"""
    M[i,j]

Define `getindex` for SymPy's `ImmutableMatrix` class.

SymMatrix is 0-based, like python, not Julia. Use Matrix{Sym} for that.

"""
Base.getindex(M::SymMatrix, i::Int, j::Int) = M.__getitem__((i,j))
Base.getindex(V::SymMatrix, i::Int) = V.__getitem__((i,0))
Base.getindex(M::SymMatrix) = M

function Base.convert(::Type{SymMatrix}, M::AbstractArray{T, N}) where {T <: Number, N}
    m,n = size(M)
    sympy.ImmutableMatrix([PyCall.PyObject.(M[i,:]) for i in 1:m])
end

function Base.convert(::Type{SymMatrix}, V::Vector{T}) where {T <: Number}
    convert(SymMatrix, hcat(V))
end

## # Functions to convert between Matrix{Sym} and SymMatrix
function Base.convert(::Type{Array{Sym,N}}, M::SymMatrix) where {N}
    reshape(M.tolist(), M.shape)
end

function Base.convert(::Type{SymMatrix}, x::PyCall.PyObject)
    SymMatrix(x)
end


text:
import Base.getindex, Base.setindex!
const .. = Val{:...}

setindex!(A::AbstractArray{T,1}, x, ::Type{Val{:...}}, n) where {T} = A[n] = x
setindex!(A::AbstractArray{T,2}, x, ::Type{Val{:...}}, n) where {T} = A[:, n] = x
setindex!(A::AbstractArray{T,3}, x, ::Type{Val{:...}}, n) where {T} = A[:, :, n] = x

setindex!(A::AbstractArray{T,2}, x, ::Type{Val{:...}}, i, j) where {T} = A[i, j] = x
setindex!(A::AbstractArray{T,3}, x, ::Type{Val{:...}}, i, j) where {T} = A[:, i, j] = x

getindex(A::AbstractArray{T,1}, ::Type{Val{:...}}, n) where {T} = A[n]
getindex(A::AbstractArray{T,2}, ::Type{Val{:...}}, n) where {T} = A[ :, n]
getindex(A::AbstractArray{T,3}, ::Type{Val{:...}}, n) where {T} = A[ :, :, n]

getindex(A::AbstractArray{T,2}, ::Type{Val{:...}}, i, j) where {T} = A[i, j]
getindex(A::AbstractArray{T,3}, ::Type{Val{:...}}, i, j) where {T} = A[:, i, j]


text:
using DrWatson
@quickactivate
using ArgParse
using CSV, DataFrames
using HierarchicalAD
using FileIO, BSON
using ValueHistories
using Flux
using CUDA
CUDA.device!(1)

include(scriptsdir("plotting_code.jl"))

s = ArgParseSettings()
@add_arg_table s begin
	"path"
		help = "either a modelfile or a dir containing multiple modelfiles"
		arg_type = String
end
args = parse_args(s)
@unpack path = args

function evaluate_model(modeldata, modelfile)
	@unpack model, training_history = modeldata
	gmodel = gpu(model)
	cat_keys = ["digit", "shape"]

	# now print some info
	println("\n")
	@info "Loaded $modelfile."
	println("  model_id => $(modeldata[:experiment_args].model_id)")
	println("  data => $(modeldata[:experiment_args].data)")
	println("  experiment settings => ")
	for (k,v) in pairs(modeldata[:experiment_args])
		println("    $k = $v")
	end
	println("  non-default filters => ")
	for f in modeldata[:non_default_filters]
		fv = modeldata[:filter_dict][f]
		if !(f in cat_keys)
			println("    $(f) $(fv[1]) $(fv[2])")
		else
			println("    $(f) $(fv)")
		end
	end
	println("\n")

	# create the plot path
	modeltype = split(dirname(modelfile), "/")[end]
	plotpath = plotsdir(modeltype)
	mkpath(plotpath)
	evalpath = datadir("eval/$modeltype")
	mkpath(evalpath)
	model_id = modeldata[:experiment_args].model_id

	# load the same data that were used for creation of the model
	dataset = modeldata[:experiment_args].data
	ratios = modeldata[:ratios]
	filter_dict = modeldata[:filter_dict]
	seed = modeldata[:experiment_args].seed
	cat_key = cat_keys[map(k->any(map(x->x==k, collect(keys(filter_dict)))), cat_keys)]
	cat_key = length(cat_key) == 0 ? nothing : cat_key[1]
	(tr_x, tr_y), (val_x, val_y), (tst_x, tst_y), (a_x, a_y) =
	    HierarchicalAD.load_train_val_test_data(dataset, filter_dict; ratios=ratios, seed=seed,
	    	categorical_key=cat_key)

	# plots
	# 01 plot training
	p = plot_training(training_history, plotsize = (600,300))
	svn = joinpath(plotpath, "$(model_id)_01_training_history.png")
	savefig(svn)
	@info "Saved $svn"

	# 02 reconstructions and generated digits
	p = compare_reconstructions(gmodel, tst_x)
	svn = joinpath(plotpath, "$(model_id)_02_reconstructions.png")
	savefig(svn)
	@info "Saved $svn"

	# 03 latent spaces
	p = plot_latents(gmodel)
	svn = joinpath(plotpath, "$(model_id)_03_latent_spaces.png")
	savefig(svn)
	@info "Saved $svn"

	### anomaly scores
	@unpack tr_scores, val_scores, tst_scores, a_scores, non_default_filters = modeldata;

	# digit anomalies
	p=plot_anomalies_cat_factor(tr_y, a_y, tr_scores, val_scores, a_scores, Symbol(cat_key))
	svn = joinpath(plotpath, "$(model_id)_04_anomalies_cat_factor.png")
	savefig(svn)
	@info "Saved $svn"

	# other anomalies
	p=plot_anomalies_other(non_default_filters, filter_dict, a_y, tr_scores, a_scores, cat_key)
	svn = joinpath(plotpath, "$(model_id)_05_anomalies_others.png")
	savefig(svn)
	@info "Saved $svn"

	### latent spaces
	γ = 0.1f0
	k = GaussianKernel(γ)
	@unpack tr_encodings, val_encodings, tst_encodings, a_encodings = modeldata;
	nl = length(model.e)
	mmd_dict = Dict()
	# subsample anomalies
	na = size(a_y, 1)
	subinds = sample(1:na, min(na, 10000), replace=false)
	a_y = a_y[subinds, :]
	a_encodings = map(x->x[:,subinds], a_encodings)

	# digit identity
	cat_vals = sort(unique(tr_y[!,Symbol(cat_key)]))
	p,mmds=plot_latent(cat_vals, Symbol(cat_key), tr_y, k, tr_encodings..., dims=[1,2])
	plot(p)
	svn = joinpath(plotpath, "$(model_id)_06_encodings_train_digits.png")
	savefig(svn)
	@info "Saved $svn"
	print_mmd_overview(Symbol(cat_key), cat_vals, mmds)
	mmd_dict[String(cat_key)] = mmds

	# other filters
	nbins = 5
	p, mmds = mmd_overview_other(nbins, non_default_filters, tr_y, tr_encodings, k, model,
		cat_key)
	plot(p)
	svn = joinpath(plotpath, "$(model_id)_07_encodings_train_other.png")
	savefig(svn)
	@info "Saved $svn"
	mmd_dict = merge(mmd_dict, mmds)

	# now compare normal vs anomalous
	p, mmds = mmd_overview_anomalies(tr_y, a_y, tr_encodings, a_encodings, non_default_filters,
		k, filter_dict, cat_key)
	print_mmd_overview_anomalies(mmds)
	plot(p)
	svn = joinpath(plotpath, "$(model_id)_08_encodings_anomalies.png")
	savefig(svn)
	@info "Saved $svn"
	mmd_dict = merge(mmd_dict, mmds)

	# save results (mmds)
	svn = joinpath(evalpath, "$(model_id)_mmds_train.bson")
	save(svn, mmd_dict)
	@info "Saved $svn"
end

### run from here
# ooening of bson files has to be done in global scope, otherwise strange things happen :(
if isfile(path)
	modeldata = try
		load(path);
	catch e
		@info "File $path could not be opened!"
		return
	end
	evaluate_model(modeldata, path)
else
	for f in readdir(path, join=true)
		modeldata = try
			load(f);
		catch e
			@info "File $f could not be opened!"
			continue
		end
		evaluate_model(modeldata, f)
	end
end

max_stars_repo_name: vitskvara/HierarchicalAD

max_issues_repo_name: vitskvara/HierarchicalAD

max_forks_repo_name: vitskvara/HierarchicalAD


text:
using JuMP

#essa função recebe um problema relaxado e os indices das variaveis que deveriam ser binarias
#apartir disso retorna uma solução viável para o problema, no caso de min isso nos da um upperbound
function fpump(mod::JuMP.Model, binI::Vector{Int64})
  m=deepcopy(mod)
  obf = JuMP.prepAffObjective(m)
  tam=m.numCols
  n=length(binI)

  v = Variable.(m, 1:tam)

  # conferir se o b&b da solve no modelo antes, se n der precisa fazer aqui
  solve(m)
  xvia=m.colVal

  xint=roundbin(xvia,binI,tam)

  if xint == false
    for i in binI
      @constraint(m,0 <= v[i])
      @constraint(m, v[i] <= 1)
    end
    xint=roundbin(xrel,binI,tam)
  end

  d=dist(xvia,xint,binI)

  cont=0
  while d>1e-7 && cont < 1e3
    cont = cont + 1

    @objective(m, Min, sum{ ifelse(xint[i] == 0  , v[i] , 0) + ifelse(xint[i] == 1  , xint[i] - v[i], 0), i in binI})

    solve(m)
    xvia = m.colVal
    d1 = m.objVal

    if d1 ≈ d
      #se o xint for igual ao anterior a distância permanecerá a mesma então criamos uma perturbação

      xint=mudaround(xvia,xint,binI,tam,n)
      d = dist(xvia,xint,binI)

    else
      xint=roundbin(xvia,binI,tam)
      d=d1
    end
  end

  objval=obf'*xvia

  if d <= 1e-7
    return xvia, objval
  end
  return false, false

end

function dist(x,xint,binI)
  dist=0
  for i  in binI
    if xint[i] == 0
      soma = x[i] - 0
    elseif xint[i] == 1
      soma = 1 - x[i]
    end
    dist=dist+soma
  end
  return dist
end

function roundbin(x::Vector, binind::Vector{Int64}, tam::Int64)
  xint=zeros(tam)
  for i  in 1:tam
    if i in binind
      xint[i] = round(x[i])
      if xint[i] != 1 && xint[i] != 0
        return false
      end
    else
      xint[i] = x[i]
    end
  end
  return xint
end

function mudaround(x::Vector, xint::Vector, binind::Vector{Int64}, tam::Int64, n::Int64)

  ind=indChange(x,xint,binind,tam,n)

  for i  in ind
    if xint[i] == 1
      xint[i] = 0
    else
      xint[i] = 1
    end
  end
  return xint
end

#essa função nos da os indices aleatorios das variaveis com maiores scores
function indChange(x::Vector, xint::Vector, binind::Vector{Int64}, tam::Int64, n::Int64)
  vet=zeros(n)

  #preenche um vetor de scores (dif entre xvia e xint das variaveis binarias)
  j=1;
  for i in binind
    vet[j] = abs(x[i] - xint[i])
    j=j+1
  end

  #p é um vetor que define a permutação dos indices de vet que o coloca em ordem decrescente
  p=sortperm(vet,rev=true)
  #como o util para mim são os índices das variaveis originais organizamos binind dessa forma
  novob=binind[p]

  #quero mudar apenas uma certa quantidade de indices, quanto menos melhor
  quant=Int.(ceil(n/400))

  if n>400
    div=0.01
  else
    div=0.1
  end

  #quero que os ind sejam das variaveis com maiores scores (uma certa proporção dos primeiros)

  percent=Int.(ceil(div*n))

  j=randperm(percent)[1:quant]
  ind=Int.(zeros(quant))
  for k in 1:quant
    ind[k]=novob[j[k]]
  end
  return ind
end


text:
using SeparatingAxisTheorem2D
using CoordinateTransformations
using StaticArrays
using Test

v = SVector(.5, .5)
w = SVector(1.5, 1.5)
B = AABB((0., 1.), (0., 1.))
L = LineSegment(v, w)
P = Polygon((0., 0.), (1.5, 0.), (0., 1.5))
C = Circle((1.4, 1.4), .2)
S = CompoundShape(B, P, C)

@test  intersecting(v, B)
@test  intersecting(v, P)
@test !intersecting(v, C)
@test  intersecting(v, S)
@test !intersecting(w, B)
@test !intersecting(w, P)
@test  intersecting(w, C)
@test  intersecting(w, S)
@test  intersecting(B, L)
@test  intersecting(B, P)
@test !intersecting(B, C)
@test  intersecting(L, P)
@test  intersecting(L, C)
@test !intersecting(P, C)


text:
module FrameFunRBF

using DomainSets, BasisFunctions, FrameFun
using StaticArrays, LinearAlgebra
using Printf
using CompactTranslatesDict

BF = BasisFunctions

import Base:
    length,
    size

import BasisFunctions:
    support,
    name,
    unsafe_eval_element,
    plotgrid

import FrameFun:
    dictionary,
    SamplingStyle,
    SolverStyle

export RBF, SmoothRBF, PiecewiseSmoothRBF
export Multiquadric,
    InverseMultiquadric,
    InverseQuadratic,
    Gaussian,
    PolyharmonicSpline,
    ThinPlateSpline

export RBFPlatform,
    RBFParLinear,
    RBFParConstant,
    RBFParSquareRoot,
    rbfselect

export RBFDictionary,
    PeriodicRBFDictionary

include("gridgenerator.jl")
include("rbf.jl")
include("rbf_dict.jl")
include("approximation.jl")
include("platform.jl")

end


text:
using Distributions: Beta, Dirichlet
using DataStructures: Queue, enqueue!, dequeue!
import Base: display, show
using Random

# losely basedon alpha zero but simpler
# bonus: uses Thompson sampling instread of uct
# winning probs come from a quick mc sampling

# ----

mutable struct NodeInfo
    num_visits::Int
    num_wins::Dict{Player, Int}
end
NodeInfo()::NodeInfo = NodeInfo(0, Dict(black => 0, white => 0))

mutable struct MCTSNode
    state::GameState
    parent::Union{Nothing, MCTSNode}
    move::Union{Nothing, Move}
    info::NodeInfo
    children::Vector{MCTSNode}
    _unvisited::Set{Move}
end
info(node::MCTSNode)::NodeInfo = node.info
num_visits(node::MCTSNode)::Int = info(node).num_visits
num_wins(node::MCTSNode)::Dict{Player, Int} = info(node).num_wins
unvisited_moves(node::MCTSNode) = node._unvisited
Base.display(io::IO, node::MCTSNode) = print(io, "Node with $(info(node).num_visits) visits and $(length(node.children)) children")
Base.show(io::IO, node::MCTSNode) = print(io, "Node with $(info(node).num_visits) visits and $(length(node.children)) children")

struct MCTSTree
    _storage::Dict{Int, NodeInfo} # hashcode -> info
end
MCTSTree()::MCTSTree = MCTSTree(Dict{Int, NodeInfo}())
storage(tree::MCTSTree) = tree._storage

function recover_nodeinfo(state, tree::MCTSTree)::NodeInfo
    id = zobristhash(state.board)
    id in keys(storage(tree)) ? storage(tree)[id] : NodeInfo()
end

function candidate_moves(state::GameState)::Vector{Move}
    not_eyes = filter(legal_moves(state)) do move
        !move.is_pass && !is_point_an_eye(state.board, move.point, state.next_player)
    end
    [not_eyes; pass_turn()]
end

MCTSNode(state::GameState, parent::Union{Nothing, MCTSNode}, move::Union{Nothing, Move}, tree::MCTSTree)::MCTSNode =
    MCTSNode(state, parent, move, recover_nodeinfo(state, tree), MCTSNode[], Set(candidate_moves(state)))
MCTSNode(state::GameState, tree::MCTSTree)::MCTSNode = MCTSNode(state, nothing, nothing, tree)

function update_tree!(tree::MCTSTree, node::MCTSNode)::Nothing
    to_visit = Queue{MCTSNode}()  # bfs traversal to update storage
    enqueue!(to_visit, node)
    while !isempty(to_visit)
        vnode = dequeue!(to_visit)
        storage(tree)[zobristhash(vnode.state.board)] = info(vnode)
        if !is_leaf(vnode)
            for child in vnode.children
                enqueue!(to_visit, child)
            end
        end
    end
end

record_win!(node::MCTSNode, winner::Player)::Nothing = (info(node).num_wins[winner] += 1; nothing)
record_visit!(node::MCTSNode)::Nothing = (info(node).num_visits += 1; nothing)

is_leaf(node::MCTSNode)::Bool = (length(node.children) == 0)
is_terminal(node::MCTSNode)::Bool = is_over(info(node).state)
winning_pct(node::MCTSNode, player::Player)::Float64 = info(node).num_wins[player] / info(node).num_visits

struct MCTSAgent <: AbstractAgent
    num_rounds::Int
    komi::Float64
end

function thompson_sampling(children::Vector{MCTSNode}, player::Player)::Int
    childvisits = [info(child).num_visits for child in children]
    childwins = [info(child).num_wins[player] for child in children]
    childvals = [rand(Beta(wins + 1, visits - wins + 1)) for (visits, wins) in zip(childvisits, childwins)]
    argmax(childvals)
end

function select_move(agent::MCTSAgent, node::MCTSNode)::MCTSNode
    # implements Thompson sampling
    children = node.children
    player = node.state.next_player
    best_idx = thompson_sampling(children, player)
    return children[best_idx]
end

function random_child!(agent::MCTSAgent, node::MCTSNode, tree::MCTSTree)::MCTSNode
    move = rand(candidate_moves(node.state))
    new_state = apply_move(node.state, move)
    child = MCTSNode(new_state, node, move, tree)
    if move in unvisited_moves(node)
        delete!(unvisited_moves(node), move)
        push!(node.children, child)
    end
    child
end
select_ramdom_move(agent::MCTSAgent, node::MCTSNode, tree::MCTSTree)::Move = rand(candidate_moves(node))

function expand_leaf!(agent::MCTSAgent, node::MCTSNode, tree::MCTSTree)::Nothing
    @assert is_leaf(node) # leaf node

    for i in 1:agent.num_rounds
        # ==== This part should be neural network/model guided =====
        # <1> choose child at random
        child = random_child!(agent, node, tree)
        # <2> play until end of game
        currstate = child.state
        while !is_over(currstate)
            move = rand(candidate_moves(currstate));
            currstate = apply_move(currstate, move);
        end
        # <3> declare winner
        result = compute_game_result(currstate, agent.komi)
        winner = winnercolor(result)
        # <4> backprop to root, if is not leaf, record winner
        currnode = child
        while currnode !== nothing
            record_win!(currnode, winner)
            record_visit!(currnode)
            currnode = currnode.parent
        end
    end
end

function search_probabilities(node::MCTSNode, mc_sims::Int, ε::Float64)::Matrix{Float64}
    num_rows = node.state.board.num_rows
    num_cols = node.state.board.num_cols
    size = num_rows * num_cols
    noise = reshape(rand(Dirichlet(size, 1.)), num_rows, num_cols)

    children = node.children
    player = node.state.next_player

    prob = ε * noise
    for i in 1:mc_sims
        idx = thompson_sampling(children, player)
        point = children[idx].move.point
        if point.row != 0
            prob[point.row, point.col] += (1.0 - ε) / mc_sims
        end
    end
    return prob
end


text:
if VERSION < v"1.1"
    using LinearAlgebra
    LinearAlgebra.mul!(C::AbstractVecOrMat, J::UniformScaling, B::AbstractVecOrMat) =
        mul!(C, J.λ, B)
end


text:
# TO DO: filename as parameter? in file N, and len given?
# test2
# we want to read a file encrypted_all_tmp.txt

N=8

A=convert( Array{Int64,2},readcsv("encrypted_all_tmp.txt"))
println(typeof(A))

len=size(A)[2]
nr_c=size(A)[1]

W=zeros(nr_c,1)
#W=zeros(8^3,1)

println("len = ", len, " bb=",bb);

for v in 1:nr_c
   tmp=0;
   for r in 1:len
     println("v=", v, ", r=", r);
     tmp=tmp + N^(r-1)*A[v,r]
   end
   println("nr_c = ", nr_c, " , tmp=",tmp);
   W[tmp+1,1]=W[tmp+1,1]+1;
end;

W=convert(Array{Int64,2},W)

W_nr_zeros=0;

for v in 1:nr_c
  if W[v,1]==0 W_nr_zeros=W_nr_zeros+1; end;
  end;

println("len = ", len, " bb=",bb);

println("sum(W)=",sum(W));
println("liczba 0 w W: = ", W_nr_zeros, " procentowo = ", W_nr_zeros/nr_c);
println("W = ", W');

x=[1:nr_c];
using PyPlot

plot(x,W);

max_stars_repo_name: PawelPamula/MarkovTools

max_issues_repo_name: PawelPamula/MarkovTools

max_forks_repo_name: PawelPamula/MarkovTools


text:
#
# Jeffrey D. Scargle et.al., 2013,
# "Studies In Astronomical Time Series Analysis. VI. Bayesian Block Representations",
# http://iopscience.iop.org/article/10.1088/0004-637X/764/2/167/meta
#
# To be used for adaptive ADWIN
#
# Bayesian blocks method joins bins in the regular histogram in an optimal way.
#
using Base.Test

"""
Eq. 19 from Scargle 2012
:param n_k: number of elements in each block
:param t_k: width of the block
:return: fitness value.

NOTE: Looks similar to the entropy function.
"""
function fitness_eq19(n_k, t_k)
    n_k * (log(n_k) - log(t_k))
end

function fit_prior(n, p0=0.05)
    4 - log(75.33 * p0 * n ^(-0.478))
end

"""
In this method the regular histogram is computed first and after that its bins
are joined using dynamic programming.

It is like a normal changedetection but along the Y-axis.
That is why last[] and best[] have lengths of the number of unique elements
(defining edges boundaries).

About the statement
 n_k = n_k = cumsum(x[end:-1:1])[end:-1:1]
 Here the twice reversing is used because e.g for the input
 [1, 1, 1, 2, 1]
the output should be
 [6, 5, 4, 3, 1]
which is a possible number of elements in the last block prior to r.
"""
function bayesian_blocks(t_input, p0_prior=0.05, delta=0.0)
    uniq_t = sort(unique(t_input))
    #... Start Calculate histogram
    edges = [uniq_t[1]-delta;
             0.5 * (uniq_t[2:end] + uniq_t[1:end-1]);
             uniq_t[end]+delta]

    # Count elements in blocks.
    x::Array{Int,1} = []
    for e in uniq_t
        push!(x, count(x -> x == e, t_input))
    end
    @assert length(x) == length(edges) - 1
    #... End Calculate histogram
    # println("...1 edges, counts:", edges, x)
    block_lengths = edges[end] - edges[1:end-1]
    last::Array{Int,1} = zeros(length(uniq_t))
    best::Array{Float64,1} = zeros(length(uniq_t))
    n = length(uniq_t)
    ## Construct a vector of possible block lengths
    ## and count the number of elements in each block.
    for r = 1:n
        if r==1
            last[r] = 1
            best[r] = 0.0
        else
            ## Possible blocks widths prior to r
            t_k = block_lengths[1:r-1] - block_lengths[r] # ??? + delta
            ## Possible numbers of elements in blocks prior to r.
            n_k = cumsum(x[1:r-1][end:-1:1])[end:-1:1]
            # if r==5 @assert length(t_k) == length(n_k) end
            fit_vec = map(x -> fitness_eq19(x[1], x[2]), zip(n_k, t_k))
            a_r = zeros(r) ## a_r[1]==0 automatically; to make sure that the size is correct
            a_r[2:r] = fit_vec - fit_prior(n, p0_prior)
            a_r[2:r] += best[1:r-1]
            v_max, i_max = findmax(a_r)
            last[r] = i_max
            best[r] = v_max
        end
    end
    # last, best
    chps = extract_changepoints(last)
    edges_c = [edges[c] for c in chps]
    if edges_c[end]!=edges[end]
        edges_c = [edges_c; edges[end]]
    end
    edges_c
end

"""
    In this method an input is a is a pre-calculated histogram.
"""
function bayesian_blocks(edges::Array{Float64,1},
                         counts::Array{Int,1},
                         p0_prior=0.05)
    x = counts
    # println("... 2 edges, counts:", edges, x)
    @assert length(edges) - 1 == length(x)
    block_lengths = edges[end] - edges[1:end-1]
    uniq_t = zeros(length(edges)-1) ## to be readable along other method
    last::Array{Int,1} = zeros(length(uniq_t))
    best::Array{Float64,1} = zeros(length(uniq_t))
    n=length(uniq_t)
    # println("...2: n=$n")
    ## Construct a vector of possible block lengths
    ## and count the number of elements in each block.
    for r = 1:n
        if r==1
            last[r] = 1
            best[r] = 0.0
        else
            ## Possible blocks widths prior to r
            t_k = block_lengths[1:r-1] - block_lengths[r] # ??? + delta
            ## Possible numbers of elements in blocks prior to r.
            n_k = cumsum(x[1:r-1][end:-1:1])[end:-1:1]
            if r==5 @assert length(t_k) == length(n_k) end
            fit_vec = map(x -> fitness_eq19(x[1], x[2]), zip(n_k, t_k))
            a_r = zeros(r) ## a_r[1]==0 automatically; to make sure that the size is correct
            a_r[2:r] = fit_vec - fit_prior(n, p0_prior)
            a_r[2:r] += best[1:r-1]
            v_max, i_max = findmax(a_r)
            last[r] = i_max
            best[r] = v_max
        end
    end
    chps=extract_changepoints(last)
    edges_c = [edges[c] for c in chps]
    if edges_c[end]!=edges[end]
        edges_c = [edges_c; edges[end]]
    end
    edges_c
    # last
end

function extract_changepoints(last::Array{Int,1})
    # cp1 = last[end]
    # cp2 = last[cp1]
    chps::Array{Int,1} = []
    next_chps = last[end]
    while next_chps != 1
        push!(chps, next_chps)
        if next_chps != last[next_chps]
            next_chps = last[next_chps]
        else
            next_chps = last[next_chps-1]
        end
    end
    push!(chps,1)
    sort(chps)
end

function runtests()
    println("... Start tests()")
    ret1 = bayesian_blocks([1, 2, 3, 4, 5, 5, 1.1, 1.1], 0.05)

    ret2 = bayesian_blocks([1.0, 1.05, 1.55, 2.5, 3.5, 4.5, 5.0],
                           [1, 2, 1, 1, 1, 2], 0.05)
    @test ret1 == ret2
    # test extract_changepoints
    @test extract_changepoints([1,1,1,2,2,2,5]) == [1,2,5]
    @test extract_changepoints([1,1,1,2,2,2,5,3]) == [1,3]
    println("... End tests()")
end
# runtests()

max_stars_repo_name: av-maslov/Recurrency

max_issues_repo_name: av-maslov/Recurrency

max_forks_repo_name: av-maslov/Recurrency