text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: emr_containers
using AWS.Compat
using AWS.UUIDs

"""
    cancel_job_run(job_run_id, virtual_cluster_id)
    cancel_job_run(job_run_id, virtual_cluster_id, params::Dict{String,<:Any})

Cancels a job run. A job run is a unit of work, such as a Spark jar, PySpark script, or
SparkSQL query, that you submit to Amazon EMR on EKS.

# Arguments
- `job_run_id`: The ID of the job run to cancel.
- `virtual_cluster_id`: The ID of the virtual cluster for which the job run will be
  canceled.

"""
cancel_job_run(jobRunId, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)/jobruns/$(jobRunId)"; aws_config=aws_config)
cancel_job_run(jobRunId, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)/jobruns/$(jobRunId)", params; aws_config=aws_config)

"""
    create_managed_endpoint(certificate_arn, client_token, execution_role_arn, name, release_label, type, virtual_cluster_id)
    create_managed_endpoint(certificate_arn, client_token, execution_role_arn, name, release_label, type, virtual_cluster_id, params::Dict{String,<:Any})

Creates a managed endpoint. A managed endpoint is a gateway that connects EMR Studio to
Amazon EMR on EKS so that EMR Studio can communicate with your virtual cluster.

# Arguments
- `certificate_arn`: The certificate ARN of the managed endpoint.
- `client_token`: The client idempotency token for this create call.
- `execution_role_arn`: The ARN of the execution role.
- `name`: The name of the managed endpoint.
- `release_label`: The Amazon EMR release version.
- `type`: The type of the managed endpoint.
- `virtual_cluster_id`: The ID of the virtual cluster for which a managed endpoint is
  created.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"configurationOverrides"`: The configuration settings that will be used to override
  existing configurations.
- `"tags"`: The tags of the managed endpoint.
"""
create_managed_endpoint(certificateArn, clientToken, executionRoleArn, name, releaseLabel, type, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters/$(virtualClusterId)/endpoints", Dict{String, Any}("certificateArn"=>certificateArn, "clientToken"=>clientToken, "executionRoleArn"=>executionRoleArn, "name"=>name, "releaseLabel"=>releaseLabel, "type"=>type); aws_config=aws_config)
create_managed_endpoint(certificateArn, clientToken, executionRoleArn, name, releaseLabel, type, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters/$(virtualClusterId)/endpoints", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("certificateArn"=>certificateArn, "clientToken"=>clientToken, "executionRoleArn"=>executionRoleArn, "name"=>name, "releaseLabel"=>releaseLabel, "type"=>type), params)); aws_config=aws_config)

"""
    create_virtual_cluster(client_token, container_provider, name)
    create_virtual_cluster(client_token, container_provider, name, params::Dict{String,<:Any})

Creates a virtual cluster. Virtual cluster is a managed entity on Amazon EMR on EKS. You
can create, describe, list and delete virtual clusters. They do not consume any additional
resource in your system. A single virtual cluster maps to a single Kubernetes namespace.
Given this relationship, you can model virtual clusters the same way you model Kubernetes
namespaces to meet your requirements.

# Arguments
- `client_token`: The client token of the virtual cluster.
- `container_provider`: The container provider of the virtual cluster.
- `name`: The specified name of the virtual cluster.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"tags"`: The tags assigned to the virtual cluster.
"""
create_virtual_cluster(clientToken, containerProvider, name; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters", Dict{String, Any}("clientToken"=>clientToken, "containerProvider"=>containerProvider, "name"=>name); aws_config=aws_config)
create_virtual_cluster(clientToken, containerProvider, name, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("clientToken"=>clientToken, "containerProvider"=>containerProvider, "name"=>name), params)); aws_config=aws_config)

"""
    delete_managed_endpoint(endpoint_id, virtual_cluster_id)
    delete_managed_endpoint(endpoint_id, virtual_cluster_id, params::Dict{String,<:Any})

Deletes a managed endpoint. A managed endpoint is a gateway that connects EMR Studio to
Amazon EMR on EKS so that EMR Studio can communicate with your virtual cluster.

# Arguments
- `endpoint_id`: The ID of the managed endpoint.
- `virtual_cluster_id`: The ID of the endpoint's virtual cluster.

"""
delete_managed_endpoint(endpointId, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)/endpoints/$(endpointId)"; aws_config=aws_config)
delete_managed_endpoint(endpointId, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)/endpoints/$(endpointId)", params; aws_config=aws_config)

"""
    delete_virtual_cluster(virtual_cluster_id)
    delete_virtual_cluster(virtual_cluster_id, params::Dict{String,<:Any})

Deletes a virtual cluster. Virtual cluster is a managed entity on Amazon EMR on EKS. You
can create, describe, list and delete virtual clusters. They do not consume any additional
resource in your system. A single virtual cluster maps to a single Kubernetes namespace.
Given this relationship, you can model virtual clusters the same way you model Kubernetes
namespaces to meet your requirements.

# Arguments
- `virtual_cluster_id`: The ID of the virtual cluster that will be deleted.

"""
delete_virtual_cluster(virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)"; aws_config=aws_config)
delete_virtual_cluster(virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/virtualclusters/$(virtualClusterId)", params; aws_config=aws_config)

"""
    describe_job_run(job_run_id, virtual_cluster_id)
    describe_job_run(job_run_id, virtual_cluster_id, params::Dict{String,<:Any})

Displays detailed information about a job run. A job run is a unit of work, such as a Spark
jar, PySpark script, or SparkSQL query, that you submit to Amazon EMR on EKS.

# Arguments
- `job_run_id`: The ID of the job run request.
- `virtual_cluster_id`: The ID of the virtual cluster for which the job run is submitted.

"""
describe_job_run(jobRunId, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/jobruns/$(jobRunId)"; aws_config=aws_config)
describe_job_run(jobRunId, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/jobruns/$(jobRunId)", params; aws_config=aws_config)

"""
    describe_managed_endpoint(endpoint_id, virtual_cluster_id)
    describe_managed_endpoint(endpoint_id, virtual_cluster_id, params::Dict{String,<:Any})

Displays detailed information about a managed endpoint. A managed endpoint is a gateway
that connects EMR Studio to Amazon EMR on EKS so that EMR Studio can communicate with your
virtual cluster.

# Arguments
- `endpoint_id`: This output displays ID of the managed endpoint.
- `virtual_cluster_id`: The ID of the endpoint's virtual cluster.

"""
describe_managed_endpoint(endpointId, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/endpoints/$(endpointId)"; aws_config=aws_config)
describe_managed_endpoint(endpointId, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/endpoints/$(endpointId)", params; aws_config=aws_config)

"""
    describe_virtual_cluster(virtual_cluster_id)
    describe_virtual_cluster(virtual_cluster_id, params::Dict{String,<:Any})

Displays detailed information about a specified virtual cluster. Virtual cluster is a
managed entity on Amazon EMR on EKS. You can create, describe, list and delete virtual
clusters. They do not consume any additional resource in your system. A single virtual
cluster maps to a single Kubernetes namespace. Given this relationship, you can model
virtual clusters the same way you model Kubernetes namespaces to meet your requirements.

# Arguments
- `virtual_cluster_id`: The ID of the virtual cluster that will be described.

"""
describe_virtual_cluster(virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)"; aws_config=aws_config)
describe_virtual_cluster(virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)", params; aws_config=aws_config)

"""
    list_job_runs(virtual_cluster_id)
    list_job_runs(virtual_cluster_id, params::Dict{String,<:Any})

Lists job runs based on a set of parameters. A job run is a unit of work, such as a Spark
jar, PySpark script, or SparkSQL query, that you submit to Amazon EMR on EKS.

# Arguments
- `virtual_cluster_id`: The ID of the virtual cluster for which to list the job run.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"createdAfter"`: The date and time after which the job runs were submitted.
- `"createdBefore"`: The date and time before which the job runs were submitted.
- `"maxResults"`: The maximum number of job runs that can be listed.
- `"name"`: The name of the job run.
- `"nextToken"`: The token for the next set of job runs to return.
- `"states"`: The states of the job run.
"""
list_job_runs(virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/jobruns"; aws_config=aws_config)
list_job_runs(virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/jobruns", params; aws_config=aws_config)

"""
    list_managed_endpoints(virtual_cluster_id)
    list_managed_endpoints(virtual_cluster_id, params::Dict{String,<:Any})

Lists managed endpoints based on a set of parameters. A managed endpoint is a gateway that
connects EMR Studio to Amazon EMR on EKS so that EMR Studio can communicate with your
virtual cluster.

# Arguments
- `virtual_cluster_id`: The ID of the virtual cluster.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"createdAfter"`:  The date and time after which the endpoints are created.
- `"createdBefore"`: The date and time before which the endpoints are created.
- `"maxResults"`: The maximum number of managed endpoints that can be listed.
- `"nextToken"`:  The token for the next set of managed endpoints to return.
- `"states"`: The states of the managed endpoints.
- `"types"`: The types of the managed endpoints.
"""
list_managed_endpoints(virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/endpoints"; aws_config=aws_config)
list_managed_endpoints(virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters/$(virtualClusterId)/endpoints", params; aws_config=aws_config)

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Lists the tags assigned to the resources.

# Arguments
- `resource_arn`: The ARN of tagged resources.

"""
list_tags_for_resource(resourceArn; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/tags/$(resourceArn)"; aws_config=aws_config)
list_tags_for_resource(resourceArn, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/tags/$(resourceArn)", params; aws_config=aws_config)

"""
    list_virtual_clusters()
    list_virtual_clusters(params::Dict{String,<:Any})

Lists information about the specified virtual cluster. Virtual cluster is a managed entity
on Amazon EMR on EKS. You can create, describe, list and delete virtual clusters. They do
not consume any additional resource in your system. A single virtual cluster maps to a
single Kubernetes namespace. Given this relationship, you can model virtual clusters the
same way you model Kubernetes namespaces to meet your requirements.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"containerProviderId"`: The container provider ID of the virtual cluster.
- `"containerProviderType"`: The container provider type of the virtual cluster. EKS is the
  only supported type as of now.
- `"createdAfter"`: The date and time after which the virtual clusters are created.
- `"createdBefore"`: The date and time before which the virtual clusters are created.
- `"maxResults"`: The maximum number of virtual clusters that can be listed.
- `"nextToken"`: The token for the next set of virtual clusters to return.
- `"states"`: The states of the requested virtual clusters.
"""
list_virtual_clusters(; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters"; aws_config=aws_config)
list_virtual_clusters(params::AbstractDict{String, Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("GET", "/virtualclusters", params; aws_config=aws_config)

"""
    start_job_run(client_token, execution_role_arn, job_driver, release_label, virtual_cluster_id)
    start_job_run(client_token, execution_role_arn, job_driver, release_label, virtual_cluster_id, params::Dict{String,<:Any})

Starts a job run. A job run is a unit of work, such as a Spark jar, PySpark script, or
SparkSQL query, that you submit to Amazon EMR on EKS.

# Arguments
- `client_token`: The client idempotency token of the job run request.
- `execution_role_arn`: The execution role ARN for the job run.
- `job_driver`: The job driver for the job run.
- `release_label`: The Amazon EMR release version to use for the job run.
- `virtual_cluster_id`: The virtual cluster ID for which the job run request is submitted.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"configurationOverrides"`: The configuration overrides for the job run.
- `"name"`: The name of the job run.
- `"tags"`: The tags assigned to job runs.
"""
start_job_run(clientToken, executionRoleArn, jobDriver, releaseLabel, virtualClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters/$(virtualClusterId)/jobruns", Dict{String, Any}("clientToken"=>clientToken, "executionRoleArn"=>executionRoleArn, "jobDriver"=>jobDriver, "releaseLabel"=>releaseLabel); aws_config=aws_config)
start_job_run(clientToken, executionRoleArn, jobDriver, releaseLabel, virtualClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/virtualclusters/$(virtualClusterId)/jobruns", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("clientToken"=>clientToken, "executionRoleArn"=>executionRoleArn, "jobDriver"=>jobDriver, "releaseLabel"=>releaseLabel), params)); aws_config=aws_config)

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Assigns tags to resources. A tag is a label that you assign to an AWS resource. Each tag
consists of a key and an optional value, both of which you define. Tags enable you to
categorize your AWS resources by attributes such as purpose, owner, or environment. When
you have many resources of the same type, you can quickly identify a specific resource
based on the tags you've assigned to it. For example, you can define a set of tags for your
Amazon EMR on EKS clusters to help you track each cluster's owner and stack level. We
recommend that you devise a consistent set of tag keys for each resource type. You can then
search and filter the resources based on the tags that you add.

# Arguments
- `resource_arn`: The ARN of resources.
- `tags`: The tags assigned to resources.

"""
tag_resource(resourceArn, tags; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/tags/$(resourceArn)", Dict{String, Any}("tags"=>tags); aws_config=aws_config)
tag_resource(resourceArn, tags, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("POST", "/tags/$(resourceArn)", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("tags"=>tags), params)); aws_config=aws_config)

"""
    untag_resource(resource_arn, tag_keys)
    untag_resource(resource_arn, tag_keys, params::Dict{String,<:Any})

Removes tags from resources.

# Arguments
- `resource_arn`: The ARN of resources.
- `tag_keys`: The tag keys of the resources.

"""
untag_resource(resourceArn, tagKeys; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/tags/$(resourceArn)", Dict{String, Any}("tagKeys"=>tagKeys); aws_config=aws_config)
untag_resource(resourceArn, tagKeys, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = emr_containers("DELETE", "/tags/$(resourceArn)", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("tagKeys"=>tagKeys), params)); aws_config=aws_config)


text:
# ---
# title: 1072. Flip Columns For Maximum Number of Equal Rows
# id: problem1072
# author: Indigo
# date: 2021-07-18
# difficulty: Medium
# categories: Hash Table
# link: <https://leetcode.com/problems/flip-columns-for-maximum-number-of-equal-rows/description/>
# hidden: true
# ---
#
# Given a `matrix` consisting of 0s and 1s, we may choose any number of columns
# in the matrix and flip **every**  cell in that column.  Flipping a cell
# changes the value of that cell from 0 to 1 or from 1 to 0.
#
# Return the maximum number of rows that have all values equal after some number
# of flips.
#
#
#
# **Example 1:**
#
#
#
#     Input: [[0,1],[1,1]]
#     Output: 1
#     Explanation: After flipping no values, 1 row has all values equal.
#
#
# **Example 2:**
#
#
#
#     Input: [[0,1],[1,0]]
#     Output: 2
#     Explanation: After flipping values in the first column, both rows have equal values.
#
#
# **Example 3:**
#
#
#
#     Input: [[0,0,0],[0,0,1],[1,1,0]]
#     Output: 2
#     Explanation: After flipping values in the first two columns, the last two rows have equal values.
#
#
#
#
# **Note:**
#
#   1. `1 <= matrix.length <= 300`
#   2. `1 <= matrix[i].length <= 300`
#   3. All `matrix[i].length`'s are equal
#   4. `matrix[i][j]` is `0` or `1`
#
#
## @lc code=start
using LeetCode

function max_equal_rows_after_flips(mat::Matrix{Int})
    mp = Dict{Vector{Int}, Int}()
    for row in eachrow(mat)
        row[1] == 1 && (row .= 1 .- row)
        mp[row] = get(mp, row, 0) + 1
    end
    return maximum(values(mp))
end
## @lc code=end

ext: jl


text:
using AutomotiveDrivingModels

const ROADWAY = gen_straight_roadway(1, 20.0)

function create_vehicle(x::Float64, y::Float64, θ::Float64 = 0.0; id::Int64=1)
    s = VehicleState(VecSE2(x, y, θ), ROADWAY, 0.0)
    return Vehicle(s, VehicleDef(), id)
end

const VEH_REF = create_vehicle(0.0, 0.0, 0.0, id=1)

function no_collisions(xspace, yspace, thetaspace)
    for (x,y,th) in zip(xspace, yspace, thetaspace)
        veh2 = create_vehicle(x,y,th,id=2)
        @assert !collision_checker(VEH_REF, veh2) "Error for veh ", veh2
    end
    return true
end

function collisions(xspace, yspace, thetaspace)
    for (x,y,th) in zip(xspace, yspace, thetaspace)
        veh2 = create_vehicle(x,y,th,id=2)
        @assert collision_checker(VEH_REF, veh2) "Error for veh ", veh2
    end
    return true
end

@testset  "collision checker" begin
    ## Series of test 1: Far field
    thetaspace = LinRange(0.0, 2*float(pi), 10)
    xspace = LinRange(15.0, 300.0, 10)
    yspace = LinRange(10.0, 300.0, 10)
    @test no_collisions(xspace, yspace, thetaspace)

    ## Series of test 2: Superposition
    thetaspace = LinRange(0.0, 2*float(pi), 10)
    xspace = LinRange(-2.0, 2.0, 10)
    yspace = LinRange(1.5, 1.5, 10)

    @test collisions(xspace,yspace,thetaspace)
end

max_stars_repo_name: MaximeBouton/AutomotivePOMDPs

max_issues_repo_name: MaximeBouton/AutomotivePOMDPs

max_forks_repo_name: MaximeBouton/AutomotivePOMDPs


text:
include("scan.jl")
include("id_error.jl")
include("token_type.jl")

using Printf
using .Scan: Scanner
using .TokenType: Token
using .IDError: UnIdentifierError

# start analysis stream
function analysis!(sc::Scanner)::Array{Tuple{Token,Union{Any,Nothing}},1}
    tokens = Array{Tuple{Token,Union{Any,Nothing}},1}(undef, 0)
    errBuf, c = "", Scan.next(sc)
    while c !== nothing
        try
            if !isempty(errBuf)
                push!(tokens, (TokenType.UNIDENTIFIER, errBuf))
                errBuf = ""
            end
            if sc.state == Scan.START
                start!(sc, c, tokens)
            # deprecated
            # elseif sc.state == Scan.PREPROCESS
            #     preprocess!(sc, c, tokens)
            elseif sc.state == Scan.IDENTIFIER
                identifier!(sc, c, tokens)
            elseif sc.state == Scan.CONSTANT
                constant!(sc, c, tokens)
            elseif sc.state == Scan.SLASH_MODE
                slash!(sc, c, tokens)
            end
        catch ex
            # if catch unidentifier, reset the sign, specifier and the state of scan mode
            if isa(ex, UnIdentifierError)
                sc.sign = nothing
                sc.specifier = nothing
                sc.state = Scan.START
                while c != '\r' && c != '\n' && c !== nothing
                    c = Scan.next(sc)
                end
                errBuf *= sprint(showerror, ex)
            else
                for (exc, bt) in Base.catch_stack()
                    showerror(stdout, exc, bt)
                    println()
                end
            end
        end
        c = Scan.next(sc, true)
    end
    # end of stream tokens
    push!(tokens, (TokenType.EOS_TOKEN, nothing))
    return tokens
end

function list_tokens(io::IO, tokens::Array{Tuple{Token,Union{Any,Nothing}},1})
    rws = filter(x->Int(x[1]) >= 0 && Int(x[1]) <= 17, tokens)
    rwTable = [(typeToString(u), count(x->x[1] == u, rws)) for u in unique(rw[1] for rw in rws)]
    lns = filter(x->Int(x[1]) == 30, tokens)
    lnTable = [(u, count(x->x[2] == u, lns)) for u in unique(ln[2] for ln in lns)]
    coms = filter(x->Int(x[1]) >= 50 && Int(x[1]) <= 51, tokens)
    comTable = [(typeToString(u), count(x->x == u, coms)) for u in unique(com for com in coms)]
    ids = filter(x->Int(x[1]) == 60, tokens)
    idTable = [(u, count(x->x[2] == u, ids)) for u in unique(id[2] for id in ids)]
    cons = filter(x->Int(x[1]) >= 80 && Int(x[1]) <= 83, tokens)
    conTable = [(u, count(x->x[2] == u, cons)) for u in unique(con[2] for con in cons)]
    ops = filter(x->Int(x[1]) >= 100 && Int(x[1]) <= 126, tokens)
    opTable = [(typeToString(u), count(x->x[1] == u, ops)) for u in unique(op[1] for op in ops)]
    comps = filter(x->Int(x[1]) >= 150 && Int(x[1]) <= 156, tokens)
    compTable = [(typeToString(u), count(x->x[1] == u, comps)) for u in unique(comp[1] for comp in comps)]
    bks = filter(x->Int(x[1]) >= 170 && Int(x[1]) <= 175, tokens)
    bkTable = [(typeToString(u), count(x->x[1] == u, bks)) for u in unique(bk[1] for bk in bks)]
    fsTokens = []
    for seq in filter(x->Int(x[1]) == 71, tokens)
        push!(fsTokens, seq[2]...)
    end
    fsTable = [(typeToString(u), count(x->x == u, fsTokens)) for u in unique(fs for fs in filter(x->begin
        if isa(x, TokenType.Token)
            return Int(x) >= 180 && Int(x) <= 213
        else
            return Int(x[1]) >= 180 && Int(x[1]) <= 213
        end
    end, fsTokens))]
    pTable = [(typeToString(u), count(x->x == u, tokens)) for u in unique(token for token in filter(x->Int(x[1]) == 61, tokens))]
    aTable = [(typeToString(u), count(x->x == u, tokens)) for u in unique(token for token in filter(x->Int(x[1]) == 62, tokens))]
    puncs = filter(x->Int(x[1]) >= 250 && Int(x[1]) <= 255, tokens)
    puncTable = [(typeToString(u), count(x->x[1] == u, puncs)) for u in unique(punc[1] for punc in puncs)]
    push!(puncTable, [(typeToString(u), count(x->x == u, fsTokens)) for u in unique(fs for fs in filter(x->isa(x, TokenType.Token) && Int(x) >= 250 && Int(x) <= 255, fsTokens))]...)
    printTokens = []
    for seq in filter(x->isa(x, Tuple) && Int(x[1]) == 70, fsTokens)
        push!(printTokens, filter(x->!isempty(x), [seq[2]...])...)
    end
    printTable = [(u, count(x->x == u, printTokens)) for u in unique(printTokens)]
    unids = filter(x->Int(x[1]) == 69, tokens)
    unidTable = [(u, count(x->x[2] == u, unids)) for u in unique(unid[2] for unid in unids)]

    println(io, @sprintf("Total: %d tokens\r\n", sum(x->x[2], [rwTable..., lnTable..., comTable..., idTable..., conTable..., opTable..., compTable..., bkTable..., fsTable..., pTable..., aTable..., puncTable..., printTable..., unidTable...])))
    printType(io, "Reserved name: ", rwTable)
    printType(io, "Library name: ", lnTable)
    printType(io, "Comment: ", comTable)
    printType(io, "Identifier: ", idTable)
    printType(io, "Constant: ", conTable)
    printType(io, "Operator: ", opTable)
    printType(io, "Comparator: ", compTable)
    printType(io, "Bracket: ", bkTable)
    printType(io, "Format specifier: ", fsTable)
    printType(io, "Pointer: ", pTable)
    printType(io, "Address: ", aTable)
    printType(io, "Punctuation: ", puncTable)
    printType(io, "Printed token: ", printTable)
    printType(io, "Unidentifier: ", unidTable)
end

function printType(io::IO, description::String, table::Array)
    if !isempty(table)
        println(io, description * @sprintf("%d\r\n%s\r\n", sum(x->x[2], table), join([string(t[1]) * "\t(x" * string(t[2]) * ')' for t in table], '\n')))
    end
end

function typeToString(type::Union{Tuple{TokenType.Token,Any},TokenType.Token})::String
    if isa(type, TokenType.Token)
        if Int(type) >= 0 && Int(type) <= 17
            return lowercase(string(type))
        elseif Int(type) == 100
            return "+"
        elseif Int(type) == 101
            return "-"
        elseif Int(type) == 102
            return "*"
        elseif Int(type) == 103
            return "/"
        elseif Int(type) == 104
            return "%"
        elseif Int(type) == 105
            return "="
        elseif Int(type) == 106
            return "!"
        elseif Int(type) == 107
            return "&"
        elseif Int(type) == 108
            return "|"
        elseif Int(type) == 109
            return "^"
        elseif Int(type) == 110
            return "~"
        elseif Int(type) == 111
            return "++"
        elseif Int(type) == 112
            return "--"
        elseif Int(type) == 113
            return "+="
        elseif Int(type) == 114
            return "-="
        elseif Int(type) == 115
            return "*="
        elseif Int(type) == 116
            return "/="
        elseif Int(type) == 117
            return "%="
        elseif Int(type) == 118
            return "&="
        elseif Int(type) == 119
            return "|="
        elseif Int(type) == 120
            return "^="
        elseif Int(type) == 121
            return "<<"
        elseif Int(type) == 122
            return ">>"
        elseif Int(type) == 123
            return "<<="
        elseif Int(type) == 124
            return ">>="
        elseif Int(type) == 125
            return "&&"
        elseif Int(type) == 126
            return "||"
        elseif Int(type) == 150
            return "=="
        elseif Int(type) == 151
            return "==="
        elseif Int(type) == 152
            return ">"
        elseif Int(type) == 153
            return "<"
        elseif Int(type) == 154
            return ">="
        elseif Int(type) == 155
            return "<="
        elseif Int(type) == 156
            return "!="
        elseif Int(type) == 170
            return "("
        elseif Int(type) == 171
            return ")"
        elseif Int(type) == 172
            return "{"
        elseif Int(type) == 173
            return "}"
        elseif Int(type) == 174
            return "["
        elseif Int(type) == 175
            return "]"
        elseif Int(type) == 200
            return "\\a"
        elseif Int(type) == 201
            return "\\b"
        elseif Int(type) == 202
            return "\\e"
        elseif Int(type) == 203
            return "\\f"
        elseif Int(type) == 204
            return "\\n"
        elseif Int(type) == 205
            return "\\r"
        elseif Int(type) == 206
            return "\\t"
        elseif Int(type) == 207
            return "\\v"
        elseif Int(type) == 208
            return "\\\\"
        elseif Int(type) == 209
            return "\\'"
        elseif Int(type) == 210
            return "\\\""
        elseif Int(type) == 211
            return "\\?"
        elseif Int(type) == 250
            return ","
        elseif Int(type) == 251
            return ";"
        elseif Int(type) == 252
            return ":"
        elseif Int(type) == 253
            return "#"
        elseif Int(type) == 254
            return "'"
        elseif Int(type) == 255
            return "\""
        end
    elseif isa(type, Tuple)
        if Int(type[1]) == 50
            return "//" * type[2]
        elseif Int(type[1]) == 51
            return "/*" * type[2] * "*/"
        elseif Int(type[1]) == 61
            return "*" * type[2]
        elseif Int(type[1]) == 62
            return "&" * type[2]
        elseif Int(type[1]) == 180
            return "%" * type[2] * "d"
        elseif Int(type[1]) == 181
            return "%" * type[2] * "l"
        elseif Int(type[1]) == 182
            return "%" * type[2] * "f"
        elseif Int(type[1]) == 183
            return "%" * type[2] * "lf"
        elseif Int(type[1]) == 184
            return "%" * type[2] * "c"
        elseif Int(type[1]) == 185
            return "%" * type[2] * "s"
        elseif Int(type[1]) == 212
            return "\\" * type[2]
        elseif Int(type[1]) == 213
            return "\\u" * type[2]
        end
    end
end

@timev for filename in ARGS
    if !isfile(filename)
        continue
    end
    println(@sprintf("Start scanning file %s...", filename))
    str = open(filename) do file
        read(file, String)
    end
    io = open("output_" * replace(filename, r"\.(txt|c)$" => s".txt"), "w+")
    tokens = analysis!(Scanner(str))
    for token in tokens
        if token[1] in [TokenType.LN_TOKEN, TokenType.EOS_TOKEN]
            println(io, token)
        else
            print(io, token)
        end
    end
    println(io, @sprintf("### LN_TOKEN is new line token; EOS_TOKEN is end of stream token ###"))
    println(io, @sprintf("%s\r\n%60s\r\n%s", '-'^100, "Result of file \"" * filename * "\"'s tokens", '-'^100))
    list_tokens(io, tokens)
    println(@sprintf("Finish file %s scan.", filename))
end


text:
"""
Main module for `GigaSOM.jl` - Huge-scale, high-performance flow cytometry clustering

The documentation is here: http://LCSB-BioCore.github.io/GigaSOM.jl

"""

module GigaSOM

    using CSV
    using DataFrames
    using Distances
    using Distributed
    using Distributions
    using FileIO
    using DistributedArrays
    using XLSX
    using NearestNeighbors

    include("structs.jl")
    include("core.jl")
    include("satellites.jl")
    include("embedding.jl")

    # include IO files
    include("io/input.jl")
    include("io/process.jl")

    # include visualization files
    # include("visualization/plotting.jl")

    export #core
        initGigaSOM,
        trainGigaSOM,
        mapToGigaSOM,
        linearRadius,
        expRadius

    export #embedding
        embedGigaSOM

    export # structs
        daFrame

    export #input
        readFlowset

    export #satellites
        cleanNames!,
        createDaFrame,
        getMarkers,
        checkDir

    export # plotting
        plotCounts,
        plotPCA

end # module


text:
"Returns a listing of all registered drivers"
function drivers()
    dlist = Dict{String,String}()
    for i=1:drivercount()
        dlist[drivershortname(i)] = driverlongname(i)
    end
    dlist
end

geotransform(raster::Raster) = getgeotransform(raster.dataset)
wktprojection(raster::Raster) = getprojection(raster.dataset)

# Raster I/O
openraster(filename::String, access::GDALAccess=GA_ReadOnly) =
    Raster(opendataset(filename, access))

function createraster(filename::Union{String,String},
                      width::Int,
                      height::Int,
                      nbands::Int,
                      dtype::DataType,
                      drivername::String,
                      options::Vector{String} = Vector{String}())
    Raster(createdataset(driverbyname(drivername), filename, width, height,
                         nbands, _gdaltype(dtype), options))
end

function writeraster(raster::Raster,
                     filename::Union{String,String},
                     strict::Bool = false,
                     options::Vector{String} = Vector{String}())
    raster.dataset == C_NULL && error("Can't read closed raster file")
    _close(createcopy(_getdatasetdriver(raster.dataset), filename,
                      raster.dataset, strict, options))
end

function writeraster(raster::Raster,
                     filename::Union{String,String},
                     drivername::String,
                     strict::Bool = false,
                     options::Vector{String} = Vector{String}())
    raster.dataset == C_NULL && error("Can't read closed raster file")
    _close(createcopy(driverbyname(drivername), filename, raster.dataset,
                      strict, options))
end

function fetch!{T <: Real}(raster::Raster, buffer::Array{T,2}, i::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer, GF_Read)
end

function fetch!{T <: Real}(raster::Raster,
                           buffer::Array{T,3},
                           indices::Vector{Cint})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, GF_Read)
end

function fetch!{T <: Real}(raster::Raster, buffer::Array{T,3})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    @assert size(buffer, 3) == raster.nband
    rasterio!(raster.dataset, buffer,
              collect(Cint(1):Cint(raster.nband)),
              GF_Read)
end

function fetch!{T <: Real}(raster::Raster,
                           buffer::Array{T,2},
                           i::Integer,
                           width::Integer,
                           height::Integer,
                           xoffset::Integer,
                           yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer,
              width, height, xoffset, yoffset, GF_Read)
end

function fetch!{T <: Real}(raster::Raster,
                           buffer::Array{T,3},
                           indices::Vector{Cint},
                           width::Integer,
                           height::Integer,
                           xoffset::Integer,
                           yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, width, height,
              xoffset, yoffset, GF_Read)
end

function fetch!{T <: Real, U <: Integer}(raster::Raster,
                                         buffer::Array{T,2},
                                         i::Integer,
                                         rows::UnitRange{U},
                                         cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer,
              width, rows, cols, GF_Read)
end

function fetch!{T <: Real, U <: Integer}(raster::Raster,
                                         buffer::Array{T,3},
                                         indices::Vector{Cint},
                                         rows::UnitRange{U},
                                         cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, rows, cols, GF_Read)
end

function fetch(raster::Raster, i::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    band = _getrasterband(raster.dataset, i)
    buffer = Array(dtype(band), _getrasterbandxsize(band),
                   _getrasterbandysize(band))
    rasterio!(band, buffer, GF_Read)
end

function fetch(raster::Raster, indices::Vector{Cint})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    buffer = Array(dtype(_getrasterband(raster.dataset, indices[1])),
                   _getrasterxsize(raster.dataset),
                   _getrasterysize(raster.dataset),
                   length(indices))
    rasterio!(raster.dataset, buffer, indices, GF_Read)
end

fetch(raster::Raster, indices::Vector{Int}) = fetch(raster, map(Cint, indices))

function fetch(raster::Raster)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    buffer = Array(dtype(_getrasterband(raster.dataset, 1)),
                   _getrasterxsize(raster.dataset),
                   _getrasterysize(raster.dataset),
                   _getrastercount(raster.dataset))
    fetch!(raster, buffer)
end

function fetch(raster::Raster,
               i::Integer,
               width::Integer,
               height::Integer,
               xoffset::Integer,
               yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    band = _getrasterband(raster.dataset, i)
    buffer = Array(dtype(band), _getrasterbandxsize(band),
                   _getrasterbandysize(band))
    rasterio!(band, buffer, width, height, xoffset, yoffset, GF_Read)
end

function fetch(raster::Raster,
               indices::Vector{Cint},
               width::Integer,
               height::Integer,
               xoffset::Integer,
               yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    buffer = Array(dtype(_getrasterband(raster.dataset, indices[1])),
                   _getrasterxsize(raster.dataset),
                   _getrasterysize(raster.dataset),
                   length(indices))
    rasterio!(raster.dataset, buffer, indices, width, height, xoffset, yoffset,
              GF_Read)
end

fetch(raster::Raster, indices::Vector{Int}, width::Integer, height::Integer,
      xoffset::Integer, yoffset::Integer) =
    fetch(raster, map(Cint,indices), width, height, xoffset, yoffset)

function fetch{U <: Integer}(raster::Raster,
                             i::Integer,
                             rows::UnitRange{U},
                             cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    band = _getrasterband(raster.dataset, i)
    buffer = Array(dtype(band), _getrasterbandxsize(band),
                   _getrasterbandysize(band))
    rasterio!(band, buffer, rows, cols, GF_Read)
end

function fetch{U <: Integer}(raster::Raster,
                             indices::Vector{Cint},
                             rows::UnitRange{U},
                             cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    buffer = Array(dtype(_getrasterband(raster.dataset, indices[1])),
                   _getrasterxsize(raster.dataset),
                   _getrasterysize(raster.dataset),
                   length(indices))
    rasterio!(raster.dataset, buffer, indices, rows, cols, GF_Read)
end

fetch{U <: Integer}(raster::Raster, indices::Vector{Int},
                    rows::UnitRange{U}, cols::UnitRange{U}) =
    fetch(raster, map(Cint,indices), rows, cols)

function update!{T <: Real}(raster::Raster, buffer::Array{T,2}, i::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer, GF_Write)
end

function update!{T <: Real}(raster::Raster,
                            buffer::Array{T,3},
                            indices::Vector{Cint})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, GF_Write)
end

update!{T <: Real}(raster::Raster, buffer::Array{T,3}, indices::Vector{Int}) =
    update!(raster, buffer, map(Cint, indices))

function update!{T <: Real}(raster::Raster,
                            buffer::Array{T,2},
                            i::Integer,
                            width::Integer,
                            height::Integer,
                            xoffset::Integer,
                            yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer, width, height,
              xoffset, yoffset, GF_Write)
end

function update!{T <: Real}(raster::Raster,
                            buffer::Array{T,3},
                            indices::Vector{Cint},
                            width::Integer,
                            height::Integer,
                            xoffset::Integer,
                            yoffset::Integer)
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, width, height,
              xoffset, yoffset, GF_Write)
end

update!{T <: Real}(raster::Raster, buffer::Array{T,3}, indices::Vector{Int},
                   width::Integer, height::Integer, xoffset::Integer,
                   yoffset::Integer) =
    update!(raster, buffer, map(Cint,indices), width, height, xoffset, yoffset)

function update!{T <: Real, U <: Integer}(raster::Raster,
                                          buffer::Array{T,2},
                                          i::Integer,
                                          rows::UnitRange{U},
                                          cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(_getrasterband(raster.dataset, i), buffer, rows, cols, GF_Write)
end

function update!{T <: Real, U <: Integer}(raster::Raster,
                                          buffer::Array{T,3},
                                          indices::Vector{Cint},
                                          rows::UnitRange{U},
                                          cols::UnitRange{U})
    raster.dataset == C_NULL && error("Can't read closed raster file")
    rasterio!(raster.dataset, buffer, indices, rows, cols, GF_Write)
end

update!{T <: Real, U <: Integer}(raster::Raster, buffer::Array{T,3},
                                 indices::Vector{Int}, rows::UnitRange{U},
                                 cols::UnitRange{U}) =
    update!(raster, buffer, map(Cint, indices), rows, cols)


text:
"""
**Checks that the queries for occurrences searches are well formatted**

This is used internally.

Everything this function does is derived from the GBIF API documentation,
including (and especially) the values for enum types. This modifies the
queryset. Filters that are not allowed are removed, and filters that have
incorrect values are dropped too.

This feels like the most conservative option -- the user can always filter the
results when they are returned.
"""
function validate_occurrence_query(query::Pair)

  # List of fields from GBIF
  allowed_fields = ["q", "basisOfRecord", "catalogNumber", "collectionCode",
    "continent", "country", "datasetKey", "decimalLatitude", "decimalLongitude",
    "depth", "elevation", "eventDate", "geometry", "hasCoordinate",
    "hasGeospatialIssue", "institutionCode", "issue", "lastInterpreted",
    "mediaType", "month", "occurrenceId", "organismId", "protocol", "license",
    "publishingCountry", "publishingOrg", "crawlId", "recordedBy", "recordNumber",
    "scientificName", "locality", "stateProvince", "waterBody", "taxonKey",
    "kingdomKey", "phylumKey", "classKey", "orderKey", "familyKey", "genusKey",
    "subGenusKey", "speciesKey", "year", "establishmentMeans", "repatriated",
    "typeStatus", "facet", "facetMincount", "facetMultiselect", "limit", "offset"]

  if !(query.first ∈ allowed_fields)
    @error "The $(query.first) parameter is not allowed by the GBIF API"
  end

  # Country must be a two-letters country code
  if query.first == "country"
    if length(query.second) != 2
      @error "$(query.second) is not a two letter country code"
    end
  end

  # Latitude and longitudes
  # TODO lat -90/90 lon -180/180, can be "min,max"

  # ENUMs
  if query.first ∈ keys(gbifenums)
    @assert query.second ∈ gbifenums[query.first]
  end

end


text:
# this is nearly an exact copy of `GenericHolidayCalendar` that BusinessDays.jl
# creates, except that it allows business dyas to be on the weekend and
# instead of a list of holidays it is a list of business days

"""
    MarketCalendar
* `bdays`: a vector of business days
* `dtmin`: minimum date allowed to check for bdays in bdays set. Defaults to `min(bdays...)`.
* `dtmax`: maximum date allowed to check for bdays in bdays set. Defaults to `max(bdays...)`.
* `cache`: instance of HolidayCalendarCache.
"""
mutable struct MarketCalendar <: BusinessDays.HolidayCalendar
    bdays::Vector{Date}
    dtmin::Date
    dtmax::Date
    isbday_array::Vector{Bool}
    bdayscounter_array::Vector{UInt32}
end

Base.:(==)(g1::MarketCalendar, g2::MarketCalendar) = g1.bdays == g2.bdays && g1.dtmin == g2.dtmin && g1.dtmax == g2.dtmax
Base.hash(g::MarketCalendar) = hash(g.bdays) + hash(g.dtmin) + hash(g.dtmax)

"""
    MarketCalendar(bdays, [dtmin], [dtmax], [_initcache_])
* `bdays`: a vector of dates
* `dtmin`: minimum date allowed to check for bdays in bdays set. Defaults to `min(bdays...)`.
* `dtmax`: maximum date allowed to check for bdays in bdays set. Defaults to `max(bdays...)`.
* `_initcache_`: initializes the cache for this calendar. Defaults to `true`.
"""
function MarketCalendar(bdays::Vector{Date}, dtmin::Date=min(bdays...), dtmax::Date=max(bdays...))
    bdays = sort(bdays)
    isbday_array = zeros(Bool, Dates.value(dtmax - dtmin)+1)
    bdayscounter_array = zeros(UInt32, length(isbday_array))

    for (i, d) in enumerate(dtmin:Day(1):dtmax)
        isbday_array[i] = d ∈ bdays
        if i > 1
            bdayscounter_array[i] = bdayscounter_array[i-1] + isbday_array[i]
        end
    end
    market_calendar = MarketCalendar(bdays, dtmin, dtmax, isbday_array, bdayscounter_array)
    return market_calendar
end

@inline BusinessDays.checkbounds(cal::MarketCalendar, dt::Date) = @assert cal.dtmin <= dt && dt <= cal.dtmax "Date out of calendar bounds: $dt. Allowed dates interval is from $(cal.dtmin) to $(cal.dtmax)."

@inline BusinessDays._linenumber(cal::MarketCalendar, dt::Date) = Dates.days(dt) - Dates.days(cal.dtmin) + 1

function BusinessDays.isholiday(cal::MarketCalendar, dt::Date)
    !isbday(cal, dt)
end

function BusinessDays.isbday(hc::MarketCalendar, dt::Date)::Bool
    BusinessDays.checkbounds(hc, dt)
    hc.isbday_array[BusinessDays._linenumber(hc, dt)]
end

function BusinessDays.bdayscount(hc::MarketCalendar, dt0::Date, dt1::Date)::Int
    dt0 = tobday(hc, dt0)
    dt1 = tobday(hc, dt1)
    Int(hc.bdayscounter_array[BusinessDays._linenumber(hc, dt1)]) - Int(hc.bdayscounter_array[BusinessDays._linenumber(hc, dt0)])
end

function BusinessDays.listbdays(hc::MarketCalendar, dt0::Date, dt1::Date)
    BusinessDays.checkbounds(hc, dt0)
    BusinessDays.checkbounds(hc, dt1)
    hc.bdays[dt0 .<= hc.bdays .<= dt1]
end

function Base.show(io::IO, cal::MarketCalendar)
    print(io, "MarketCalendar: $(cal.dtmin) .. $(cal.dtmax) with $(sum(cal.isbday_array)) business days")
end

cal_dt_min(x::MarketCalendar) = x.dtmin
cal_dt_max(x::MarketCalendar) = x.dtmax


text:
@unionise begin

"""
    Radial{Tα<:Real, Tβ<:Real, Tγ<:RealVec} <: Invertible

A Radial Flow as parameterised in [1].

[1] - Trippe, B. L., & Turner, R. E. (2018). Conditional Density Estimation with Bayesian
    Normalising Flows.
"""
struct Radial{Tα<:Real, Tβ<:Real, Tγ<:RealVec} <: Invertible
    α::Tα
    β::Tβ
    γ::Tγ
    Radial(α::Tα, β::Tβ, γ::Tγ) where {Tα, Tβ, Tγ} =
        new{Tα, Tβ, Tγ}(softplus(α), exp(β) - 1, γ)
end

dim(r::Radial) = length(r.γ)
params(r::Radial) = [r.α, r.β, r.γ]
nparams(::Type{<:Radial}, D::Int) = D + 2
function Radial(θ::RealVec, D::Int)
    @assert length(θ) == nparams(Radial, D)
    return Radial(θ[1], θ[2], θ[3:D+2])
end

"""
    apply(r::Radial, z)

Compute the Radial transform of `z` specified by `f`.
"""
function apply(f::Radial, z::AbstractVecOrMat{<:Real})
    @assert dim(f) == size(z, 1)
    δ = z .- f.γ
    c = (f.α * f.β) ./ (f.α .+ sqrt.(mapreducedim(abs2, +, δ, 1)))
    return z .+ c .* δ
end

function logdetJ(f::Radial, z::AbstractVector{<:Real})
    @assert dim(f) == length(z)
    return log(abs(1 + f.α^2 * f.β / (f.α + norm(z - f.γ))^2) + eps())
end

function logdetJ(f::Radial, z::AbstractMatrix{<:Real})
    @assert dim(f) == size(z, 1)
    c, δnorms = f.α^2 * f.β, reshape(sqrt.(mapreducedim(abs2, +, z .- f.γ, 1)), size(z, 2))
    return log.(abs.(1 .+ c ./ (f.α .+ δnorms).^2) .+ eps())
end

end # @unionise

function identity_init!(θ::AbstractVector{<:Real}, ::Type{<:Radial}, ::AbstractRNG, D::Int)
    @assert length(θ) == nparams(Radial, D)
    θ[1] = log(expm1(1))
    θ[2] = zero(eltype(θ))
    fill!(view(θ, 3:D+2), zero(eltype(θ)))
    return θ
end


text:
module Hooking

typealias MachTask Ptr{Void}
typealias KernReturn UInt32

immutable MemoryRegion
    @osx_only task::MachTask
    addr::Ptr{Void}
    size::UInt64
end

region_to_array(region::MemoryRegion) =
    pointer_to_array(convert(Ptr{UInt8}, region.addr), (region.size,), false)

# mach vm wrappers
@osx_only begin

    mach_task_self() = ccall(:mach_task_self,MachTask,())

    to_page(addr, size) =
        (@assert size <= 4096;
        MemoryRegion(mach_task_self(),
            addr-(reinterpret(UInt, addr)%4096),1))

    const KERN_SUCCESS     = 0x0

    const VM_PROT_NONE     = 0x0
    const VM_PROT_READ     = 0x1
    const VM_PROT_WRITE    = 0x2
    const VM_PROT_EXECUTE  = 0x4
    const VM_PROT_COPY     = 0x8

    function mach_vm_protect(task::MachTask, addr::Ptr{Void}, size::UInt64,
        prots; set_maximum::Bool = false)

        ccall(:mach_vm_protect, KernReturn,
            (MachTask, Ptr{Void}, UInt64, Bool, UInt32),
            task, addr, size, set_maximum, prots)
    end
    mach_vm_protect(region, prots) = mach_vm_protect(region.task,
        region.addr, region.size, prots)

    const VM_FLAGS_FIXED    = 0x0000
    const VM_FLAGS_ANYWHERE = 0x0001
    const VM_FLAGS_PURGABLE = 0x0002
    const VM_FLAGS_NO_CACHE = 0x0010

    function mach_vm_allocate(task, size; addr = C_NULL)
        x = Ref{Ptr{Void}}()
        x[] = addr
        ret = ccall(:mach_vm_allocate, KernReturn,
            (MachTask, Ref{Ptr{Void}}, Csize_t, Cint), task, x, size,
            addr == C_NULL ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
        (ret, MemoryRegion(task,x[],size))
    end
    mach_vm_allocate(size) = mach_vm_allocate(mach_task_self(),size)

    function mach_check(status,args...)
        if status != KERN_SUCCESS
            error("Mach system call failed (error code $status)")
        end
        @assert length(args) <= 1
        length(args) == 1 ? args[1] : nothing
    end

    const VM_REGION_BASIC_INFO_64 = 9
    const KERN_INVALID_ADDRESS = 1
    function mem_validate(addr, length)
        x = Array(UInt8,2)
        addr_info = Ref{Ptr{Void}}(addr)
        size_info = Ref{Csize_t}(length)
        flavor = VM_REGION_BASIC_INFO_64
        buf = Array(UInt,1000)
        buf_size = Ref{Csize_t}(1000)
        object_name = Ref{Ptr{Void}}(C_NULL)
        ret = ccall(:mach_vm_region, KernReturn, (Ptr{Void}, Ptr{Ptr{Void}}, Ptr{Csize_t}, Csize_t, Ptr{Void}, Ptr{Csize_t}, Ptr{Void}),
            mach_task_self(), addr_info, size_info, flavor, buf, buf_size, object_name)
        ret == 0 &&
            addr_info[] <= Ptr{Void}(addr) &&
            Ptr{Void}(addr + length) < (addr_info[] + size_info[])
    end

end

@linux_only begin
    to_page(addr, size) = (@assert size <= 4096;
        MemoryRegion(addr-(reinterpret(UInt, addr)%4096),1))

    mprotect(region, flags) = ccall(:mprotect, Cint,
        (Ptr{Void}, Csize_t, Cint), region.addr, region.size, flags)

    function mem_validate(addr, length)
        x = Array(UInt8,2)
        ret = ccall(:mincore, Cint, (Ptr{Void}, Csize_t, Ptr{UInt8}), to_page(addr, length).addr, length, x)
        ret == 0
    end

    const PROT_READ	 =  0x1
    const PROT_WRITE =  0x2
    const PROT_EXEC	 =  0x4
    const PROT_NONE	 =  0x0
end

include("backtraces.jl")

# Register save implementation

@osx_only const RegisterMap = Dict(
    :rsp => 8,
    :rip => 17
)

@linux_only const RegisterMap = Dict(
    :rip => div(UC_MCONTEXT_GREGS_RIP,sizeof(Ptr{Void}))+1,
    :rsp => div(UC_MCONTEXT_GREGS_RSP,sizeof(Ptr{Void}))+1
)

@osx_only const RC_SIZE = 20*8
@linux_only const RC_SIZE = 0xb0

immutable RegisterContext
    data::Array{UInt}
end
RegisterContext() = RegisterContext(Array(UInt,RC_SIZE))
Base.copy(RC::RegisterContext) = RegisterContext(copy(RC.data))
get_ip(RC::RegisterContext) = RC.data[RegisterMap[:rip]]-14

# Actual hooking

immutable Hook
    addr::Ptr{Void}
    orig_data::Vector{UInt8}
    callback::Function
end

using Base.llvmcall
hooks = Dict{Ptr{Void},Hook}()

immutable Deopt
    addr::Ptr{Void}
end

# The text section of jumpto-x86_64-macho.o
@osx_only const resume_length = 91
@linux_only const resume_length = 0x71

# Split this out to avoid constructing a gc frame in the callback directly
@noinline function _callback(x::Ptr{Void})
    RC = RegisterContext(reinterpret(UInt,
        copy(pointer_to_array(convert(Ptr{UInt8}, x), (RC_SIZE,), false))))
    hook_addr = RC.data[RegisterMap[:rip]]-14
    hook = hooks[reinterpret(Ptr{Void},hook_addr)]
    ret = hook.callback(hook, copy(RC))
    if isa(ret, Deopt)
        ret_addr = ret.addr
        extra_instructions = []
    else
        ret_addr = hook_addr+length(hook.orig_data)
        extra_instructions = hook.orig_data
    end
    addr_bytes = reinterpret(UInt8,[ret_addr])
    resume_data = [
        resume_instructions...,
        # Counteract the pushq %rip in the resume code
        0x48, 0x83, 0xc4, 0x8, # addq $8, %rsp
        # Is this a good idea? Probably not
        extra_instructions...,
        0x66, 0x68, addr_bytes[7:8]...,
        0x66, 0x68, addr_bytes[5:6]...,
        0x66, 0x68, addr_bytes[3:4]...,
        0x66, 0x68, addr_bytes[1:2]...,
        0xc3
    ]
    global callback_rwx
    callback_rwx[1:length(resume_data)] = resume_data

    # invalidate instruction cache here if ever ported to other
    # architectures

    ptr = convert(Ptr{Void},pointer(callback_rwx))::Ptr{Void}
    ptr, pointer(RC.data)::Ptr{UInt64}
end
function callback(x::Ptr{Void})
    ptr, data = _callback(x)
    # jump to resume code
    ccall(ptr,Void,(Ptr{Void},),data)
    nothing
end

const hooking_lib = joinpath(dirname(@__FILE__),"hooking")

function __init__()
    global resume
    global thehook
    global callback_rwx
    global resume_instructions
    here = dirname(@__FILE__)
    function resume(RC::RegisterContext)
        ccall((:hooking_jl_jumpto, hooking_lib),Void,(Ptr{UInt8},),pointer(RC.data))
    end
    theresume = cglobal((:hooking_jl_jumpto, hooking_lib), Ptr{UInt8})
    resume_instructions = pointer_to_array(convert(Ptr{UInt8}, theresume),
        (resume_length,), false)
    # Allocate an RWX page for the callback return
    @osx_only callback_rwx = begin
        region = mach_check(mach_vm_allocate(4096)...)
        mach_check(mach_vm_protect(region,
            VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE))
        region_to_array(region)
    end
    @linux_only callback_rwx = begin
        region = MemoryRegion(ccall(:mmap, Ptr{Void},
            (Ptr{Void}, Csize_t, Cint, Cint, Cint, Csize_t),
            C_NULL, 4096, PROT_EXEC | PROT_READ | PROT_WRITE,
            Base.Mmap.MAP_ANONYMOUS | Base.Mmap.MAP_PRIVATE,
            -1, 0), 4096)
        Base.systemerror("mmap", reinterpret(Int, region.addr) == -1)
        region_to_array(region)
    end
    ccall((:hooking_jl_set_callback, hooking_lib), Void, (Ptr{Void},),
        Base.cfunction(callback,Void,Tuple{Ptr{Void}})::Ptr{Void})
    thehook = cglobal((:hooking_jl_savecontext, hooking_lib), Ptr{UInt8})
end

# High Level Implementation

# Temporarily allow writing to an executable page.
# It would be nice to have a general version of this, but unfortunately, it
# seems the only reliable version to write something to a protected page on
# linux is to either parse the /proc mappings file or to use ptrace, neither
# of which sounds like a lot of fun. For now, just do this and assume the page
# is executable to begin with.
function allow_writing(f, region)
    # On OS X, make sure that the page is mapped as COW
    @osx_only mach_check(
        mach_vm_protect(region, VM_PROT_READ | VM_PROT_WRITE))
    @linux_only mprotect(region, PROT_READ | PROT_WRITE | PROT_EXEC)
    f()
    @osx_only mach_check(
        mach_vm_protect(region, VM_PROT_EXECUTE | VM_PROT_READ))
    @linux_only mprotect(region, PROT_READ | PROT_EXEC)
end

function hook(callback::Function, addr)
    # Compute number of bytes by disassembly
    # Ideally we would also check for uses of rip and branches here and error
    # out if any are found, but for now we don't need to
    triple = "x86_64-apple-darwin15.0.0"
    DC = ccall(:jl_LLVMCreateDisasm, Ptr{Void},
        (Ptr{UInt8},Ptr{Void},Cint,Ptr{Void},Ptr{Void}),
        triple, C_NULL, 0, C_NULL, C_NULL)
    @assert DC != C_NULL

    hook_asm_template = [
        0x90; #0xcc;
        0x50; #pushq   %rax
        # movq $hookto, %rax
        0x48; 0xb8; reinterpret(UInt8, [thehook]);
        0xff; 0xd0; #callq *%rax
    ]

    nbytes = 0
    while nbytes < length(hook_asm_template)
        outs = Ref{UInt8}()
        nbytes += ccall(:jl_LLVMDisasmInstruction, Csize_t,
            (Ptr{Void}, Ptr{UInt8}, Csize_t, UInt64, Ptr{UInt8}, Csize_t),
            DC,          # Disassembler
            addr+nbytes, # bytes
            30,          # Size
            addr+nbytes, # PC
            outs, 1      # OutString
            )
    end

    # Record the instructions that were there originally
    dest = pointer_to_array(convert(Ptr{UInt8}, addr), (nbytes,), false)
    orig_data = copy(dest)

    hook_asm = [ hook_asm_template; zeros(UInt8,nbytes-length(hook_asm_template)) ]# Pad to nbytes

    allow_writing(to_page(addr,nbytes)) do
        dest[:] = hook_asm
    end

    hooks[addr] = Hook(addr,orig_data,callback)
end

function get_function_addr(f, t)
    t = Tuple{typeof(f), Base.to_tuple_type(t).parameters...}
    llvmf = ccall(:jl_get_llvmf, Ptr{Void}, (Any, Any, Bool, Bool), f, t, false, true)
    @assert llvmf != C_NULL
    reinterpret(Ptr{Void},ccall(:jl_get_llvm_fptr, UInt64, (Ptr{Void},), llvmf))
end

hook(callback, f, t) = hook(callback, get_function_addr(f, t))

function unhook(addr)
    hook = pop!(hooks, addr)

    nbytes = length(hook.orig_data)
    dest = pointer_to_array(convert(Ptr{UInt8}, addr),
        (nbytes,), false)

    allow_writing(to_page(addr,nbytes)) do
        dest[:] = hook.orig_data
    end
end

end # module


text:
module CrossValidators

using Statistics: mean, std

using MLBase: StratifiedKfold, LOOCV, Kfold,
              RandomSub, StratifiedRandomSub

# standard included modules
using ..Utils
using ..AbsTypes

using Random
using DataFrames: DataFrame

export crossvalidate, pipe_performance

macc(X,Y) = score(:accuracy,X,Y)

"""
    crossvalidate(pl::Machine,X::DataFrame,Y::Vector,pfunc::Function,kfolds=10)

Run K-fold crossvalidation where:
- `pfunc` is a performance metric
- `X` and `Y` are input and target
"""
function crossvalidate(pl::Machine,X::DataFrame,Y::Vector,pfunc::Function,nfolds::Int,verbose::Bool)
   ## flatten arrays
   @assert size(X)[1] == length(Y)
   rowsize = size(X)[1]
   folds = Kfold(length(Y),nfolds) |> collect
   pacc = Float64[]
   fold = 0
   error = 0
   for trndx in folds
      ppl = deepcopy(pl)
      rlist = collect(1:rowsize) # nasty bug if not collected and placed inside setdiff
      tsndx = setdiff(rlist,trndx)
      trX = X[trndx,:]
      trY = Y[trndx] |> collect
      tstX = X[tsndx,:]
      tstY = Y[tsndx] |> collect
      res = 0.0
      try
         res = pipe_performance(ppl,pfunc,trX,trY,tstX,tstY)
         push!(pacc,res)
         fold += 1
         if verbose == true
            println("fold: ",fold,", ",res)
         end
      catch e
         Base.invokelatest(Base.display_error, Base.catch_stack())
         @error e
         error += 1
      end
   end
   if verbose == true
      println("errors: ",error)
   end
   (mean=mean(pacc),std=std(pacc),folds=nfolds,errors=error)
end

function crossvalidate(pl::Machine,X::DataFrame,Y::Vector; metric::Function=macc, nfolds=10,verbose=true)
   crossvalidate(pl,X,Y,metric,nfolds,verbose)
end

function crossvalidate(pl::Machine,X::DataFrame,Y::Vector,met::Function,nfolds=10,verbose=true)
   crossvalidate(pl,X,Y,met,nfolds,verbose)
end

function crossvalidate(pl::Machine,X::DataFrame,Y::Vector,met::Function,folds::Int,verbose=true)
   crossvalidate(pl,X,Y,met,folds,versbose)
end

function pipe_performance(plearner::Machine,perf::Function,trX::DataFrame, trY::Vector,tstX::DataFrame,tstY::Vector)
   learner = deepcopy(plearner)
   fit!(learner,trX,trY)
   pred = transform!(learner,tstX)
   res = perf(pred,tstY)
   return res
end

end


text:
module PythonInterop

#=
Links:
    - https://github.com/JuliaPy/PyCall.jl
    - https://en.wikipedia.org/wiki/Fibonacci_number

Notes:
    - When py"..." is used it evaluates the code in a Python namespace dedicated
    to your module (e.g. PythonInterop). Thus, you don't have to worry about
    name clash, but Python functions declared cannot be accessed outside your module.
    - Python functions must be defined in __init__ of your Julia module
=#

using Pkg
Pkg.add("PyCall")
using PyCall

function __init__()
end

# This function will return the sin of x pi
function py_sinpi(x)
end

function get_pyfib()
end

#=
This fibonacci function is a bit different of what is usually done,
it expects to receive the function it will call to do its recursion.
The general idea is to do something like this: jlfib -> pyfib -> jlfib -> ...
=#
function jlfib(n, fib)
end

end


text:
using Test

@time @testset "Lines" begin include("test_lines.jl") end
@time @testset "Quadrilaterals" begin include("test_quadrilaterals.jl") end
@time @testset "Search elements" begin include("test_eselect.jl") end
@time @testset "Boxes" begin include("test_boxes.jl") end
@time @testset "Search vertices" begin include("test_vselect.jl") end

@time @testset "Mesh import/export" begin include("test_io.jl") end

@time @testset "Triangles" begin include("test_triangles.jl") end
@time @testset "Tetrahedra" begin include("test_tetrahedra.jl") end

@time @testset "Modification" begin include("test_modification.jl") end

@time @testset "High-level" begin include("test_mesh.jl") end


text:
"""
Internal data structure to partition a box

`domain`:       box defining the entire domain

`left`:         leftmost / bottom edge of the domain

`scale`:        1 / diameter of each box in the new partition (componentwise)

`dimsprod`:     for indexing the partition:
                key is counted up in first dimension first, then second dimension, etc... ie

                 1st dim →
                  * — * — * — *
            2nd   | 1 | 2 | 3 |
            dim   * — * — * — *
            ↓     | 4 | 5 | 6 |
                  * — * — * — *
.
"""
struct BoxPartition{N,T} <: AbstractBoxPartition{Box{N,T}}
    domain::Box{N,T}
    left::SVector{N,T}
    scale::SVector{N,T}
    dims::SVector{N,Int}
    dimsprod::SVector{N,Int}
end

function BoxPartition(domain::Box{N,T}, dims::NTuple{N,Int}) where {N,T}
    dims = SVector{N,Int}(dims)
    left = domain.center .- domain.radius
    scale = dims ./ (2*domain.radius)
    # nr. of boxes / diameter of the domain == 1 / diameter of each box
    dimsprod_ = [SVector(1); cumprod(dims)]
    dimsprod = dimsprod_[SOneTo(N)]

    return BoxPartition(domain, left, scale, dims, dimsprod)
end

function BoxPartition(domain::Box{N,T}) where {N,T}
    dims = tuple(ones(Int64,N)...)
    BoxPartition(domain, dims)
end

BoxPartition(domain::Box{1,T}, dims::Int) where {T} = BoxPartition(domain, (dims,))

dimension(partition::BoxPartition{N,T}) where {N,T} = N

function subdivide(P::BoxPartition{N,T}, dim::Int) where {N,T}
    new_dims = ntuple(i -> P.dims[i]*(i==dim ? 2 : 1), N)
    return BoxPartition(P.domain, new_dims)
end

keytype(::Type{<:BoxPartition}) = Int
keys_all(partition::BoxPartition) = 1:prod(partition.dims)
# == 1 : partition.dimsprod[end] * partition.dims[end]

Base.size(partition::BoxPartition) = partition.dims.data # .data returns as tuple

function Base.show(io::IO, partition::BoxPartition)
    print(io, "$(partition.dims[1])")
    for k = 2:length(partition.left)
        print(io, " x $(partition.dims[k])")
    end
    print(io, " BoxPartition")
end

# TODO: replace with overloaded getindex
function key_to_box(partition::BoxPartition{N,T}, key::M) where M <: Union{Int, NTuple{N, Int}} where {N,T}
    dims = size(partition)
    radius = partition.domain.radius ./ dims
    left = partition.domain.center .- partition.domain.radius
    center = left .+ radius .+ (2 .* radius) .* (CartesianIndices(dims)[key].I .- 1)
    # start at leftmost box in the partition and move $key boxes right
    return Box(center, radius)
end

function unsafe_point_to_ints(partition::BoxPartition, point)
    x = (point .- partition.left) .* partition.scale
    # counts how many boxes x is away from left (componentwise)
    return map(xi -> Base.unsafe_trunc(Int, xi), x)
end

function point_to_key(partition::BoxPartition, point)
    x_ints = unsafe_point_to_ints(partition, point)

    if any(x_ints .< zero(eltype(x_ints))) || any(x_ints .>= partition.dims)
        @debug "point does not lie in the domain" point partition.domain
        return nothing
    end

    return sum(x_ints .* partition.dimsprod) + 1
end


text:
@time @testset "block_operations" begin
    @testset begin
        A = [1 2; 3 4]
        B = [5 6; 7 8]
        C = Matrix{Int}(undef, 2, 2)
        Gaius.block_covec_vec_mul!(Gaius.multithreaded, C, A, B, 1)
        @test C == A * B
    end

    @testset begin
        A = [1 2; 3 4]
        B = [5, 6]
        C = Vector{Int}(undef, 2)
        Gaius.block_covec_vec_mul!(Gaius.multithreaded, C, A, B, 1)
        @test C == A * B
    end
end


text:
using Plots
using jInv.Mesh
using Flux
using Printf
using Statistics
using LinearAlgebra
using JLD
using MFGnet
using MAT

include("../ROLNWF2019/viewers.jl")
include("../ROLNWF2019/runOMThelpers.jl")

for d=[2,10,50,100]
	for level=[1,2]
		for iter=[100,200,300,400,500]
			dir = pwd()
			fname ="Obstacle-Multilevel-d-$d-level-$level-iter-$iter"

imgdir = dir * "/" * fname *"/"
mkdir(imgdir)
println("fname=$fname")
file = fname *".jld"

res  = load(file)
settings = res["settings"]
(m,α,nTrain,R,d,mu0,sig0,mu1,sig1,domain,nTh,m,nt,T,Qheight,sigQ,muFe,muFp,Tth) = settings
println("nTrain=$nTrain")
tspan = [0.0, T]
stepper = RK4Step()

ar = x-> R.(x)
Θbest = res["Θbest"]
println("number of model parameters: $(length(MFGnet.param2vec(Θbest)))")

rho0 = Gaussian(d,sig0,mu0)
rho1 = Gaussian(d,sig1,mu1)

Q1   = Gaussian(2,sigQ,zeros(R,2),R(Qheight))
Q    = x -> vec(Q1(x[1:2,:]))

domain = [-3.0 3.0 -4.5 4.5]

M      = getRegularMesh(domain,[128, 128])
X0     = Matrix(getCellCenteredGrid(M)')
X0  = ar([X0; zeros(d-2,size(X0,2))])

Φ = getPotentialResNet(nTh, T, nTh, R)

# setup validation
rho0x = rho0(X0)
rho1x = rho1(X0)

vol  = (domain[4]-domain[3])^d
w    = (vol/M.nc) * rho0x
sum(w)

F1  = Fp(Q,rho0,rho0x,ar(muFp))
F2  = Fe(rho0,rho0x,ar(muFe))
F   = Fcomb([F1;F2])
G = Gkl(rho0,rho1,rho0x,rho1x,ar(1.0))
J = MeanFieldGame(F,G,X0,rho0,w,Φ=Φ,stepper=stepper,nt=16,α=α,tspan=tspan)

Jc  = J(Θbest)

## plots:
Qx = Q(X0)
p0 =  viewImage2D(Qx,M,aspect_ratio=:equal)
title!("Qx(x)")
savefig(p0,imgdir * "Qx-n-$(M.n[1])x$(M.n[2]).png")

p1= viewImage2D(J.rho0x,M,aspect_ratio=:equal)
title!("rho0(x)")
savefig(p1,imgdir * "rho0x-n-$(M.n[1])x$(M.n[2]).png")

p2= viewImage2D(J.G.rho1x,M,aspect_ratio=:equal)
title!("rho1(x)")
savefig(p2,imgdir * "rho1x-n-$(M.n[1])x$(M.n[2]).png")

detDy = exp.(J.UN[d+1,:])
rho1y = (J.G.rho1(J.UN[1:d,:]).*detDy)
p3= viewImage2D(rho1y ,M,aspect_ratio=:equal,clims=(minimum(J.rho0x),maximum(J.rho0x)))
# p3= viewImage2D(rho1y ,M,aspect_ratio=:equal)
title!("rho1(y).*det")
savefig(p3,imgdir * "rho1y-n-$(M.n[1])x$(M.n[2]).png")

res0 = J.rho0x - rho1y
p4 = viewImage2D(abs.(res0),M,aspect_ratio=:equal,clims=(minimum(J.rho0x),maximum(J.rho0x)))
title!("rho0x-rho1y.*det") # where's the determinant
savefig(p4,imgdir * "absDiffRho-n-$(M.n[1])x$(M.n[2]).png")

p5 = viewImage2D(detDy,M,aspect_ratio=:equal)
title!("det")
savefig(p5,imgdir * "det-n-$(M.n[1])x$(M.n[2]).png")

xc = [ J.X0; fill(0.0,1,size(J.X0,2))]
Φ0 = J.Φ(xc,Θbest)
p6 = viewImage2D(Φ0, M, aspect_ratio=:equal)
title!("Phi(x,0)")
savefig(p6,imgdir * "phi0-n-$(M.n[1])x$(M.n[2]).png")

xc = [ J.UN[1:d,:]; fill(1.0,1,size(J.X0,2))]
Φ1 = vec(J.Φ(xc,Θbest) )
p7 = viewImage2D(Φ1,M,aspect_ratio=:equal)
title!("Phi(z,1)")
savefig(p7,imgdir * "phi1-n-$(M.n[1])x$(M.n[2]).png")

deltaG = J.α[3]*MFGnet.getDeltaG(J.G,J.UN)
p8 = viewImage2D(deltaG,M,aspect_ratio=:equal)
title!("deltaG(z)")
savefig(p8,imgdir * "deltaG-n-$(M.n[1])x$(M.n[2]).png")

resHJB = abs.(Φ1-deltaG)
p9 = viewImage2D(resHJB,M,aspect_ratio=:equal)
title!("|Phi1 - deltaG(z)|")
savefig(p9,imgdir * "absDiffHJBfinal-n-$(M.n[1])x$(M.n[2]).png")

################################################################################
####### Create new MFG for characteristics
################################################################################
Xt = load("Obstacle-X0.jld")["Xt"]
if d>2
	Xt = [Xt; zeros(R,d-2,size(Xt,2))]
end
rho0xt = rho0(Xt)
rho1xt = rho1(Xt)
wt     = 1/size(Xt,2) * ones(size(Xt,2))
F1t  = Fp(Q,rho0,rho0xt,1.0)
F2t  = Fe(rho0,rho0xt,1.0)
Ft   = Fcomb([F1t;F2t])
Gt = Gkl(rho0,rho1,rho0xt,rho1xt,1.0)
Jt = MeanFieldGame(Ft,Gt,Xt,rho0,wt,Φ=Φ,stepper=RK4Step(),nt=2*nt,α=α)

ntv = Jt.nt
Ut = MFGnet.integrate2(Jt.stepper,MFGnet.odefun,Jt,ar([Xt;zeros(4,size(Xt,2))]),Θbest,ar([0.0 T]),ntv)
charFwd = copy(Ut);

Y0  = ar([Ut[1:d,:,end];zeros(4,size(Xt,2))])
U0 = MFGnet.integrate2(Jt.stepper,MFGnet.odefun,Jt,Y0,Θbest,ar([T 0.0]),ntv)
charBwd = copy(U0);

p10 = viewImage2D(J.rho0x ,M,aspect_ratio=:equal)
for j=1:size(Xt,2)
    uj = Ut[1:d,j,:]
    vj = U0[1:d,j,:]
    plot!(p10,uj[1,:],uj[2,:],legend=false,linecolor=:white,linewidth=1.5)
    plot!(p10,vj[1,:],vj[2,:],legend=false,linecolor=:red,linewidth=1.5)
end
title!("characteristics, fwd+inv")
savefig(p10,imgdir * "characteristics-n-$(M.n[1])x$(M.n[2]).png")
################################################################################
################################################################################

U0 = MFGnet.integrate(J.stepper,MFGnet.odefun,J,ar([J.X0;zeros(4,size(J.X0,2))]),Θbest,[T 0.0],ntv)
detDyInv = exp.(U0[d+1,:])
rho0z = rho0(U0[1:d,:]).*detDyInv
p11= viewImage2D(rho0z ,M,aspect_ratio=:equal)
title!("rho0(z).*detInv")
savefig(p11,imgdir * "rho0z-n-$(M.n[1])x$(M.n[2]).png")

res1 = rho1x - rho0z
p4 = viewImage2D(abs.(res1),M,aspect_ratio=:equal)
title!("| rho1(x)-rho0(z).*detInv |") # where's the determinant
savefig(p4,imgdir * "absDiffRho1-n-$(M.n[1])x$(M.n[2]).png")

X1,X2 = getFaceGrids(M)
XC = getCellCenteredGrid(M)
tt = range(0,stop=T,length=64)
X1t = t -> [Matrix(X1'); zeros(d-2,size(X1,1)); fill(t,1,size(X1,1))]
X2t = t -> [Matrix(X2'); zeros(d-2,size(X2,1)); fill(t,1,size(X2,1))]
XCt = t -> [Matrix(XC'); zeros(d-2,size(XC,1)); fill(t,1,size(XC,1))]
V1 = zeros(M.n[1]+1,M.n[2],length(tt))
V2 = zeros(M.n[1],M.n[2]+1,length(tt))
ΦOpt  = zeros(M.n[1],M.n[2],length(tt))
for k=1:length(tt)
    tk = tt[k]
    println("tk=$tk")
    Phi = J.Φ(X1t(tk),Θbest) # run fwd prop to populate N.tmp
    gradPhi = MFGnet.getGradPotential(J.Φ,X1t(tk),Θbest)
    V1[:,:,k] = reshape(-(1/J.α[1])*gradPhi[1,:],M.n[1]+1,M.n[2])
    Phi = J.Φ(X2t(tk),Θbest) # run fwd prop to populate N.tmp
    gradPhi = MFGnet.getGradPotential(J.Φ,X2t(tk),Θbest)
    V2[:,:,k] = reshape(-(1/J.α[1])*gradPhi[2,:],M.n[1],M.n[2]+1)

	ΦOpt[:,:,k] = reshape(J.Φ(XCt(tk),Θbest),M.n[1],M.n[2])
end

matwrite( fname * ".mat", Dict(
	"Qx" => Qx,
    "rho0x" => rho0x,
    "rho1x" => rho1x,
    "rho1y" => rho1y,
    "rho0z" => rho0z,
    "domain" => M.domain,
    "detDy" => detDy,
    "detDyInv" => detDyInv,
    "res0" => res0,
    "res1" => res1,
    "charFwd" => charFwd,
    "charBwd" => charBwd,
    "deltaG" => deltaG,
    "n" => M.n,
    "Phi0" => Φ0,
    "Phi1" => Φ1,
	"V1" => V1,
	"V2" => V2,
	"PhiOpt" => ΦOpt,
    "His" => res["His"]))
end

end
end

max_stars_repo_name: EmoryMLIP/MFGnet

max_issues_repo_name: EmoryMLIP/MFGnet

max_forks_repo_name: EmoryMLIP/MFGnet


text:
using OrderedBinning
using Test

@testset "non-strinct, zero tolerance" begin
    boundaries = 0:3
    ob = ordered_bins(boundaries; strict = false)
    @test ob(-1) == 0
    @test ob(0) == 1
    @test ob(0.5) == 1
    @test ob(3) == 3
    @test ob(4) == 4
    for _ in 1:100
        x = rand(Bool) ? rand(0:3) : rand() * 3.0
        i = ob(x)
        @test boundaries[i] ≤ x ≤ boundaries[i + 1]
        if x < 3
            @test x < boundaries[i + 1]
        end
    end
end

@testset "strint, 0.5 tolerance" begin
    boundaries = 0:3
    ob = ordered_bins(boundaries; strict = true, tolerance = 0.5)
    @test_throws DomainError ob(-1)
    @test ob(0) == 1
    @test ob(0.5) == 1
    @test ob(3) == 3
    @test ob(3.5) == 3
    @test_throws DomainError ob(4)
    for _ in 1:100
        x = rand(Bool) ? rand(0:3) : rand() * 3.0
        i = ob(x)
        @test boundaries[i] ≤ x ≤ boundaries[i + 1]
        if x < 3
            @test x < boundaries[i + 1]
        end
    end
end