text:
using KernelAbstractions, Test

# Not passing in typelist because CuArrays will not support passing in
# non-inlined types
@kernel function convert_kernel!(A, B)
    tid = @index(Global, Linear)

    # Int -> Int
    tid = Int64(Int32(tid))

    @inbounds B[tid, 1] = ceil(Int8, A[tid])
    @inbounds B[tid, 2] = ceil(Int16, A[tid])
    @inbounds B[tid, 3] = ceil(Int32, A[tid])
    @inbounds B[tid, 4] = ceil(Int64, A[tid])
    @inbounds B[tid, 5] = ceil(Int128, A[tid])
    @inbounds B[tid, 6] = ceil(UInt8, A[tid])
    @inbounds B[tid, 7] = ceil(UInt16, A[tid])
    @inbounds B[tid, 8] = ceil(UInt32, A[tid])
    @inbounds B[tid, 9] = ceil(UInt64, A[tid])
    @inbounds B[tid, 10] = ceil(UInt128, A[tid])

    @inbounds B[tid, 11] = floor(Int8, A[tid])
    @inbounds B[tid, 12] = floor(Int16, A[tid])
    @inbounds B[tid, 13] = floor(Int32, A[tid])
    @inbounds B[tid, 14] = floor(Int64, A[tid])
    @inbounds B[tid, 15] = floor(Int128, A[tid])
    @inbounds B[tid, 16] = floor(UInt8, A[tid])
    @inbounds B[tid, 17] = floor(UInt16, A[tid])
    @inbounds B[tid, 18] = floor(UInt32, A[tid])
    @inbounds B[tid, 19] = floor(UInt64, A[tid])
    @inbounds B[tid, 20] = floor(UInt128, A[tid])

    @inbounds B[tid, 21] = round(Int8, A[tid])
    @inbounds B[tid, 22] = round(Int16, A[tid])
    @inbounds B[tid, 23] = round(Int32, A[tid])
    @inbounds B[tid, 24] = round(Int64, A[tid])
    @inbounds B[tid, 25] = round(Int128, A[tid])
    @inbounds B[tid, 26] = round(UInt8, A[tid])
    @inbounds B[tid, 27] = round(UInt16, A[tid])
    @inbounds B[tid, 28] = round(UInt32, A[tid])
    @inbounds B[tid, 29] = round(UInt64, A[tid])
    @inbounds B[tid, 30] = round(UInt128, A[tid])

end

function convert_testsuite(backend, ArrayT)

    N = 32
    d_A = ArrayT([rand()*3 for i = 1:N])

    # 30 because we have 10 integer types and we have 3 operations
    d_B = ArrayT(zeros(N, 30))

    @testset "convert test" begin
        kernel = convert_kernel!(backend(), 4)
        wait(kernel(d_A, d_B, ndrange=(N),))

        for i = 1:10
            @test d_B[:,i] == ceil.(d_A)
            @test d_B[:,i+10] == floor.(d_A)
            @test d_B[:,i+20] == round.(d_A)
        end
    end
end


text:
using NewtonsMethod3
using Test

@testset "NewtonsMethod3.jl" begin
    # Write your own tests here.
end

ext: jl


text:
#### Tolerances

export ResidualTolerance, SolutionTolerance

abstract type AbstractTolerance{FT <: AbstractFloat} end

"""
    ResidualTolerance

A tolerance type based on the residual of the equation ``f(x) = 0``
"""
struct ResidualTolerance{FT} <: AbstractTolerance{FT}
    tol::FT
end

"""
    (tol::ResidualTolerance)(x1, x2, y)

Evaluates residual tolerance, based on ``|f(x)|``
"""
(tol::ResidualTolerance)(x1, x2, y) = all(abs.(y) .< tol.tol)

"""
    SolutionTolerance

A tolerance type based on the solution ``x`` of the equation ``f(x) = 0``
"""
struct SolutionTolerance{FT} <: AbstractTolerance{FT}
    tol::FT
end

"""
    (tol::SolutionTolerance)(x1, x2, y)

Evaluates solution tolerance, based on ``|x2-x1|``
"""
(tol::SolutionTolerance)(x1, x2, y) = all(abs.(x2 .- x1) .< tol.tol)


text:
module Types

using UUIDs
using Random
using Dates
import LibGit2
import REPL
using REPL.TerminalMenus

using ..TOML
import ..Pkg
import Pkg: GitTools, depots, logdir

import Base: SHA1
using SHA

export UUID, pkgID, SHA1, VersionRange, VersionSpec, empty_versionspec,
    Requires, Fixed, merge_requires!, satisfies, ResolverError,
    PackageSpec, EnvCache, Context, Context!,
    CommandError, cmderror, has_name, has_uuid, write_env, parse_toml, find_registered!,
    project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
    manifest_info, registered_uuids, registered_paths, registered_uuid, registered_name,
    read_project, read_manifest, pathrepr, registries,
    PackageMode, PKGMODE_MANIFEST, PKGMODE_PROJECT, PKGMODE_COMBINED,
    UpgradeLevel, UPLEVEL_FIXED, UPLEVEL_PATCH, UPLEVEL_MINOR, UPLEVEL_MAJOR,
    PackageSpecialAction, PKGSPEC_NOTHING, PKGSPEC_PINNED, PKGSPEC_FREED, PKGSPEC_DEVELOPED, PKGSPEC_TESTED, PKGSPEC_REPO_ADDED,
    printpkgstyle,
    projectfile_path

include("versions.jl")

## ordering of UUIDs ##

Base.isless(a::UUID, b::UUID) = a.value < b.value

## Computing UUID5 values from (namespace, key) pairs ##
function uuid5(namespace::UUID, key::String)
    data = [reinterpret(UInt8, [namespace.value]); codeunits(key)]
    u = reinterpret(UInt128, sha1(data)[1:16])[1]
    u &= 0xffffffffffff0fff3fffffffffffffff
    u |= 0x00000000000050008000000000000000
    return UUID(u)
end
uuid5(namespace::UUID, key::AbstractString) = uuid5(namespace, String(key))

const uuid_dns = UUID(0x6ba7b810_9dad_11d1_80b4_00c04fd430c8)
const uuid_julia_project = uuid5(uuid_dns, "julialang.org")
const uuid_package = uuid5(uuid_julia_project, "package")
const uuid_registry = uuid5(uuid_julia_project, "registry")
const uuid_julia = uuid5(uuid_package, "julia")

## user-friendly representation of package IDs ##
function pkgID(p::UUID, uuid_to_name::Dict{UUID,String})
    name = get(uuid_to_name, p, "")
    isempty(name) && (name = "(unknown)")
    uuid_short = string(p)[1:8]
    return "$name [$uuid_short]"
end

####################
# Requires / Fixed #
####################
const Requires = Dict{UUID,VersionSpec}

function merge_requires!(A::Requires, B::Requires)
    for (pkg, vers) in B
        A[pkg] = haskey(A, pkg) ? intersect(A[pkg], vers) : vers
    end
    return A
end

satisfies(pkg::UUID, ver::VersionNumber, reqs::Requires) =
    !haskey(reqs, pkg) || in(ver, reqs[pkg])

struct Fixed
    version::VersionNumber
    requires::Requires
end
Fixed(v::VersionNumber) = Fixed(v, Requires())

Base.:(==)(a::Fixed, b::Fixed) = a.version == b.version && a.requires == b.requires
Base.hash(f::Fixed, h::UInt) = hash((f.version, f.requires), h + (0x68628b809fd417ca % UInt))

Base.show(io::IO, f::Fixed) = isempty(f.requires) ?
    print(io, "Fixed(", repr(f.version), ")") :
    print(io, "Fixed(", repr(f.version), ",", f.requires, ")")

struct ResolverError <: Exception
    msg::AbstractString
    ex::Union{Exception,Nothing}
end
ResolverError(msg::AbstractString) = ResolverError(msg, nothing)

function Base.showerror(io::IO, pkgerr::ResolverError)
    print(io, pkgerr.msg)
    if pkgerr.ex !== nothing
        pkgex = pkgerr.ex
        if isa(pkgex, CompositeException)
            for cex in pkgex
                print(io, "\n=> ")
                showerror(io, cex)
            end
        else
            print(io, "\n")
            showerror(io, pkgex)
        end
    end
end

#################
# Command Error #
#################
struct CommandError <: Exception
    msg::String
end
cmderror(msg::String...) = throw(CommandError(join(msg)))
Base.show(io::IO, err::CommandError) = print(io, err.msg)

###############
# PackageSpec #
###############
@enum(UpgradeLevel, UPLEVEL_FIXED, UPLEVEL_PATCH, UPLEVEL_MINOR, UPLEVEL_MAJOR)

function UpgradeLevel(s::Symbol)
    s == :fixed ? UPLEVEL_FIXED :
    s == :patch ? UPLEVEL_PATCH :
    s == :minor ? UPLEVEL_MINOR :
    s == :major ? UPLEVEL_MAJOR :
    throw(ArgumentError("invalid upgrade bound: $s"))
end

@enum(PackageMode, PKGMODE_PROJECT, PKGMODE_MANIFEST, PKGMODE_COMBINED)
@enum(PackageSpecialAction, PKGSPEC_NOTHING, PKGSPEC_PINNED, PKGSPEC_FREED,
                            PKGSPEC_DEVELOPED, PKGSPEC_TESTED, PKGSPEC_REPO_ADDED)

const VersionTypes = Union{VersionNumber,VersionSpec,UpgradeLevel}

mutable struct GitRepo
    url::String
    rev::String
    git_tree_sha1::Union{Nothing,SHA1}
end

GitRepo(url::String, revspec) = GitRepo(url, revspec, nothing)
GitRepo(url::String) = GitRepo(url, "", nothing)
Base.:(==)(repo1::GitRepo, repo2::GitRepo) = (repo1.url == repo2.url && repo1.rev == repo2.rev && repo1.git_tree_sha1 == repo2.git_tree_sha1)

mutable struct PackageSpec
    name::String
    uuid::UUID
    version::VersionTypes
    mode::PackageMode
    path::Union{Nothing,String}
    special_action::PackageSpecialAction # If the package is currently being pinned, freed etc
    repo::Union{Nothing,GitRepo}
    PackageSpec() = new("", UUID(zero(UInt128)), VersionSpec(), PKGMODE_PROJECT, nothing, PKGSPEC_NOTHING, nothing)
    PackageSpec(name::AbstractString, uuid::UUID, version::VersionTypes,
                mode::PackageMode=PKGMODE_PROJECT, path=nothing, special_action=PKGSPEC_NOTHING,
                repo=nothing) =
        new(String(name), uuid, version, mode, path, special_action, repo)
end
PackageSpec(name::AbstractString, uuid::UUID) =
    PackageSpec(name, uuid, VersionSpec())
PackageSpec(name::AbstractString, version::VersionTypes=VersionSpec()) =
    PackageSpec(name, UUID(zero(UInt128)), version)
PackageSpec(uuid::UUID, version::VersionTypes=VersionSpec()) =
    PackageSpec("", uuid, version)
function PackageSpec(repo::GitRepo)
    pkg = PackageSpec()
    pkg.repo = repo
    return pkg
end

has_name(pkg::PackageSpec) = !isempty(pkg.name)
has_uuid(pkg::PackageSpec) = pkg.uuid != UUID(zero(UInt128))

function Base.show(io::IO, pkg::PackageSpec)
    print(io, "PackageSpec(")
    has_name(pkg) && show(io, pkg.name)
    has_name(pkg) && has_uuid(pkg) && print(io, ", ")
    has_uuid(pkg) && show(io, pkg.uuid)
    vstr = repr(pkg.version)
    if vstr != "VersionSpec(\"*\")"
        (has_name(pkg) || has_uuid(pkg)) && print(io, ", ")
        print(io, vstr)
    end
    print(io, ")")
end

############
# EnvCache #
############

function parse_toml(path::String...; fakeit::Bool=false)
    p = joinpath(path...)
    !fakeit || isfile(p) ? TOML.parsefile(p) : Dict{String,Any}()
end

let trynames(names) = begin
    return root_path::AbstractString -> begin
        for x in names
            maybe_file = joinpath(root_path, x)
            if isfile(maybe_file)
                return maybe_file
            end
        end
    end
end # trynames
    global projectfile_path = trynames(Base.project_names)
    global manifestfile_path = trynames(Base.manifest_names)
end # let

function find_project_file(env::Union{Nothing,String}=nothing)
    project_file = nothing
    if env isa Nothing
        project_file = Base.active_project()
        project_file == nothing && error("no active project")
    elseif startswith(env, '@')
        project_file = Base.load_path_expand(env)
        project_file === nothing && error("package environment does not exist: $env")
    elseif env isa String
        if isdir(env)
            isempty(readdir(env)) || error("environment is a package directory: $env")
            project_file = joinpath(env, Base.project_names[end])
        else
            project_file = endswith(env, ".toml") ? abspath(env) :
                abspath(env, Base.project_names[end])
        end
    end
    @assert project_file isa String &&
        (isfile(project_file) || !ispath(project_file) ||
         isdir(project_file) && isempty(readdir(project_file)))
     return project_file
end

mutable struct EnvCache
    # environment info:
    env::Union{Nothing,String}
    git::Union{Nothing,LibGit2.GitRepo}

    # paths for files:
    project_file::String
    manifest_file::String

    # name / uuid of the project
    pkg::Union{PackageSpec, Nothing}

    # cache of metadata:
    project::Dict
    manifest::Dict

    # registered package info:
    uuids::Dict{String,Vector{UUID}}
    paths::Dict{UUID,Vector{String}}

    function EnvCache(env::Union{Nothing,String}=nothing)
        project_file = find_project_file(env)
        project_dir = dirname(project_file)
        git = ispath(joinpath(project_dir, ".git")) ? LibGit2.GitRepo(project_dir) : nothing

        project = read_project(project_file)
        if any(haskey.((project,), ["name", "uuid", "version"]))
            project_package = PackageSpec(
                get(project, "name", ""),
                UUID(get(project, "uuid", 0)),
                VersionNumber(get(project, "version", "0.0")),
            )
        else
            project_package = nothing
        end
        # determine manifest_file name
        dir = abspath(dirname(project_file))
        manifest_file = haskey(project, "manifest") ?
            abspath(project["manifest"]) :
            manifestfile_path(dir)
        # use default name if still not determined
        (manifest_file === nothing) && (manifest_file = joinpath(dir, "Manifest.toml"))
        write_env_usage(manifest_file)
        manifest = read_manifest(manifest_file)
        uuids = Dict{String,Vector{UUID}}()
        paths = Dict{UUID,Vector{String}}()
        return new(env,
            git,
            project_file,
            manifest_file,
            project_package,
            project,
            manifest,
            uuids,
            paths,)
    end
end

collides_with_project(env::EnvCache, pkg::PackageSpec) =
    is_project_name(env, pkg.name) || is_project_uuid(env, pkg.uuid)
is_project(env::EnvCache, pkg::PackageSpec) = is_project_uuid(env, pkg.uuid)
is_project_name(env::EnvCache, name::String) =
    env.pkg !== nothing && env.pkg.name == name
is_project_uuid(env::EnvCache, uuid::UUID) =
    env.pkg !== nothing && env.pkg.uuid == uuid

###########
# Context #
###########
stdlib_dir() = joinpath(Sys.BINDIR, "..", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)")
stdlib_path(stdlib::String) = joinpath(stdlib_dir(), stdlib)
function gather_stdlib_uuids()
    stdlibs = Dict{UUID,String}()
    for stdlib in readdir(stdlib_dir())
        projfile = projectfile_path(stdlib_path(stdlib))
        if nothing !== projfile
            proj = TOML.parsefile(projfile)
            if haskey(proj, "uuid")
                stdlibs[UUID(proj["uuid"])] = stdlib
            end
        end
    end
    return stdlibs
end

# ENV variables to set some of these defaults?
Base.@kwdef mutable struct Context
    env::EnvCache = EnvCache()
    preview::Bool = false
    use_libgit2_for_all_downloads::Bool = false
    num_concurrent_downloads::Int = 8
    graph_verbose::Bool = false
    stdlibs::Dict{UUID,String} = gather_stdlib_uuids()
    # Remove next field when support for Pkg2 CI scripts is removed
    old_pkg2_clone_name::String = ""
end

function Context!(ctx::Context; kwargs...)
    for (k, v) in kwargs
        setfield!(ctx, k, v)
    end
end

function project_compatibility(ctx::Context, name::String)
    v = VersionSpec()
    project = ctx.env.project
    compat = get(project, "compat", Dict())
    if haskey(compat, name)
        v = VersionSpec(semver_spec(compat[name]))
    end
    return v
end

function write_env_usage(manifest_file::AbstractString)
    !ispath(logdir()) && mkpath(logdir())
    usage_file = joinpath(logdir(), "manifest_usage.toml")
    touch(usage_file)
    !isfile(manifest_file) && return
    # Do not rewrite as do syntax (no longer precompilable)
    io = open(usage_file, "a")
    println(io, "[[\"", escape_string(manifest_file), "\"]]")
    print(io, "time = ", now()); println(io, 'Z')
    close(io)
end

function read_project(io::IO)
    project = TOML.parse(io)
    if !haskey(project, "deps")
        project["deps"] = Dict{String,Any}()
    end
    return project
end
function read_project(file::String)
    isfile(file) ? open(read_project, file) : read_project(devnull)
end

function read_manifest(io::IO)
    manifest = TOML.parse(io)
    for (name, infos) in manifest, info in infos
        haskey(info, "deps") || continue
        info["deps"] isa AbstractVector || continue
        for dep in info["deps"]
            length(manifest[dep]) == 1 ||
                error("ambiguious dependency for $name: $dep")
        end
        new_dict = Dict()
        for d in info["deps"]
            new_dict[d] = manifest[d][1]["uuid"]
        end
        info["deps"] = new_dict
    end
    return manifest
end
function read_manifest(file::String)
    try isfile(file) ? open(read_manifest, file) : read_manifest(devnull)
    catch err
        err isa ErrorException && startswith(err.msg, "ambiguious dependency") || rethrow(err)
        err.msg *= "In manifest file: $file"
        rethrow(err)
    end
end

const refspecs = ["+refs/*:refs/remotes/cache/*"]
const reg_pkg = r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?(?:\/)?$"

# Windows sometimes throw on `isdir`...
function isdir_windows_workaround(path::String)
    try isdir(path)
    catch e
        false
    end
end

casesensitive_isdir(dir::String) = isdir_windows_workaround(dir) && dir in readdir(joinpath(dir, ".."))

function handle_repos_develop!(ctx::Context, pkgs::AbstractVector{PackageSpec})
    Base.shred!(LibGit2.CachedCredentials()) do creds
        env = ctx.env
        new_uuids = UUID[]
        for pkg in pkgs
            pkg.repo == nothing && continue
            pkg.special_action = PKGSPEC_DEVELOPED
            isempty(pkg.repo.url) && set_repo_for_pkg!(env, pkg)

            if isdir_windows_workaround(pkg.repo.url)
                # Developing a local package, just point `pkg.path` to it
                pkg.path = abspath(pkg.repo.url)
                folder_already_downloaded = true
                project_path = pkg.repo.url
                parse_package!(ctx, pkg, project_path)
            else
                # We save the repo in case another environement wants to
                # develop from the same repo, this avoids having to reclone it
                # from scratch.
                clone_path = joinpath(depots()[1], "clones")
                mkpath(clone_path)
                repo_path = joinpath(clone_path, string(hash(pkg.repo.url), "_full"))
                repo, just_cloned = ispath(repo_path) ? (LibGit2.GitRepo(repo_path), false) : begin
                    r = GitTools.clone(pkg.repo.url, repo_path)
                    GitTools.fetch(r, pkg.repo.url; refspecs=refspecs, credentials=creds)
                    r, true
                end
                if !just_cloned
                    GitTools.fetch(repo, pkg.repo.url; refspecs=refspecs, credentials=creds)
                end
                close(repo)

                # Copy the repo to a temporary place and check out the rev
                project_path = mktempdir()
                cp(repo_path, project_path; force=true)
                repo = LibGit2.GitRepo(project_path)
                rev = pkg.repo.rev
                if isempty(rev)
                    if LibGit2.isattached(repo)
                        rev = LibGit2.branch(repo)
                    else
                        rev = string(LibGit2.GitHash(LibGit2.head(repo)))
                    end
                end
                gitobject, isbranch = get_object_branch(repo, rev)
                LibGit2.transact(repo) do r
                    if isbranch
                        LibGit2.branch!(r, rev, track=LibGit2.Consts.REMOTE_ORIGIN)
                    else
                        LibGit2.checkout!(r, string(LibGit2.GitHash(gitobject)))
                    end
                end
                close(repo); close(gitobject)

                parse_package!(ctx, pkg, project_path)
                dev_pkg_path = joinpath(Pkg.devdir(), pkg.name)
                if isdir(dev_pkg_path)
                    if !isfile(joinpath(dev_pkg_path, "src", pkg.name * ".jl"))
                        cmderror("Path `$(dev_pkg_path)` exists but it does not contain `src/$(pkg.name).jl")
                    else
                        @info "Path `$(dev_pkg_path)` exists and looks like the correct package, using existing path instead of cloning"
                    end
                else
                    mkpath(dev_pkg_path)
                    mv(project_path, dev_pkg_path; force=true)
                    push!(new_uuids, pkg.uuid)
                end
                pkg.path = dev_pkg_path
            end
            @assert pkg.path != nothing
        end
        return new_uuids
    end
end

function handle_repos_add!(ctx::Context, pkgs::AbstractVector{PackageSpec}; upgrade_or_add::Bool=true)
    Base.shred!(LibGit2.CachedCredentials()) do creds
        env = ctx.env
        new_uuids = UUID[]
        for pkg in pkgs
            pkg.repo == nothing && continue
            pkg.special_action = PKGSPEC_REPO_ADDED
            isempty(pkg.repo.url) && set_repo_for_pkg!(env, pkg)
            clones_dir = joinpath(depots()[1], "clones")
            mkpath(clones_dir)
            repo_path = joinpath(clones_dir, string(hash(pkg.repo.url)))
            repo, just_cloned = ispath(repo_path) ? (LibGit2.GitRepo(repo_path), false) : begin
                r = GitTools.clone(pkg.repo.url, repo_path, isbare=true, credentials=creds)
                GitTools.fetch(r, pkg.repo.url; refspecs=refspecs, credentials=creds)
                r, true
            end
            info = manifest_info(env, pkg.uuid)
            pinned = (info != nothing && get(info, "pinned", false))
            if upgrade_or_add && !pinned && !just_cloned
                rev = pkg.repo.rev
                GitTools.fetch(repo, pkg.repo.url; refspecs=refspecs, credentials=creds)
            end
            if upgrade_or_add && !pinned
                rev = pkg.repo.rev
            else
                # Not upgrading so the rev should be the current git-tree-sha
                rev = info["git-tree-sha1"]
                pkg.version = VersionNumber(info["version"])
            end

            # see if we can get rev as a branch
            if isempty(rev)
                if LibGit2.isattached(repo)
                    rev = LibGit2.branch(repo)
                else
                    rev = string(LibGit2.GitHash(LibGit2.head(repo)))
                end
            end
            gitobject, isbranch = get_object_branch(repo, rev)
            # If the user gave a shortened commit SHA, might as well update it to the full one
            pkg.repo.rev = isbranch ? rev : string(LibGit2.GitHash(gitobject))
            git_tree = LibGit2.peel(LibGit2.GitTree, gitobject)
            @assert git_tree isa LibGit2.GitTree
            pkg.repo.git_tree_sha1 = SHA1(string(LibGit2.GitHash(git_tree)))
            version_path = nothing
            folder_already_downloaded = false
            if has_uuid(pkg) && has_name(pkg)
                version_path = Pkg.Operations.find_installed(pkg.name, pkg.uuid, pkg.repo.git_tree_sha1)
                isdir(version_path) && (folder_already_downloaded = true)
                info = manifest_info(env, pkg.uuid)
                if info != nothing && get(info, "git-tree-sha1", "") == string(pkg.repo.git_tree_sha1) && folder_already_downloaded
                    # Same tree sha and this version already downloaded, nothing left to do
                    pkg.version = VersionNumber(info["version"])
                    continue
                end
            end
            if folder_already_downloaded
                project_path = version_path
            else
                project_path = mktempdir()
                opts = LibGit2.CheckoutOptions(checkout_strategy=LibGit2.Consts.CHECKOUT_FORCE,
                    target_directory=Base.unsafe_convert(Cstring, project_path))
                LibGit2.checkout_tree(repo, git_tree, options=opts)
            end
            close(repo); close(git_tree); close(gitobject)
            parse_package!(ctx, pkg, project_path)
            if !folder_already_downloaded
                version_path = Pkg.Operations.find_installed(pkg.name, pkg.uuid, pkg.repo.git_tree_sha1)
                mkpath(version_path)
                mv(project_path, version_path; force=true)
                push!(new_uuids, pkg.uuid)
            end
            @assert pkg.version isa VersionNumber
        end
        return new_uuids
    end
end

function parse_package!(ctx, pkg, project_path)
    env = ctx.env
    project_file = projectfile_path(project_path)
    if project_file !== nothing
        project_data = parse_toml(project_file)
        pkg.uuid = UUID(project_data["uuid"])
        pkg.name = project_data["name"]
        if haskey(project_data, "version")
            pkg.version = VersionNumber(project_data["version"])
        else
            @warn "project file for $(pkg.name) is missing a `version` entry"
            Pkg.Operations.set_maximum_version_registry!(env, pkg)
        end
    else
        @warn "packages will need to have a [Julia]Project.toml file in the future"
        if !isempty(ctx.old_pkg2_clone_name) # remove when legacy CI script support is removed
            pkg.name = ctx.old_pkg2_clone_name
        else
            # This is an old style package, get the name from src/PackageName
            m = match(reg_pkg, pkg.repo.url)
            m === nothing && cmderror("cannot determine package name from URL: $(pkg.repo.url)")
            pkg.name = m.captures[1]
        end
        reg_uuids = registered_uuids(env, pkg.name)
        is_registered = !isempty(reg_uuids)
        if !is_registered
            # This is an unregistered old style package, give it a UUID and a version
            if !has_uuid(pkg)
                uuid_unreg_pkg = UUID(0xa9a2672e746f11e833ef119c5b888869)
                pkg.uuid = uuid5(uuid_unreg_pkg, pkg.name)
                @info "Assigning UUID $(pkg.uuid) to $(pkg.name)"
            end
            pkg.version = v"0.0"
        else
            # TODO: Fix
            @assert length(reg_uuids) == 1
            pkg.uuid = reg_uuids[1]
            # Old style registered package
            # What version does this package have? We have no idea... let's give it the latest one with a `+`...
            Pkg.Operations.set_maximum_version_registry!(env, pkg)
        end
    end
end

function set_repo_for_pkg!(env, pkg)
    if !has_uuid(pkg)
        registry_resolve!(env, [pkg])
        ensure_resolved(env, [pkg]; registry=true)
    end
    # TODO: look into [1]
    _, pkg.repo.url = Types.registered_info(env, pkg.uuid, "repo")[1]
end

function get_object_branch(repo, rev)
    gitobject = nothing
    isbranch = false
    try
        gitobject = LibGit2.GitObject(repo, "remotes/cache/heads/" * rev)
        isbranch = true
    catch err
        err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow(err)
    end
    if gitobject == nothing
        try
            gitobject = LibGit2.GitObject(repo, rev)
        catch err
            err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow(err)
            cmderror("git object $(rev) could not be found")
        end
    end
    return gitobject, isbranch
end

########################################
# Resolving packages from name or uuid #
########################################

function project_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    for pkg in pkgs
        if has_uuid(pkg) && !has_name(pkg) && Types.is_project_uuid(env, pkg.uuid)
            pkg.name = env.pkg.name
        end
        if has_name(pkg) && !has_uuid(pkg) && Types.is_project_name(env, pkg.name)
            pkg.uuid = env.pkg.uuid
        end
    end
end

# Disambiguate name/uuid package specifications using project info.
function project_deps_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    uuids = env.project["deps"]
    names = Dict(uuid => name for (uuid, name) in uuids)
    length(uuids) < length(names) && # TODO: handle this somehow?
        cmderror("duplicate UUID found in project file's [deps] section")
    for pkg in pkgs
        pkg.mode == PKGMODE_PROJECT || continue
        if has_name(pkg) && !has_uuid(pkg) && pkg.name in keys(uuids)
            pkg.uuid = UUID(uuids[pkg.name])
        end
        if has_uuid(pkg) && !has_name(pkg) && pkg.uuid in keys(names)
            pkg.name = names[pkg.uuid]
        end
    end
    return pkgs
end

# Disambiguate name/uuid package specifications using manifest info.
function manifest_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    uuids = Dict{String,Vector{String}}()
    names = Dict{String,String}()
    for (name, infos) in env.manifest, info in infos
        haskey(info, "uuid") || continue
        uuid = info["uuid"]
        push!(get!(uuids, name, String[]), uuid)
        names[uuid] = name # can be duplicate but doesn't matter
    end
    for pkg in pkgs
        pkg.mode == PKGMODE_MANIFEST || continue
        if has_name(pkg) && !has_uuid(pkg) && pkg.name in keys(uuids)
            length(uuids[pkg.name]) == 1 && (pkg.uuid = UUID(uuids[pkg.name][1]))
        end
        if has_uuid(pkg) && !has_name(pkg) && pkg.uuid in keys(names)
            pkg.name = names[pkg.uuid]
        end
    end
    return pkgs
end

# Disambiguate name/uuid package specifications using registry info.
function registry_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    # if there are no half-specified packages, return early
    any(pkg -> has_name(pkg) ⊻ has_uuid(pkg), pkgs) || return
    # collect all names and uuids since we're looking anyway
    names = [pkg.name for pkg in pkgs if has_name(pkg)]
    uuids = [pkg.uuid for pkg in pkgs if has_uuid(pkg)]
    find_registered!(env, names, uuids)
    for pkg in pkgs
        @assert has_name(pkg) || has_uuid(pkg)
        if has_name(pkg) && !has_uuid(pkg)
            pkg.uuid = registered_uuid(env, pkg.name)
        end
        if has_uuid(pkg) && !has_name(pkg)
            pkg.name = registered_name(env, pkg.uuid)
        end
    end
    return pkgs
end

function stdlib_resolve!(ctx::Context, pkgs::AbstractVector{PackageSpec})
    for pkg in pkgs
        @assert has_name(pkg) || has_uuid(pkg)
        if has_name(pkg) && !has_uuid(pkg)
            for (uuid, name) in ctx.stdlibs
                name == pkg.name && (pkg.uuid = uuid)
            end
        end
        if has_uuid(pkg) && !has_name(pkg)
            haskey(ctx.stdlibs, pkg.uuid) && (pkg.name = ctx.stdlibs[pkg.uuid])
        end
    end
end

# Ensure that all packages are fully resolved
function ensure_resolved(env::EnvCache,
    pkgs::AbstractVector{PackageSpec};
    registry::Bool=false,)::Nothing
    unresolved = Dict{String,Vector{UUID}}()
    for pkg in pkgs
        has_uuid(pkg) && continue
        uuids = UUID[]
        for (name, infos) in env.manifest, info in infos
            name == pkg.name && haskey(info, "uuid") || continue
            uuid = UUID(info["uuid"])
            uuid in uuids || push!(uuids, uuid)
        end
        sort!(uuids, by=uuid -> uuid.value)
        unresolved[pkg.name] = uuids
    end
    isempty(unresolved) && return
    msg = sprint() do io
        println(io, "The following package names could not be resolved:")
        for (name, uuids) in sort!(collect(unresolved), by=lowercase ∘ first)
        print(io, " * $name (")
        if length(uuids) == 0
            what = ["project", "manifest"]
            registry && push!(what, "registry")
            print(io, "not found in ")
            join(io, what, ", ", " or ")
        else
            join(io, uuids, ", ", " or ")
            print(io, " in manifest but not in project")
        end
        println(io, ")")
    end
        print(io, "Please specify by known `name=uuid`.")
    end
    cmderror(msg)
end

const DEFAULT_REGISTRIES = Dict("Uncurated" => "https://github.com/JuliaRegistries/Uncurated.git")

# Return paths of all registries in a depot
function registries(depot::String)::Vector{String}
    d = joinpath(depot, "registries")
    ispath(d) || return String[]
    regs = filter!(readdir(d)) do r
        isfile(joinpath(d, r, "Registry.toml"))
    end
    String[joinpath(depot, "registries", r) for r in regs]
end

# Return paths of all registries in all depots
function registries(; clone_default=true)::Vector{String}
    isempty(depots()) && return String[]
    user_regs = abspath(depots()[1], "registries")
    if clone_default
        if !ispath(user_regs)
            mkpath(user_regs)
            Base.shred!(LibGit2.CachedCredentials()) do creds
                printpkgstyle(stdout, :Cloning, "default registries into $user_regs")
                for (reg, url) in DEFAULT_REGISTRIES
                    path = joinpath(user_regs, reg)
                    repo = GitTools.clone(url, path; header = "registry $reg from $(repr(url))", credentials = creds)
                    close(repo)
                end
            end
        end
    end
    return [r for d in depots() for r in registries(d)]
end

const line_re = r"""
    ^ \s*
    ([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})
    \s* = \s* \{
    \s* name \s* = \s* "([^"]*)" \s*,
    \s* path \s* = \s* "([^"]*)" \s*,?
    \s* \} \s* $
"""x

# Lookup package names & uuids in a single pass through registries
function find_registered!(env::EnvCache,
    names::Vector{String},
    uuids::Vector{UUID}=UUID[];
    force::Bool=false,
)::Nothing
    # only look if there's something new to see (or force == true)
    names = filter(name -> !haskey(env.uuids, name), names)
    uuids = filter(uuid -> !haskey(env.paths, uuid), uuids)
    !force && isempty(names) && isempty(uuids) && return

    # since we're looking anyway, look for everything
    save(name::String) =
        name in names || haskey(env.uuids, name) || push!(names, name)
    save(uuid::UUID) =
        uuid in uuids || haskey(env.paths, uuid) || push!(uuids, uuid)

    # lookup any dependency in the project file
    for (name, uuid) in env.project["deps"]
        save(name); save(UUID(uuid))
    end
    # lookup anything mentioned in the manifest file
    for (name, infos) in env.manifest, info in infos
        save(name)
        haskey(info, "uuid") && save(UUID(info["uuid"]))
        haskey(info, "deps") || continue
        for (n, u) in info["deps"]
            save(n); save(UUID(u))
        end
    end
    # if there's still nothing to look for, return early
    isempty(names) && isempty(uuids) && return

    # build regexs for names and uuids
    uuid_re = sprint() do io
        if !isempty(uuids)
        print(io, raw"^( ")
        for (i, uuid) in enumerate(uuids)
            1 < i && print(io, " | ")
            print(io, raw"\Q", uuid, raw"\E")
        end
        print(io, raw" )\b")
    end
    end
    name_re = sprint() do io
        if !isempty(names)
        print(io, raw"\bname \s* = \s* \"( ")
        for (i, name) in enumerate(names)
            1 < i && print(io, " | ")
            print(io, raw"\Q", name, raw"\E")
        end
        print(io, raw" )\"")
    end
    end
    regex = if !isempty(uuids) && !isempty(names)
        Regex("( $uuid_re | $name_re )", "x")
    elseif !isempty(uuids)
        Regex(uuid_re, "x")
    elseif !isempty(names)
        Regex(name_re, "x")
    else
        error("this should not happen")
    end

    # initialize env entries for names and uuids
    for name in names; env.uuids[name] = UUID[]; end
    for uuid in uuids; env.paths[uuid] = String[]; end
    # note: empty vectors will be left for names & uuids that aren't found

    # search through all registries
    for registry in registries()
        open(joinpath(registry, "Registry.toml")) do io
            # skip forward until [packages] section
            for line in eachline(io)
            occursin(r"^ \s* \[ \s* packages \s* \] \s* $"x, line) && break
        end
            # find lines with uuid or name we're looking for
            for line in eachline(io)
            occursin(regex, line) || continue
            m = match(line_re, line)
            m == nothing &&
                    error("misformatted registry.toml package entry: $line")
            uuid = UUID(m.captures[1])
            name = Base.unescape_string(m.captures[2])
            path = abspath(registry, Base.unescape_string(m.captures[3]))
            push!(get!(env.uuids, name, typeof(uuid)[]), uuid)
            push!(get!(env.paths, uuid, typeof(path)[]), path)
        end
        end
    end
end

find_registered!(env::EnvCache, uuids::Vector{UUID}; force::Bool=false)::Nothing =
    find_registered!(env, String[], uuids, force=force)

# Lookup all packages in project & manifest files
find_registered!(env::EnvCache)::Nothing =
    find_registered!(env, String[], UUID[], force=true)

# Get registered uuids associated with a package name
function registered_uuids(env::EnvCache, name::String)::Vector{UUID}
    find_registered!(env, [name], UUID[])
    return unique(env.uuids[name])
end

# Get registered paths associated with a package uuid
function registered_paths(env::EnvCache, uuid::UUID)::Vector{String}
    find_registered!(env, String[], [uuid])
    return env.paths[uuid]
end

#Get registered names associated with a package uuid
function registered_names(env::EnvCache, uuid::UUID)::Vector{String}
    find_registered!(env, String[], [uuid])
    String[n for (n, uuids) in env.uuids for u in uuids if u == uuid]
end

# Determine a single UUID for a given name, prompting if needed
function registered_uuid(env::EnvCache, name::String)::UUID
    uuids = registered_uuids(env, name)
    length(uuids) == 0 && return UUID(zero(UInt128))
    choices::Vector{String} = []
    choices_cache::Vector{Tuple{UUID,String}} = []
    for uuid in uuids
        values = registered_info(env, uuid, "repo")
        for value in values
            depot = "(unknown)"
            for d in depots()
                r = joinpath(d, "registries")
                startswith(value[1], r) || continue
                depot = split(relpath(value[1], r), Base.Filesystem.path_separator_re)[1]
                break
            end
            push!(choices, "Registry: $depot - Path: $(value[2])")
            push!(choices_cache, (uuid, value[1]))
        end
    end
    length(choices_cache) == 1 && return choices_cache[1][1]
    # prompt for which UUID was intended:
    menu = RadioMenu(choices)
    choice = request("There are multiple registered `$name` packages, choose one:", menu)
    choice == -1 && return UUID(zero(UInt128))
    env.paths[choices_cache[choice][1]] = [choices_cache[choice][2]]
    return choices_cache[choice][1]
end

# Determine current name for a given package UUID
function registered_name(env::EnvCache, uuid::UUID)::String
    names = registered_names(env, uuid)
    length(names) == 0 && return ""
    length(names) == 1 && return names[1]
    values = registered_info(env, uuid, "name")
    name = nothing
    for value in values
        name  == nothing && (name = value[2])
        name != value[2] && cmderror("package `$uuid` has multiple registered name values: $name, $(value[2])")
    end
    return name
end

# Return most current package info for a registered UUID
function registered_info(env::EnvCache, uuid::UUID, key::String)
    paths = env.paths[uuid]
    isempty(paths) && cmderror("`$uuid` is not registered")
    values = []
    for path in paths
        info = parse_toml(path, "Package.toml")
        value = get(info, key, nothing)
        push!(values, (path, value))
    end
    return values
end

# Find package by UUID in the manifest file
function manifest_info(env::EnvCache, uuid::UUID)::Union{Dict{String,Any},Nothing}
    uuid in values(env.uuids) || find_registered!(env, [uuid])
    for (name, infos) in env.manifest, info in infos
        haskey(info, "uuid") && uuid == UUID(info["uuid"]) || continue
        return merge!(Dict{String,Any}("name" => name), info)
    end
    return nothing
end

# TODO: redirect to ctx stream
function printpkgstyle(io::IO, cmd::Symbol, text::String, ignore_indent::Bool=false)
    indent = textwidth(string(:Downloaded))
    ignore_indent && (indent = 0)
    printstyled(io, lpad(string(cmd), indent), color=:green, bold=true)
    println(io, " ", text)
end

# TODO: use ctx specific context
function printpkgstyle(ctx::Context, cmd::Symbol, text::String, ignore_indent::Bool=false)
    printpkgstyle(stdout, cmd, text)
end

function pathrepr(ctx::Union{Nothing, Context}, path::String, base::String=pwd())
    project_path = dirname(ctx.env.project_file)
    path = joinpath(project_path, path)
    if startswith(path, project_path) && startswith(base, project_path)
        # We are in project and path is in project
        path = relpath(path, base)
    end
    if !Sys.iswindows() && isabspath(path)
        home = joinpath(homedir(), "")
        if startswith(path, home)
            path = joinpath("~", path[nextind(path, lastindex(home)):end])
        end
    end
    return "`" * path * "`"
end

function project_key_order(key::String)
    key == "name"     && return 1
    key == "uuid"     && return 2
    key == "keywords" && return 3
    key == "license"  && return 4
    key == "desc"     && return 5
    key == "deps"     && return 6
    key == "compat"   && return 7
    return 8
end

function write_env(ctx::Context; display_diff=true)
    env = ctx.env
    # load old environment for comparison
    old_env = EnvCache(env.env)
    # update the project file
    project = deepcopy(env.project)
    isempty(project["deps"]) && delete!(project, "deps")
    if !isempty(project) || ispath(env.project_file)
        if display_diff
            printpkgstyle(ctx, :Updating, pathrepr(ctx, env.project_file))
            Pkg.Display.print_project_diff(ctx, old_env, env)
        end
        if !ctx.preview
            mkpath(dirname(env.project_file))
            open(env.project_file, "w") do io
                TOML.print(io, project, sorted=true, by=key -> (project_key_order(key), key))
            end
        end
    end
    # update the manifest file
    if !isempty(env.manifest) || ispath(env.manifest_file)
        if display_diff
            printpkgstyle(ctx, :Updating, pathrepr(ctx, env.manifest_file))
            Pkg.Display.print_manifest_diff(ctx, old_env, env)
        end
        manifest = deepcopy(env.manifest)
        uniques = sort!(collect(keys(manifest)), by=lowercase)
        filter!(name -> length(manifest[name]) == 1, uniques)
        uuids = Dict(name => UUID(manifest[name][1]["uuid"]) for name in uniques)
        for (name, infos) in manifest, info in infos
            haskey(info, "deps") || continue
            deps = Dict{String,UUID}(n => UUID(u) for (n, u) in info["deps"])
            all(d in uniques && uuids[d] == u for (d, u) in deps) || continue
            info["deps"] = sort!(collect(keys(deps)))
        end
        if !ctx.preview
            open(env.manifest_file, "w") do io
                TOML.print(io, manifest, sorted=true)
            end
        end
    end
end

end # module


text:
# DWT operators: joDWT

## helper module
module joSWT_etc
    using JOLI: jo_convert
    using PyCall
    # 1D
    function apply_swt(v::Vector{vdt}, wt::String, L::Integer,rdt::DataType,pad::Integer) where vdt<:Union{AbstractFloat,Complex}
        pywt = pyimport("pywt")
        v = [v;zeros(vdt, pad)]
        rv = pywt.swt(v, wt, level=L, start_level=0, norm=true, trim_approx=true)
        rv = vcat(rv...)
        rv = jo_convert(rdt, rv, false)
        return rv
    end
    function apply_iswt(v::Vector{vdt},wt::String,L::Integer,rdt::DataType,pad::Integer) where vdt<:Union{AbstractFloat,Complex}
        pywt = pyimport("pywt")
        v = reshape(v, :, L+1)
        v = [v[:, i] for i=1:L+1]
        rv = pywt.iswt(v, wt, norm=true)
        # Convert and remove pad
        rv = jo_convert(rdt, rv[1:end-pad], false)
        return rv
    end
end
using .joSWT_etc

export joSWT
"""
    julia> op = joSWT(m,[wt,];[L=...,][DDT=...,][RDT=...])

1-dimensional SWT transform based on PyWavelets. Only 1D is supported currently in JOLI but n-dimensional supported in PyWavelets.
Choice of wavelet is fixed to "db20"

# Signature

    function joSWT(m::Integer, wt::String="db20";
        L::Integer=maxtransformlevels(min(m,n)),
        DDT::DataType=joFloat,RDT::DataType=DDT,
        name::String="joDWT")

# Arguments

- `m`: dimention
- optional
    - `wt`: wavelet filter ("db20" by default) see for options:
    https://github.com/PyWavelets/pywt/blob/master/pywt/_extensions/wavelets_list.pxi
- keywords
    - `L`: number of levels
    - `DDT`: domain data type
    - `RDT`: range data type
    - `name`: custom name

# Notes

- only 1D arrays are supported for now due to limitations of author time.
- only orthogonal wavelets should be used from ["haar", "db", "sym", "coif"] to have access to adjoint via inverse.

# Examples

1D DWT with default wavelet (Daubechie20 wavelet)

    joSWT(m)

examples with DDT/RDT

    joDWT(m,"db20"; DDT=Float32)
    joDWT(m,"sym5"; DDT=Float32,RDT=Float64)

"""
function joSWT(m::Integer,wt::String="db20";
    L::Integer=maxtransformlevels(m + m%2),
    DDT::DataType=joFloat,RDT::DataType=DDT,
    name::String="joSWT")

    # Only work with even number of points
    pad = m%2
    n = (L+1)*(m + pad)
    return joLinearFunction_A(n,m,
        v1->joSWT_etc.apply_swt(v1, wt, L, RDT, pad),
        v2->joSWT_etc.apply_iswt(v2, wt, L, DDT, pad),
        v3->joSWT_etc.apply_iswt(v3, wt, L, DDT, pad),
        v4->joSWT_etc.apply_swt(v4, wt, L, RDT, pad),
        DDT,RDT;
        name=name)
end


text:
"""
    readsize(io::IO, c)

Return the number of bytes that would be read if `c` were read from `io`.
"""
readsize(_::IO, x::Union{Base.BitInteger, Float16, Float32, Float64}) =
    sizeof(x)
readsize(_::IO, c::Char) = @static VERSION < v"1.1.0" ? Base.codelen(c) : ncodeunits(c)

"""
    peek(io::IO, T::Type)

Read a `T` (`UInt8` or `Char`) from `io` without changing its state. This
function may be implemented by putting the read value back to the stream `io`.
"""
function peek(io::IO, T::Type)
    char = read(io, T)
    skip(io, -readsize(io, char))
    char
end


text:
module GameData

using JSON
using HTTP
using DataFrames
using Dates

using AKCalc

export GDMem, GDKengxxiao, GDAceShip

export resource_types, factory_recipes, workshop_recipes, operator_bases
export enemy_bases, enemies, tiles, maps, stages

include("gamedata_types.jl")

abstract type Source end

# Source accessors
function resource_types end
function factory_recipes end
function workshop_recipes end
function operator_bases end
function enemy_bases end
function enemies end
function tiles end
function maps end
function stages end

"""
  GameData Source that simply stores objects in memory.
"""
mutable struct SourceInMem <: Source
  resource_types::Union{Array{ResourceType}, Nothing}
  factory_recipes::Union{Array{FactoryRecipe}, Nothing}
  workshop_recipes::Union{Array{WorkshopRecipe}, Nothing}
  operator_bases::Union{Array{OperatorBase}, Nothing}
  enemy_bases::Union{Array{EnemyBase}, Nothing}
  enemies::Union{Array{EnemyStats}, Nothing}
  tiles::Union{Array{Tile}, Nothing}
  maps::Union{Array{Map}, Nothing}
  stages::Union{Array{Stage}, Nothing}
end
SourceInMem() = SourceInMem(nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing)

_comb(arr...) = vcat(filter(x -> !isnothing(x), arr)...)

resource_types(s::SourceInMem) = s.resource_types
factory_recipes(s::SourceInMem) = s.factory_recipes
workshop_recipes(s::SourceInMem) = s.workshop_recipes
operator_bases(s::SourceInMem) = s.operator_bases
Base.append!(s::SourceInMem, bases::Array{OperatorBase}) =
  s.operator_bases = _comb(s.operator_bases, bases)
enemy_bases(s::SourceInMem) = s.enemy_bases
enemies(s::SourceInMem) = s.enemies
tiles(s::SourceInMem) = s.tiles
maps(s::SourceInMem) = s.maps
stages(s::SourceInMem) = s.stages

"""
  GameData Source that caches results in memory, or falls back on another provided source.
"""
mutable struct SourceCache <: Source
  source::Source
  cache::SourceInMem

  function SourceCache(source::Source)
    return new(source, SourceInMem())
  end
end

function Base.show(io::IO, sc::SourceCache)
  print(io, "SourceCache($(sc.source))")
end

function _load_and_cache(sc::SourceCache, name::Symbol, accessor::Function)
  if isnothing(Base.getproperty(sc.cache, name))
    val = accessor(sc.source)
    Base.setproperty!(sc.cache, name, val)
    return val
  else
    return Base.getproperty(sc.cache, name)
  end
end

resource_types(sc::SourceCache)   = _load_and_cache(sc, :resource_types, resource_types)
factory_recipes(sc::SourceCache)  = _load_and_cache(sc, :factory_recipes, factory_recipes)
workshop_recipes(sc::SourceCache) = _load_and_cache(sc, :workshop_recipes, workshop_recipes)
operator_bases(sc::SourceCache)   = _load_and_cache(sc, :operator_bases, operator_bases)
enemy_bases(sc::SourceCache)      = _load_and_cache(sc, :enemy_bases, enemy_bases)
enemies(sc::SourceCache)          = _load_and_cache(sc, :enemies, enemies)
tiles(sc::SourceCache)            = _load_and_cache(sc, :tiles, tiles)
maps(sc::SourceCache)             = _load_and_cache(sc, :maps, maps)
stages(sc::SourceCache)           = _load_and_cache(sc, :stages, stages)

"""
  Returns the contents of a gamedata file, parsed out of JSON into a Juila type.
"""
function _fetch_file end

"""
  GameData Source that pulls data from a URL.
  Caches intermediate JSON results.
"""
struct SourceByURL <: Source
  base_url::String
  lang::String
  cache::Dict{AbstractString, Any}
end
SourceByURL(base_url::String, lang::String) = SourceByURL(base_url, lang, Dict())

function Base.show(io::IO, su::SourceByURL)
  print(io, "SourceByURL($(su.base_url), $(su.lang))")
end

function _fetch_file(source::SourceByURL, file::String)
  url = "$(source.base_url)/$(source.lang)/$file"
  return get!(source.cache, url) do
    JSON.parse(String(HTTP.get(url).body))
  end
end

struct SourceByPath <: Source
  base_path::String
  lang::String
  cache::Union{Dict{AbstractString, AbstractString}, Nothing}
end

function Base.show(io::IO, su::SourceByPath)
  print(io, "SourceByPath($(su.base_path), $(su.lang))")
end

function _fetch_file(source::SourceByPath, file::String)
end

const LANGS = Dict(
  :en => "en_US",
  :ja => "ja_JP",
  :ko => "ko_KR",
  :zhcn => "zh_CN",
  :zhtw => "zh_TW",
)

const FILES = Dict(
  :buffs => "gamedata/buff_table.json",

  :activities       => "gamedata/excel/activity_table.json",
  :audio            => "gamedata/excel/audio_data.json",
  :buildings        => "gamedata/excel/building_data.json",
  :characters       => "gamedata/excel/character_table.json",
  :char_words       => "gamedata/excel/charword_table.json",
  :checkins         => "gamedata/excel/checkin_table.json",
  :clues            => "gamedata/excel/clue_data.json",
  :crisises         => "gamedata/excel/crisis_table.json",
  :version          => "gamedata/excel/data_version.txt",
  :handbook_enemies => "gamedata/excel/enemy_handbook_table.json",
  :favors           => "gamedata/excel/favor_table.json",
  :gatcha           => "gamedata/excel/gacha_table.json",
  :consts           => "gamedata/excel/gamedata_const.json",
  :handbook_info    => "gamedata/excel/handbook_info_table.json",
  :hanbook          => "gamedata/excel/handbook_table.json",
  :handboox_teams   => "gamedata/excel/handbook_team_table.json",
  :items            => "gamedata/excel/item_table.json",
  :medals           => "gamedata/excel/medal_table.json",
  :missions         => "gamedata/excel/mission_table.json",
  :open_servers     => "gamedata/excel/open_server_table.json",
  :ranges           => "gamedata/excel/range_table.json",
  :replicates       => "gamedata/excel/replicate_table.json",
  :roguelike        => "gamedata/excel/roguelike_table.json",
  :runes            => "gamedata/excel/rune_table.json",
  :shop             => "gamedata/excel/shop_client_table.json",
  :skills           => "gamedata/excel/skill_table.json",
  :skins            => "gamedata/excel/skin_table.json",
  :stages           => "gamedata/excel/stage_table.json",
  :story_reviews    => "gamedata/excel/story_review_table.json",
  :story            => "gamedata/excel/story_table.json",
  :tips             => "gamedata/excel/tip_table.json",
  :tokens           => "gamedata/excel/token_table.json",
  :zones            => "gamedata/excel/zone_table.json",

  :enemies => "gamedata/levels/enemydata/enemy_database.json",
)

const RESOURCE_ITEM_TYPES = Dict(
  "MATERIAL" => i_material,
  "CARD_EXP" => i_card_exp,
  "TKT_TRY" => i_ticket_try,
  "TKT_RECRUIT" => i_ticket_recruit,
  "TKT_GACHA" => i_ticket,
  "TKT_GACHA_10" => i_ticket_10,
  "TKT_INST_FIN" => i_ticket_inst_fin,
  "TKT_GACHA_PRSV" => i_ticket_rand_prsv,
  "LIMITED_TKT_GACHA_10" => i_lim_ticket_10,
  "ACTIVITY_ITEM" => i_act_item,
  "DIAMOND" => i_diamond,
  "DIAMOND_SHD" => i_diamond_shard,
  "EXP_PLAYER" => i_exp_player,
  "AP_ITEM" => i_ap_item,
  "AP_BASE" => i_ap_base,
  "AP_SUPPLY" => i_ap_supply,
  "AP_GAMEPLAY" => i_ap_gameplay,
  "VOUCHER_MGACHA" => i_voucher_rand_m,
  "VOUCHER_CGACHA" => i_voucher_rand_c,
  "VOUCHER_PICK" => i_voucher_pick,
  "GOLD" => i_gold,
  "ET_STAGE" => i_et_stage,
  "EPGS_COIN" => i_epgs,
  "REP_COIN" => i_rep_coin,
  "CRS_SHOP_COIN" => i_crs_shop_coin,
  "CRS_RUNE_COIN" => i_crs_rune_coin,
  "LMTGS_COIN" => i_lmtgs_coin,
  "ACTIVITY_COIN" => i_activity_coin,
  "SOCIAL_PT" => i_social_pt,
  "HGG_SHD" => i_hgg_shd,
  "LGG_SHD" => i_lgg_shd,
)

const RESOURCE_CLASS_TYPES = Dict(
  "MATERIAL" => c_material,
  "NORMAL" => c_normal,
  "CONSUME" => c_consume,
  "NONE" => c_none,
)

function resource_types(source::Source)
  json = _fetch_file(source, FILES[:items])
  items = ResourceType[]
  sizehint!(items, 400)

  for (id, val) in pairs(json["items"])
    name = val["name"]
    item_type = get(RESOURCE_ITEM_TYPES, val["itemType"]) do
      println("Unknown itemType $(val["itemType"])")
      i_unknown
    end
    class_type = get(RESOURCE_CLASS_TYPES, val["classifyType"]) do
      println("Unknown classifyType $(val["classifyType"])")
      c_unknown
    end
    abbrev = nothing
    usage = val["usage"]
    desc = val["description"]
    obtaining = val["obtainApproach"]
    rarity = val["rarity"]
    sort_id = val["sortId"]

    rt = ResourceType(id, name, item_type, class_type, abbrev, usage, desc, obtaining, rarity, sort_id)
    push!(items, rt)
  end

  return items
end

const GDMem = SourceInMem
const GDKengxxiao = SourceCache(SourceByURL("https://raw.githubusercontent.com/Kengxxiao/ArknightsGameData/master", LANGS[:en]))
const GDAceShip = SourceByURL("https://raw.githubusercontent.com/Aceship/AN-EN-Tags/master/json/gamedata", LANGS[:en])

end


text:
"""
    sparse_pflap(args; kwargs)

Find a transformation flow (complex displacment field), that transforms image `source` to closer to image `target`.
Returns the transformation a [`Flow`](@ref) and the registered `source` image.

# Arguments:
- `target::Image`: target/fixed grayscale image.
- `source::Image`: source/moving grayscale image.

# Keyword Arguments:
- `filter_count::Integer=3`: the number of basis filters used (so far only =3 implemented).
- `timer::TimerOutput=TimerOutput("sparse pflap")`: provide a timer which times certain blocks in the function.
- `display::Bool=false`: verbose and debug prints.
- `point_count::Int=500`: the number of points attempted to be found at the edges of the target image.
- `spacing::Int=10`: the minimal distance between two points.
- `match_source_histogram::Bool=true`: choose to match the `source` image histogram to the `target` image histogram.

## Note:
If `display=true` is chosen the function will also return an array of figures demonstrating the
work of the registration.

See also: [`sparse_lap`](@ref), [`pflap`](@ref), [`showflow`](@ref), [`single_lap`](@ref), [`sparse_pflap`](@ref).
"""
function sparse_pflap_psnr(target::Image,
                           source::Image;
                           filter_count::Integer=3,
                           max_repeats::Integer=3,
                           display::Bool=false,
                           point_count::Int=500,
                           spacing::Int=13,
                           timer::TimerOutput=TimerOutput("sparse pflap"),
                           match_source_histogram::Bool=true,
                           rescale_intensities::Bool=false)

    @timeit_debug timer "setup" begin
        # convert images to floats
        source = Float64.(source)
        target = Float64.(target)

        # rescale images to have the whole [0, 1] spectrum.
        if rescale_intensities
            target, source = rescale!(target, source)
        end

        # adjust the source histogram to be like target
        if match_source_histogram
            @timeit_debug timer "hist match" begin
                source = adjust_histogram(source, Matching(targetimg = target)) # takes 8ms on average
            end
        end

        # pad with zeros if sizes difer.
        target, source = pad_images(target, source)

        image_size = size(target)

        # set number of layers in the filter pyramid.
        level_count = floor(Int64, log2(minimum(size(target))/8)+1)+1

        @assert ((2^(level_count)+1) <= minimum(image_size)) "level number results in a filter larger than the size of the input images."

        # displacement init.
        u_est = zeros(ComplexF64, size(target))

        # filter half sizes array eg. [16, 8, 4, 2, 1]
        half_size_pyramid = Int64.(2 .^ range(level_count-1, stop=0, length=level_count))

        if display
            num_plots = 4
            figs = Array{Figure}(undef, level_count, max_repeats, num_plots)
        end

        psnr_threshold = 0.3
        psnr_before = 0
        source_reg = source

        #TODO edit
        # get edge points
        mask = falses(size(target))
        fhs = 3
        mask[fhs+1:end-fhs, fhs+1:end-fhs] .= true
        @timeit_debug timer "find edge points" begin
            inds = find_edge_points(target, spacing=spacing, point_count=point_count, mask=mask)
        end
        if display
            println("ind count: ", length(inds))
        end

    end # "setup"

    for level in 1:level_count

        @timeit_debug timer "single filter pyramid level" begin
            if display
                println("###################")
                println("ITERATION: ", level)
                println("filter_half_size: ", half_size_pyramid[level])
            end

            fhs = half_size_pyramid[level]
            window_size = Int64.(2 .* fhs .+ (1, 1))

            for iter_repeat in 1:max_repeats

                @timeit_debug timer "filter inds" begin
                    current_inds = filter_border_inds(inds, size(target), fhs)
                    if display
                        println("\tnon border inds: $(length(current_inds))/$(length(inds)) ")
                    end
                    if level != 1
                        current_inds = LAP_julia.mapped_out(current_inds, target)
                    end
                end
                if display
                    println("inner loop iter: $iter_repeat")
                    println("\tnon mapped out inds: $(length(current_inds))/$(length(inds)) ")
                end
                @timeit_debug timer "sparse lap" begin
                    new_estim_at_inds, current_inds = single_lap_at_points(target, source_reg, fhs, window_size, current_inds, timer=timer, display=display)
                end

                # interpolate flow
                if all(isnan, new_estim_at_inds)
                    @timeit_debug timer "interpolate flow" begin
                        if display
                            println("\tIMPORTANT: all new estim vectors are NaN.")
                        end
                    end
                    break
                else
                    @timeit_debug timer "flow interpolation" begin
                        # extract u_est_vecs from the previous estimate and add them to the newly estimated ones
                        # then interpolate a new flow.
                        if display
                            println("\tnon NaN new estim flow vector count: ", length(new_estim_at_inds))
                        end
                        u_est_at_inds = map(ind -> u_est[ind], current_inds) .+ new_estim_at_inds
                        # @assert any(.!isnan.(u_est_at_inds)) "new estim nans", count(isnan, u_est_at_inds), length(u_est_at_inds)
                        u_est_adept = interpolate_flow(u_est_at_inds, current_inds, size(u_est))
                    end
                end

                # if the interpolation failes and returns u_est_adept full of NaNs go next level.
                if any(isnan.(u_est_adept))
                    if display
                        println("\tIMPORTANT: u_est_adept full of NaNs, skipping")
                    end
                    break
                end

                # if the interpolation estimated a flow 20% larger than the fhs then go to next level
                # if any(vec_len.(u_est_adept) .> (fhs*1.3))
                #     if display
                #         println("\tIMPORTANT: u_est_adept is above size threshold, skipping (max: $(maximum(vec_len.(u_est_adept))), fhs: $fhs)")
                #     end
                #     break
                # end
                @assert any(.!isnan.(u_est_adept)) count(isnan, u_est_adept), length(u_est_adept)

                # linear interpolation
                @timeit_debug timer "image interpolation" begin
                    # source_reg = warp_img(source, -real(u_est_adept), -imag(u_est_adept), target)
                    source_reg = warp_img(source, -real(u_est_adept), -imag(u_est_adept))
                end

                # calculate the psnr after warping the source by u_est_adept
                psnr_after = assess_psnr(source_reg, target)

                # compare psnr before and after to see if there was an improvement larger than psnr_threshold
                if psnr_after - psnr_before > psnr_threshold
                    # nice, the improvement is large -> try to run again with the same filter size.
                    u_est = u_est_adept
                    psnr_before = psnr_after
                    if display
                        println("\tstatus: success, repeating filter size")
                    end
                else
                    # the improvement isn't larger than psnr_threshold -> next filter size
                    # check to see if it was even an improvement:
                    if psnr_after - psnr_before > 0
                        # it was an improvement, save the variables:
                        u_est = u_est_adept
                        psnr_before = psnr_after
                        if display
                            println("\tstatus: success, next filter size")
                            add_figs_sparse_pflap(figs, level, iter_repeat, u_est, new_estim_at_inds, source_reg, level_count, max_repeats, current_inds, fhs)
                        end
                    else
                        # it wasn't an improvement, skip
                        if display
                            println("\tstatus: failure, trashing")
                        end
                    end
                    break # go for next filter size.
                end

                if display
                    add_figs_sparse_pflap(figs, level, iter_repeat, u_est, new_estim_at_inds, source_reg, level_count, max_repeats, current_inds, fhs)
                end
            end
        end # "single filter pyramid level"
    end

    if display; return u_est, source_reg, figs; end
    return u_est, source_reg
end


text:
f_1p(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 # [100., missing, missing, missing]

f_2p_1im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + 0.0*x[2]  # [3., missing, missing, missing]

f_2p(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (x[2]-4.0)^2  # [3., 4., missing, missing]

f_3p_1im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (x[2]/x[3]-4.0)^2 # [3., missing, missing, missing]

f_3p_1im_dep(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (x[1]-x[2]-1.0)^2 + 0*x[3]^2

f_4p_2im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (x[2]-4.0)^2 + 0.0*x[3] + 0.0*x[4] # [3., 4., missing, missing]

f_4p_3im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (x[2]/x[3]-4.0)^2 + 0.0*x[4] # [3., missing, missing, missing]

f_1p_ex(x::Vector{Float64}) = 5.0 + (x[1]-1e-8)^2 # [1e-8, missing, missing, missing]

f_5p_3im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (exp(x[2])-1.0)^2 + (x[3]/x[4]-4.0)^2 + 0.0*x[5]

f_3p_im(x::Vector{Float64}) = 5.0 + (x[1]-3.0)^2 + (exp(x[2])-1.0)^2 + 0.0*x[3]


text:
using RLEVectors

function searchsortedfirst1(v::AbstractVector, x::AbstractVector, lo::Int, hi::Int)
    indices = similar(x)
    n = hi
    min = lo - 1
    max = hi + 1
    hi = hi + 1
    @inbounds for (i,query) in enumerate(x)
        indices[i] = searchsortedfirst(v, query, 1, n)
    end
    return(indices)
end

function searchsortedfirst2(v::AbstractVector, x::AbstractVector, lo::Int, hi::Int)
    indices = similar(x)
    min = lo - 1
    max = hi + 1
    hi = hi + 1
    @inbounds for (i,query) in enumerate(x)
        # unsorted x, restart left side
        if lo <= min || query <= v[lo]
            lo = min
        end
        # cast out exponentially to get hi to the right of query
        jump = 1
        while true
            if hi >= max
                hi = max
                break
            end
            if query <= v[hi]
                break
            end
            lo = hi
            hi = hi + jump
            jump = jump * 2
        end
        # binary search for the exact bin
        while hi - lo > 1
            m = (lo+hi)>>>1
            if query > v[m]
                lo = m
            else
                hi = m
            end
        end
        indices[i] = hi
    end
    return(indices)
end

function searchsortedfirst3(v::AbstractVector, x::AbstractVector, lo::Int, hi::Int)
    indices = similar(x)
    min = lo - 1
    max = hi + 1
    @inbounds for (i,query) in enumerate(x)
        hi = hi + 1 # 2X speedup with this *inside* the loop for sorted x
        # unsorted x, restart left side
        if lo <= min || query <= v[lo]
            lo = min
        end
        if hi >= max || query >= v[hi]
            hi = max
        end
        # binary search for the exact bin
        while hi - lo > 1
            m = (lo+hi)>>>1
            if query > v[m]
                lo = m
            else
                hi = m
            end
        end
        indices[i] = hi
    end
    return(indices)
end

macro timeit(ex)
# like @time, but returning the timing rather than the computed value
  return quote
    #gc_disable()
    local val = $ex # compile
    local t0 = time()
    for i in 1:1e5 val = $ex end
    local t1 = time()
    #gc_enable()
    t1-t0
  end
end

foo = RLEVector(collect(1:1000), collect(5:5:5000))
re = rlast(foo)
x = rand(1:1000, 1000)
sx = sort(x)

@timeit searchsortedfirst1(re, x, 1, 1000)
@timeit searchsortedfirst1(re, sx, 1, 1000)

@timeit searchsortedfirst2(re, x, 1, 1000)
@timeit searchsortedfirst2(re, sx, 1, 1000)

@timeit searchsortedfirst3(re, x, 1, 1000)
@timeit searchsortedfirst3(re, sx, 1, 1000)

using ProfileView
searchsortedfirst3(re, sx, 1, 1000); Profile.clear(); @profile for i in 1:1e4 foo + foo end; ProfileView.view()


text:
using Flux, Metalhead, Statistics, LinearAlgebra, Zygote, Random
using Flux: onehotbatch, onecold, onehot, crossentropy, throttle, NNlib
using Images: channelview
using Statistics: mean
using Base.Iterators: partition

# # for gpu
# using CUDAapi, CuArrays, CUDAdrv, CUDAnative
# if has_cuda()
#     @info "CUDA is on"
#     CuArrays.allowscalar(false)
# end

STNkd(k::Int) = Chain(
    Conv((1,), k=>64, relu),
    BatchNorm(64),
    Conv((1,), 64=>128, relu),
    BatchNorm(128),
    Conv((1,), 128=>1024, relu),
    BatchNorm(1024),
    x -> maximum(x, dims=1),
    x -> reshape(x, :, size(x, 3)),
    Dense(1024, 512, relu),
#     BatchNorm(512),
    Dense(512, 256, relu),
    BatchNorm(256),
    Dense(256,k*k),
    x -> reshape(x, k, k, size(x, 2)),
    x -> x .+ reshape(Matrix{Float64}(I, k, k), k, k, 1),
    x -> PermutedDimsArray(x, (2,1,3)),
)

PointNet(k::Int) = Chain(
    x -> batched_mul(x, stn(x)),
    Conv((1,), 3=>64),
    BatchNorm(64),
    x -> batched_mul(x, fstn(x)),
    Conv((1,), 64=>128, relu),
    BatchNorm(128),
    Conv((1,), 128=>1024),
    BatchNorm(1024),
    x -> maximum(x, dims=1),
    x -> reshape(x, 1024, :),
    Dense(1024, 512, relu),
    BatchNorm(512),
    Dense(512, 256, relu),
    Dropout(0.4),
    BatchNorm(256),
    Dense(256, k, relu),
    x -> softmax(x, dims=1),
)

function pcNormalize(pc)
    centroid = mean((x->x), pc, dims = 1)
    pc = pc .- centroid
    m = max((sum(pc.^2, dims=2).^.5)...)
    pc = pc ./ m
    return pc
end

function datapoint(fn)
    #fn = datapath[index]
    cls = [classes[fn[1]]]
    Pointset = Array{Float64}(undef, npoints, 3)
    stream = open(fn[2], "r")
    for i in 1:npoints
        tmp = readline(stream, keep=false)
        Pointset[i, :] = map((x->parse(Float64, x)), split(tmp, ",")[1:3])
    end
    Pointset = pcNormalize(Pointset)
    return (Pointset,cls)
end

root = "./data/"
batch_size = 4
num_classes = 10 #possible values {10,40}
npoints = 1024
catFile = joinpath(root, "modelnet$(num_classes)_shape_names.txt")
categories = [line::String for line in readlines(catFile)]

classes = Dict{String, Int}()
for i in 1:length(categories)
    classes[categories[i]] = i
end

shapeids = Dict()
shape_names = Dict()
datapath = Dict()

shapeids["train"] = [line::String for line in readlines(joinpath(root, "modelnet$(num_classes)_train.txt"))]
shapeids["test"] = [line::String for line in readlines(joinpath(root, "modelnet$(num_classes)_test.txt"))]

split_ = "train"
shape_names[split_] = [join(split(shapeids[split_][i], "_")[1:end-1], "_") for i in 1:length(shapeids[split_])]
datapath[split_] = [(shape_names[split_][i], joinpath(root, shape_names[split_][i], (shapeids[split_][i])*".txt")) for i in 1:length(shapeids[split_])]

split_ = "test"
shape_names[split_] = [join(split(shapeids[split_][i], "_")[1:end-1], "_") for i in 1:length(shapeids[split_])]
datapath[split_] = [(shape_names[split_][i], joinpath(root, shape_names[split_][i], (shapeids[split_][i])*".txt")) for i in 1:length(shapeids[split_])]

#shuffling trainset Point
rng = MersenneTwister(1234);
shuffle!(rng, datapath["train"])

# input: width*height*channel*minibatch

# Fetching the train and validation data and getting them into proper shape
X = [datapoint(p) for p in datapath["train"]]
data = [X[i][1] for i in 1:length(X)]
labels = onehotbatch(cat([X[i][2] for i in 1:length(X)]..., dims=1), 1:num_classes)
train = [(cat(data[i]..., dims = 3), labels[:,i]) for i in partition(1:length(data), batch_size)]

VAL = [datapoint(p) for p in datapath["test"]]
valX = cat([VAL[i][1] for i in 1:length(VAL)]..., dims=3)
valY = onehotbatch(cat([VAL[i][2] for i in 1:length(VAL)]..., dims=1), 1:num_classes)

# Defining the loss and accuracy functions

stn = STNkd(3)
fstn = STNkd(64)
m = PointNet(num_classes)

loss(x, y) = crossentropy(m(x), y)

accuracy(x, y) = mean(onecold(cpu(m(x)), 1:num_classes) .== onecold(cpu(y), 1:num_classes))

# Defining the callback and the optimizer

evalcb = throttle(() -> @show(accuracy(valX, valY)), 10)

opt = ADAM()

# Starting to train models

print("Training Start")
Flux.train!(loss, params(m), train, opt, cb = evalcb)
print("Training Finished")

# Accuracy of valset of ModelNet dataset
@show(accuracy(valX, valY))

max_stars_repo_name: nirmal-suthar/PointNet

max_issues_repo_name: nirmal-suthar/PointNet

max_forks_repo_name: nirmal-suthar/PointNet


text:
@testset "Schwefel CMA-ES" begin

    # Schwefel's ellipsoid test function:
    # a moderately conditioned ellipsoid with a dominating isolated eigenvalue
    function schwefel(x::AbstractVector{T}) where {T<:AbstractFloat}
        out = zero(T)
        for i = 1:length(x)
            out += sum(abs2, x[1:i])
        end
        return out
    end

    # Parameters
    N = 30
    function Evolutionary.trace!(record::Dict{String,Any}, objfun, state, population, method::CMAES, options)
        record["σ"] = state.σ
    end

    # Testing: CMA-ES
    result = Evolutionary.optimize(schwefel, ()->rand(N), CMAES(μ = 3, λ = 12))
    println("(3/3,12)-CMA-ES (Schwefel) => F: $(minimum(result)), C: $(Evolutionary.iterations(result))")
    @test Evolutionary.converged(result)
    @test Evolutionary.minimizer(result) ≈ zeros(N) atol=1e-5
    @test minimum(result) ≈ 0.0 atol=1e-5

    bounds = Evolutionary.ConstraintBounds(fill(-1.0f0,N),fill(1.0f0,N),[],[])
    opts = Evolutionary.Options(store_trace=true, iterations=10)
    result = Evolutionary.optimize(schwefel, bounds, CMAES(μ = 3, λ = 12), opts)
    @test Evolutionary.iterations(result) == 10
    @test !Evolutionary.converged(result)
    @test haskey(Evolutionary.trace(result)[end].metadata, "σ")
end


text:
using LinearAlgebra

struct ValueOne; end
ValueOne()

# Compute X <- a X + b I.
function matfun_axpby!(X,a,b,Y::UniformScaling)
    m,n=size(X)
    if ~(a isa ValueOne)
        rmul!(X,a)
    end
    @inbounds for i=1:n
        X[i,i]+=(b isa ValueOne) ? 1 : b
    end
end

@inline function exp_ps_m9(A)
    T=promote_type(eltype(A),Float64)
    A_copy=similar(A,T); A_copy .= A;
    return exp_ps_m9!(A_copy)
end

@inline function exp_ps_m9!(A)
    T=promote_type(eltype(A),Float64) # Make it work for many 'bigger' types (matrices and scalars)
    # max_memslots=8
    n=size(A,1)
    # The first slots are precomputed nodes [:A]
    memslots2 = similar(A,T)
    memslots3 = similar(A,T)
    memslots4 = similar(A,T)
    memslots5 = similar(A,T)
    memslots6 = similar(A,T)
    memslots7 = similar(A,T)
    memslots8 = similar(A,T)
    # Assign precomputed nodes memslots
    memslots1=A # overwrite A
    # Uniform scaling is exploited.
    # No matrix I explicitly allocated.
    value_one=ValueOne()
    # Computation order: A2 A3 A4 A5 A6 P4 C3 P3 C2 P2 C1 P1 C0 P0
    # Computing A2 with operation: mult
    mul!(memslots2,memslots1,memslots1)
    # Computing A3 with operation: mult
    mul!(memslots3,memslots1,memslots2)
    # Computing A4 with operation: mult
    mul!(memslots4,memslots1,memslots3)
    # Computing A5 with operation: mult
    mul!(memslots5,memslots1,memslots4)
    # Computing A6 with operation: mult
    mul!(memslots6,memslots1,memslots5)
    # Computing P4 = x*I+x*A+x*A2+x*A3+x*A4+x*A5+x*A6
    coeff1=1.6117375710961184e-24
    coeff2=6.446950284384474e-26
    coeff3=2.4795962632247976e-27
    coeff4=9.183689863795546e-29
    coeff5=3.279889237069838e-30
    coeff6=1.1309962886447716e-31
    coeff7=3.7699876288159054e-33
    memslots7 .= coeff2.*memslots1 .+ coeff3.*memslots2 .+ coeff4.*memslots3 .+ coeff5.*memslots4 .+ coeff6.*memslots5 .+ coeff7.*memslots6
    mul!(memslots7,true,I*coeff1,true,true)
    # Computing C3 with operation: mult
    mul!(memslots8,memslots7,memslots6)
    # Deallocating P4 in slot 7
    # Computing P3 = x*I+x*A+x*A2+x*A3+x*A4+x*A5+x*C3
    coeff1=1.5619206968586225e-16
    coeff2=8.22063524662433e-18
    coeff3=4.110317623312165e-19
    coeff4=1.9572941063391263e-20
    coeff5=8.896791392450574e-22
    coeff6=3.868170170630684e-23
    coeff7=1.0
    # Smart lincomb recycle C3
    memslots8 .= coeff2.*memslots1 .+ coeff3.*memslots2 .+ coeff4.*memslots3 .+ coeff5.*memslots4 .+ coeff6.*memslots5 .+ coeff7.*memslots8
    mul!(memslots8,true,I*coeff1,true,true)
    # Computing C2 with operation: mult
    mul!(memslots7,memslots8,memslots6)
    # Deallocating P3 in slot 8
    # Computing P2 = x*I+x*A+x*A2+x*A3+x*A4+x*A5+x*C2
    coeff1=2.08767569878681e-9
    coeff2=1.6059043836821613e-10
    coeff3=1.1470745597729725e-11
    coeff4=7.647163731819816e-13
    coeff5=4.779477332387385e-14
    coeff6=2.8114572543455206e-15
    coeff7=1.0
    # Smart lincomb recycle C2
    memslots7 .= coeff2.*memslots1 .+ coeff3.*memslots2 .+ coeff4.*memslots3 .+ coeff5.*memslots4 .+ coeff6.*memslots5 .+ coeff7.*memslots7
    mul!(memslots7,true,I*coeff1,true,true)
    # Computing C1 with operation: mult
    mul!(memslots8,memslots7,memslots6)
    # Deallocating P2 in slot 7
    # Computing P1 = x*I+x*A+x*A2+x*A3+x*A4+x*A5+x*C1
    coeff1=0.001388888888888889
    coeff2=0.0001984126984126984
    coeff3=2.48015873015873e-5
    coeff4=2.7557319223985893e-6
    coeff5=2.755731922398589e-7
    coeff6=2.505210838544172e-8
    coeff7=1.0
    # Smart lincomb recycle C1
    memslots8 .= coeff2.*memslots1 .+ coeff3.*memslots2 .+ coeff4.*memslots3 .+ coeff5.*memslots4 .+ coeff6.*memslots5 .+ coeff7.*memslots8
    mul!(memslots8,true,I*coeff1,true,true)
    # Computing C0 with operation: mult
    mul!(memslots7,memslots8,memslots6)
    # Deallocating P1 in slot 8
    # Deallocating A6 in slot 6
    # Computing P0 = x*I+x*A+x*A2+x*A3+x*A4+x*A5+x*C0
    coeff1=1.0
    coeff2=1.0
    coeff3=0.5
    coeff4=0.16666666666666666
    coeff5=0.041666666666666664
    coeff6=0.008333333333333333
    coeff7=1.0
    # Smart lincomb recycle A
    memslots1 .= coeff2.*memslots1 .+ coeff3.*memslots2 .+ coeff4.*memslots3 .+ coeff5.*memslots4 .+ coeff6.*memslots5 .+ coeff7.*memslots7
    mul!(memslots1,true,I*coeff1,true,true)
    # Deallocating A2 in slot 2
    # Deallocating A3 in slot 3
    # Deallocating A4 in slot 4
    # Deallocating A5 in slot 5
    # Deallocating C0 in slot 7
    return memslots1 # Returning P0
end

max_stars_repo_name: matrixfunctions/GraphMatFunData

max_issues_repo_name: matrixfunctions/GraphMatFunData

max_forks_repo_name: matrixfunctions/GraphMatFunData


text:
nload{T<:Bus}(buses::Vector{T}) = length([b for b in buses if isa(b, LoadBus)])

ngen{T<:Bus}(buses::Vector{T}) = length([b for b in buses if isa(b, GenBus)])

count_bus_type(grid::Grid) = (nload(buses(grid)), ngen(buses(grid)))

#######################
### CHECK FUNCTIONS ###
#######################
# Everything here uses connectivty matrices as input instead of grids because
# all functions are supposed to work both for buses and for substations,
# effectively giving us two representations of the network (fine and coarse)
"""
    laplacian(con_mat::AbstractMatrix{<:Real})

Return the Laplacian matrix for a graph defined by adjacency matrix `con_mat`.
"""
function laplacian(con_mat::AbstractMatrix{<:Real})
    lap = zeros(Int, size(con_mat))
    conmat = con_mat .> 0 # adjusting for the case of substation connectivty matrix
    degree = sum(conmat, 1) # this gives the node degrees
    n = length(degree)
    for j in 1:(n - 1), i in (j + 1):n
        if conmat[i,j] > 0
            lap[i,j] = -1
        end
        lap[j,j] = degree[j]
    end
    lap[n,n] = degree[n]
    return Symmetric(lap, :L)
end

"""
    total_links(con_mat::AbstractMatrix{<:Real})

Return the total number of links in a system with connectivity matrix `con_mat`.
"""
function total_links(con_mat::AbstractMatrix{<:Real})
    return (1 / 2) * trace(laplacian(con_mat))
end

"""
    mean_node_deg(con_mat::AbstractMatrix{<:Real})

Return the average nodal degree of a system with connectivity matrix `con_mat`.
"""
function mean_node_deg{T}(con_mat::AbstractArray{T,2})
    return (1 / size(con_mat, 1)) * trace(laplacian(con_mat))
end

"""
    cluster_coeff(con_mat::AbstractMatrix{<:Real})

Return the clustering coefficient of a system with connectivity matrix `con_mat`.
"""
function cluster_coeff(con_mat::AbstractMatrix{<:Real})
    conmat = con_mat .> 0 # adjusting for the case of substation connectivty matrix
    degree = sum(conmat, 1) # this gives the node degrees
    clust_coeff = Vector{Float64}(length(degree))
    for i in 1:length(degree)
        maxedges = degree[i] * (degree[i] - 1) / 2
        cumul = 0
        for j in 1:length(degree)
            if conmat[i,j]
                cumul += dot(conmat[i,:],conmat[:,j])
            end
        end
        push!(clust_coeff, maxedges > 0 ? cumul/maxedges : 0)
    end
    return mean(clust_coeff)
end

"""
    cluster_coeff_degw(con_mat::AbstractMatrix{<:Real})

Return the degree-weighted clustering coefficient of a system with connectivity
matrix `con_mat`.
"""
function cluster_coeff_degw(con_mat::AbstractMatrix{<:Real})
    conmat = con_mat .> 0 # adjusting for the case of substation connectivty matrix
    degree = sum(conmat, 1) # this gives the node degrees
    clust_coeff = Vector{Float64}(length(degree))
    for i in 1:length(degree)
        maxedges = degree[i] * (degree[i] - 1) / 2
        cumul = 0
        for j in 1:length(degree)
            if conmat[i, j]
                cumul += dot(conmat[i,:], conmat[:,j])
            end
        end
        push!(clust_coeff, maxedges > 0 ? degree[i] * cumul / maxedges : 0)
    end
    return mean(clust_coeff) / mean(degree)
end

"""
    test_connectivity(con_mat::AbstractMatrix{<:Real}, verb=true)

Return `true` if the system with connectivity matrix `con_mat`is connected and `false`
otherwise. If `verb=true` the result will be printed onscreen together with the Fiedler
eigenvalue.
"""
function test_connectivity(con_mat::AbstractMatrix{<:Real}, verb=true)
    lap = laplacian(con_mat)
    evals = eigvals(lap)
    unique_vals = [v for v in Set(evals)]
    sort!(unique_vals)
    if length(unique_vals) > 1 && unique_vals[2] > 0
        fiedler = unique_vals[2]
        if verb
            println("Fiedler eigenvalue: $fiedler. System is connected.")
        end
        return true
    else
        if verb
            println("Fiedler eigenvalue equals zero. Network is NOT connected!")
        end
        return false
    end
end

function net_adj_mat(con_mat::AbstractMatrix{<:Real})
    lap = laplacian(con_mat)
    return Diagonal(lap) - lap
end

"""
    dijkstra(con_mat::AbstractMatrix{<:Real}, v1)

Compute minimum distances to each node in the network with connectivity matrix
con_mat, starting from node v1, using Dijkstra's algorithm.
"""
function dijkstra(con_mat::AbstractMatrix{<:Real}, v1)
    n = size(con_mat, 1)
    c = v1
    visited = [v1]
    dist = [i == v1 ? 0 : Inf for i in 1:n]
    r_dist = [Inf for d in dist]
    while length(visited) < n
        neigh = [i for i in 1:n if con_mat[c, i] > 0 && !(i in visited)]
        for i in neigh
            if dist[i] > (dist[c] + con_mat[c, i])
                dist[i] = (dist[c] + con_mat[c, i])
                r_dist[i] = dist[i]
            end
        end
        c = indmin(r_dist)
        push!(visited, c)
        r_dist[c] = Inf
    end
    return dist
end

"""
    mean_shortest_path(conmat::AbstractMatrix{<:Real})

Compute the mean shortest distance between nodes for a network represented by `conmat`.
The `conmat` matrix should specify the distance between each connected node and be equal to
zero whenever two nodes are not connected. Distances are computed using Dijkstra's algorithm.
"""
function mean_shortest_path(conmat::AbstractMatrix{<:Real})
    n = size(conmat, 1)
    mdist = [mean(dijkstra(conmat, i)) for i in 1:n]
    return mean(mdist)
end

"""
    mean_shortest_path(conmat::AbstractMatrix{<:Real}, distmat::AbstractMatrix{<:Real})

Compute the mean shortest distance between nodes for a network with connectivity matrix
`conmat` and distances between nodes given by `distmat`. Distances are computed using
Dijkstra's algorithm.
"""
function mean_shortest_path(
        conmat::AbstractMatrix{<:Real},
        distmat::AbstractMatrix{<:Real}
    )
    return mean_shortest_path(conmat.*distmat)
end

"""
    robustness_line(con_mat::AbstractMatrix{<:Real}, n=1000)

Return the average number of transmission lines that have to be randomly removed from a
system with connectivity matrix `con_mat` before it becomes disconnected. Average is
computed over n iterations.
"""
function robustness_line(con_mat::AbstractMatrix{<:Real}, n=1000)
    s = size(con_mat, 1)
    conns = []
    for i in 1:(s - 1), j in (i + 1):s
        con_mat[j, i] > 0 ? push!(conns, (i, j)) : nothing
    end
    total = 0
    for rep in 1:n
        connscopy = copy(conns)
        conmat = con_mat .> 0 # adjusting for the case of substation connectivty matrix
        step = 0
        while test_connectivity(conmat, false)
            k = rand(1:length(connscopy))
            ij = connscopy[k]
            conmat[ij[1],ij[2]] = false
            conmat[ij[2],ij[1]] = false
            deleteat!(connscopy, k)
            step += 1
        end
        total += step
    end
    return total / n
end

"""
    robustness_node(con_mat::AbstractMatrix{<:Real}, n=1000)

Return the average number of nodes that have to be randomly removed from a system with
connectivity matrix `con_mat` before it becomes disconnected. Average is computed over n
iterations.
"""
function robustness_node(con_mat::AbstractMatrix{<:Real}, n=1000)
    s = size(con_mat, 1)
    total = 0
    for rep in 1:n
        nodes = collect(1:s)
        conmat = con_mat .> 0 # adjusting for the case of substation connectivty matrix
        step = 0
        while test_connectivity(conmat, false)
            k = rand(1:length(nodes))
            i = nodes[k]
            for j in 1:s
                conmat[i,j] = false
                conmat[j,i] = false
            end
            deleteat!(nodes, k)
            step += 1
        end
        total += step
    end
    return total / n
end


text:
cd(Pkg.dir("JellyFish", "deps"))
run(`make`)


text:
using LinearAlgebra

module JuliaPetra

# Internal Utilities
include("Utils.jl")

# Communication interface
include("Comm.jl")
include("LocalComm.jl")
include("Distributor.jl")
include("Directory.jl")

include("BlockMapData.jl")
include("BlockMap.jl")

include("BasicDirectory.jl")
include("DirectoryMethods.jl")

# Communication implementations
include("SerialComm.jl")

# Data interface
include("ImportExportData.jl")
include("Import.jl")
include("Export.jl")

include("DistObject.jl")

# Dense Data types
include("MultiVector.jl")
include("DenseMultiVector.jl")

# Sparse Data types
include("Operator.jl")
include("RowGraph.jl")
include("RowMatrix.jl")

include("SparseRowView.jl")
include("LocalCSRGraph.jl")
include("LocalCSRMatrix.jl")

include("CSRGraphConstructors.jl")
include("CSRGraphInternalMethods.jl")
include("CSRGraphExternalMethods.jl")

include("CSRMatrix.jl")

function __init__()
    #code dependant on MPI.jl can't be precompiled
    sourcedir = dirname(@__FILE__)
    include(joinpath(sourcedir, "MPIUtil.jl"))
    include(joinpath(sourcedir, "MPIComm.jl"))
    include(joinpath(sourcedir, "MPIDistributor.jl"))
end

end # module


text:
# %% imports

using PyCall
gym = pyimport("gym")

using Cambrian

include("../Modules/Networks/net_arch.jl")
include("../Modules/Poisson/PoissonStateSpike.jl")

# %% function utils

function norm_cartpole(obs::Array{Float64})
    pos = obs[1]
    theta = mod(obs[3]+pi,2pi)-pi
    vel = [obs[2], obs[4]]
    pos_scaled = (pos + 2.4)/4.8
    cone = 12*pi/180
    theta_scaled = (theta + cone)/(2*cone)
    vel_scaled = (vel .+ 7.5)/15
    obs_scaled = [pos_scaled, vel_scaled[1], theta_scaled, vel_scaled[2]]
    return obs_scaled
end

function count_spikes(arch::Array{Int64}, spiked::Array{Int64}, layer::Int64)
    count_layer = zeros(arch[layer])
    interval = [sum(arch[1:layer-1])+1, sum(arch[1:layer])]
    spikes_in = filter(x-> interval[1] <= x <= interval[2], spiked)
    spikes_in .-= sum(arch[1:layer-1])
    count_layer[spikes_in] .+= 1
    return count_layer
end

function left_right(count_spikes::Array{Float64}, n_left::Int64, n_right::Int64)
    sl = sum(count_spikes[1:n_left])
    sr = sum(count_spikes[n_left+1:end])
    (sl >= sr) ? action = 0 : action = 1
    return action, sl, sr
end

function borne(x,a,b)
    y = a + 0.5*(b-a)*(1+cos(pi*x/10))
    return y
end

# %% play_episode

function play_episode(net::Network, matalpha::Array{Float64,2}, n_steps::Int64 = 200; rng_seed::Bool = true)
    env = gym.make("CartPole-v1")
    if rng_seed
        env.seed(0)
        rng_action = Random.MersenneTwister(0)
    end
    win_size = 40 # ms
    tot_reward = 0
    N_rand = 0
    N_tot = 0

    obs = env.reset()

    @inbounds for j in 1:n_steps

        #Input computation
        obs_net = matalpha*norm_cartpole(obs)
        spikes = int_spikes(obs_net)

        count_tot = zeros(net.arch[end])
        net.v .= net.param["c"]

        # Count last layer spikes for win_size msec
        @inbounds for msec in 1:win_size
            spiked = step_LIF!(net, spikes, false)
            if msec >= length(net.arch)-1
                count_tot .+= count_spikes(net.arch, spiked, length(net.arch))
            end
        end

        # Decide action
        n_left = div(net.arch[end],2)
        n_right = net.arch[end] - n_left
        action, sl, sr = left_right(count_tot, n_left, n_right)

        # if indecision reduce reward
        if sl == sr
            if rng_seed
                action = rand(rng_action, 0:1)
            else
                action = rand(0:1)
            end
            tot_reward -= 0.9
            N_rand += 1
        end
        N_tot += 1

        # gym env step
        obs_new, reward, done, _ = env.step(action)
        tot_reward += 1

        if done
            break
        end

        obs = obs_new

    end
    println("Rand = ", N_rand, "/", N_tot)
    env.close()
    env = nothing
    Base.GC.gc()
    return tot_reward
end

# %% Weight setting functions

function ei_idx(arch::Array{Int64})
    inh = Int64[]
    exc = Int64[]
    n_inh = [div(arch[i],4) for i in 1:length(arch)-1]
    push!(n_inh, 0)
    for l in 1:length(arch)
        for neuron in 1:arch[l]
            offset = sum(arch[1:l-1])
            if neuron <= n_inh[l]
                push!(inh, neuron+offset)
            else
                push!(exc, neuron+offset)
            end
        end
    end
    return exc, inh
end

function set_weight!(net::Network, genes_w::Array{Float64}, exc::Array{Int64}, inh::Array{Int64})
    w = net.s.e
    inh_count = 0
    for mat_idx in 1:length(w)
        for idx in eachindex(w[mat_idx])
            offset = sum([length(mat) for mat in w[1:mat_idx-1]]) - inh_count
            offset_neuron = sum(net.arch[1:mat_idx-1])
            (idx%size(w[mat_idx])[1]==0) ? line = size(w[mat_idx])[1] : line = idx%size(w[mat_idx])[1]
            current_neuron = line+offset_neuron
            if current_neuron in exc
                net.s.e[mat_idx][idx] = borne(genes_w[idx+offset], 0.0, 1.1)
            elseif current_neuron in inh
                net.s.e[mat_idx][idx] = borne(genes_w[end], -2.0, 0.0)
                inh_count += 1
            end
        end
    end
end

# %% fitness function

function fitness(i::Individual)
    genes = i.genes
    # load the optimisation parameters
    params = YAML.load(open("Julia/Actors/cartpole_cfglif.yml"))
    arch = params["arch"]

    # Input linear combination
    genes_alpha = map(x->borne(x,-1.0, 1.0),genes[1:arch[1]*4])
    matalpha = reshape(genes_alpha, (arch[1],4))

    # Weights of the network
    genes_w = genes[arch[1]*4+1:end]

    # Initialize network
    n = Network(arch, params)
    exc, inh = ei_idx(arch)
    set_weight!(n,genes_w, exc, inh)

    # Compute the fitness
    reward = play_episode(n, matalpha)
    return [reward] # (put - if ES try to minimize by default)
end

max_stars_repo_name: SuReLI/RL-STDP

max_issues_repo_name: SuReLI/RL-STDP

max_forks_repo_name: SuReLI/RL-STDP


text:
struct TODO
    indicator::String
    text::String
    file::String
    line_number::UInt32
    project::String
end

# TODO: define json method for it
function Base.show(io::IO, todo::TODO)
    println(io,
        """
        $(todo.project):
        in $(todo.file) : line $(Int(todo.line_number))
        \e[1m$(todo.indicator)\e[0m $(todo.text)
        """
    )
end # function

function find_todos()
    todos = TODO[]
    for project in configuration["paths"]
        for (root, dirs, files) in walkdir(project)
            if startswith(splitdir(root)[2], '.')
                continue
            end
            filter!(f->any(endswith.(f,configuration["file-endings"])), files)
            for file in files
                open( joinpath(root,file) ) do io
                    line_number = 0
                    for line in eachline(io)
                        line_number += 1
                        the_match = match( Regex("($(join(configuration["identifiers"],"|")))\\s*:\\s*(.*)"), line )
                        if the_match !== nothing
                            push!( todos, TODO(the_match.captures[1], the_match.captures[2], file, line_number, project) )
                        end
                    end
                end
            end
        end
    end
    return todos
end # function