text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

module REPLTests

using Pkg
import Pkg.Types.PkgError
using UUIDs
using Test
import LibGit2

include("utils.jl")

const TEST_SIG = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time()), 0)
const TEST_PKG = (name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a"))

function git_init_package(tmp, path)
    base = basename(path)
    pkgpath = joinpath(tmp, base)
    cp(path, pkgpath)
    LibGit2.with(LibGit2.init(pkgpath)) do repo
        LibGit2.add!(repo, "*")
        LibGit2.commit(repo, "initial commit"; author=TEST_SIG, committer=TEST_SIG)
    end
    return pkgpath
end

@testset "generate args" begin
    @test_throws PkgError pkg"generate"
end

temp_pkg_dir() do project_path
    with_pkg_env(project_path; change_dir=true) do;
        pkg"generate HelloWorld"
        LibGit2.close((LibGit2.init(".")))
        cd("HelloWorld")
        with_current_env() do
            pkg"st"
            @eval using HelloWorld
            Base.invokelatest(HelloWorld.greet)
            @test isfile("Project.toml")
            Pkg.REPLMode.pkgstr("develop $(joinpath(@__DIR__, "test_packages", "PackageWithBuildSpecificTestDeps"))")
            Pkg.test("PackageWithBuildSpecificTestDeps")
        end

        @test_throws PkgError pkg"dev Example#blergh"

        pkg"generate Foo"
        pkg"dev Foo"
        mv(joinpath("Foo", "src", "Foo.jl"), joinpath("Foo", "src", "Foo2.jl"))
        @test_throws PkgError pkg"dev Foo"
        mv(joinpath("Foo", "src", "Foo2.jl"), joinpath("Foo", "src", "Foo.jl"))
        write(joinpath("Foo", "Project.toml"), """
            name = "Foo"
        """
        )
        @test_throws PkgError pkg"dev Foo"
        write(joinpath("Foo", "Project.toml"), """
            uuid = "b7b78b08-812d-11e8-33cd-11188e330cbe"
        """
        )
        @test_throws PkgError pkg"dev Foo"
    end
end

@testset "tokens" begin
    statement = Pkg.REPLMode.parse("?dev")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_HELP
    @test length(statement.arguments) == 1
    @test statement.arguments[1] == "dev"
    statement = Pkg.REPLMode.parse("add git@github.com:JuliaLang/Example.jl.git")[1]
    @test "add" in statement.command.names
    @test statement.arguments[1] == "git@github.com:JuliaLang/Example.jl.git"
    statement = Pkg.REPLMode.parse("add git@github.com:JuliaLang/Example.jl.git#master")[1]
    @test "add" in statement.command.names
    @test length(statement.arguments) == 2
    @test statement.arguments[1] == "git@github.com:JuliaLang/Example.jl.git"
    @test statement.arguments[2] == "#master"
    statement = Pkg.REPLMode.parse("add git@github.com:JuliaLang/Example.jl.git#c37b675")[1]
    @test "add" in statement.command.names
    @test length(statement.arguments) == 2
    @test statement.arguments[1] == "git@github.com:JuliaLang/Example.jl.git"
    @test statement.arguments[2] == "#c37b675"
    statement = Pkg.REPLMode.parse("add git@github.com:JuliaLang/Example.jl.git@v0.5.0")[1]
    @test statement.arguments[1] == "git@github.com:JuliaLang/Example.jl.git"
    @test statement.arguments[2] == "@v0.5.0"
    statement = Pkg.REPLMode.parse("add git@gitlab-fsl.jsc.näsan.guvv:drats/URGA2010.jl.git@0.5.0")[1]
    @test "add" in statement.command.names
    @test length(statement.arguments) == 2
    @test statement.arguments[1] == "git@gitlab-fsl.jsc.näsan.guvv:drats/URGA2010.jl.git"
    @test statement.arguments[2] == "@0.5.0"
end

temp_pkg_dir() do project_path; cd(project_path) do; mktempdir() do tmp_pkg_path
    pkg"activate ."
    pkg"add Example@0.5"
    @test isinstalled(TEST_PKG)
    v = Pkg.API.__installed()[TEST_PKG.name]
    pkg"rm Example"
    pkg"add Example#master"

    # Test upgrade --fixed doesn't change the tracking (https://github.com/JuliaLang/Pkg.jl/issues/434)
    info = Pkg.Types.manifest_info(Pkg.Types.EnvCache(), TEST_PKG.uuid)
    @test info["repo-rev"] == "master"
    pkg"up --fixed"
    info = Pkg.Types.manifest_info(Pkg.Types.EnvCache(), TEST_PKG.uuid)
    @test info["repo-rev"] == "master"

    pkg"test Example"
    @test isinstalled(TEST_PKG)
    @test Pkg.API.__installed()[TEST_PKG.name] > v
    pkg = "UnregisteredWithoutProject"
    p = git_init_package(tmp_pkg_path, joinpath(@__DIR__, "test_packages/$pkg"))
    Pkg.REPLMode.pkgstr("add $p; precompile")
    @eval import $(Symbol(pkg))
    @test Pkg.API.__installed()[pkg] == v"0.0"
    Pkg.test("UnregisteredWithoutProject")

    pkg2 = "UnregisteredWithProject"
    p2 = git_init_package(tmp_pkg_path, joinpath(@__DIR__, "test_packages/$pkg2"))
    Pkg.REPLMode.pkgstr("add $p2")
    Pkg.REPLMode.pkgstr("pin $pkg2")
    # FIXME: this confuses the precompile logic to know what is going on with the user
    # FIXME: why isn't this testing the Pkg after importing, rather than after freeing it
    #@eval import Example
    #@eval import $(Symbol(pkg2))
    @test Pkg.API.__installed()[pkg2] == v"0.1.0"
    Pkg.REPLMode.pkgstr("free $pkg2")
    @test_throws PkgError Pkg.REPLMode.pkgstr("free $pkg2")
    Pkg.test("UnregisteredWithProject")

    write(joinpath(p2, "Project.toml"), """
        name = "UnregisteredWithProject"
        uuid = "58262bb0-2073-11e8-3727-4fe182c12249"
        version = "0.2.0"
        """
    )
    LibGit2.with(LibGit2.GitRepo, p2) do repo
        LibGit2.add!(repo, "*")
        LibGit2.commit(repo, "bump version"; author = TEST_SIG, committer=TEST_SIG)
        pkg"update"
        @test Pkg.API.__installed()[pkg2] == v"0.2.0"
        Pkg.REPLMode.pkgstr("rm $pkg2")

        c = LibGit2.commit(repo, "empty commit"; author = TEST_SIG, committer=TEST_SIG)
        c_hash = LibGit2.GitHash(c)
        Pkg.REPLMode.pkgstr("add $p2#$c")
    end

    mktempdir() do tmp_dev_dir
    withenv("JULIA_PKG_DEVDIR" => tmp_dev_dir) do
        pkg"develop Example"

        # Copy the manifest + project and see that we can resolve it in a new environment
        # and get all the packages installed
        proj = read("Project.toml", String)
        manifest = read("Manifest.toml", String)
        cd_tempdir() do tmp
            old_depot = copy(DEPOT_PATH)
            try
                empty!(DEPOT_PATH)
                write("Project.toml", proj)
                write("Manifest.toml", manifest)
                mktempdir() do depot_dir
                    pushfirst!(DEPOT_PATH, depot_dir)
                    pkg"instantiate"
                    @test Pkg.API.__installed()[pkg2] == v"0.2.0"
                end
            finally
                empty!(DEPOT_PATH)
                append!(DEPOT_PATH, old_depot)
            end
        end # cd_tempdir
    end # withenv
    end # mktempdir
end # mktempdir
end # cd
end # temp_pkg_dir

temp_pkg_dir() do project_path; cd(project_path) do
    mktempdir() do tmp
        mktempdir() do depot_dir
            old_depot = copy(DEPOT_PATH)
            try
                empty!(DEPOT_PATH)
                pushfirst!(DEPOT_PATH, depot_dir)
                withenv("JULIA_PKG_DEVDIR" => tmp) do
                    # Test an unregistered package
                    p1_path = joinpath(@__DIR__, "test_packages", "UnregisteredWithProject")
                    p2_path = joinpath(@__DIR__, "test_packages", "UnregisteredWithoutProject")
                    p1_new_path = joinpath(tmp, "UnregisteredWithProject")
                    p2_new_path = joinpath(tmp, "UnregisteredWithoutProject")
                    cp(p1_path, p1_new_path)
                    cp(p2_path, p2_new_path)
                    Pkg.REPLMode.pkgstr("develop $(p1_new_path)")
                    Pkg.REPLMode.pkgstr("develop $(p2_new_path)")
                    Pkg.REPLMode.pkgstr("build; precompile")
                    @test Base.find_package("UnregisteredWithProject") == joinpath(p1_new_path, "src", "UnregisteredWithProject.jl")
                    @test Base.find_package("UnregisteredWithoutProject") == joinpath(p2_new_path, "src", "UnregisteredWithoutProject.jl")
                    @test Pkg.API.__installed()["UnregisteredWithProject"] == v"0.1.0"
                    @test Pkg.API.__installed()["UnregisteredWithoutProject"] == v"0.0.0"
                    Pkg.test("UnregisteredWithoutProject")
                    Pkg.test("UnregisteredWithProject")
                end
            finally
                empty!(DEPOT_PATH)
                append!(DEPOT_PATH, old_depot)
            end
        end # withenv
    end # mktempdir
    # nested
    mktempdir() do other_dir
        mktempdir() do tmp;
            cd(tmp)
            pkg"generate HelloWorld"
            cd("HelloWorld") do
                with_current_env() do
                    pkg"generate SubModule1"
                    pkg"generate SubModule2"
                    pkg"develop SubModule1"
                    mkdir("tests")
                    cd("tests")
                    pkg"develop ../SubModule2"
                    @test Pkg.API.__installed()["SubModule1"] == v"0.1.0"
                    @test Pkg.API.__installed()["SubModule2"] == v"0.1.0"
                    # make sure paths to SubModule1 and SubModule2 are relative
                    manifest = Pkg.Types.Context().env.manifest
                    @test manifest["SubModule1"][1]["path"] == "SubModule1"
                    @test manifest["SubModule2"][1]["path"] == "SubModule2"
                end
            end
            cp("HelloWorld", joinpath(other_dir, "HelloWorld"))
            cd(joinpath(other_dir, "HelloWorld"))
            with_current_env() do
                # Check that these didn't generate absolute paths in the Manifest by copying
                # to another directory
                @test Base.find_package("SubModule1") == joinpath(pwd(), "SubModule1", "src", "SubModule1.jl")
                @test Base.find_package("SubModule2") == joinpath(pwd(), "SubModule2", "src", "SubModule2.jl")
            end
        end
    end
end # cd
end # temp_pkg_dir

# activate
temp_pkg_dir() do project_path
    cd(mktempdir()) do
        path = pwd()
        pkg"activate ."
        @test Base.active_project() == joinpath(path, "Project.toml")
        # tests illegal names for shared environments
        @test_throws Pkg.Types.PkgError pkg"activate --shared ."
        @test_throws Pkg.Types.PkgError pkg"activate --shared ./Foo"
        @test_throws Pkg.Types.PkgError pkg"activate --shared Foo/Bar"
        @test_throws Pkg.Types.PkgError pkg"activate --shared ../Bar"
        # check that those didn't change te enviroment
        @test Base.active_project() == joinpath(path, "Project.toml")
        mkdir("Foo")
        cd(mkdir("modules")) do
            pkg"generate Foo"
        end
        pkg"develop modules/Foo"
        pkg"activate Foo" # activate path Foo over deps Foo
        @test Base.active_project() == joinpath(path, "Foo", "Project.toml")
        pkg"activate ."
        @test_logs (:info, r"new shared environment") pkg"activate --shared Foo" # activate shared Foo
        @test Base.active_project() == joinpath(Pkg.envdir(), "Foo", "Project.toml")
        pkg"activate ."
        rm("Foo"; force=true, recursive=true)
        pkg"activate Foo" # activate path from developed Foo
        @test Base.active_project() == joinpath(path, "modules", "Foo", "Project.toml")
        pkg"activate ."
        @test_logs (:info, r"new environment") pkg"activate ./Foo" # activate empty directory Foo (sidestep the developed Foo)
        @test Base.active_project() == joinpath(path, "Foo", "Project.toml")
        pkg"activate ."
        @test_logs (:info, r"new environment") pkg"activate Bar" # activate empty directory Bar
        @test Base.active_project() == joinpath(path, "Bar", "Project.toml")
        pkg"activate ."
        pkg"add Example" # non-deved deps should not be activated
        @test_logs (:info, r"new environment") pkg"activate Example"
        @test Base.active_project() == joinpath(path, "Example", "Project.toml")
        pkg"activate ."
        cd(mkdir("tests"))
        pkg"activate Foo" # activate developed Foo from another directory
        @test Base.active_project() == joinpath(path, "modules", "Foo", "Project.toml")
        tmpdepot = mktempdir()
        tmpdir = mkpath(joinpath(tmpdepot, "environments", "Foo"))
        push!(Base.DEPOT_PATH, tmpdepot)
        pkg"activate --shared Foo" # activate existing shared Foo
        @test Base.active_project() == joinpath(tmpdir, "Project.toml")
        pop!(Base.DEPOT_PATH)
        pkg"activate" # activate home project
        @test Base.ACTIVE_PROJECT[] === nothing
    end
end

# test relative dev paths (#490)
cd(mktempdir()) do
    pkg"generate HelloWorld"
    cd("HelloWorld")
    pkg"generate SubModule"
    cd(mkdir("tests"))
    pkg"activate ."
    pkg"develop .." # HelloWorld
    pkg"develop ../SubModule"
    @test Pkg.installed()["HelloWorld"] == v"0.1.0"
    @test Pkg.installed()["SubModule"] == v"0.1.0"
    manifest = Pkg.Types.Context().env.manifest
    @test manifest["HelloWorld"][1]["path"] == ".."
    @test manifest["SubModule"][1]["path"] == joinpath("..", "SubModule")
end

# develop with --shared and --local
using Pkg.Types: manifest_info, EnvCache
cd(mktempdir()) do
    uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a") # Example
    pkg"activate ."
    pkg"develop Example" # test default
    @test manifest_info(EnvCache(), uuid)["path"] == joinpath(Pkg.devdir(), "Example")
    pkg"develop --shared Example"
    @test manifest_info(EnvCache(), uuid)["path"] == joinpath(Pkg.devdir(), "Example")
    pkg"develop --local Example"
    @test manifest_info(EnvCache(), uuid)["path"] == joinpath("dev", "Example")
end

test_complete(s) = Pkg.REPLMode.completions(s,lastindex(s))
apply_completion(str) = begin
    c, r, s = test_complete(str)
    @test s == true
    str[1:prevind(str, first(r))]*first(c)
end

# Autocompletions
temp_pkg_dir() do project_path; cd(project_path) do
    Pkg.Types.registries()
    pkg"activate ."
    c, r = test_complete("add Exam")
    @test "Example" in c
    c, r = test_complete("rm Exam")
    @test isempty(c)
    Pkg.REPLMode.pkgstr("develop $(joinpath(@__DIR__, "test_packages", "RequireDependency"))")

    c, r = test_complete("rm RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm -p RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm --project RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm Exam")
    @test isempty(c)
    c, r = test_complete("rm -p Exam")
    @test isempty(c)
    c, r = test_complete("rm --project Exam")
    @test isempty(c)

    c, r = test_complete("rm -m RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm --manifest RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm -m Exam")
    @test "Example" in c
    c, r = test_complete("rm --manifest Exam")
    @test "Example" in c

    c, r = test_complete("rm RequireDep")
    @test "RequireDependency" in c
    c, r = test_complete("rm Exam")
    @test isempty(c)
    c, r = test_complete("rm -m Exam")
    c, r = test_complete("rm -m Exam")
    @test "Example" in c

    pkg"add Example"
    c, r = test_complete("rm Exam")
    @test "Example" in c
    c, r = test_complete("add --man")
    @test "--manifest" in c
    c, r = test_complete("rem")
    @test "remove" in c
    @test apply_completion("rm E") == "rm Example"
    @test apply_completion("add Exampl") == "add Example"

    c, r = test_complete("preview r")
    @test "remove" in c
    c, r = test_complete("help r")
    @test "remove" in c
    @test !("rm" in c)

    c, r = test_complete("add REPL")
    # Filtered by version
    @test !("REPL" in c)
end end

temp_pkg_dir() do project_path; cd(project_path) do
    mktempdir() do tmp
        cp(joinpath(@__DIR__, "test_packages", "BigProject"), joinpath(tmp, "BigProject"))
        cd(joinpath(tmp, "BigProject"))
        with_current_env() do
            pkg"dev RecursiveDep2"
            pkg"dev RecursiveDep"
            pkg"dev SubModule"
            pkg"dev SubModule2"
            pkg"add Random"
            pkg"add Example"
            pkg"add JSON"
            pkg"build"
            @eval using BigProject
            pkg"build BigProject"
            @test_throws PkgError pkg"add BigProject"
            # the command below also tests multiline input
            Pkg.REPLMode.pkgstr("""
                test SubModule
                test SubModule2
                test BigProject
                test
                """)
            current_json = Pkg.API.installed()["JSON"]
            old_project = read("Project.toml", String)
            open("Project.toml"; append=true) do io
                print(io, """

                [compat]
                JSON = "0.16.0"
                """
                )
            end
            pkg"up"
            @test Pkg.API.installed()["JSON"].minor == 16
            write("Project.toml", old_project)
            pkg"up"
            @test Pkg.API.installed()["JSON"] == current_json
        end
    end
end; end

temp_pkg_dir() do project_path
    cd(project_path) do
        @testset "add/remove using quoted local path" begin
            # utils
            setup_package(parent_dir, pkg_name) = begin
                mkdir(parent_dir)
                cd(parent_dir) do
                    withenv("USER" => "Test User") do
                        Pkg.generate(pkg_name)
                    end
                    cd(pkg_name) do
                        LibGit2.with(LibGit2.init(joinpath(project_path, parent_dir, pkg_name))) do repo
                            LibGit2.add!(repo, "*")
                            LibGit2.commit(repo, "initial commit"; author=TEST_SIG, committer=TEST_SIG)
                        end
                    end #cd pkg_name
                end # cd parent_dir
            end

            # extract uuid from a Project.toml file
            extract_uuid(toml_path) = begin
                uuid = ""
                for line in eachline(toml_path)
                    m = match(r"uuid = \"(.+)\"", line)
                    if m !== nothing
                        uuid = m.captures[1]
                        break
                    end
                end
                return uuid
            end

            # testing local dir with space in name
            dir_name = "space dir"
            pkg_name = "WeirdName77"
            setup_package(dir_name, pkg_name)
            uuid = extract_uuid("$dir_name/$pkg_name/Project.toml")
            Pkg.REPLMode.pkgstr("add \"$dir_name/$pkg_name\"")
            @test isinstalled((name=pkg_name, uuid = UUID(uuid)))
            Pkg.REPLMode.pkgstr("remove \"$pkg_name\"")
            @test !isinstalled((name=pkg_name, uuid = UUID(uuid)))

            # testing dir name with significant characters
            dir_name = "some@d;ir#"
            pkg_name = "WeirdName77"
            setup_package(dir_name, pkg_name)
            uuid = extract_uuid("$dir_name/$pkg_name/Project.toml")
            Pkg.REPLMode.pkgstr("add \"$dir_name/$pkg_name\"")
            @test isinstalled((name=pkg_name, uuid = UUID(uuid)))
            Pkg.REPLMode.pkgstr("remove '$pkg_name'")
            @test !isinstalled((name=pkg_name, uuid = UUID(uuid)))

            # more complicated input
            ## pkg1
            dir1 = "two space dir"
            pkg_name1 = "name1"
            setup_package(dir1, pkg_name1)
            uuid1 = extract_uuid("$dir1/$pkg_name1/Project.toml")

            ## pkg2
            dir2 = "two'quote'dir"
            pkg_name2 = "name2"
            setup_package(dir2, pkg_name2)
            uuid2 = extract_uuid("$dir2/$pkg_name2/Project.toml")

            Pkg.REPLMode.pkgstr("add '$dir1/$pkg_name1' \"$dir2/$pkg_name2\"")
            @test isinstalled((name=pkg_name1, uuid = UUID(uuid1)))
            @test isinstalled((name=pkg_name2, uuid = UUID(uuid2)))
            Pkg.REPLMode.pkgstr("remove '$pkg_name1' $pkg_name2")
            @test !isinstalled((name=pkg_name1, uuid = UUID(uuid1)))
            @test !isinstalled((name=pkg_name2, uuid = UUID(uuid2)))

            Pkg.REPLMode.pkgstr("add '$dir1/$pkg_name1' \"$dir2/$pkg_name2\"")
            @test isinstalled((name=pkg_name1, uuid = UUID(uuid1)))
            @test isinstalled((name=pkg_name2, uuid = UUID(uuid2)))
            Pkg.REPLMode.pkgstr("remove '$pkg_name1' \"$pkg_name2\"")
            @test !isinstalled((name=pkg_name1, uuid = UUID(uuid1)))
            @test !isinstalled((name=pkg_name2, uuid = UUID(uuid2)))
        end
    end
end

@testset "uint test `parse_package`" begin
    name = "FooBar"
    uuid = "7876af07-990d-54b4-ab0e-23690620f79a"
    url = "https://github.com/JuliaLang/Example.jl"
    path = "./Foobar"
    # valid input
    pkg = Pkg.REPLMode.parse_package(name)
    @test pkg.name == name
    pkg = Pkg.REPLMode.parse_package(uuid)
    @test pkg.uuid == UUID(uuid)
    pkg = Pkg.REPLMode.parse_package("$name=$uuid")
    @test (pkg.name == name) && (pkg.uuid == UUID(uuid))
    pkg = Pkg.REPLMode.parse_package(url; add_or_develop=true)
    @test (pkg.repo.url == url)
    pkg = Pkg.REPLMode.parse_package(path; add_or_develop=true)
    @test (pkg.repo.url == path)
    # errors
    @test_throws PkgError Pkg.REPLMode.parse_package(url)
    @test_throws PkgError Pkg.REPLMode.parse_package(path)
end

@testset "unit test for REPLMode.promptf" begin
    function set_name(projfile_path, newname)
        sleep(1.1)
        project = Pkg.TOML.parsefile(projfile_path)
        project["name"] = newname
        open(projfile_path, "w") do io
            Pkg.TOML.print(io, project)
        end
    end

    with_temp_env("SomeEnv") do
        @test Pkg.REPLMode.promptf() == "(SomeEnv) pkg> "
    end

    env_name = "Test2"
    with_temp_env(env_name) do env_path
        projfile_path = joinpath(env_path, "Project.toml")
        @test Pkg.REPLMode.promptf() == "($env_name) pkg> "

        newname = "NewName"
        set_name(projfile_path, newname)
        @test Pkg.REPLMode.promptf() == "($env_name) pkg> "
        cd(env_path) do
            @test Pkg.REPLMode.promptf() == "($env_name) pkg> "
        end
        @test Pkg.REPLMode.promptf() == "($env_name) pkg> "

        newname = "NewNameII"
        set_name(projfile_path, newname)
        cd(env_path) do
            @test Pkg.REPLMode.promptf() == "($newname) pkg> "
        end
        @test Pkg.REPLMode.promptf() == "($newname) pkg> "
    end
end

@testset "Argument order" begin
    with_temp_env() do
        @test_throws PkgError Pkg.REPLMode.pkgstr("add FooBar Example#foobar#foobar")
        @test_throws PkgError Pkg.REPLMode.pkgstr("up Example#foobar@0.0.0")
        @test_throws PkgError Pkg.REPLMode.pkgstr("pin Example@0.0.0@0.0.1")
        @test_throws PkgError Pkg.REPLMode.pkgstr("up #foobar")
        @test_throws PkgError Pkg.REPLMode.pkgstr("add @0.0.1")
    end
end

@testset "`do_generate!` error paths" begin
    with_temp_env() do
        @test_throws PkgError Pkg.REPLMode.pkgstr("generate Example Example2")
        @test_throws PkgError Pkg.REPLMode.pkgstr("generate")
    end
end

@testset "`parse_option` unit tests" begin
    opt = Pkg.REPLMode.parse_option("-x")
    @test opt.val == "x"
    @test opt.argument === nothing
    opt = Pkg.REPLMode.parse_option("--hello")
    @test opt.val == "hello"
    @test opt.argument === nothing
    opt = Pkg.REPLMode.parse_option("--env=some")
    @test opt.val == "env"
    @test opt.argument == "some"
end

@testset "`parse` integration tests" begin
    @test isempty(Pkg.REPLMode.parse(""))

    statement = Pkg.REPLMode.parse("up")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_UP
    @test isempty(statement.meta_options)
    @test isempty(statement.options)
    @test isempty(statement.arguments)

    statement = Pkg.REPLMode.parse("dev Example")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_DEVELOP
    @test isempty(statement.meta_options)
    @test isempty(statement.options)
    @test statement.arguments == ["Example"]

    statement = Pkg.REPLMode.parse("dev Example#foo #bar")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_DEVELOP
    @test isempty(statement.meta_options)
    @test isempty(statement.options)
    @test statement.arguments == ["Example", "#foo", "#bar"]

    statement = Pkg.REPLMode.parse("dev Example#foo Example@v0.0.1")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_DEVELOP
    @test isempty(statement.meta_options)
    @test isempty(statement.options)
    @test statement.arguments == ["Example", "#foo", "Example", "@v0.0.1"]

    statement = Pkg.REPLMode.parse("--one -t add --first --second arg1")[1]
    @test statement.command.kind == Pkg.REPLMode.CMD_ADD
    @test statement.meta_options == ["--one", "-t"]
    @test statement.options == ["--first", "--second"]
    @test statement.arguments == ["arg1"]

    statements = Pkg.REPLMode.parse("--one -t add --first -o arg1; --meta pin -x -a arg0 Example")
    @test statements[1].command.kind == Pkg.REPLMode.CMD_ADD
    @test statements[1].meta_options == ["--one", "-t"]
    @test statements[1].options == ["--first", "-o"]
    @test statements[1].arguments == ["arg1"]
    @test statements[2].command.kind == Pkg.REPLMode.CMD_PIN
    @test statements[2].meta_options == ["--meta"]
    @test statements[2].options == ["-x", "-a"]
    @test statements[2].arguments == ["arg0", "Example"]

    statements = Pkg.REPLMode.parse("up; --meta -x pin --first; dev")
    @test statements[1].command.kind == Pkg.REPLMode.CMD_UP
    @test isempty(statements[1].meta_options)
    @test isempty(statements[1].options)
    @test isempty(statements[1].arguments)
    @test statements[2].command.kind == Pkg.REPLMode.CMD_PIN
    @test statements[2].meta_options == ["--meta", "-x"]
    @test statements[2].options == ["--first"]
    @test isempty(statements[2].arguments)
    @test statements[3].command.kind == Pkg.REPLMode.CMD_DEVELOP
    @test isempty(statements[3].meta_options)
    @test isempty(statements[3].options)
    @test isempty(statements[3].arguments)
end

@testset "argument count errors" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("activate one two")
        @test_throws PkgError Pkg.REPLMode.pkgstr("activate one two three")
        @test_throws PkgError Pkg.REPLMode.pkgstr("precompile Example")
    end
    end
    end
end

@testset "invalid options" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("rm --minor Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("pin --project Example")
    end
    end
    end
end

@testset "Argument order" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("add FooBar Example#foobar#foobar")
        @test_throws PkgError Pkg.REPLMode.pkgstr("up Example#foobar@0.0.0")
        @test_throws PkgError Pkg.REPLMode.pkgstr("pin Example@0.0.0@0.0.1")
        @test_throws PkgError Pkg.REPLMode.pkgstr("up #foobar")
        @test_throws PkgError Pkg.REPLMode.pkgstr("add @0.0.1")
    end
    end
    end
end

@testset "conflicting options" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("up --major --minor")
        @test_throws PkgError Pkg.REPLMode.pkgstr("rm --project --manifest")
    end
    end
    end
end

@testset "gc" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("gc --project")
        @test_throws PkgError Pkg.REPLMode.pkgstr("gc --minor")
        @test_throws PkgError Pkg.REPLMode.pkgstr("gc Example")
        Pkg.REPLMode.pkgstr("gc")
    end
    end
    end
end

@testset "precompile" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("precompile --project")
        @test_throws PkgError Pkg.REPLMode.pkgstr("precompile Example")
        Pkg.REPLMode.pkgstr("precompile")
    end
    end
    end
end

@testset "generate" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("generate --major Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("generate --foobar Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("generate Example1 Example2")
        Pkg.REPLMode.pkgstr("generate Example")
    end
    end
    end
end

@testset "test" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        Pkg.add("Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("test --project Example")
        Pkg.REPLMode.pkgstr("test --coverage Example")
        Pkg.REPLMode.pkgstr("test Example")
    end
    end
    end
end

@testset "build" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("build --project")
        @test_throws PkgError Pkg.REPLMode.pkgstr("build --minor")
    end
    end
    end
end

@testset "free" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError Pkg.REPLMode.pkgstr("free --project")
        @test_throws PkgError Pkg.REPLMode.pkgstr("free --major")
    end
    end
    end
end

@testset "unit tests for `group_words`" begin
    # simple
    groups = Pkg.REPLMode.group_words(["add", "Example"])
    @test length(groups) == 1
    @test groups[1][1] == "add"
    @test groups[1][2] == "Example"
    # statement break
    groups = Pkg.REPLMode.group_words(["a", "b", "c", ";", "a", "b"])
    @test length(groups) == 2
    groups = Pkg.REPLMode.group_words(["a", "b", "c", ";", "a", "b", ";", "d"])
    @test length(groups) == 3
    # trailing statement break
    groups = Pkg.REPLMode.group_words(["a", "b", "c", ";", "a", "b", ";"])
    @test length(groups) == 2
    # errors
    @test_throws PkgError Pkg.REPLMode.group_words(["a", "b", ";", ";", "a", "b"])
    @test_throws PkgError Pkg.REPLMode.group_words([";", "add", "Example"])
end

@testset "tests for api opts" begin
    specs = Pkg.REPLMode.OptionSpecs(Pkg.REPLMode.OptionDeclaration[
        (["project", "p"], Pkg.REPLMode.OPT_SWITCH, :mode => Pkg.Types.PKGMODE_PROJECT),
        (["manifest", "m"], Pkg.REPLMode.OPT_SWITCH, :mode => Pkg.Types.PKGMODE_MANIFEST),
        ("major", Pkg.REPLMode.OPT_SWITCH, :level => Pkg.Types.UPLEVEL_MAJOR),
        ("minor", Pkg.REPLMode.OPT_SWITCH, :level => Pkg.Types.UPLEVEL_MINOR),
        ("patch", Pkg.REPLMode.OPT_SWITCH, :level => Pkg.Types.UPLEVEL_PATCH),
        ("fixed", Pkg.REPLMode.OPT_SWITCH, :level => Pkg.Types.UPLEVEL_FIXED),
        ("rawnum", Pkg.REPLMode.OPT_ARG, :num => nothing),
        ("plus", Pkg.REPLMode.OPT_ARG, :num => x->parse(Int,x)+1),
    ])

    api_opts = Pkg.REPLMode.APIOptions([
        Pkg.REPLMode.Option("manifest"),
        Pkg.REPLMode.Option("patch"),
        Pkg.REPLMode.Option("rawnum", "5"),
    ], specs)

    @test get(api_opts,:foo,nothing) === nothing
    @test get(api_opts,:mode,nothing) == Pkg.Types.PKGMODE_MANIFEST
    @test get(api_opts,:level,nothing) == Pkg.Types.UPLEVEL_PATCH
    @test get(api_opts,:num,nothing) == "5"

    api_opts = Pkg.REPLMode.APIOptions([
        Pkg.REPLMode.Option("project"),
        Pkg.REPLMode.Option("patch"),
        Pkg.REPLMode.Option("plus", "5"),
    ], specs)

    @test get(api_opts,:mode,nothing) == Pkg.Types.PKGMODE_PROJECT
    @test get(api_opts,:level,nothing) == Pkg.Types.UPLEVEL_PATCH
    @test get(api_opts,:num,nothing) == 6
end

@testset "meta option errors" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        # unregistered meta options
        @test_throws PkgError Pkg.REPLMode.pkgstr("--foo=foo add Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("--bar add Example")
        @test_throws PkgError Pkg.REPLMode.pkgstr("-x add Example")
        # malformed, but registered meta option
        @test_throws PkgError Pkg.REPLMode.pkgstr("--env Example")
    end end end
end

@testset "activate" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        mkdir("Foo")
        pkg"activate"
        default = Base.active_project()
        pkg"activate Foo"
        @test Base.active_project() == joinpath(pwd(), "Foo", "Project.toml")
        pkg"activate"
        @test Base.active_project() == default
    end end end
end

@testset "subcommands" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do
        Pkg.REPLMode.pkg"package add Example"
        @test isinstalled(TEST_PKG)
        Pkg.REPLMode.pkg"package rm Example"
        @test !isinstalled(TEST_PKG)
    end end end
end

@testset "`parse_quotes` unit tests" begin
    qwords = Pkg.REPLMode.parse_quotes("\"Don't\" forget to '\"test\"'")
    @test qwords[1].isquoted
    @test qwords[1].word == "Don't"
    @test !qwords[2].isquoted
    @test qwords[2].word == " forget to "
    @test qwords[3].isquoted
    @test qwords[3].word == "\"test\""
    @test_throws PkgError Pkg.REPLMode.parse_quotes("Don't")
    @test_throws PkgError Pkg.REPLMode.parse_quotes("Unterminated \"quot")
end

@testset "argument kinds" begin
    temp_pkg_dir() do project_path; cd_tempdir() do tmpdir; with_temp_env() do;
        @test_throws PkgError pkg"pin Example#foo"
        @test_throws PkgError pkg"test Example#foo"
        @test_throws PkgError pkg"test Example@v0.0.1"
    end end end
end

end # module

max_stars_repo_name: ylxdzsw/julia

max_issues_repo_name: ylxdzsw/julia

max_forks_repo_name: ylxdzsw/julia


text:
@testset "Quantities" begin

@testset "Singular" begin
    @test string(ItemQuantity(1, "apple")) == "1 apple"
    @test string(ItemQuantity(1, "web page")) == "1 web page"
    @test string(ItemQuantity(1, "Julia library")) == "1 Julia library"
end

@testset "Plural" begin
    @test string(ItemQuantity(2, "apple")) == "2 apples"
    @test string(ItemQuantity(3, "web page")) == "3 web pages"
    @test string(ItemQuantity(0, "Julia library")) == "0 Julia libraries"
end

@testset "Collection-like Interface" begin
    @test length(ItemQuantity(0, "human")) == 0
    @test length(ItemQuantity(4, "line")) == 4
    @test isempty(ItemQuantity(0, "train"))
    @test !isempty(ItemQuantity(1, "train"))
end

end

ext: jl


text:
struct NoObjective <: AbstractObjective
    name::String
    NoObjective() = new("feasibility model")
end


text:
x = create_var(3, 4)
y = x'
test_expr(y, (x,))

x = create_var(10, 1)
y = x''
test_expr(y, (x,))

info("All transpose tests passed")


text:
using DataFrames
using Query
using PyPlot
using LaTeXStrings

include("schema.jl")
include("plotting.jl")

const LEGEND_FIGURE_SIZE = (3.2, 5)
const PLOT_FIGURE_SIZE = (3, 2.5)
const COLOR_LIKELIHOOD_WEIGHTING_ONE_PARTICLE = "white"
const COLOR_GOLD_STANDARD = "gold"
const ANNOTATION_FONTSIZE = 11
const TITLE_FONTSIZE=14

function annotate_num_clusters_plot(data::DataFrame)

    # draw filled circle for gold-standard
    gold_standard = @from i in data begin
            @where ((i.proposal_name == OPTIMAL_PROPOSAL_NAME) &&
                    (i.num_sweeps == 4) &&
                    (i.num_particles == 400))
            @select {i.num_particles, i.num_clusters_estimate}
            @collect DataFrame
        end
    add_filled_circle(gold_standard[COL_NUM_PARTICLES],
                      gold_standard[COL_NUM_CLUSTERS_ESTIMATE],
                      COLOR_GOLD_STANDARD)

    # draw horizontal dashed line for gold standard
    xlim = get_xlim(data)
    gold_standard_value = gold_standard[COL_NUM_CLUSTERS_ESTIMATE]
    plt[:plot](xlim, [gold_standard_value, gold_standard_value], "k--")
end

function plot_num_clusters(data::DataFrame, fname::String, to_plot::Vector{Tuple{String,Int}})
    plt[:figure](figsize=PLOT_FIGURE_SIZE)
    for (proposal_name, num_sweeps) in to_plot
        result = @from i in data begin
            @where ((i.proposal_name == proposal_name) &&
                    (i.num_sweeps == num_sweeps))
            @select {i.num_particles, i.num_clusters_estimate, i.num_clusters_stderr}
            @collect DataFrame
        end
        plt[:plot](result[COL_NUM_PARTICLES], result[COL_NUM_CLUSTERS_ESTIMATE])
        lower = result[COL_NUM_CLUSTERS_ESTIMATE] - result[COL_NUM_CLUSTERS_STDERR]
        upper = result[COL_NUM_CLUSTERS_ESTIMATE] + result[COL_NUM_CLUSTERS_STDERR]
        plt[:fill_between](result[COL_NUM_PARTICLES], lower, upper, alpha=0.3)
    end

    annotate_num_clusters_plot(data)

    ax = plt[:gca]()
    ax[:set_xscale]("log")
    ax[:set_xlim](get_xlim(data))
    plt[:ylabel]("Average number\nof clusters")
    plt[:xlabel]("Number of particles")
    plt[:title]("Heuristic diagnostic", fontsize=TITLE_FONTSIZE)
    plt[:tight_layout](pad=0)
    plt[:savefig](fname)
end

function annotate_aide_plot(data::DataFrame)

    # draw filled circle for gold-standard
    gold_standard = @from i in data begin
            @where ((i.proposal_name == OPTIMAL_PROPOSAL_NAME) &&
                    (i.num_sweeps == 4) &&
                    (i.num_particles == 400))
            @select {i.num_particles, i.aide_estimate}
            @collect DataFrame
        end
    add_filled_circle(gold_standard[COL_NUM_PARTICLES],
                      gold_standard[COL_AIDE_ESTIMATE],
                      COLOR_GOLD_STANDARD)

    # draw horizontal dashed line for gold standard
    xlim = get_xlim(data)
    gold_standard_value = gold_standard[COL_AIDE_ESTIMATE]
    plt[:plot](xlim, [gold_standard_value, gold_standard_value], "k--")
end

function plot_aide(data::DataFrame, fname::String, to_plot::Vector{Tuple{String,Int}})
    plt[:figure](figsize=PLOT_FIGURE_SIZE)
    for (proposal_name, num_sweeps) in to_plot
        result = @from i in data begin
            @where ((i.proposal_name == proposal_name) &&
                    (i.num_sweeps == num_sweeps))
            @select {i.num_particles, i.aide_estimate, i.aide_stderr}
            @collect DataFrame
        end
        plt[:plot](result[COL_NUM_PARTICLES], result[COL_AIDE_ESTIMATE])
        lower = result[COL_AIDE_ESTIMATE] - result[COL_AIDE_STDERR]
        upper = result[COL_AIDE_ESTIMATE] + result[COL_AIDE_STDERR]
        plt[:fill_between](result[COL_NUM_PARTICLES], lower, upper, alpha=0.3)
    end

    annotate_aide_plot(data)

    ax = plt[:gca]()
    ax[:set_xscale]("log")
    ax[:set_yscale]("log")
    ax[:set_xlim](get_xlim(data))
    plt[:ylabel]("nats")
    plt[:xlabel]("Number of particles")
    plt[:title]("AIDE estimates", fontsize=TITLE_FONTSIZE)
    plt[:tight_layout](pad=0)
    plt[:savefig](fname)
end

function plot_legend(data::DataFrame, fname::String, to_plot::Vector{Tuple{String,Int}})
    plt[:figure](figsize=LEGEND_FIGURE_SIZE)
    for (proposal, num_sweeps) in to_plot
        plt[:plot]([], [], label="SMC, $proposal proposal\n$num_sweeps rejuvenation sweep(s)")
    end
    plt[:scatter]([], [], color="gold", s=70, edgecolor="black", label="Gold-standard")
    plt[:legend](fontsize=13, ncol=1, frameon=false, borderpad=0, borderaxespad=0, loc="lower left", labelspacing=2)
    plt[:axis]("off")
    plt[:tight_layout](pad=0)
    plt[:savefig](fname)
end

function plot_histogram()
    data_full = readcsv("$data_dir/galaxy_velocities.txt")[:]
    plt[:figure](figsize=(5, 2))
    plt[:hist](data_full, bins=50)
    plt[:xlabel]("Recessional velocity (redshift, km/s)")
    plt[:gca]()[:set_xlim](0, 50000)
    plt[:tight_layout](pad=0)
    plt[:savefig]("$plot_dir/galaxies_histogram.pdf")
end

data_dir = "data"
data = readtable("$data_dir/galaxy_sampler_data.csv")

plot_dir = "plots"
if !Base.Filesystem.isdir(plot_dir)
    Base.Filesystem.mkdir(plot_dir)
end

# which proposals, and which number of sweeps to plot, as a function of number
# of particles
to_plot =Tuple{String,Int}[]
for proposal in (PRIOR_PROPOSAL_NAME, OPTIMAL_PROPOSAL_NAME)
    for num_sweeps in [0, 1, 4]
        push!(to_plot, (proposal, num_sweeps))
    end
end
plot_aide(data, "$plot_dir/galaxies_aide.pdf", to_plot)
plot_num_clusters(data, "$plot_dir/galaxies_num_clusters.pdf", to_plot)
plot_legend(data, "$plot_dir/galaxies_legend.pdf", to_plot)
plot_histogram()


text:
import Base: broadcasted

LoopVectorization.vmap(f, x::ReverseDiff.TrackedArray) = ReverseDiff.track(vmap, f, x)

drelu(x, Δ) = ifelse(x > 0, Δ, zero(x))

ReverseDiff.@grad function broadcasted(::typeof(relu), x::ReverseDiff.TrackedArray)
    relu.(x), Δ -> (nothing, drelu.(x, Δ))
end

ReverseDiff.@grad function vmap(::typeof(σ), x)
    y = vmap(σ, ReverseDiff.value(x))
    y, Δ -> (nothing, Δ .* (y .* (1 .- y)))
end

ReverseDiff.@grad function vmap(::typeof(tanh), x)
    y = vmap(tanh, ReverseDiff.value(x))
    y, Δ -> (nothing, Δ .* (1 .- y.^2))
end


text:
using JSON
using WebIO: AbstractConnection

export Scope,
       AbstractConnection,
       withscope,
       Observable,
       @private,
       setobservable!,
       on, onjs,
       evaljs,
       onimport,
       ondependencies,
       adddeps!,
       import!,
       addconnection!

import Compat.Sockets: send
import Observables: Observable, AbstractObservable, listeners

"""
    ConnectionPool(outbox::Channel, connections::Set{AbstractConnection}=Set())

Manages the distribution of messages from the `outbox` channel to a set of
connections. The ConnectionPool asynchronously takes messages from its outbox
and sends each message to every connection in the pool. Any closed connections
are automatically removed from the pool.
"""
struct ConnectionPool
    outbox::Channel
    connections::Set{AbstractConnection}
    new_connections::Channel{AbstractConnection}

    function ConnectionPool(outbox::Channel, connections=Set{AbstractConnection}())
        new_connections = Channel{AbstractConnection}(32)
        pool = new(outbox, connections, new_connections)
        @async process_messages(pool)
        pool
    end
end

addconnection!(pool::ConnectionPool, c::AbstractConnection) = put!(pool.new_connections, c)
send(pool::ConnectionPool, msg) = put!(pool.outbox, msg)

"""
Ensure that the pool has at least one connection, potentially blocking
the current task until that is the case.
"""
function ensure_connection(pool::ConnectionPool)
    if isempty(pool.connections)
        wait(pool.new_connections)
    end
    while isready(pool.new_connections)
        push!(pool.connections, take!(pool.new_connections))
    end
end

function process_messages(pool::ConnectionPool)
    while true
        msg = fetch(pool.outbox)  # don't take! yet, since we're not sure msg can be sent
        ensure_connection(pool)
        msg_sent = false
        @sync begin
            for connection in pool.connections
                @async begin
                    if isopen(connection)
                        send(connection, msg)
                        msg_sent = true
                    else
                        delete!(pool.connections, connection)
                    end
                end
            end
        end
        if msg_sent
            # Message was sent, so remove it from the channel
            take!(pool.outbox)
        end
    end
end

"""
    Scope(id; kwargs...)

An object which can send and receive messages.

Fields:
- `id::String`: A unique ID
- `imports`: An array of js/html/css assets to load
  before rendering the contents of a scope.
"""
mutable struct Scope
    id::AbstractString
    dom::Any
    observs::Dict{String, Tuple{AbstractObservable, Union{Nothing,Bool}}} # bool marks if it is synced
    private_obs::Set{String}
    systemjs_options
    imports

    # A collection of handler functions associated with various observables in
    # this scope. Of the form
    # "observable-name" => ["array", "of", "JS", "strings"]
    # where each JS-string is a function that is evoked when the observable
    # changes.
    jshandlers
    pool::ConnectionPool
end

const scopes = Dict{String, Scope}()

function Scope(id::String=newid("scope");
        dom=dom"span"(),
        outbox::Channel=Channel{Any}(Inf),
        observs::Dict=Dict(),
        private_obs::Set{String}=Set{String}(),
        dependencies=nothing,
        systemjs_options=nothing,
        imports=[],
        jshandlers::Dict=Dict()
    )

    if dependencies !== nothing
        @warn("dependencies key word argument is deprecated, use imports instead")
        imports = dependencies
    end

    if haskey(scopes, id)
        @warn("A scope by the id $id already exists. Overwriting.")
    end

    pool = ConnectionPool(outbox)

    scopes[id] = Scope(id, dom, observs, private_obs, systemjs_options, imports, jshandlers, pool)
end
Base.@deprecate Scope(id::AbstractString; kwargs...) Scope(; id=id, kwargs...)

(w::Scope)(arg) = (w.dom = arg; w)

function Observables.on(f, w::Scope, key)
    key = string(key)
    listener, _ = Base.@get! w.observs key (Observable{Any}(w, key, nothing), nothing)
    on(f, listener)
end

private(scope::Scope, props...) = foreach(p->push!(scope.private_obs, string(p)), props)

macro private(ex)
    if ex.head == :(=)
        ref, val = ex.args
        scope, key = ref.args
        quote
            x=$(esc(ex))
            private($(esc(scope)), $(esc(key)))
            x
        end
    else
        error("Wrong usage of @private. Use as in `@private scope[\"key\"] = observable`")
    end
end

# we need to maintain mapping from an ID to observable
# in order to allow interpolation of observables.
const observ_id_dict = WeakKeyDict()

function setobservable!(ctx, key, obs; sync=nothing)
    key = string(key)
    if haskey(ctx.observs, key)
        @warn("An observable named $key already exists in scope $(ctx.id).
             Overwriting.")
    end

    ctx.observs[key] = (obs, sync)

    # the following metadata is stored for use in interpolation
    # of observables into DOM trees and `@js` expressions
    observ_id_dict[obs]  = (WeakRef(ctx), key)
    obs
end

# Ask JS to send stuff
function setup_comm(f, ob::AbstractObservable)
    if haskey(observ_id_dict, ob)
        scope, key = observ_id_dict[ob]
        # if !(key in scope.value.private_obs)
        #     evaljs(scope.value, js"""
        #            console.log(this)
        #            this.observables[$key].sync = true
        #     """)
        # end
    end
end

# TODO: hook `off` up

function Base.getindex(w::Scope, key)
    key = string(key)
    if haskey(w.observs, key)
        w.observs[key][1]
    else
        Observable{Any}(w, key, nothing)
    end
end

function Base.setindex!(w::Scope, obs, key)
    setobservable!(w, key, obs)
end

function Observable{T}(ctx::Scope, key, value; sync=nothing) where {T}
    setobservable!(ctx, key, Observable{T}(value), sync=sync)
end

function Observable(ctx::Scope, key, val::T; sync=nothing) where {T}
    Observable{T}(ctx, key, val; sync=sync)
end

const Observ = Observable

include("imports.jl")

function JSON.lower(x::Scope)
    obs_dict = Dict()
    for (k, ob_) in x.observs
        ob, sync = ob_
        if k in x.private_obs
            continue
        end
        if sync === nothing
            # by default, we sync if there are any listeners
            # other than the JS back edge
            sync = any(f-> !isa(f, SyncCallback), listeners(ob))
        end
        obs_dict[k] = Dict("sync" => sync,
             "value" => ob[],
             "id" => obsid(ob))
    end
    Dict(
        "id" => x.id,
        "systemjs_options" => x.systemjs_options,
        "imports" => lowerdeps(x.imports),
        "handlers" => x.jshandlers,
        "observables" => obs_dict)
end

function render(s::Scope)
    node(s, s.dom)
end

function send(ctx::Scope, key, data)
    command_data = Dict(
      "type" => "command",
      "scope" => ctx.id,
      "command" => key,
      "data" => data,
    )
    send(ctx.pool, command_data)
    nothing
end

macro evaljs(ctx, expr)
    @warn("@evaljs is deprecated, use evaljs function instead")
    :(send($(esc(ctx)), "Basics.eval", $(esc(expr))))
end

function evaljs(ctx, expr)
    send(ctx, "Basics.eval", expr)
end

function onimport(scope::Scope, f)
    promise_name = "importsLoaded"
    jshandlers = scope.jshandlers
    if !haskey(jshandlers, "_promises")
        jshandlers["_promises"] = Dict()
    end
    if !haskey(jshandlers["_promises"], promise_name)
        jshandlers["_promises"][promise_name] = []
    end
    push!(jshandlers["_promises"][promise_name], f)
end

Base.@deprecate ondependencies(ctx, jsf) onimport(ctx, jsf)

const waiting_messages = Dict{String, Condition}()

function send_sync(ctx::Scope, key, data)
    msgid = string(rand(UInt128))
    command_data = Dict(
      "type" => "command",
      "scope" => ctx.id,
      "messageId" => msgid,
      "command" => key,
      "data" => data,
      "sync" => true,
    )
    cond = Condition()
    waiting_messages[msgid] = cond
    send(ctx.outbox, command_data)
    wait(cond)
end

"""
A callable which updates the frontend
"""
struct SyncCallback
    ctx
    f
end

(s::SyncCallback)(xs...) = s.f(xs...)
"""
Set observable without synchronizing with the counterpart on the browser
"""
function set_nosync(ob, val)
    Observables.setexcludinghandlers(ob, val, x -> !(x isa SyncCallback))
end

const lifecycle_commands = ["scope_created"]
function dispatch(ctx, key, data)
    if haskey(ctx.observs, string(key))
        # this message has come from the browser
        # so don't update the browser back!
        set_nosync(ctx.observs[key][1], data)
    else
        key ∈ lifecycle_commands ||
            @warn("$key does not have a handler for scope id $(ctx.id)")
    end
end

function onjs(ctx, key, f)
    push!(Base.@get!(ctx.jshandlers, key, []), f)
end

function offjs(ctx, key, f)
    if f in get(ctx.jshandlers, key, [])
        keys = ctx.jshandlers[key]
        deleteat!(keys, findall(in(f), keys))
    end
    nothing
end

function ensure_sync(ctx, key)
    ob = ctx.observs[key][1]
    # have at most one synchronizing handler per observable
    if !any(x->isa(x, SyncCallback) && x.ctx==ctx, listeners(ob))
        f = SyncCallback(ctx, (msg) -> send(ctx, key, msg))
        on(SyncCallback(ctx, f), ob)
    end
end

function onjs(ob::AbstractObservable, f)
    if haskey(observ_id_dict, ob)
        ctx, key = observ_id_dict[ob]
        ctx = ctx.value
        # make sure updates are set up to propagate to JS
        ensure_sync(ctx, key)
        onjs(ctx, key, f)
    else
        error("This observable is not associated with any scope.")
    end
end

function Base.show(io::IO, ::WEBIO_NODE_MIME, x::Scope)
    write(io, JSON.json(render_internal(x)))
end
function Base.show(io::IO, m::MIME"text/html", x::Scope)
    show(io, m, render(x))
end

function _show(io::IO, el::Scope, indent_level=0)
    showindent(io, indent_level)
    _show(io, el.dom, indent_level)
end

Base.@deprecate_binding Context Scope
Base.@deprecate handle! on
Base.@deprecate handlejs! onjs


text:
using Documenter, {{ app }}

makedocs(
  modules = [{{ app }}],
  format = :html,
  sitename = "{{ app }}.jl",
  pages = Any[
    "Home" => "index.md",
    "Code" => "code.md",
    "Glossary" => "glossary.md"
  ]
)

deploydocs(
  repo = "github.com/{{ user }}/{{ app }}.jl.git",
  target = "build",
  deps = nothing,
  make = nothing
)


text:
"""
    struct UnstructuredGrid{Dc,Dp,Tp,Ti,O} <: Grid{Dc,Dp}
      node_coordinates::Vector{Point{Dp,Tp}}
      cell_nodes::Table{Ti,Int32}
      reffes::Vector{<:LagrangianRefFE{Dc}}
      cell_types::Vector{Int8}
    end
"""
struct UnstructuredGrid{Dc,Dp,Tp,O} <: Grid{Dc,Dp}
  node_coordinates::Vector{Point{Dp,Tp}}
  cell_nodes::Table{Int,Vector{Int},Vector{Int32}}
  reffes::Vector{LagrangianRefFE{Dc}}
  cell_types::Vector{Int8}
  @doc """
      function UnstructuredGrid(
        node_coordinates::Vector{Point{Dp,Tp}},
        cell_nodes::Table{Ti},
        reffes::Vector{<:LagrangianRefFE{Dc}},
        cell_types::Vector,
        ::Val{B}=Val{false}()) where {Dc,Dp,Tp,Ti,B}
      end

  Low-level inner constructor.
  """
  function UnstructuredGrid(
    node_coordinates::Vector{Point{Dp,Tp}},
    cell_nodes::Table{Ti},
    reffes::Vector{<:LagrangianRefFE{Dc}},
    cell_types::Vector,
    ::Val{B}=Val{false}()) where {Dc,Dp,Tp,Ti,B}
    new{Dc,Dp,Tp,B}(node_coordinates,cell_nodes,reffes,cell_types)
  end
end

"""
    UnstructuredGrid(trian::Grid)
"""
function UnstructuredGrid(trian::Grid)
  @assert is_regular(trian) "UnstructuredGrid constructor only for regular grids"
  node_coordinates = collect1d(get_node_coordinates(trian))
  cell_nodes = Table(get_cell_nodes(trian))
  reffes = get_reffes(trian)
  cell_types = collect1d(get_cell_type(trian))
  orien = OrientationStyle(trian)
  UnstructuredGrid(node_coordinates,cell_nodes,reffes,cell_types,orien)
end

function UnstructuredGrid(trian::UnstructuredGrid)
  trian
end

OrientationStyle(
  ::Type{UnstructuredGrid{Dc,Dp,Tp,B}}) where {Dc,Dp,Tp,B} = Val{B}()

get_reffes(g::UnstructuredGrid) = g.reffes

get_cell_type(g::UnstructuredGrid) = g.cell_types

get_node_coordinates(g::UnstructuredGrid) = g.node_coordinates

get_cell_nodes(g::UnstructuredGrid) = g.cell_nodes

# From ReferenceFE

"""
    UnstructuredGrid(reffe::LagrangianRefFE)

Build a grid with a single cell that is the given reference FE itself
"""
function UnstructuredGrid(reffe::LagrangianRefFE)
  node_coordinates = get_node_coordinates(reffe)
  cell_nodes = Table([collect(1:num_nodes(reffe)),])
  reffes = [reffe,]
  cell_types = [1,]
  UnstructuredGrid(node_coordinates,cell_nodes,reffes,cell_types)
end

# From Polytope

function UnstructuredGrid(::Type{ReferenceFE{D}},p::Polytope{D}) where D
  order = 1
  reffe = LagrangianRefFE(Float64,p,order)
  UnstructuredGrid(reffe)
end

"""
    UnstructuredGrid(::Type{ReferenceFE{d}},p::Polytope) where d
"""
function UnstructuredGrid(::Type{ReferenceFE{d}},p::Polytope) where d
  node_coordinates = get_vertex_coordinates(p)
  cell_nodes = Table(get_faces(p,d,0))
  reffaces = get_reffaces(Polytope{d},p)
  cell_type = get_face_type(p,d)
  order = 1
  reffes = map( (f)-> LagrangianRefFE(Float64,f,order), reffaces)
  UnstructuredGrid(
    node_coordinates,
    cell_nodes,
    reffes,
    cell_type)
end

# From coordinates

"""
    UnstructuredGrid(x::AbstractArray{<:Point})
"""
function UnstructuredGrid(x::AbstractArray{<:Point})
  np = length(x)
  node_coords = collect1d(x)
  cell_nodes = identity_table(Int,Int32,np)
  cell_type = fill(1,np)
  order = 1
  reffes = [LagrangianRefFE(Float64,VERTEX,order),]
  UnstructuredGrid(
    node_coords,
    cell_nodes,
    reffes,
    cell_type)
end

# IO

function to_dict(grid::UnstructuredGrid)
  dict = Dict{Symbol,Any}()
  x = get_node_coordinates(grid)
  dict[:node_coordinates] = reinterpret(eltype(eltype(x)),x)
  dict[:Dp] = num_point_dims(grid)
  dict[:cell_nodes] = to_dict(get_cell_nodes(grid))
  dict[:reffes] = map(to_dict, get_reffes(grid))
  dict[:cell_type] = get_cell_type(grid)
  dict[:orientation] = is_oriented(grid)
  dict
end

function from_dict(::Type{UnstructuredGrid},dict::Dict{Symbol,Any})
  x = collect1d(dict[:node_coordinates])
  T = eltype(x)
  Dp = dict[:Dp]
  node_coordinates::Vector{Point{Dp,T}} = reinterpret(Point{Dp,T},x)
  cell_nodes = from_dict(Table{Int,Vector{Int},Vector{Int32}},dict[:cell_nodes])
  reffes = [ from_dict(LagrangianRefFE,reffe) for reffe in dict[:reffes]]
  cell_type::Vector{Int8} = dict[:cell_type]
  O::Bool = dict[:orientation]
  UnstructuredGrid(
    node_coordinates,
    cell_nodes,
    reffes,
    cell_type,
    Val(O))
end

# Extract grid topology

"""
    UnstructuredGridTopology(grid::UnstructuredGrid)

    UnstructuredGridTopology(
      grid::UnstructuredGrid,
      cell_to_vertices::Table,
      vertex_to_node::AbstractVector)
"""
function UnstructuredGridTopology(grid::UnstructuredGrid)
  cell_to_vertices, vertex_to_node, = _generate_cell_to_vertices_from_grid(grid)
  _generate_grid_topology_from_grid(grid,cell_to_vertices,vertex_to_node)
end

function UnstructuredGridTopology(grid::UnstructuredGrid, cell_to_vertices::Table, vertex_to_node::AbstractVector)
  _generate_grid_topology_from_grid(grid,cell_to_vertices,vertex_to_node)
end

function _generate_grid_topology_from_grid(grid::UnstructuredGrid,cell_to_vertices,vertex_to_node)

  @notimplementedif (! is_regular(grid)) "Extrtacting the GridTopology form a Grid only implemented for the regular case"

  node_to_coords = get_node_coordinates(grid)
  if vertex_to_node == 1:num_nodes(grid)
    vertex_to_coords = node_to_coords
  else
    vertex_to_coords = node_to_coords[vertex_to_node]
  end

  cell_to_type = get_cell_type(grid)
  polytopes = map(get_polytope, get_reffes(grid))

  UnstructuredGridTopology(
    vertex_to_coords,
    cell_to_vertices,
    cell_to_type,
    polytopes,
    OrientationStyle(grid))

end

function _generate_cell_to_vertices_from_grid(grid::UnstructuredGrid)
  if is_first_order(grid)
    cell_to_vertices = Table(get_cell_nodes(grid))
    vertex_to_node = collect(1:num_nodes(grid))
    node_to_vertex = vertex_to_node
  else
    cell_to_nodes = get_cell_nodes(grid)
    cell_to_cell_type = get_cell_type(grid)
    reffes = get_reffes(grid)
    cell_type_to_lvertex_to_lnode = map(get_vertex_node, reffes)
    cell_to_vertices, vertex_to_node, node_to_vertex = _generate_cell_to_vertices(
      cell_to_nodes,
      cell_to_cell_type,
      cell_type_to_lvertex_to_lnode,
      num_nodes(grid))
  end
  (cell_to_vertices, vertex_to_node, node_to_vertex)
end

function _generate_cell_to_vertices(
  cell_to_nodes::Table,
  cell_to_cell_type::AbstractVector{<:Integer},
  cell_type_to_lvertex_to_lnode::Vector{Vector{Int}},
  nnodes::Int=maximum(cell_to_nodes.data))

  data, ptrs, vertex_to_node, node_to_vertex = _generate_cell_to_vertices(
    cell_to_nodes.data,
    cell_to_nodes.ptrs,
    cell_to_cell_type,
    cell_type_to_lvertex_to_lnode,
    nnodes)

  (Table(data,ptrs), vertex_to_node)
end

function _generate_cell_to_vertices(
  cell_to_nodes_data,
  cell_to_nodes_ptrs,
  cell_to_cell_type,
  cell_type_to_lvertex_to_lnode,
  nnodes)

  cell_to_vertices_ptrs = similar(cell_to_nodes_ptrs)

  cell_type_to_nlvertices = map(length,cell_type_to_lvertex_to_lnode)

  _generate_cell_to_vertices_count!(
    cell_to_vertices_ptrs,
    cell_to_cell_type,
    cell_type_to_nlvertices)

  T = eltype(cell_to_nodes_data)

  node_to_vertex = fill(T(UNSET),nnodes)

  length_to_ptrs!(cell_to_vertices_ptrs)

  ndata = cell_to_vertices_ptrs[end]-1
  cell_to_vertices_data = zeros(T,ndata)

  _generate_cell_to_vertices_fill!(
    cell_to_vertices_data,
    cell_to_vertices_ptrs,
    cell_to_nodes_data,
    cell_to_nodes_ptrs,
    node_to_vertex,
    cell_to_cell_type,
    cell_type_to_lvertex_to_lnode)

  vertex_to_node = find_inverse_index_map(node_to_vertex)

  (cell_to_vertices_data, cell_to_vertices_ptrs, vertex_to_node, node_to_vertex)

end

function  _generate_cell_to_vertices_count!(
    cell_to_vertices_ptrs,
    cell_to_cell_type,
    cell_type_to_nlvertices)

  cells = 1:length(cell_to_cell_type)
  for cell in cells
    cell_type = cell_to_cell_type[cell]
    nlvertices = cell_type_to_nlvertices[cell_type]
    cell_to_vertices_ptrs[1+cell] = nlvertices
  end
end

function  _generate_cell_to_vertices_fill!(
    cell_to_vertices_data,
    cell_to_vertices_ptrs,
    cell_to_nodes_data,
    cell_to_nodes_ptrs,
    node_to_vertex,
    cell_to_cell_type,
    cell_type_to_lvertex_to_lnode)

  cells = 1:length(cell_to_cell_type)

  vertex = 1

  for cell in cells
    cell_type = cell_to_cell_type[cell]
    a = cell_to_nodes_ptrs[cell]-1
    b = cell_to_vertices_ptrs[cell]-1

    lvertex_to_lnode = cell_type_to_lvertex_to_lnode[cell_type]
    for (lvertex, lnode) in enumerate(lvertex_to_lnode)
      node = cell_to_nodes_data[a+lnode]
      if node_to_vertex[node] == UNSET
        node_to_vertex[node] = vertex
        vertex += 1
      end
      cell_to_vertices_data[b+lvertex] = node_to_vertex[node]
    end

  end
end

function simplexify(grid::UnstructuredGrid)
  reffes = get_reffes(grid)
  @notimplementedif length(reffes) != 1
  reffe = first(reffes)
  order = 1
  @notimplementedif get_order(reffe) != order
  p = get_polytope(reffe)
  ltcell_to_lpoints, simplex = simplexify(p)
  cell_to_points = get_cell_nodes(grid)
  tcell_to_points = _refine_grid_connectivity(cell_to_points, ltcell_to_lpoints)
  ctype_to_reffe = [LagrangianRefFE(Float64,simplex,order),]
  tcell_to_ctype = fill(Int8(1),length(tcell_to_points))
  point_to_coords = get_node_coordinates(grid)
  UnstructuredGrid(
    point_to_coords,
    tcell_to_points,
    ctype_to_reffe,
    tcell_to_ctype,
    Val{true}())
end

function _refine_grid_connectivity(cell_to_points::Table, ltcell_to_lpoints)
  data, ptrs = _refine_grid_connectivity(
    cell_to_points.data,
    cell_to_points.ptrs,
    ltcell_to_lpoints)
  Table(data,ptrs)
end


text:
using CauseMap

nboots = 50  # use bootstrap libraries instead of sliding windows (reduces effect of secular trends)

E     = 4
tau_s = 1
tau_p = 0

ds   = readdlm("vr.raw_fixed.txt", '\t', Float64, header=false)  # read in data
para = ds[:, 2]
didi = ds[:, 3]

println("Starting bootstrap test")
makesingleplot(para, didi,
                            E, tau_s, tau_p,
                            "para", "didi"; nboots=nboots, plot=false)

E_vals     = 2:10  # range to test of system dimensionality
tau_s_vals = 1:1 # range for lag length for manifold reconstruction
tau_p_vals = 0:15  # range to test for time lag of causal effect

println("Starting optimization plot test")
makeoptimizationplots(para, didi,
                                        E_vals, tau_s_vals, tau_p_vals,
                                        "Para.", "Didi."; nreps=1, plot=false)


text:
using Suppressor
using StableRNGs
using Term
using Test
import Term: console_width

const RNG = StableRNG(1337)

include("__test_utils.jl")

using TimerOutputs: TimerOutputs, @timeit
const TIMEROUTPUT = TimerOutputs.TimerOutput()

dotest = console_width() ≥ 88

# ? 1  - text utils
tprint("{bold green}Running: '01_test_text_utils.jl' {/bold green}")
@time @timeit_include("01_test_text_utils.jl")

# ? 2 ansi
tprint("\n\n{bold green}Running: '02_test_ansi.jl' {/bold green}")  # ansi & col' {/bold green}")
@time @timeit_include("02_test_ansi.jl")

# ? 3 measure
tprint("\n\n{bold green}Running: '03_test_measure.jl' {/bold green}")
@time @timeit_include("03_test_measure.jl")

# ? 4 style
tprint("\n\n{bold green}Running: '04_test_style.jl' {/bold green}")
@time @timeit_include("04_test_style.jl")

# ? 5 macros
tprint("\n\n{bold green}Running: '05_test_macros.jl' {/bold green}")
@time @timeit_include("05_test_macros.jl")

# ? 6a box
tprint("\n\n{bold green}Running: '6a_test_box.jl' {/bold green}")
@time @timeit_include("6a_test_box.jl")

# ? 6 renderables
tprint("\n\n{bold green}Running: '06_test_renderables.jl' {/bold green}")
@time @timeit_include("06_test_renderables.jl")

# ? 7 panel
tprint("\n\n{bold green}Running: '07_test_panel.jl' {/bold green}")
@time @timeit_include("07_test_panel.jl")

# ? 8 layout
tprint("\n\n{bold green}Running: '08_test_layout.jl' {/bold green}")
@time @timeit_include("08_test_layout.jl")

#  ? 9 inspect
tprint("\n\n{bold green}Running: '09_test_inspect.jl' {/bold green}")
@time @timeit_include("09_test_inspect.jl")

# ? 11 console
tprint("\n\n{bold green}Running: '11_test_console.jl' {/bold green}")
@time @timeit_include("11_test_console.jl")

# ? 12 logging
tprint("\n\n{bold green}Running: '12_test_logging.jl' {/bold green}")
@time @timeit_include("12_test_logging.jl")

# ? 14 highlight
tprint("\n\n{bold green}Running: '14_test_highlight.jl' {/bold green}")
@time @timeit_include("14_test_highlight.jl")

# ? 15 progress
tprint("\n\n{bold green}Running: '15_test_progress.jl' {/bold green}")
@time @timeit_include("15_test_progress.jl")

# ? 16 Tree
tprint("\n\n{bold green}Running: '16_test_tree.jl' {/bold green}")
@time @timeit_include("16_test_tree.jl")

# ? 17 Dendogram
tprint("\n\n{bold green}Running: '17_test_dendogram.jl' {/bold green}")
@time @timeit_include("17_test_dendogram.jl")

# ? 18 Table
tprint("\n\n{bold green}Running: '18_test_table.jl' {/bold green}")
@time @timeit_include("18_test_table.jl")

# ? 19 Repr
tprint("\n\n{bold green}Running: '19_test_repr.jl' {/bold green}")
@time @timeit_include("19_test_repr.jl")

# ? 20 Compositor
tprint("\n\n{bold green}Running: '20_test_compositor.jl' {/bold green}")
@time @timeit_include("20_test_compositor.jl")

# ? 21 Markdown
tprint("\n\n{bold green}Running: '21_test_markdown.jl' {/bold green}")
@time @timeit_include("21_test_markdown.jl")

# ? 22 Theme
tprint("\n\n{bold green}Running: '22_test_theme.jl' {/bold green}")
@time @timeit_include("22_test_theme.jl")

# ? 999 EXAMPLES
tprint("\n\n{bold green}Running: '999_test_examples.jl' {/bold green}")
@time @timeit_include("999_test_examples.jl")

# ? 99 ERRORS
tprint("\n\n{bold green}Running: '99_test_errors.jl' {/bold green}")
@time @timeit_include("99_test_errors.jl")

show(TIMEROUTPUT; compact = true, sortby = :firstexec)
println("\n")


text:
module JpegTurbo

using ImageCore
using TOML
const project_info = TOML.parsefile(joinpath(@__DIR__, "..", "Project.toml"))

include("../lib/LibJpeg.jl")
using .LibJpeg
include("libjpeg_utils.jl")

include("common.jl")
include("encode.jl")
include("decode.jl")

include("fileio.jl")

export jpeg_encode, jpeg_decode

end


text:
"""
Dpar - input as either 5.0M or 5.5M case
"""
function init_track_len(;calc_type="Dpar",Dpar=1.65,OH=0.0,Cl=0.0,rmr0=0.0,
  a_vol=0.0,Ketcham07_5_5m=false,Ketcham07_5_0m=false,Donelick99=false,
  orig_calc=true,custom=false,custom_intercept=0.0,custom_slope=0.0,
  custom_param=0.0,custom_c_intercept=0.0,custom_c_slope=0.0)

  num_args = Ketcham07_5_5m+Ketcham07_5_0m+Donelick99+custom
  if isone(num_args)==false
    error("COASTerror: must select only one calculation method
          (probably need to set Ketcham07_5_5m=false)")
  end

  if Ketcham07_5_5m==true
    (l0_m,l0_c_m) = ketcham5_5_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,
                                        a_vol)
  elseif Ketcham07_5_0m==true
    (l0_m,l0_c_m) = ketcham5_0_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,
                                        a_vol)
  elseif Donelick99==true
    (l0_m,l0_c_m) = donelick99_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,
                                        a_vol)
  elseif custom==true
    l0_m = custom_slope*custom_param + custom_intercept
    l0_c_m = custom_c_slope*custom_param + custom_c_intercept

  else
    error("COASTerror: Must select at least one l0 calc method!")
  end

  return l0_m,l0_c_m
end

function ketcham5_5_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,a_vol)

  # Dicts w raw data
  # 1st val = unproj slope
  # 2nd val = unproj intercept
  # 3rd val = c proj slope
  # 4th val = c proj intercept
  raw_data_ketcham5_5_hefty = Dict("Dpar"=>[0.283,15.63,0.35,15.72],
                                  "rmr0"=>[0.0,16.14,0.0,16.35],
                                  "OH_apfu"=>[0.638,16.04,0.932,16.18],
                                  "Cl_apfu"=>[1.004,16.14,1.197,16.35],
                                  "a_vol"=>[5.519,-35.6,6.4,-43.635])
  raw_data_ketcham5_5_original = Dict("Dpar"=>[0.283,15.63,0.35,15.72],
                                  "rmr0"=>[0.0,16.14,0.0,16.35],
                                  "OH_apfu"=>[0.638,16.04,0.932,16.18],
                                  "Cl_apfu"=>[1.004,16.14,1.197,16.35],
                                  "a_vol"=>[5.519,-35.6,6.4,-43.635])

  if orig_calc==true # use as originally in paper
     (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
     )= extract_raw_l0_data(raw_data_ketcham5_5_original,calc_type,
                                         Dpar,OH,Cl,rmr0,a_vol)
  elseif (orig_calc==false)  # HeFTy defaults (unpublished)
    (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
    )= extract_raw_l0_data(raw_data_ketcham5_5_hefty,calc_type,
                                        Dpar,OH,Cl,rmr0,a_vol)
  else
    error("COASTerror: received non-boolean for initial len calculations!")
  end

  l0_m = unproj_slope*param_in + unproj_intercept
  l0_c_m = c_slope*param_in + c_intercept

  return l0_m,l0_c_m
end

function ketcham5_0_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,a_vol)

  # Dicts w raw data
  # 1st val = unproj slope
  # 2nd val = unproj intercept
  # 3rd val = c proj slope
  # 4th val = c proj intercept
  raw_data_ketcham5_0_hefty = Dict("Dpar"=>[0.258,15.391,0.287,15.582],
                                  "rmr0"=>[0.0,15.936,0.0,16.187],
                                  "OH_apfu"=>[0.0,15.936,0.0,16.187],
                                  "Cl_apfu"=>[0.538,15.936,0.604,16.187],
                                  "a_vol"=>[7.094,-50.702,7.184,-51.287])
  raw_data_ketcham5_0_original = Dict("Dpar"=>[0.258,15.391,0.287,15.582],
                                  "rmr0"=>[0.0,15.936,0.0,16.187],
                                  "OH_apfu"=>[0.0,15.936,0.0,16.187],
                                  "Cl_apfu"=>[0.538,15.936,0.604,16.187],
                                  "a_vol"=>[7.094,-50.702,7.184,-51.287])

  if orig_calc==true # use as originally in paper
     (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
     )= extract_raw_l0_data(raw_data_ketcham5_0_original,calc_type,
                                         Dpar,OH,Cl,rmr0,a_vol)
  elseif (orig_calc==false)  # HeFTy defaults (unpublished)
    (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
    )= extract_raw_l0_data(raw_data_ketcham5_0_hefty,calc_type,
                                        Dpar,OH,Cl,rmr0,a_vol)
  else
    error("COASTerror: received non-boolean for initial len calculations!")
  end

  l0_m = unproj_slope*param_in + unproj_intercept
  l0_c_m = c_slope*param_in + c_intercept

  return l0_m,l0_c_m
end

function donelick99_init_len(calc_type,orig_calc,Dpar,OH,Cl,rmr0,a_vol)

  # Dicts w raw data
  # 1st val = unproj slope
  # 2nd val = unproj intercept
  # 3rd val = c proj slope
  # 4th val = c proj intercept
  raw_data_donelick99_hefty = Dict("Dpar"=>[0.283,15.63,0.35,15.72],
                                  "rmr0"=>[0.0,16.14,0.0,16.35],
                                  "OH_apfu"=>[0.638,16.04,0.932,16.18],
                                  "Cl_apfu"=>[1.004,16.14,1.197,16.35],
                                  "a_vol"=>[5.519,-35.6,6.4,-43.635])
  raw_data_donelick99_original = Dict("Dpar"=>[0.283,15.63,0.35,15.72],
                                  "rmr0"=>[0.0,16.14,0.0,16.35],
                                  "OH_apfu"=>[0.638,16.04,0.932,16.18],
                                  "Cl_apfu"=>[1.004,16.14,1.197,16.35],
                                  "a_vol"=>[5.519,-35.6,6.4,-43.635])

  if orig_calc==true # use as originally in paper
     (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
     )= extract_raw_l0_data(raw_data_donelick99_original,calc_type,
                                         Dpar,OH,Cl,rmr0,a_vol)
  elseif (orig_calc==false)  # HeFTy defaults (unpublished)
    (param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
    )= extract_raw_l0_data(raw_data_donelick99_hefty,calc_type,
                                        Dpar,OH,Cl,rmr0,a_vol)
  else
    error("COASTerror: received non-boolean for initial len calculations!")
  end

  l0_m = unproj_slope*param_in + unproj_intercept
  l0_c_m = c_slope*param_in + c_intercept

  return l0_m,l0_c_m
end

function extract_raw_l0_data(data_dict,calc_type,Dpar,OH,Cl,rmr0,a_vol)

  if calc_type=="Dpar"
    param_in = Dpar
    unproj_slope = data_dict["Dpar"][1]
    unproj_intercept = data_dict["Dpar"][2]
    c_slope = data_dict["Dpar"][3]
    c_intercept = data_dict["Dpar"][4]
  elseif calc_type=="Cl_apfu"
    param_in = Cl
    unproj_slope = data_dict["Cl_apfu"][1]
    unproj_intercept = data_dict["Cl_apfu"][2]
    c_slope = data_dict["Cl_apfu"][3]
    c_intercept = data_dict["Cl_apfu"][4]
  elseif calc_type=="OH_apfu"
    param_in = OH
    unproj_slope = data_dict["OH_apfu"][1]
    unproj_intercept = data_dict["OH_apfu"][2]
    c_slope = data_dict["OH_apfu"][3]
    c_intercept = data_dict["OH_apfu"][4]
  elseif calc_type=="rmr0"
    param_in = rmr0
    unproj_slope = data_dict["rmr0"][1]
    unproj_intercept = data_dict["rmr0"][2]
    c_slope = data_dict["rmr0"][3]
    c_intercept = data_dict["rmr0"][4]
  elseif calc_type=="a_len"
    param_in = a_vol
    unproj_slope = data_dict["a_vol"][1]
    unproj_intercept = data_dict["a_vol"][2]
    c_slope = data_dict["a_vol"][3]
    c_intercept = data_dict["a_vol"][4]
  else
    error("COASTerror: choose calculation type from the following:
          Dpar, Cl_apfu, OH_apfu, rmr0, a_len or specify custom values")
  end

  return param_in,unproj_slope,unproj_intercept,c_slope,c_intercept
end


text:
using Manifolds
using LinearAlgebra
using StaticArrays

## The factors
struct MeasurementOnTangent end

function PointPoint_distance(M, m, p, q)
    q̂ = compose(M, p, m)
    return distance(M, q, q̂)
end

function grad_PointPoint_distance(M, m, p, q)
    q̂ = compose(M, p, m)
    return grad_distance(M, q, q̂)
end

function PointPoint_distance(M, X, p, q, ::MeasurementOnTangent)
    q̂ = compose(M, p, exp(M, identity_element(M, p), X))
    return distance(M, q, q̂)
end

function PointPoint_velocity_distance(M, X, dt, p, q, ::MeasurementOnTangent)
    q̂ = compose(M, p, group_exp(M, X*dt))
    return distance(M, q, q̂)
end

function Prior_distance(M, meas, p)
#
    return distance(M, meas, p)
end

# Pose
# Pose2Pose2(m, p, q) = PointPoint_distance(getManifold(Pose2), m, p, q)
Pose2Pose2(m, p, q) = PointPoint_distance(SpecialEuclidean(2), m, p, q)
Pose3Pose3(m, p, q) = PointPoint_distance(SpecialEuclidean(3), m, p, q)
PriorPose2(m, p) = Prior_distance(SpecialEuclidean(2), m, p)
PriorPose3(m, p) = Prior_distance(SpecialEuclidean(3), m, p)

#Point
Point1Point1(m, p, q) = PointPoint_distance(TranslationGroup(1), m, p, q)
Point2Point2(m, p, q) = PointPoint_distance(TranslationGroup(2), m, p, q)
Point3Point3(m, p, q) = PointPoint_distance(TranslationGroup(3), m, p, q)
PriorPoint2(m, p) = Prior_distance(TranslationGroup(2), m, p)
PriorPoint3(m, p) = Prior_distance(TranslationGroup(3), m, p)

## Testing the Factors

ϵSE2 = ProductRepr(SA[0.,0] , SA[1. 0; 0 1])
ϵSE3 = ProductRepr(SA[0.,0,0] , SA[1. 0 0; 0 1 0; 0 0 1])

M = SpecialEuclidean(2)

X = hat(M, ϵSE2, [1, 0, pi/4+0.1])
p = compose(M, ϵSE2, exp(M, ϵSE2, X))

X = hat(M, ϵSE2, [1, 0, pi/8])
q = compose(M, p, exp(M, ϵSE2, X))

X = hat(M, ϵSE2, [1-1e-3, 0+1e-3, pi/8+1e-3])
m = compose(M, ϵSE2, exp(M, ϵSE2, X))

Pose2Pose2(m, p, q)

PointPoint_distance(M, X, p, q, MeasurementOnTangent())

PriorPose2(m, q)
PriorPose2(q, q)

## Testing the Factor

## and with matrices
# using LinearAlgebra

# ϵSE2 = Matrix(1.0I(3))
# ϵSE3 = Matrix(1.0I(4))

# M = SpecialEuclidean(2)

# X = hat(M, ϵSE2, [1, 0, pi/4])
# # exp does not work with affine matrix
# p = compose(M, ϵSE2, exp(M, ϵSE2, X))


text:
using CircuitSimulator
using Test

include("readcardstest.jl")
include("parsertest.jl")


text:
function run_Ipopt_para_ac(itema, network_data, load_distrn, load_keys, gen_keys, time_lim)

    network_data_new = deepcopy(network_data)
    #relax the inf term
    for i in 1:length(gen_keys)
        if network_data_new["gen"][gen_keys[i]]["qmax"] == Inf
            network_data_new["gen"][gen_keys[i]]["qmax"] = 10^20
        end

        if network_data_new["gen"][gen_keys[i]]["qmax"] == -Inf
            network_data_new["gen"][gen_keys[i]]["qmax"] = -10^20
        end

        if network_data_new["gen"][gen_keys[i]]["qmin"] == Inf
            network_data_new["gen"][gen_keys[i]]["qmin"] = 10^20
        end

        if network_data_new["gen"][gen_keys[i]]["qmin"] == -Inf
            network_data_new["gen"][gen_keys[i]]["qmin"] = -10^20
        end
    end

    if haskey(network_data_new, "dcline")
        dcline_keys = [i for i in keys(network_data["dcline"])]

        for i in 1:length(dcline_keys)
            if network_data_new["dcline"][dcline_keys[i]]["qmaxf"] == Inf
                network_data_new["dcline"][dcline_keys[i]]["qmaxf"] = 10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qmaxf"] == -Inf
                network_data_new["dcline"][dcline_keys[i]]["qmaxf"] = -10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qmint"] == Inf
                network_data_new["dcline"][dcline_keys[i]]["qmint"] = 10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qmint"] == -Inf
                network_data_new["dcline"][dcline_keys[i]]["qmint"] = -10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qminf"] == Inf
                network_data_new["dcline"][dcline_keys[i]]["qminf"] = 10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qminf"] == -Inf
                network_data_new["dcline"][dcline_keys[i]]["qminf"] = -10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qmaxt"] == Inf
                network_data_new["dcline"][dcline_keys[i]]["qmaxt"] = 10^20
            end

            if network_data_new["dcline"][dcline_keys[i]]["qmaxt"] == -Inf
                network_data_new["dcline"][dcline_keys[i]]["qmaxt"] = -10^20
            end
        end
    end

    for i in 1:length(load_keys)
        network_data_new["load"][load_keys[i]]["pd"] += rand(load_distrn[load_keys[i]])
    end

    out_file_name = "./data/perturb_output/" * itema[1:end-2] * "perturb_output" * ".csv"

    # run the test with Ipopt
    r_ipopt = run_ac_opf(network_data_new,
        optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0, "max_wall_time" => time_lim, "file_print_level" => 3,
            "output_file" => out_file_name))

    obj_val = r_ipopt["objective"]
    Optimal = r_ipopt["termination_status"]
    solve_time = r_ipopt["solve_time"]
    primal_feasibility = r_ipopt["primal_status"]
    dual_feasibility = r_ipopt["dual_status"]
    time_limit = time_lim

    # initialize variables
    esti_obj_val = 0
    dual_infeasibility = 0
    constraint_violation = 0
    overall_nlp = 0
    complementarity = 0
    var_bound_violation = 0
    esti_Optimal = 0

    open(out_file_name, "r") do file_object
        for line in eachline(file_object)

            if occursin("Objective...............", line)
                esti_obj_val = line[55:end]
            end

            if occursin("Dual infeasibility....", line)
                dual_infeasibility = line[55:end]
            end

            if occursin("Constraint violation....", line)
                constraint_violation = line[55:end]
            end

            if occursin("Overall NLP error....", line)
                overall_nlp = line[55:end]
            end

            if occursin("Complementarity..", line)
                complementarity = line[55:end]
            end

            if occursin("Variable bound violation", line)
                var_bound_violation = line[55:end]
            end

            if occursin("EXIT:", line)
                esti_Optimal = line[7:end]
            end

        end

    end

    return obj_val, Optimal, solve_time, primal_feasibility, dual_feasibility, time_limit, network_data_new["load"], r_ipopt["solution"]["bus"], r_ipopt["solution"]["gen"], r_ipopt["solution"]["branch"], esti_obj_val, dual_infeasibility, constraint_violation, overall_nlp, complementarity, var_bound_violation,esti_Optimal
end

max_stars_repo_name: zhouliang-yu/Grid-ML

max_issues_repo_name: zhouliang-yu/Grid-ML

max_forks_repo_name: zhouliang-yu/Grid-ML


text:
"""
    PatchEmbedding(imsize::NTuple{2} = (224, 224); inchannels = 3, patch_size = (16, 16),
                   embedplanes = 768, norm_layer = planes -> identity, flatten = true)

Patch embedding layer used by many vision transformer-like models to split the input image into
patches.

# Arguments:
- `imsize`: the size of the input image
- `inchannels`: the number of channels in the input image
- `patch_size`: the size of the patches
- `embedplanes`: the number of channels in the embedding
- `norm_layer`: the normalization layer - by default the identity function but otherwise takes a
                single argument constructor for a normalization layer like LayerNorm or BatchNorm
- `flatten`: set true to flatten the input spatial dimensions after the embedding
"""
function PatchEmbedding(imsize::NTuple{2} = (224, 224); inchannels = 3, patch_size = (16, 16),
                        embedplanes = 768, norm_layer = planes -> identity, flatten = true)

  im_height, im_width = imsize
  patch_height, patch_width = patch_size

  @assert (im_height % patch_height == 0) && (im_width % patch_width == 0)
  "Image dimensions must be divisible by the patch size."

  return Chain(Conv(patch_size, inchannels => embedplanes; stride = patch_size),
               flatten ? x -> permutedims(reshape(x, (:, size(x, 3), size(x, 4))), (2, 1, 3))
                       : identity,
               norm_layer(embedplanes))
end

"""
    ViPosEmbedding(embedsize, npatches; init = (dims) -> rand(Float32, dims))

Positional embedding layer used by many vision transformer-like models.
"""
struct ViPosEmbedding{T}
  vectors::T
end

ViPosEmbedding(embedsize, npatches; init = (dims::NTuple{2, Int}) -> rand(Float32, dims)) =
  ViPosEmbedding(init((embedsize, npatches)))

(p::ViPosEmbedding)(x) = x .+ p.vectors

@functor ViPosEmbedding

"""
    ClassTokens(dim; init = Flux.zeros32)

Appends class tokens to an input with embedding dimension `dim` for use in many vision transformer models.
"""
struct ClassTokens{T}
  token::T
end

ClassTokens(dim::Integer; init = Flux.zeros32) = ClassTokens(init(dim, 1, 1))

function (m::ClassTokens)(x)
  tokens = repeat(m.token, 1, 1, size(x, 3))
  return hcat(tokens, x)
end

@functor ClassTokens


text:
using CCallHelp
using Test

@testset "CCallHelp.jl" begin
    # Write your tests here.
end


text:
using Documenter:
    DocMeta,
    HTML,
    MathJax3,
    asset,
    deploydocs,
    makedocs
using PlutoStaticHTML

const NOTEBOOK_DIR = joinpath(@__DIR__, "src", "notebooks")

"""
    build()

Run all Pluto notebooks (".jl" files) in `NOTEBOOK_DIR`.
"""
function build()
    println("Building notebooks")
    hopts = HTMLOptions(; append_build_context=true)
    output_format = documenter_output
    bopts = BuildOptions(NOTEBOOK_DIR; output_format)
    build_notebooks(bopts, hopts)
    return nothing
end

# Build the notebooks; defaults to true.
if get(ENV, "BUILD_DOCS_NOTEBOOKS", "true") == "true"
    build()
end

sitename = "PlutoStaticHTML.jl"
pages = [
    "PlutoStaticHTML" => "index.md",
    "Example notebook" => "notebooks/example.md",
    "`with_terminal`" => "with_terminal.md"
]

# Using MathJax3 since Pluto uses that engine too.
mathengine = MathJax3()
prettyurls = get(ENV, "CI", nothing) == "true"
format = HTML(; mathengine, prettyurls)
modules = [PlutoStaticHTML]
strict = true
checkdocs = :none
makedocs(; sitename, pages, format, modules, strict, checkdocs)

deploydocs(;
    branch="docs-output",
    devbranch="main",
    repo="github.com/rikhuijzer/PlutoStaticHTML.jl.git",
    push_preview=false
)


text:
@testset "Masking" begin
    @testset "circle" begin
        X = ones(3, 3)
        out = mask_circle(X, 1, fill=NaN)
        @test out[2, 2] === NaN
        @test all(i == 5 ? true : out[i] !== NaN for i in eachindex(out))
        @test all(mask_circle(X, 2, fill=NaN) .=== NaN)

        # interface
        A = @inferred mask_circle(ones(100, 100), 5, fill=NaN)
        B = @inferred mask_circle!(ones(100, 100), 5, fill=NaN)
        @test filter(!isnan, A) == filter(!isnan, B)

        # cubes
        A = @inferred mask_circle(ones(3, 100, 100), 5, fill=NaN)
        B = @inferred mask_circle!(ones(3, 100, 100), 5, fill=NaN)
        @test filter(!isnan, A) == filter(!isnan, B)

    end

    @testset "annulus" begin
        X = ones(3, 3)
        out = mask_annulus(X, 0, 1, fill=NaN)
        @test out[2, 2] === NaN
        @test all(i == 5 ? true : out[i] !== NaN for i in eachindex(out))
        @test all(mask_annulus(X, 0, 2, fill=NaN) .=== NaN)

        # interface
        A = @inferred mask_annulus(ones(100, 100), 3, 6, fill=NaN)
        B = @inferred mask_annulus!(ones(100, 100), 3, 6, fill=NaN)
        @test filter(!isnan, A) == filter(!isnan, B)
        @test_throws MethodError mask_annulus!(ones(3, 100, 100), 5)
    end

    # regression check: annulus with inner 0 should be same as circle
    data = ones(100, 100)
    @test filter(!isnan, mask_circle(data, 10, fill=NaN)) == filter(!isnan, mask_annulus(data, 0, 10, fill=NaN))
end


text:
#@generated function thetainc(::Val{T}, θ) where T
#    z = Expr(:tuple, (:(θ[$i]) for i in 1:length(T) if  T[i])...)
#    return z
#end

function pseudo_conjugate_draw(θ, XX, PT, prior, updtIdx, α = 1.0)
    𝓦 = mustart(updtIdx)*mustart(updtIdx)'
    𝓦 = _pseudo_conjugate_draw(𝓦, XX, PT, updtIdx)

    Σ = inv(𝓦 + inv(Matrix(prior.Σ)))
    Σ = (Σ + Σ')/2 # eliminates numerical inconsistencies
    ϑ = thetainc(updtIdx, θ)
    ϑᵒ = rand(Gaussian(ϑ, α * Σ))
    move_to_proper_place(ϑᵒ, θ, updtIdx), Σ
end

function _pseudo_conjugate_draw(𝓦, XX, PT, updtIdx)
    for X in XX
        Γ⁻¹ = hypo_a_inv(PT, X.tt[end], X.yy[end])
        for i in 1:length(X)-1
            φₜ = φ(updtIdx, X.tt[i], X.yy[i], PT)
            dt = X.tt[i+1] - X.tt[i]
            𝓦 = 𝓦 + φₜ'*Γ⁻¹*φₜ*dt
        end
    end
    𝓦
end


text:
include("./imports.jl")

@testset ExtendedTestSet "Protected" begin
    mutable struct C
        x::Int
        y::Int
    end

    mutable struct B
        c::C
    end

    mutable struct A
        b1::B
        b2::B
    end

    makea() = A(
        B(C(1, 2)),
        B(C(3, 4)),
    )

    @testset ExtendedTestSet "fully protected" begin
        a_p = protect(makea())  # protect all child structs of `a`
        @test a_p.b1.c isa Protected
        @test_throws ProtectedException a_p.b1.c = C(5, 6)
    end

    @testset ExtendedTestSet "write direct child" begin
        a_p2 = protect(makea(), (:b1,))  # protect everything but `b1`
        @test a_p2.b1 isa B
        @test a_p2.b2 isa Protected
    end
    @testset ExtendedTestSet "write nested" begin
        a_p3 = protect(makea(), (;b1 = (;c = (:x,))))  # allow mutating only a.b1.c.x
        @test a_p3.b1.c isa Protected
        @test_nowarn a_p3.b1.c.x = 2
        @test_throws ProtectedException a_p3.b1.c.y = 2
    end
end


text:
using Documenter, Kmers

makedocs(
    format = Documenter.HTML(),
    sitename = "Kmers.jl",
    pages = [
        "Home"                           => "index.md",
        "Kmer types"                     => "kmer_types.md",
        "Constructing kmers"             => "construction.md"#,
        #"Indexing & modifying kmers"     => "transforms.md",
        #"Predicates"                     => "predicates.md",
        #"Random kmers"                   => "random.md",
        #"Pattern matching and searching" => "sequence_search.md",
        #"Iteration"                      => "iteration.md",
        #"Counting"                       => "counting.md",
        #"I/O"                            => "io.md",
        #"Interfaces"                     => "interfaces.md"
    ],
    authors = "Ben J. Ward, The BioJulia Organisation and other contributors."
)

deploydocs(
    repo = "github.com/BioJulia/Kmer.jl.git",
    push_preview = true,
    deps = nothing,
    make = nothing
)


text:
# JIT compilation of Julia code to PTX

export cufunction

#
# main code generation functions
#

function module_setup(mod::LLVM.Module)
    # NOTE: NVPTX::TargetMachine's data layout doesn't match the NVPTX user guide,
    #       so we specify it ourselves
    if Int === Int64
        triple!(mod, "nvptx64-nvidia-cuda")
        datalayout!(mod, "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64")
    else
        triple!(mod, "nvptx-nvidia-cuda")
        datalayout!(mod, "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64")
    end

    # debug info metadata
    push!(metadata(mod), "llvm.module.flags",
          MDNode([ConstantInt(Int32(1)),    # llvm::Module::Error
                  MDString("Debug Info Version"), ConstantInt(DEBUG_METADATA_VERSION())]))
end

# make function names safe for ptxas
sanitize_fn(fn::String) = replace(fn, r"[^aA-zZ0-9_]", "_")

function raise_exception(insblock::BasicBlock, ex::Value)
    fun = LLVM.parent(insblock)
    mod = LLVM.parent(fun)
    ctx = context(mod)

    builder = Builder(ctx)
    position!(builder, insblock)

    trap = if haskey(functions(mod), "llvm.trap")
        get(functions(mod), "llvm.trap")
    else
        LLVM.Function(mod, "llvm.trap", LLVM.FunctionType(LLVM.VoidType(ctx)))
    end
    call!(builder, trap)
end

function irgen(func::ANY, tt::ANY)
    # sanity checks
    Base.JLOptions().can_inline == 0 &&
        Base.warn_once("inlining disabled, CUDA code generation will almost certainly fail")

    fn = String(typeof(func).name.mt.name)
    mod = LLVM.Module(sanitize_fn(fn))
    module_setup(mod)

    # collect all modules of IR
    # TODO: emit into module instead of parsing
    # TODO: make codegen pure
    hook_module_setup(ref::Ptr{Void}) =
        module_setup(LLVM.Module(convert(LLVM.API.LLVMModuleRef, ref)))
    hook_raise_exception(insblock::Ptr{Void}, ex::Ptr{Void}) =
        raise_exception(BasicBlock(convert(LLVM.API.LLVMValueRef, insblock)),
                        Value(convert(LLVM.API.LLVMValueRef, ex)))
    modrefs = Vector{Ptr{Void}}()
    hook_module_activation(ref::Ptr{Void}) = push!(modrefs, ref)
    hooks = Base.CodegenHooks(module_setup=hook_module_setup,
                              module_activation=hook_module_activation,
                              raise_exception=hook_raise_exception)
    params = Base.CodegenParams(cached=false,
                                runtime=false, exceptions=false,
                                track_allocations=false, code_coverage=false,
                                static_alloc=false, dynamic_alloc=false,
                                hooks=hooks)
    entry_irmod = Base._dump_function(func, tt,
                                      #=native=#false, #=wrapper=#false, #=strip=#false,
                                      #=dump_module=#true, #=syntax=#:att, #=optimize=#false,
                                      params)
    irmods = map(ref->convert(String, LLVM.Module(ref)), modrefs)
    unshift!(irmods, entry_irmod)

    # find the entry function
    # TODO: let Julia report this
    entry_fn = Nullable{String}()
    let entry_mod = parse(LLVM.Module, entry_irmod)
        for ir_f in functions(entry_mod)
            ir_fn = LLVM.name(ir_f)
            if startswith(ir_fn, "julia_$fn") # FIXME: LLVM might have mangled this
                entry_fn = Nullable(ir_fn)
                break
            end
        end

        isnull(entry_fn) && error("could not find entry-point function")
    end

    # link all the modules
    for irmod in irmods
        partial_mod = parse(LLVM.Module, irmod)

        name!(partial_mod, "Julia IR")
        triple!(partial_mod, triple(mod))
        datalayout!(partial_mod, datalayout(mod))

        link!(mod, partial_mod)
    end

    # FIXME: clean up incompatibilities
    for f in functions(mod)
        if startswith(LLVM.name(f), "jlcall_")
            unsafe_delete!(mod, f)
        else
            # only occurs in debug builds
            delete!(function_attributes(f), EnumAttribute("sspreq"))

            # make function names safe for ptxas
            # (LLVM ought to do this, see eg. D17738 and D19126), but fails
            # TODO: fix all globals?
            if !isdeclaration(f)
                orig_fn = LLVM.name(f)
                safe_fn = sanitize_fn(orig_fn)
                if orig_fn != safe_fn
                    LLVM.name!(f, safe_fn)

                    # take care if we're renaming the entry-point function
                    if orig_fn == get(entry_fn)
                        entry_fn = Nullable(safe_fn)
                    end
                end
            end
        end
    end

    verify(mod)

    return mod, get(functions(mod), get(entry_fn))
end

const libdevices = Dict{VersionNumber, LLVM.Module}()
function link_libdevice!(mod::LLVM.Module, cap::VersionNumber)
    # select the most recent & compatible libdevice
    const vers = keys(CUDAnative.libdevice_libraries)
    compat_vers = Set(ver for ver in vers if ver <= cap)
    isempty(compat_vers) && error("No compatible CUDA device library available")
    ver = maximum(compat_vers)
    path = libdevice_libraries[ver]

    # load the library, once
    if !haskey(libdevices, ver)
        open(path) do io
            libdevice_mod = parse(LLVM.Module, read(io))
            name!(libdevice_mod, "libdevice")
            libdevices[ver] = libdevice_mod
        end
    end
    libdevice_mod = LLVM.Module(libdevices[ver])

    # override libdevice's triple and datalayout to avoid warnings
    triple!(libdevice_mod, triple(mod))
    datalayout!(libdevice_mod, datalayout(mod))

    # 1. Save list of external functions
    exports = map(LLVM.name, functions(mod))
    filter!(fn->!haskey(functions(libdevice_mod), fn), exports)

    # 2. Link with libdevice
    link!(mod, libdevice_mod)

    ModulePassManager() do pm
        # 3. Internalize all functions not in list from (1)
        internalize!(pm, exports)

        # 4. Eliminate all unused internal functions
        global_optimizer!(pm)
        global_dce!(pm)
        strip_dead_prototypes!(pm)

        # 5. Run NVVMReflect pass
        push!(metadata(mod), "nvvm-reflect-ftz",
              MDNode([ConstantInt(Int32(1))]))

        # FIXME: we shouldn't queue this manually,
        #        but use the TM's `addEarlyAsPossiblePasses` instead

        # 6. Run standard optimization pipeline
        always_inliner!(pm)

        run!(pm, mod)
    end
end

function machine(cap::VersionNumber, triple::String)
    InitializeNVPTXTarget()
    InitializeNVPTXTargetInfo()
    t = Target(triple)

    InitializeNVPTXTargetMC()
    cpu = "sm_$(cap.major)$(cap.minor)"
    tm = TargetMachine(t, triple, cpu)

    return tm
end

function optimize!(mod::LLVM.Module, cap::VersionNumber)
    tm = machine(cap, triple(mod))

    ModulePassManager() do pm
        ccall(:LLVMAddLowerGCFramePass, Void,
              (LLVM.API.LLVMPassManagerRef,), LLVM.ref(pm))
        populate!(pm, tm)
        ccall(:LLVMAddLowerPTLSPass, Void,
              (LLVM.API.LLVMPassManagerRef, Cint), LLVM.ref(pm), 0)

        PassManagerBuilder() do pmb
            always_inliner!(pm) # TODO: set it as the builder's inliner
            populate!(pm, pmb)
        end

        run!(pm, mod)
    end
end

function mcgen(mod::LLVM.Module, func::LLVM.Function, cap::VersionNumber;
               kernel::Bool=true)
    tm = machine(cap, triple(mod))

    # kernel metadata
    if kernel
        push!(metadata(mod), "nvvm.annotations",
             MDNode([func, MDString("kernel"), ConstantInt(Int32(1))]))
    end

    InitializeNVPTXAsmPrinter()
    return convert(String, emit(tm, mod, LLVM.API.LLVMAssemblyFile))
end

"""
Compile a function to PTX, returning the assembly and an entry point.
Not to be used directly, see `cufunction` instead.

The `kernel` argument indicates whether we are compiling a kernel entry-point function,
in which case extra metadata needs to be attached.
"""
function compile_function(func::ANY, tt::ANY, cap::VersionNumber; kernel::Bool=true)
    sig = """$func($(join(tt.parameters, ", ")))"""
    debug("Compiling $sig for capability level $cap")

    # generate LLVM IR
    mod, entry = irgen(func, tt)
    trace("Module entry point: ", LLVM.name(entry))

    # link libdevice, if it might be necessary
    if any(f->isdeclaration(f) && intrinsic_id(f)==0, functions(mod))
        link_libdevice!(mod, cap)
    end

    # generate (PTX) assembly
    optimize!(mod, cap)
    module_asm = mcgen(mod, entry, cap; kernel=kernel)

    return module_asm, LLVM.name(entry)
end

# check whether a (function, tuple of types) yields a valid kernel method
function check_kernel(func::ANY, tt::ANY)
    sig = """$func($(join(tt.parameters, ", ")))"""

    ml = Base.methods(func, tt)
    if length(ml) == 0
        error("no method found for kernel $sig")
    elseif length(ml) > 1
        # TODO: when does this happen?
        error("ambiguous call to kernel $sig")
    end
    rt = Base.return_types(func, tt)[1]
    if rt != Void
        error("$sig is not a valid kernel as it returns $rt")
    end
end

# Main entry-point for compiling a Julia function + argtypes to a callable CUDA function
function cufunction(dev::CuDevice, func::ANY, types::ANY)
    @assert isa(func, Core.Function)
    tt = Base.to_tuple_type(types)
    check_kernel(func, tt)

    # select a capability level
    dev_cap = capability(dev)
    compat_caps = filter(cap -> cap <= dev_cap, supported_capabilities)
    isempty(compat_caps) &&
        error("Device capability v$dev_cap not supported by available toolchain")
    cap = maximum(compat_caps)

    (module_asm, module_entry) = compile_function(func, tt, cap)

    # enable debug options based on Julia's debug setting
    jit_options = Dict{CUDAdrv.CUjit_option,Any}()
    if DEBUG || Base.JLOptions().debug_level >= 1
        jit_options[CUDAdrv.GENERATE_LINE_INFO] = true
    end
    if DEBUG || Base.JLOptions().debug_level >= 2
        # TODO: detect cuda-gdb
        jit_options[CUDAdrv.GENERATE_DEBUG_INFO] = true
    end
    cuda_mod = CuModule(module_asm, jit_options)
    cuda_fun = CuFunction(cuda_mod, module_entry)

    return cuda_fun, cuda_mod
end

function init_jit()
    llvm_args = [
        # Program name; can be left blank.
        "",
        # Enable generation of FMA instructions to mimic behavior of nvcc.
        "--nvptx-fma-level=1",
    ]
    LLVM.API.LLVMParseCommandLineOptions(Int32(length(llvm_args)),
        [Base.unsafe_convert(Cstring, llvm_arg) for llvm_arg in llvm_args], C_NULL)
end