text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

#############################################
# Create some temporary files & directories #
#############################################
starttime = time_ns() / 1e9
pwd_ = pwd()
dir = mktempdir()
file = joinpath(dir, "afile.txt")
# like touch, but lets the operating system update the timestamp
# for greater precision on some platforms (windows)
@test close(open(file,"w")) === nothing

subdir = joinpath(dir, "adir")
mkdir(subdir)
subdir2 = joinpath(dir, "adir2")
mkdir(subdir2)
@test_throws Base._UVError("mkdir", Base.UV_EEXIST) mkdir(file)
let err = nothing
    try
        mkdir(file)
    catch err
        io = IOBuffer()
        showerror(io, err)
        @test startswith(String(take!(io)), "IOError: mkdir: file already exists (EEXIST)")
    end
end

if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER
    dirlink = joinpath(dir, "dirlink")
    symlink(subdir, dirlink)
    # relative link
    cd(subdir)
    relsubdirlink = joinpath(subdir, "rel_subdirlink")
    reldir = joinpath("..", "adir2")
    symlink(reldir, relsubdirlink)
    cd(pwd_)
end

if !Sys.iswindows()
    link = joinpath(dir, "afilelink.txt")
    symlink(file, link)
    # relative link
    cd(subdir)
    rellink = joinpath(subdir, "rel_afilelink.txt")
    relfile = joinpath("..", "afile.txt")
    symlink(relfile, rellink)
    cd(pwd_)
end

using Random

@testset "that temp names are actually unique" begin
    temps = [tempname(cleanup=false) for _ = 1:100]
    @test allunique(temps)
    temps = map(1:100) do _
        path, io = mktemp(cleanup=false)
        close(io)
        rm(path, force=true)
        return path
    end
    @test allunique(temps)
end

@testset "tempname with parent" begin
    t = tempname()
    @test dirname(t) == tempdir()
    mktempdir() do d
        t = tempname(d)
        @test dirname(t) == d
    end
    @test_throws ArgumentError tempname(randstring())
end

child_eval(code::String) = eval(Meta.parse(readchomp(`$(Base.julia_cmd()) -E $code`)))

@testset "mktemp/dir basic cleanup" begin
    # mktemp without cleanup
    t = child_eval("t = mktemp(cleanup=false)[1]; @assert isfile(t); t")
    @test isfile(t)
    rm(t, force=true)
    @test !ispath(t)
    # mktemp with cleanup
    t = child_eval("t = mktemp()[1]; @assert isfile(t); t")
    @test !ispath(t)
    # mktempdir without cleanup
    t = child_eval("t = mktempdir(cleanup=false); touch(joinpath(t, \"file.txt\")); t")
    @test isfile("$t/file.txt")
    rm(t, recursive=true, force=true)
    @test !ispath(t)
    # mktempdir with cleanup
    t = child_eval("t = mktempdir(); touch(joinpath(t, \"file.txt\")); t")
    @test !ispath(t)
    # tempname without cleanup
    t = child_eval("t = tempname(cleanup=false); touch(t); t")
    @test isfile(t)
    rm(t, force=true)
    @test !ispath(t)
    # tempname with cleanup
    t = child_eval("t = tempname(); touch(t); t")
    @test !ispath(t)
end

import Base.Filesystem: TEMP_CLEANUP_MIN, TEMP_CLEANUP_MAX, TEMP_CLEANUP

function with_temp_cleanup(f::Function, n::Int)
    SAVE_TEMP_CLEANUP_MIN = TEMP_CLEANUP_MIN[]
    SAVE_TEMP_CLEANUP_MAX = TEMP_CLEANUP_MAX[]
    SAVE_TEMP_CLEANUP = copy(TEMP_CLEANUP)
    empty!(TEMP_CLEANUP)
    TEMP_CLEANUP_MIN[] = n
    TEMP_CLEANUP_MAX[] = n
    try f()
    finally
        Sys.iswindows() && GC.gc(true)
        for t in keys(TEMP_CLEANUP)
            rm(t, recursive=true, force=true)
        end
        copy!(TEMP_CLEANUP, SAVE_TEMP_CLEANUP)
        TEMP_CLEANUP_MAX[] = SAVE_TEMP_CLEANUP_MAX
        TEMP_CLEANUP_MIN[] = SAVE_TEMP_CLEANUP_MIN
    end
end

function mktempfile(; cleanup=true)
    (file, io) = mktemp(cleanup=cleanup)
    Sys.iswindows() && close(io)
    return file
end

@testset "mktemp/dir cleanup list purging" begin
    n = 12 # cleanup min & max
    @assert n % 2 == n % 3 == 0 # otherwise tests won't work
    with_temp_cleanup(n) do
        @test length(TEMP_CLEANUP) == 0
        @test TEMP_CLEANUP_MAX[] == n
        # for n mktemps, no purging is triggered
        temps = String[]
        for i = 1:n
            t = i % 2 == 0 ? mktempfile() : mktempdir()
            push!(temps, t)
            @test ispath(t)
            @test length(TEMP_CLEANUP) == i
            @test TEMP_CLEANUP_MAX[] == n
            # delete 1/3 of the temp paths
            i % 3 == 0 && rm(t, recursive=true, force=true)
        end
        # without cleanup no purge is triggered
        t = mktempdir(cleanup=false)
        @test isdir(t)
        @test length(TEMP_CLEANUP) == n
        @test TEMP_CLEANUP_MAX[] == n
        rm(t, recursive=true, force=true)
        # purge triggered by next mktemp with cleanup
        t = mktempfile()
        push!(temps, t)
        n′ = 2n÷3 + 1
        @test 2n′ > n
        @test isfile(t)
        @test length(TEMP_CLEANUP) == n′
        @test TEMP_CLEANUP_MAX[] == 2n′
        # remove all temp files
        for t in temps
            rm(t, recursive=true, force=true)
        end
        # for n′ mktemps, no purging is triggered
        for i = 1:n′
            t = i % 2 == 0 ? mktempfile() : mktempdir()
            push!(temps, t)
            @test ispath(t)
            @test length(TEMP_CLEANUP) == n′ + i
            @test TEMP_CLEANUP_MAX[] == 2n′
            # delete 2/3 of the temp paths
            i % 3 != 0 && rm(t, recursive=true, force=true)
        end
        # without cleanup no purge is triggered
        t = mktempfile(cleanup=false)
        @test isfile(t)
        @test length(TEMP_CLEANUP) == 2n′
        @test TEMP_CLEANUP_MAX[] == 2n′
        rm(t, force=true)
        # purge triggered by next mktemp
        t = mktempdir()
        push!(temps, t)
        n′′ = n′÷3 + 1
        @test 2n′′ < n
        @test isdir(t)
        @test length(TEMP_CLEANUP) == n′′
        @test TEMP_CLEANUP_MAX[] == n
    end
end

no_error_logging(f::Function) =
    Base.CoreLogging.with_logger(f, Base.CoreLogging.NullLogger())

@testset "hof mktemp/dir when cleanup is prevented" begin
    d = mktempdir()
    with_temp_cleanup(3) do
        @test length(TEMP_CLEANUP) == 0
        @test TEMP_CLEANUP_MAX[] == 3
        local t, f
        temps = String[]
        # mktemp is normally cleaned up on completion
        mktemp(d) do path, _
            @test isfile(path)
            t = path
        end
        @test !ispath(t)
        @test length(TEMP_CLEANUP) == 0
        @test TEMP_CLEANUP_MAX[] == 3
        # mktemp when cleanup is prevented
        no_error_logging() do
            mktemp(d) do path, _
                @test isfile(path)
                f = open(path) # make undeletable on Windows
                chmod(d, 0o400) # make undeletable on UNIX
                t = path
            end
        end
        chmod(d, 0o700)
        close(f)
        @test isfile(t)
        @test length(TEMP_CLEANUP) == 1
        @test TEMP_CLEANUP_MAX[] == 3
        push!(temps, t)
        # mktempdir is normally cleaned up on completion
        mktempdir(d) do path
            @test isdir(path)
            t = path
        end
        @test !ispath(t)
        @test length(TEMP_CLEANUP) == 1
        @test TEMP_CLEANUP_MAX[] == 3
        # mktempdir when cleanup is prevented
        no_error_logging() do
            mktempdir(d) do path
                @test isdir(path)
                # make undeletable on Windows:
                f = open(joinpath(path, "file.txt"), "w+")
                chmod(d, 0o400) # make undeletable on UNIX
                t = path
            end
        end
        chmod(d, 0o700)
        close(f)
        @test isdir(t)
        @test length(TEMP_CLEANUP) == 2
        @test TEMP_CLEANUP_MAX[] == 3
        push!(temps, t)
        # make one more temp file
        t = mktemp()[1]
        @test isfile(t)
        @test length(TEMP_CLEANUP) == 3
        @test TEMP_CLEANUP_MAX[] == 3
        # nothing has been deleted yet
        for t in temps
            @test ispath(t)
        end
        # another temp file triggers purge
        t = mktempdir()
        @test isdir(t)
        @test length(TEMP_CLEANUP) == 2
        @test TEMP_CLEANUP_MAX[] == 4
        # now all the temps are gone
        for t in temps
            @test !ispath(t)
        end
    end
end

#######################################################################
# This section tests some of the features of the stat-based file info #
#######################################################################
@test !isfile(Base.Filesystem.StatStruct())
@test isdir(dir)
@test !isfile(dir)
@test !islink(dir)
@test !isdir(file)
@test isfile(file)
@test !islink(file)

@test filemode(file) & 0o444 > 0 # readable
@test filemode(file) & 0o222 > 0 # writable
@test chmod(file, filemode(file) & 0o7555) == file
@test filemode(file) & 0o222 == 0
chmod(file, filemode(file) | 0o222)
@test filemode(file) & 0o111 == 0
@test filesize(file) == 0

# issue #26685
@test !isfile("http://google.com")

if Sys.iswindows()
    permissions = 0o444
    @test filemode(dir) & 0o777 != permissions
    @test filemode(subdir) & 0o777 != permissions
    @test filemode(file) & 0o777 != permissions
    chmod(dir, permissions, recursive=true)
    @test filemode(dir) & 0o777 == permissions
    @test filemode(subdir) & 0o777 == permissions
    @test filemode(file) & 0o777 == permissions
    chmod(dir, 0o666, recursive=true)  # Reset permissions in case someone wants to use these later
else
    function get_umask()
        umask = ccall(:umask, UInt32, (UInt32,), 0)
        ccall(:umask, UInt32, (UInt32,), umask)
        return umask
    end

    mktempdir() do tmpdir
        umask = get_umask()
        tmpfile=joinpath(tmpdir, "tempfile.txt")
        tmpfile2=joinpath(tmpdir, "tempfile2.txt")
        touch(tmpfile)
        cp(tmpfile, tmpfile2)
        @test filemode(tmpfile) & (~umask) == filemode(tmpfile2)
        rm(tmpfile2)
        chmod(tmpfile, 0o777)
        cp(tmpfile, tmpfile2)
        @test filemode(tmpfile) & (~umask) == filemode(tmpfile2)
        rm(tmpfile2)
        chmod(tmpfile, 0o707)
        cp(tmpfile, tmpfile2)
        @test filemode(tmpfile) & (~umask) == filemode(tmpfile2)
        rm(tmpfile2)
        linkfile=joinpath(dir, "tempfile.txt")
        symlink(tmpfile, linkfile)
        permissions=0o776
        @test filemode(dir) & 0o777 != permissions
        @test filemode(subdir) & 0o777 != permissions
        @test filemode(file) & 0o777 != permissions
        @test filemode(linkfile) & 0o777 != permissions
        @test filemode(tmpfile) & 0o777 != permissions
        chmod(dir, permissions, recursive=true)
        @test filemode(dir) & 0o777 == permissions
        @test filemode(subdir) & 0o777 == permissions
        @test filemode(file) & 0o777 == permissions
        @test lstat(link).mode & 0o777 != permissions  # Symbolic links are not modified.
        @test filemode(linkfile) & 0o777 != permissions  # Symbolic links are not followed.
        @test filemode(tmpfile) & 0o777 != permissions
        rm(linkfile)
    end
end

function test_stat_error(stat::Function, pth)
    if stat === lstat && !(pth isa AbstractString)
        return # no lstat for fd handles
    end
    ex = try; stat(pth); false; catch ex; ex; end::Base.IOError
    @test ex.code == (pth isa AbstractString ? Base.UV_EACCES : Base.UV_EBADF)
    @test startswith(ex.msg, "stat: ")
    pth isa AbstractString || (pth = Base.INVALID_OS_HANDLE)
    @test endswith(ex.msg, repr(pth))
    nothing
end
@testset "stat errors" begin # PR 32031
    mktempdir() do dir
        cd(dir) do
            touch("afile")
            try
                # remove permission to access this folder
                # to cause cause EACCESS-denied errors
                @static if Sys.iswindows()
                    @test ccall((:ImpersonateAnonymousToken, "Advapi32.dll"), stdcall, Cint, (Libc.WindowsRawSocket,),
                                ccall(:GetCurrentThread, Libc.WindowsRawSocket, ())) != 0
                else
                    chmod(dir, 0o000)
                end
                for pth in ("afile",
                            joinpath("afile", "not_file"),
                            SubString(joinpath(dir, "afile")),
                            Base.RawFD(-1),
                            -1)
                    test_stat_error(stat, pth)
                    test_stat_error(lstat, pth)
                end
            finally
                # restore permissions
                # to let us cleanup afile
                @static if Sys.iswindows()
                    ccall((:RevertToSelf, "advapi32.dll"), stdcall, Cint, ()) == 0 && exit(1)
                else
                    chmod(dir, 0o777)
                end
            end
        end
    end
end

# On windows the filesize of a folder is the accumulation of all the contained
# files and is thus zero in this case.
if Sys.iswindows()
    @test filesize(dir) == 0
else
    @test filesize(dir) > 0
end
# We need both: one to check passed time, one to comapare file's mtime()
nowtime = time_ns() / 1e9
nowwall = time()
# Allow 10s skew in addition to the time it took us to actually execute this code
let skew = 10 + (nowtime - starttime)
    mfile = mtime(file)
    mdir  = mtime(dir)
    @test abs(nowwall - mfile) <= skew && abs(nowwall - mdir) <= skew && abs(mfile - mdir) <= skew
end
#@test Int(time()) >= Int(mtime(file)) >= Int(mtime(dir)) >= 0 # 1 second accuracy should be sufficient

# test links
if Sys.isunix()
    @test islink(link) == true
    @test readlink(link) == file
end

if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER
    @test islink(dirlink) == true
    @test isdir(dirlink) == true
    @test readlink(dirlink) == subdir * (Sys.iswindows() ? "\\" : "")
end

# rm recursive TODO add links
newfile = joinpath(dir, "bfile.txt")
mv(file, newfile)
file = newfile
c_tmpdir = mktempdir()
c_subdir = joinpath(c_tmpdir, "c_subdir")
mkdir(c_subdir)
c_file = joinpath(c_tmpdir, "cfile.txt")
cp(newfile, c_file)

@test isdir(c_subdir)
@test isfile(c_file)
@test_throws SystemError rm(c_tmpdir)
@test_throws SystemError rm(c_tmpdir, force=true)

# create temp dir in specific directory
d_tmpdir = mktempdir(c_tmpdir)
@test isdir(d_tmpdir)
@test Base.samefile(dirname(d_tmpdir), c_tmpdir)

# create temp file in specific directory
d_tmpfile,f = mktemp(c_tmpdir)
close(f)
@test isfile(d_tmpfile)
@test Base.samefile(dirname(d_tmpfile), c_tmpdir)

rm(c_tmpdir, recursive=true)
@test !isdir(c_tmpdir)
@test_throws Base._UVError(Sys.iswindows() ? "chmod" : "unlink", Base.UV_ENOENT) rm(c_tmpdir)
@test rm(c_tmpdir, force=true) === nothing
@test_throws Base._UVError(Sys.iswindows() ? "chmod" : "unlink", Base.UV_ENOENT) rm(c_tmpdir, recursive=true)
@test rm(c_tmpdir, force=true, recursive=true) === nothing

if !Sys.iswindows()
    # chown will give an error if the user does not have permissions to change files
    if get(ENV, "USER", "") == "root" || get(ENV, "HOME", "") == "/root"
        chown(file, -2, -1)  # Change the file owner to nobody
        @test stat(file).uid != 0
        chown(file, 0, -2)  # Change the file group to nogroup (and owner back to root)
        @test stat(file).gid != 0
        @test stat(file).uid == 0
        @test chown(file, -1, 0) == file
        @test stat(file).gid == 0
        @test stat(file).uid == 0
    else
        @test_throws Base._UVError("chown", Base.UV_EPERM) chown(file, -2, -1)  # Non-root user cannot change ownership to another user
        @test_throws Base._UVError("chown", Base.UV_EPERM) chown(file, -1, -2)  # Non-root user cannot change group to a group they are not a member of (eg: nogroup)
    end
else
    # test that chown doesn't cause any errors for Windows
    @test chown(file, -2, -2) == file
end

##############
# mark/reset #
##############

s = open(file, "w")
write(s, "Marked!\n")
write(s, "Hello world!\n")
write(s, "Goodbye world!\n")
close(s)
s = open(file)
mark(s)
str = readline(s)
@test startswith(str, "Marked!")
@test ismarked(s)
reset(s)
@test !ismarked(s)
str = readline(s)
@test startswith(str, "Marked!")
mark(s)
@test readline(s) == "Hello world!"
@test ismarked(s)
unmark(s)
@test !ismarked(s)
@test_throws ArgumentError reset(s)
@test !unmark(s)
@test !ismarked(s)
close(s)

#######################################################################
# This section tests temporary file and directory creation.           #
#######################################################################

@testset "quoting filenames" begin
    @test try
        open("this file is not expected to exist")
        false
    catch e
        isa(e, SystemError) || rethrow()
        @test sprint(showerror, e) == "SystemError: opening file \"this file is not expected to exist\": No such file or directory"
        true
    end
end

@testset "tempdir" begin
    my_tempdir = tempdir()
    @test isdir(my_tempdir)
    @test my_tempdir[end] != '/'
    @test my_tempdir[end] != '\\'

    var =  Sys.iswindows() ? "TMP" : "TMPDIR"
    PATH_PREFIX = Sys.iswindows() ? "C:\\" : "/tmp/" * "x"^255   # we want a long path on UNIX so that we test buffer resizing in `tempdir`
    # Warning: On Windows uv_os_tmpdir internally calls GetTempPathW. The max string length for
    # GetTempPathW is 261 (including the implied trailing backslash), not the typical length 259.
    # We thus use 260 (with implied trailing slash backlash this then gives 261 chars)
    # NOTE: not the actual max path on UNIX, but true in the Windows case for this function.
    # NOTE: we subtract 9 to account for i = 0:9.
    MAX_PATH = (Sys.iswindows() ? 260 - length(PATH_PREFIX) : 255)  - 9
    for i = 0:8
        local tmp = joinpath(PATH_PREFIX, "x"^MAX_PATH * "123456789"[1:i])
        @test withenv(var => tmp) do
            tempdir()
        end == (tmp)
    end
    for i = 9
        local tmp = joinpath(PATH_PREFIX, "x"^MAX_PATH * "123456789"[1:i])
        if Sys.iswindows()
            # libuv bug
            @test_broken withenv(var => tmp) do
                tempdir()
            end == tmp
        else
            @test withenv(var => tmp) do
                tempdir()
            end == tmp
        end
    end
end

let path = tempname()
    # issue #9053
    @test !ispath(path)
end

(p, f) = mktemp()
print(f, "Here is some text")
close(f)
@test isfile(p) == true
@test read(p, String) == "Here is some text"
rm(p)

let
    tmp_path = mktemp() do p, io
        @test isfile(p)
        print(io, "鴨かも？")
        p
    end
    @test tmp_path != ""
    @test !isfile(tmp_path)
end

let
    tmpdir = mktempdir() do d
        @test isdir(d)
        d
    end
    @test tmpdir != ""
    @test !isdir(tmpdir)
end

emptyfile = joinpath(dir, "empty")
touch(emptyfile)
emptyf = open(emptyfile)
@test isempty(readlines(emptyf, keep=true))
close(emptyf)
rm(emptyfile)

########################################################################################
## This section tests cp & mv(rename) files, directories, absolute and relative links. #
########################################################################################
function check_dir(orig_path::AbstractString, copied_path::AbstractString, follow_symlinks::Bool)
    isdir(orig_path) || throw(ArgumentError("'$orig_path' is not a directory."))
    # copied_path must also be a dir.
    @test isdir(copied_path)
    readir_orig = readdir(orig_path)
    readir_copied = readdir(copied_path)
    @test readir_orig == readir_copied
    # check recursive
    for name in readir_orig
        @test name in readir_copied
        check_cp(joinpath(orig_path, name), joinpath(copied_path, name), follow_symlinks)
    end
end

function check_cp(orig_path::AbstractString, copied_path::AbstractString, follow_symlinks::Bool)
    if islink(orig_path)
        if !follow_symlinks
            # copied_path must be a link
            @test islink(copied_path)
            readlink_orig = readlink(orig_path)
            # copied_path must have the same link value:
            #    this is true for absolute and relative links
            @test readlink_orig == readlink(copied_path)
            if isabspath(readlink_orig)
                @test isabspath(readlink(copied_path))
            end
        else
            # copied_path may not be a link if follow_symlinks=true
            @test islink(orig_path) == !islink(copied_path)
            if isdir(orig_path)
                check_dir(orig_path, copied_path, follow_symlinks)
            else
                # copied_path must also be a file.
                @test isfile(copied_path)
                # copied_path must have same content
                @test read(orig_path, String) == read(copied_path, String)
            end
        end
    elseif isdir(orig_path)
        check_cp_main(orig_path, copied_path, follow_symlinks)
    else
        # copied_path must also be a file.
        @test isfile(copied_path)
        # copied_path must have same content
        @test read(orig_path, String) == read(copied_path, String)
    end
end

function check_cp_main(orig::AbstractString, copied::AbstractString, follow_symlinks::Bool)
    if isdir(orig)
        check_dir(orig, copied, follow_symlinks)
    else
        check_cp(orig, copied, follow_symlinks)
    end
end

function cp_and_test(src::AbstractString, dst::AbstractString, follow_symlinks::Bool)
    cp(src, dst; follow_symlinks=follow_symlinks)
    check_cp_main(src, dst, follow_symlinks)
end

## cp ----------------------------------------------------
# issue #8698
# Test copy file
let
    afile = joinpath(dir, "a.txt")
    touch(afile)
    af = open(afile, "r+")
    write(af, "This is indeed a test")

    bfile = joinpath(dir, "b.txt")
    cp(afile, bfile)

    cfile = joinpath(dir, "c.txt")
    write(cfile, "This is longer than the contents of afile")
    cp(afile, cfile; force=true)

    a_stat = stat(afile)
    b_stat = stat(bfile)
    c_stat = stat(cfile)
    @test a_stat.mode == b_stat.mode
    @test a_stat.size == b_stat.size
    @test a_stat.size == c_stat.size

    @test parse(Int, match(r"mode=(.*),", sprint(show, a_stat)).captures[1]) == a_stat.mode

    close(af)
    rm(afile)
    rm(bfile)
    rm(cfile)
end

## mv ----------------------------------------------------
mktempdir() do tmpdir
    # rename file
    file = joinpath(tmpdir, "afile.txt")
    files_stat = stat(file)
    close(open(file, "w")) # like touch, but lets the operating system update
    # the timestamp for greater precision on some platforms (windows)

    newfile = joinpath(tmpdir, "bfile.txt")
    mv(file, newfile)
    newfile_stat = stat(file)

    @test !ispath(file)
    @test isfile(newfile)
    @test Base.samefile(files_stat, newfile_stat)

    file = newfile

    # Test renaming directories
    a_tmpdir = mktempdir()
    b_tmpdir = joinpath(tmpdir, "b_tmpdir")

    # grab a_tmpdir's file info before renaming
    a_stat = stat(a_tmpdir)

    # rename, then make sure b_tmpdir does exist and a_tmpdir doesn't
    mv(a_tmpdir, b_tmpdir)
    @test isdir(b_tmpdir)
    @test !ispath(a_tmpdir)

    # get b_tmpdir's file info and compare with a_tmpdir
    b_stat = stat(b_tmpdir)
    @test Base.samefile(a_stat, b_stat)

    rm(b_tmpdir)
end

# issue #10506 #10434
## Tests for directories and links to directories
if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER
    function setup_dirs(tmpdir)
        srcdir = joinpath(tmpdir, "src")
        hidden_srcdir = joinpath(tmpdir, ".hidden_srcdir")
        hidden_srcsubdir = joinpath(hidden_srcdir, ".hidden_srcsubdir")
        srcdir_cp = joinpath(tmpdir, "srcdir_cp")
        mkdir(srcdir)
        mkdir(hidden_srcdir)
        mkdir(hidden_srcsubdir)
        abs_dirlink = joinpath(tmpdir, "abs_dirlink")
        symlink(abspath(srcdir), abs_dirlink)
        cd(tmpdir)
        rel_dirlink = "rel_dirlink"
        symlink("src", rel_dirlink)
        cd(pwd_)

        cfile = joinpath(srcdir, "c.txt")
        file_txt = "This is some text with unicode - 这是一个文件"
        write(cfile, file_txt)
        hidden_cfile = joinpath(hidden_srcsubdir, "c.txt")
        write(hidden_cfile, file_txt)

        abs_dirlink_cp = joinpath(tmpdir, "abs_dirlink_cp")
        hidden_srcsubdir_cp = joinpath(tmpdir, ".hidden_srcsubdir_cp")
        path_rel_dirlink = joinpath(tmpdir, rel_dirlink)
        path_rel_dirlink_cp = joinpath(tmpdir, "rel_dirlink_cp")

        test_src_paths = [srcdir, hidden_srcsubdir, abs_dirlink, path_rel_dirlink]
        test_cp_paths = [srcdir_cp, hidden_srcsubdir_cp, abs_dirlink_cp, path_rel_dirlink_cp]
        return test_src_paths, test_cp_paths
    end

    function cp_follow_symlinks_false_check(s, d; force=false)
        cp(s, d; force=force, follow_symlinks=false)
        @test isdir(s) == isdir(d)
        @test islink(s) == islink(d)
        islink(s) && @test readlink(s) == readlink(d)
        islink(s) && @test isabspath(readlink(s)) == isabspath(readlink(d))
        # all should contain 1 file named  "c.txt"
        @test "c.txt" in readdir(d)
        @test length(readdir(d)) == 1
    end

    function mv_check(s, d, d_mv; force=true)
        # setup dest
        cp(s, d; force=true, follow_symlinks=false)
        stat_d = stat(d)
        # mv(rename) dst to dst_mv
        mv(d, d_mv; force=force)
        stat_d_mv = stat(d_mv)
        # make sure d does not exist anymore
        @test !ispath(d)
        # compare s, with d_mv
        @test isdir(s) == isdir(d_mv)
        @test islink(s) == islink(d_mv)
        islink(s) && @test readlink(s) == readlink(d_mv)
        islink(s) && @test isabspath(readlink(s)) == isabspath(readlink(d_mv))
        # all should contain 1 file named  "c.txt"
        @test "c.txt" in readdir(d_mv)
        @test length(readdir(d_mv)) == 1
        # d => d_mv same file/dir
        @test Base.samefile(stat_d, stat_d_mv)
    end

    ## Test require `force=true` (remove destination first) for existing
    #  directories and existing links to directories
    # cp ----------------------------------------------------
    mktempdir() do tmpdir
        # Setup new copies for the test
        maindir1 = joinpath(tmpdir, "maindir1")
        maindir2 = joinpath(tmpdir, "maindir2")
        mkdir(maindir1)
        mkdir(maindir2)
        test_src_paths1, test_new_paths1 = setup_dirs(maindir1)
        test_src_paths2, test_new_paths2 = setup_dirs(maindir2)
        for (s, d) in zip(test_src_paths1, test_new_paths1)
            cp_follow_symlinks_false_check(s, d)
        end
        for (s, d) in zip(test_src_paths2, test_new_paths2)
            cp_follow_symlinks_false_check(s, d)
        end
        # Test require `force=true`
        for s in test_src_paths1
            for d in test_new_paths2
                @test_throws ArgumentError cp(s, d; force=false)
                @test_throws ArgumentError cp(s, d; force=false, follow_symlinks=true)
            end
        end
        # Test remove the existing path first and copy
        # need to use here the test_src_paths2:
        # otherwise ArgumentError: 'src' and 'dst' refer to the same file/dir.
        for (s, d) in zip(test_src_paths2, test_new_paths1)
            cp_follow_symlinks_false_check(s, d; force=true)
        end
        # Test remove the existing path first and copy an empty dir
        emptydir = joinpath(maindir1, "emptydir")
        mkdir(emptydir)
        for d in test_new_paths1
            cp(emptydir, d; force=true, follow_symlinks=false)
            # Expect no link because a dir is copied (follow_symlinks=false does not effect this)
            @test isdir(d) && !islink(d)
            # none should contain any file
            @test isempty(readdir(d))
        end
    end
    # mv ----------------------------------------------------
    mktempdir() do tmpdir
        # Setup new copies for the test
        maindir1 = joinpath(tmpdir, "maindir1")
        maindir2 = joinpath(tmpdir, "maindir2")
        mkdir(maindir1)
        mkdir(maindir2)
        test_src_paths1, test_new_paths1 = setup_dirs(maindir1)
        test_src_paths2, test_new_paths2 = setup_dirs(maindir2)
        for (s, d) in zip(test_src_paths1, test_new_paths1)
            cp_follow_symlinks_false_check(s, d; force=true)
        end
        for (s, d) in zip(test_src_paths2, test_new_paths2)
            cp_follow_symlinks_false_check(s, d; force=true)
        end

        # Test require `force=true`
        for s in test_src_paths1
            for d in test_new_paths2
                @test_throws ArgumentError mv(s, d; force=false)
            end
        end
        # Test remove the existing path first and move
        # need to use here the test_src_paths2:
        # otherwise ArgumentError: 'src' and 'dst' refer to the same file/dir.This is not supported.
        for (s, d) in zip(test_src_paths2, test_new_paths1)
            d_mv = joinpath(dirname(d), "$(basename(d))_mv")
            mv_check(s, d, d_mv; force=true)
        end
    end

    # Test full: absolute and relative directory links
    # cp / mv ----------------------------------------------------
    mktempdir() do tmpdir
        maindir = joinpath(tmpdir, "mytestdir")
        mkdir(maindir)
        targetdir = abspath(joinpath(maindir, "targetdir"))
        mkdir(targetdir)
        subdir1 = joinpath(maindir, "subdir1")
        mkdir(subdir1)

        cfile = abspath(joinpath(maindir, "c.txt"))
        write(cfile, "This is c.txt - 这是一个文件")
        write(abspath(joinpath(targetdir, "file1.txt")),
              "This is file1.txt - 这是一个文件")

        abs_dl = joinpath(maindir, "abs_linkto_targetdir")
        symlink(targetdir, abs_dl)
        # Setup relative links
        cd(subdir1)
        rel_dl = "rel_linkto_targetdir"
        rel_dir = joinpath("..", "targetdir")
        symlink(rel_dir, rel_dl)
        cd(pwd_)
        # TEST: Directory with links: Test each option
        maindir_new = joinpath(dirname(maindir),"maindir_new")
        maindir_new_keepsym = joinpath(dirname(maindir),"maindir_new_keepsym")
        cp_and_test(maindir, maindir_new, true)
        cp_and_test(maindir, maindir_new_keepsym, false)

        # mv ----------------------------------------------------
        # move the 3 maindirs
        for d in [maindir_new, maindir_new_keepsym, maindir]
            d_mv = joinpath(dirname(d), "$(basename(d))_mv")
            mv(d, d_mv; force=true)
        end
    end

    # issue  ----------------------------------------------------
    # Check for issue when: (src == dst) or when one is a link to the other
    # https://github.com/JuliaLang/julia/pull/11172#issuecomment-100391076
    mktempdir() do tmpdir
        test_src_paths1, test_new_paths1 = setup_dirs(tmpdir)
        dirs = [joinpath(tmpdir, "src"), joinpath(tmpdir, "abs_dirlink"), joinpath(tmpdir, "rel_dirlink")]
        for src in dirs
            for dst in dirs
                # cptree
                @test_throws ArgumentError Base.cptree(src, dst; force=true, follow_symlinks=false)
                @test_throws ArgumentError Base.cptree(src, dst; force=true, follow_symlinks=true)
                # cp
                @test_throws ArgumentError cp(src, dst; force=true, follow_symlinks=false)
                @test_throws ArgumentError cp(src, dst; force=true, follow_symlinks=true)
                # mv
                @test_throws ArgumentError mv(src, dst; force=true)
            end
        end
    end
    # None existing src
    mktempdir() do tmpdir
        nonexisting_src = joinpath(tmpdir, "nonexisting_src")
        dst = joinpath(tmpdir, "dst")
        @test !ispath(nonexisting_src)
        # cptree
        @test_throws(ArgumentError("'$nonexisting_src' is not a directory. Use `cp(src, dst)`"),
                     Base.cptree(nonexisting_src, dst; force=true, follow_symlinks=false))
        @test_throws(ArgumentError("'$nonexisting_src' is not a directory. Use `cp(src, dst)`"),
                     Base.cptree(nonexisting_src, dst; force=true, follow_symlinks=true))
        # cp
        @test_throws Base._UVError("open", Base.UV_ENOENT) cp(nonexisting_src, dst; force=true, follow_symlinks=false)
        @test_throws Base._UVError("open", Base.UV_ENOENT) cp(nonexisting_src, dst; force=true, follow_symlinks=true)
        # mv
        @test_throws Base._UVError("open", Base.UV_ENOENT) mv(nonexisting_src, dst; force=true)
    end
end

# issue #10506 #10434
## Tests for files and links to files as well as directories and links to directories
if !Sys.iswindows()
    function setup_files(tmpdir)
        srcfile = joinpath(tmpdir, "srcfile.txt")
        hidden_srcfile = joinpath(tmpdir, ".hidden_srcfile.txt")
        srcfile_new = joinpath(tmpdir, "srcfile_new.txt")
        hidden_srcfile_new = joinpath(tmpdir, ".hidden_srcfile_new.txt")
        file_txt = "This is some text with unicode - 这是一个文件"
        write(srcfile, file_txt)
        write(hidden_srcfile, file_txt)
        abs_filelink = joinpath(tmpdir, "abs_filelink")
        symlink(abspath(srcfile), abs_filelink)
        cd(tmpdir)
        rel_filelink = "rel_filelink"
        symlink("srcfile.txt", rel_filelink)
        cd(pwd_)

        abs_filelink_new = joinpath(tmpdir, "abs_filelink_new")
        path_rel_filelink = joinpath(tmpdir, rel_filelink)
        path_rel_filelink_new = joinpath(tmpdir, "rel_filelink_new")

        test_src_paths = [srcfile, hidden_srcfile, abs_filelink, path_rel_filelink]
        test_new_paths = [srcfile_new, hidden_srcfile_new, abs_filelink_new, path_rel_filelink_new]
        return test_src_paths, test_new_paths, file_txt
    end

    function cp_follow_symlinks_false_check(s, d, file_txt; force=false)
        cp(s, d; force=force, follow_symlinks=false)
        @test isfile(s) == isfile(d)
        @test islink(s) == islink(d)
        islink(s) && @test readlink(s) == readlink(d)
        islink(s) && @test isabspath(readlink(s)) == isabspath(readlink(d))
        # all should contain the same
        @test read(s, String) == read(d, String) == file_txt
    end

    function mv_check(s, d, d_mv, file_txt; force=true)
        # setup dest
        cp(s, d; force=true, follow_symlinks=false)
        stat_d = stat(d)
        # mv(rename) dst to dst_mv
        mv(d, d_mv; force=force)
        stat_d_mv = stat(d_mv)
        # make sure d does not exist anymore
        @test !ispath(d)
        # comare s, with d_mv
        @test isfile(s) == isfile(d_mv)
        @test islink(s) == islink(d_mv)
        islink(s) && @test readlink(s) == readlink(d_mv)
        islink(s) && @test isabspath(readlink(s)) == isabspath(readlink(d_mv))
        # all should contain the same
        @test read(s, String) == read(d_mv, String) == file_txt
        # d => d_mv same file/dir
        @test Base.samefile(stat_d, stat_d_mv)
    end

    ## Test require `force=true` (remove destination first) for existing
    #  files and existing links to files
    # cp ----------------------------------------------------
    mktempdir() do tmpdir
        # Setup new copies for the test
        maindir1 = joinpath(tmpdir, "maindir1")
        maindir2 = joinpath(tmpdir, "maindir2")
        mkdir(maindir1)
        mkdir(maindir2)
        test_src_paths1, test_new_paths1, file_txt1 = setup_files(maindir1)
        test_src_paths2, test_new_paths2, file_txt2 = setup_files(maindir2)
        for (s, d) in zip(test_src_paths1, test_new_paths1)
            cp_follow_symlinks_false_check(s, d, file_txt1)
        end
        for (s, d) in zip(test_src_paths2, test_new_paths2)
            cp_follow_symlinks_false_check(s, d, file_txt2)
        end
        # Test require `force=true`
        for s in test_src_paths1
            for d in test_new_paths2
                @test_throws ArgumentError cp(s, d; force=false)
                @test_throws ArgumentError cp(s, d; force=false, follow_symlinks=true)
            end
        end
        # Test remove the existing path first and copy
        # need to use here the test_src_paths2:
        # otherwise ArgumentError: 'src' and 'dst' refer to the same file/dir.This is not supported.
        for (s, d) in zip(test_src_paths2, test_new_paths1)
            cp_follow_symlinks_false_check(s, d, file_txt2; force=true)
        end
        # Test remove the existing path first and copy an other file
        otherfile = joinpath(tmpdir, "otherfile.txt")
        otherfile_content = "This is otherfile.txt with unicode - 这是一个文件"
        write(otherfile, otherfile_content)
        for d in test_new_paths1
            cp(otherfile, d; force=true, follow_symlinks=false)
            # Expect no link because a file is copied (follow_symlinks=false does not effect this)
            @test isfile(d) && !islink(d)
            # all should contain otherfile_content
            @test read(d, String) == otherfile_content
        end
    end
    # mv ----------------------------------------------------
    mktempdir() do tmpdir
        # Setup new copies for the test
        maindir1 = joinpath(tmpdir, "maindir1")
        maindir2 = joinpath(tmpdir, "maindir2")
        mkdir(maindir1)
        mkdir(maindir2)
        test_src_paths1, test_new_paths1, file_txt1 = setup_files(maindir1)
        test_src_paths2, test_new_paths2, file_txt2 = setup_files(maindir2)
        for (s, d) in zip(test_src_paths1, test_new_paths1)
            cp_follow_symlinks_false_check(s, d, file_txt1)
        end
        for (s, d) in zip(test_src_paths2, test_new_paths2)
            cp_follow_symlinks_false_check(s, d, file_txt2)
        end
        # Test require `force=true`
        for s in test_src_paths1
            for d in test_new_paths2
                @test_throws ArgumentError mv(s, d; force=false)
            end
        end
        # Test remove the existing path first and move
        # need to use here the test_src_paths2:
        # otherwise ArgumentError: 'src' and 'dst' refer to the same file/dir.This is not supported.
        for (s, d) in zip(test_src_paths2, test_new_paths1)
            d_mv = joinpath(dirname(d), "$(basename(d))_mv")
            mv_check(s, d, d_mv, file_txt2; force=true)
        end
    end

    # Test full: absolute and relative file links and absolute and relative directory links
    # cp / mv ----------------------------------------------------
    mktempdir() do tmpdir
        maindir = joinpath(tmpdir, "mytestdir")
        mkdir(maindir)
        targetdir = abspath(joinpath(maindir, "targetdir"))
        mkdir(targetdir)
        subdir1 = joinpath(maindir, "subdir1")
        mkdir(subdir1)

        cfile = abspath(joinpath(maindir, "c.txt"))
        write(cfile, "This is c.txt - 这是一个文件")
        write(abspath(joinpath(targetdir, "file1.txt")),
                      "This is file1.txt - 这是一个文件")

        abs_fl = joinpath(maindir, "abs_linkto_c.txt")
        symlink(cfile, abs_fl)
        abs_dl = joinpath(maindir, "abs_linkto_targetdir")
        symlink(targetdir, abs_dl)
        # Setup relative links
        cd(subdir1)
        rel_fl = "rel_linkto_c.txt"
        rel_file = joinpath("..", "c.txt")
        symlink(rel_file, rel_fl)
        rel_dl = "rel_linkto_targetdir"
        rel_dir = joinpath("..", "targetdir")
        symlink(rel_dir, rel_dl)
        rel_file_read_txt = read(rel_file, String)
        cd(pwd_)
        # Setup copytodir
        copytodir = joinpath(tmpdir, "copytodir")
        mkdir(copytodir)
        cp(cfile, joinpath(copytodir, basename(cfile)))
        subdir_test = joinpath(copytodir, "subdir_test")
        mkdir(subdir_test)
        cp(targetdir, joinpath(copytodir, basename(targetdir)); follow_symlinks=false)
        # TEST: Directory with links: Test each option
        maindir_new =  joinpath(dirname(maindir),"maindir_new")
        maindir_new_keepsym =  joinpath(dirname(maindir),"maindir_new_keepsym")
        cp_and_test(maindir, maindir_new, true)
        cp_and_test(maindir, maindir_new_keepsym, false)

        ## Tests single Files, File Links
        rel_flpath = joinpath(subdir1, rel_fl)
        # `cp file`
        cp_and_test(cfile, joinpath(copytodir,"cfile_new.txt"), true)
        cp_and_test(cfile, joinpath(copytodir,"cfile_new_keepsym.txt"), false)
        # `cp absolute file link`
        cp_and_test(abs_fl, joinpath(copytodir,"abs_fl_new.txt"), true)
        cp_and_test(abs_fl, joinpath(copytodir,"abs_fl_new_keepsym.txt"), false)
        # `cp relative file link`
        cp_and_test(rel_flpath, joinpath(subdir_test,"rel_fl_new.txt"), true)
        cp_and_test(rel_flpath, joinpath(subdir_test,"rel_fl_new_keepsym.txt"), false)

        # mv ----------------------------------------------------
        # move all 4 existing dirs
        # As expected this will leave some absolute links broken #11145#issuecomment-99315168
        for d in [copytodir, maindir_new, maindir_new_keepsym, maindir]
            d_mv = joinpath(dirname(d), "$(basename(d))_mv")
            mv(d, d_mv; force=true)
        end
    end
    # issue  ----------------------------------------------------
    # Check for issue when: (src == dst) or when one is a link to the other
    # https://github.com/JuliaLang/julia/pull/11172#issuecomment-100391076
    mktempdir() do tmpdir
        test_src_paths, test_new_paths, file_txt = setup_files(tmpdir)
        files = [joinpath(tmpdir, "srcfile.txt"), joinpath(tmpdir, "abs_filelink"), joinpath(tmpdir, "rel_filelink")]
        for src in files
            for dst in files
                # cptree
                @test_throws ArgumentError Base.cptree(src, dst; force=true, follow_symlinks=false)
                @test_throws ArgumentError Base.cptree(src, dst; force=true, follow_symlinks=true)
                # cp
                @test_throws ArgumentError cp(src, dst; force=true, follow_symlinks=false)
                @test_throws ArgumentError cp(src, dst; force=true, follow_symlinks=true)
                # mv
                @test_throws ArgumentError mv(src, dst; force=true)
            end
        end
    end
    # None existing src: not needed here as it is done above with the directories.
end

###################
# FILE* interface #
###################

function test_LibcFILE(FILEp)
    buf = Vector{UInt8}(undef, 8)
    str = ccall(:fread, Csize_t, (Ptr{Cvoid}, Csize_t, Csize_t, Ptr{Cvoid}), buf, 1, 8, FILEp)
    @test String(buf) == "Hello, w"
    @test position(FILEp) == 8
    seek(FILEp, 5)
    @test position(FILEp) == 5
    close(FILEp)
end

let f = open(file, "w")
    write(f, "Hello, world!")
    close(f)
    f = open(file, "r")
    test_LibcFILE(Libc.FILE(f))
    close(f)
    if Sys.iswindows()
        f = RawFD(ccall(:_open, Cint, (Cstring, Cint), file, Base.Filesystem.JL_O_RDONLY))
    else
        f = RawFD(ccall(:open, Cint, (Cstring, Cint), file, Base.Filesystem.JL_O_RDONLY))
    end
    test_LibcFILE(Libc.FILE(f, Libc.modestr(true, false)))
end

# issue #10994: pathnames cannot contain embedded NUL chars
for f in (mkdir, cd, Base.Filesystem.unlink, readlink, rm, touch, readdir, mkpath,
        stat, lstat, ctime, mtime, filemode, filesize, uperm, gperm, operm, touch,
        isblockdev, ischardev, isdir, isfifo, isfile, islink, ispath, issetgid,
        issetuid, issocket, issticky, realpath)
    local f
    @test_throws ArgumentError f("adir\0bad")
end
@test_throws ArgumentError chmod("ba\0d", 0o222)
@test_throws ArgumentError open("ba\0d", "w")
@test_throws ArgumentError cp(file, "ba\0d")
@test_throws ArgumentError mv(file, "ba\0d")
if !Sys.iswindows() || (Sys.windows_version() >= Sys.WINDOWS_VISTA_VER)
    @test_throws ArgumentError symlink(file, "ba\0d")
else
    @test_throws ErrorException symlink(file, "ba\0d")
end
@test_throws ArgumentError download("good", "ba\0d")
@test_throws ArgumentError download("ba\0d", "good")

###################
#     walkdir     #
###################

dirwalk = mktempdir()
cd(dirwalk) do
    for i=1:2
        mkdir("sub_dir$i")
        open("file$i", "w") do f end

        mkdir(joinpath("sub_dir1", "subsub_dir$i"))
        touch(joinpath("sub_dir1", "file$i"))
    end
    touch(joinpath("sub_dir2", "file_dir2"))
    has_symlinks = !Sys.iswindows() || (Sys.windows_version() >= Sys.WINDOWS_VISTA_VER)
    follow_symlink_vec = has_symlinks ? [true, false] : [false]
    has_symlinks && symlink(abspath("sub_dir2"), joinpath("sub_dir1", "link"))
    for follow_symlinks in follow_symlink_vec
        chnl = walkdir(".", follow_symlinks=follow_symlinks)
        root, dirs, files = take!(chnl)
        @test root == "."
        @test dirs == ["sub_dir1", "sub_dir2"]
        @test files == ["file1", "file2"]

        root, dirs, files = take!(chnl)
        @test root == joinpath(".", "sub_dir1")
        if has_symlinks
            if follow_symlinks
                @test dirs ==  ["link", "subsub_dir1", "subsub_dir2"]
                @test files == ["file1", "file2"]
            else
                @test dirs ==  ["subsub_dir1", "subsub_dir2"]
                @test files == ["file1", "file2", "link"]
            end
        else
            @test dirs ==  ["subsub_dir1", "subsub_dir2"]
            @test files == ["file1", "file2"]
        end

        root, dirs, files = take!(chnl)
        if follow_symlinks
            @test root == joinpath(".", "sub_dir1", "link")
            @test dirs == []
            @test files == ["file_dir2"]
            root, dirs, files = take!(chnl)
        end
        for i=1:2
            @test root == joinpath(".", "sub_dir1", "subsub_dir$i")
            @test dirs == []
            @test files == []
            root, dirs, files = take!(chnl)
        end

        @test root == joinpath(".", "sub_dir2")
        @test dirs == []
        @test files == ["file_dir2"]
    end

    for follow_symlinks in follow_symlink_vec
        chnl = walkdir(".", follow_symlinks=follow_symlinks, topdown=false)
        root, dirs, files = take!(chnl)
        if follow_symlinks
            @test root == joinpath(".", "sub_dir1", "link")
            @test dirs == []
            @test files == ["file_dir2"]
            root, dirs, files = take!(chnl)
        end
        for i=1:2
            @test root == joinpath(".", "sub_dir1", "subsub_dir$i")
            @test dirs == []
            @test files == []
            root, dirs, files = take!(chnl)
        end
        @test root == joinpath(".", "sub_dir1")
        if has_symlinks
            if follow_symlinks
                @test dirs ==  ["link", "subsub_dir1", "subsub_dir2"]
                @test files == ["file1", "file2"]
            else
                @test dirs ==  ["subsub_dir1", "subsub_dir2"]
                @test files == ["file1", "file2", "link"]
            end
        else
            @test dirs ==  ["subsub_dir1", "subsub_dir2"]
            @test files == ["file1", "file2"]
        end

        root, dirs, files = take!(chnl)
        @test root == joinpath(".", "sub_dir2")
        @test dirs == []
        @test files == ["file_dir2"]

        root, dirs, files = take!(chnl)
        @test root == "."
        @test dirs == ["sub_dir1", "sub_dir2"]
        @test files == ["file1", "file2"]
    end
    #test of error handling
    chnl_error = walkdir(".")
    chnl_noerror = walkdir(".", onerror=x->x)
    root, dirs, files = take!(chnl_error)
    @test root == "."
    @test dirs == ["sub_dir1", "sub_dir2"]
    @test files == ["file1", "file2"]

    rm(joinpath("sub_dir1"), recursive=true)
    @test_throws(Base._UVError("readdir", Base.UV_ENOENT, "with ", repr(joinpath(".", "sub_dir1"))),
                 take!(chnl_error)) # throws an error because sub_dir1 do not exist

    root, dirs, files = take!(chnl_noerror)
    @test root == "."
    @test dirs == ["sub_dir1", "sub_dir2"]
    @test files == ["file1", "file2"]

    root, dirs, files = take!(chnl_noerror) # skips sub_dir1 as it no longer exist
    @test root == joinpath(".", "sub_dir2")
    @test dirs == []
    @test files == ["file_dir2"]

    # Test that symlink loops don't cause errors
    if has_symlinks
        mkdir(joinpath(".", "sub_dir3"))
        foo = joinpath(".", "sub_dir3", "foo")
        symlink("foo", foo)

        let files = walkdir(joinpath(".", "sub_dir3"); follow_symlinks=true)
            @test_throws Base._UVError("stat", Base.UV_ELOOP, "for file ", repr(foo)) take!(files)
        end
        root, dirs, files = take!(walkdir(joinpath(".", "sub_dir3"); follow_symlinks=false))
        @test root == joinpath(".", "sub_dir3")
        @test dirs == []
        @test files == ["foo"]
    end
end
rm(dirwalk, recursive=true)

###################
#     readdir     #
###################
@testset "readdir is sorted" begin
    mktempdir() do dir
        cd(dir) do
            for k in 1:10
                touch(randstring())
            end
            @test issorted(readdir())
        end
    end
end

############
# Clean up #
############
if !Sys.iswindows()
    rm(link)
    rm(rellink)
end
if !Sys.iswindows() || (Sys.windows_version() >= Sys.WINDOWS_VISTA_VER)
    rm(dirlink)
    rm(relsubdirlink)
end
rm(file)
rm(subdir)
rm(subdir2)
rm(dir)

@test !ispath(file)
@test !ispath(dir)

##################
# Return values of mkpath, mkdir, cp, mv and touch
####################
mktempdir() do dir
    name1 = joinpath(dir, "apples")
    name2 = joinpath(dir, "bannanas")
    @test !ispath(name1)
    @test touch(name1) == name1
    @test isfile(name1)
    @test touch(name1) == name1
    @test isfile(name1)
    @test !ispath(name2)
    @test mv(name1, name2) == name2
    @test !ispath(name1)
    @test isfile(name2)
    @test cp(name2, name1) == name1
    @test isfile(name1)
    @test isfile(name2)
    namedir = joinpath(dir, "chalk")
    namepath = joinpath(dir, "chalk","cheese","fresh")
    @test !ispath(namedir)
    @test mkdir(namedir) == namedir
    @test isdir(namedir)
    @test !ispath(namepath)
    @test mkpath(namepath) == namepath
    @test isdir(namepath)
    @test mkpath(namepath) == namepath
    @test isdir(namepath)
end

# issue #30588
@test realpath(".") == realpath(pwd())
mktempdir() do dir
    cd(dir) do
        path = touch("FooBar.txt")
        @test ispath(realpath(path))
        if ispath(uppercase(path)) # case-insensitive filesystem
            @test realpath(path) == realpath(uppercase(path)) == realpath(lowercase(path)) ==
                  realpath(uppercase(realpath(path))) == realpath(lowercase(realpath(path)))
            @test basename(realpath(uppercase(path))) == path
        end
        rm(path)
        @test_throws Base.IOError realpath(path)
    end
end

# issue #9687
let n = tempname()
    w = open(n, "a")
    io = open(n)
    write(w, "A"); flush(w)
    @test read(io) == UInt8[0x41]
    @test read(io) == UInt8[]
    write(w, "A"); flush(w)
    @test read(io) == UInt8[0x41]
    close(io); close(w)
    rm(n)
end

@test_throws ArgumentError mkpath("fakepath", mode = -1)

@testset "mktempdir 'prefix' argument" begin
    tmpdirbase = joinpath(tempdir(), "")
    def_prefix = "jl_"
    mktempdir() do tmpdir
        @test isdir(tmpdir)
        @test startswith(tmpdir, tmpdirbase * def_prefix)
        @test sizeof(tmpdir) == sizeof(tmpdirbase) + sizeof(def_prefix) + 6
        @test sizeof(basename(tmpdir)) == sizeof(def_prefix) + 6
        cd(tmpdir) do
            Sys.iswindows() || mkdir(".\\")
            for relpath in (".", "./", ".\\", "")
                mktempdir(relpath) do tmpdir2
                    pfx = joinpath(relpath, def_prefix)
                    @test sizeof(tmpdir2) == sizeof(pfx) + 6
                    @test startswith(tmpdir2, pfx)
                end
            end
        end
    end
    # Special character prefix tests
    for tst_prefix in ("ABCDEF", "./pfx", ".\\pfx", "", "#!@%^&()-", "/", "\\", "////abc", "\\\\\\\\abc", "∃x∀y")
        mktempdir(; prefix=tst_prefix) do tmpdir
            @test isdir(tmpdir)
            @test startswith(tmpdir, tmpdirbase * tst_prefix)
            @test sizeof(basename(tmpdir)) == 6 + sizeof(basename(tst_prefix))
        end
    end

    @test_throws Base.IOError mktempdir(; prefix="dir_notexisting/bar")
    @test_throws Base.IOError mktempdir(; prefix="dir_notexisting/")
    @test_throws Base.IOError mktempdir("dir_notexisting/")

    # Behavioral differences across OS types
    if Sys.iswindows()
        # invalid file name
        @test_throws Base.IOError mktempdir(; prefix="a*b")
        @test_throws Base.IOError mktempdir("a*b")
    end

    mktempdir(""; prefix=tmpdirbase) do tmpdir
        @test startswith(tmpdir, tmpdirbase)
        @test sizeof(tmpdir) == 6 + sizeof(tmpdirbase)
        @test sizeof(basename(tmpdir)) == 6
    end
end

@testset "readdir tests" begin
    ≛(a, b) = sort(a) == sort(b)
    mktempdir() do dir
        d = cd(pwd, dir) # might resolve symlinks
        @test isempty(readdir(d))
        @test isempty(readdir(d, join=true))
        cd(d) do
            @test isempty(readdir())
            @test isempty(readdir(join=true))
        end
        touch(joinpath(d, "file"))
        mkdir(joinpath(d, "dir"))
        names = ["dir", "file"]
        paths = [joinpath(d, x) for x in names]
        @test readdir(d) ≛ names
        @test readdir(d, join=true) ≛ paths
        cd(d) do
            @test readdir() ≛ names
            @test readdir(join=true) ≛ paths
        end
        t, b = splitdir(d)
        cd(t) do
            @test readdir(b) ≛ names
            @test readdir(b, join=true) ≛ [joinpath(b, x) for x in names]
        end
    end
    if !Sys.iswindows()
        mktempdir() do dir
            cd(dir) do
                d = pwd() # might resolve symlinks
                @test isdir(d)
                @test Base.Filesystem.samefile(d, ".")
                @test isempty(readdir())
                @test isempty(readdir(d))
                @test isempty(readdir(join=true))
                rm(d, recursive=true)
                @test !ispath(d)
                @test isempty(readdir())
                @test_throws Base._UVError("readdir", Base.UV_ENOENT, "with ", repr(d)) readdir(d)
                @test_throws Base._UVError("cwd", Base.UV_ENOENT) readdir(join=true)
            end
        end
    end
end

@testset "chmod/isexecutable" begin
    mktempdir() do dir
        mkdir(joinpath(dir, "subdir"))
        fpath = joinpath(dir, "subdir", "foo")

        # Test that we can actually set the executable bit on all platforms.
        touch(fpath)
        chmod(fpath, 0o644)
        @test !Sys.isexecutable(fpath)
        chmod(fpath, 0o755)
        @test Sys.isexecutable(fpath)

        # Ensure that, on Windows, where inheritance is default,
        # chmod still behaves as we expect.
        if Sys.iswindows()
            chmod(joinpath(dir, "subdir"), 0o666)
            @test Sys.isexecutable(fpath)
        end

        # Reset permissions to all at the end, so it can be deleted properly.
        chmod(dir, 0o777; recursive=true)
    end
end

max_stars_repo_name: goropikari/julia

max_issues_repo_name: Navachethan-Murugeppa/julia

max_forks_repo_name: Navachethan-Murugeppa/julia


text:
module problem131

description = """
Prime cube partnership

There are some prime values, p, for which there exists a positive integer, n, such that the expression n^3 + n^2 * p is a perfect cube.
For example, when p = 19, 8^3 + 8^2 × 19 = 12^3.
What is perhaps most surprising is that for each prime with this property the value of n is unique, and there are only four such primes below one-hundred.

How many primes below one million have this remarkable property?
"""

# By exhaustive search, I found the first few primes are 7, 19, 37, 61
# These are known as Cuban primes: primes of the form n^3 - (n-1)^3
#
# Insight after reading solutions from the ProjectEuler forums:
#
# n^3 + p * n^2 = k^3
# n^3 * (p/n + 1) = k^3
# n^3 * ((p+n)/n) = k^3
# n * cuberoot((p+n)/n) = k
# n * cuberoot(p+n) / cuberoot(n) = k
# For the cuberoot expressions to have integer results, p+n and n must be cubes.
# let n = x^3, p+n = y^3
# p = y^3 - x^3
# p = (y - x)(y^2 + xy + x^2)
# But p is required to be prime, so (y-x) must be 1 - otherwise (y-x) would be a factor of p.
# Therefore, p is the difference of consecutive cubes:
# p = y^3 - (y-1)^3
function cubans(n = 1000000)
  c = 0
  for y = 2:Inf
    p = y^3 - (y-1)^3
    if p > n
      return c
    elseif isprime(p)
      c += 1
    end
  end
end

using Base.Test
@test cubans(100) == 4

end

ext: jl

max_stars_repo_name: mbuhot/mbuhot-euler-solutions

max_issues_repo_name: mbuhot/mbuhot-euler-solutions

max_forks_repo_name: mbuhot/mbuhot-euler-solutions


text:
function entropy(q::AbstractVector)
    η = x -> x <= 0 ? zero(x) : -x * log2(x)
    return sum(η, q)
end

entropy(ρ::AbstractMatrix) = entropy(eigvals(ρ))

renyi_entropy(α::Real) = r -> renyi_entropy(r, α)

function renyi_entropy(q::AbstractVector, α)
    α == 1 && return entropy(q)
    α == Inf && return -log2(maximum(q))
    α == 0 && return sum(q .> 0)
    φ = x -> x <= 0 ? zero(x) : x^α
    return inv(1 - α) * log2(sum(φ, q))
end

tsallis_entropy(q::Real) = r -> tsallis_entropy(r, q)

function tsallis_entropy(r::AbstractVector, q)
    q == 1 && return entropy(r)
    φ = x -> x <= 0 ? zero(x) : x^q
    return (sum(φ, r) - 1) * inv(1 - q)
end

subentropy(r::AbstractVector) = subentropy(SortedProbVecMult(r))

function subentropy(spvm::SortedProbVecMult)
    μs = spvm.distinct_entries
    ks = spvm.multiplicities
    n = sum(ks)
    s = zero(eltype(μs))
    for i in eachindex(μs)
        iszero(μs[i]) && continue
        s -= _subentropy_fi(spvm, i, ks[i] - 1, μs[i]) / factorial(ks[i] - 1)
    end
    return s
end

# this is slow for large `n`
# computes the `n`th derivative of `f` at `x`
function _deriv(f, n, x)
    if n == 0
        return f(x)
    end
    D = f -> (s -> ForwardDiff.derivative(f, s))
    Dn = foldl(∘, (D for _ = 1:n))
    Dn(f)(x)
end

function _subentropy_fi(spvm, i, k, x)
    μs = spvm.distinct_entries
    ks = spvm.multiplicities
    d = sum(ks)
    fi = x -> begin
        itr = ((x - μs[j])^ks[j] for j = eachindex(μs) if j ≠ i)
        prod_others = isempty(itr) ? one(x) : prod(itr)
        x^d * log(x) / prod_others
    end
    _deriv(fi, k, x)
end


text:
using TaylorExponentialMatrix
using Test

@testset " simplest implementation " begin

A = rand( 5, 5)

@test all( expm2(A) .≈ exp(A))

end

@testset " sophisticated implementation " begin

A = rand( 5, 5)

@test all( expm3(A) .≈ exp(A))

end


text:
using HackerNewsAnalysis

"John" |> logx("Hello, {}")
[1,2,3] |> logx("Array size is {}", length)
#=
julia> "John" |> logx("Hello, {}")
2019-12-01T12:49:36.947 Hello, John
"John"

julia> [1,2,3] |> logx("Array size is {}", length)
2019-12-01T12:49:40.099 Array size is 3
3-element Array{Int64,1}:
 1
 2
 3
=#


text:
using ElectronicStructurePySCF
using Test

@testset "ElectronicStructurePySCF.jl" begin
    # Write your tests here.
end


text:
export EntropyVector
export PrimalEntropy, cardminusentropy, cardentropy, invalidfentropy, matusrentropy, entropyfrompdf, subpdf, toTikz
export DualEntropy, DualEntropyLift, nonnegative, nondecreasing, submodular, submodulareq, ingleton, setequality
import Base.isless

# Entropy Vector

function ntodim(n::Int)
    # It will works with bitsets which are unsigned
    fullset(n)
end
function ntodim(n::Vector{Int})
    map(ntodim, n)
end

function indset(n::Int)
    setsto(ntodim(n))
end

function dimton(N)
    n = Int(round(log2(N + 1)))
    if ntodim(n) != N
        error("Illegal size of entropic constraint")
    end
    n
end

abstract type EntropyVector{T<:Real} end # <: AbstractVector{T} end

function indset(h::EntropyVector, id::Int)
    indset(h.n[id])
end

#Store vectors as tuple to reuse their `isless'
function Base.isless(h::EntropyVector, g::EntropyVector)
    @assert Polyhedra.fulldim(h) == Polyhedra.fulldim(g)
    for i in 1:length(h.h)
        if h.h[i] < g.h[i]
            return true
        elseif h.h[i] > g.h[i]
            return false
        end
    end
    return false
end

Base.size(h::EntropyVector) = (Polyhedra.fulldim(h),)
Base.IndexStyle(::Type{EntropyVector}) = Base.LinearFast()
#Base.getindex(h::EntropyVector, i::Int) = h.h[i]
#Base.getindex{T}(h::EntropyVector, i::AbstractArray{T,1}) = EntropyVector(h.h[i])
Base.getindex(h::EntropyVector, i) = h.h[i]
Base.setindex!(h::EntropyVector{T}, v::T, i::Integer) where {T} = h.h[i] = v

#function *(x, h::EntropyVector)
#  EntropyVector(x * h.h)
#end

#function entropyof(p::AbstractArray{Real, n})
#  println(n)
#  for i in indset(n)
#  end
#end

abstract type AbstractDualEntropy{L, T<:Real} <: EntropyVector{T} end

mutable struct DualEntropy{L, T<:Real, AT<:AbstractVector{T}} <: AbstractDualEntropy{L, T}
    n::Int
    h::AT
    liftid::Int

    function DualEntropy{L, T}(n::Int, h::AbstractVector{T}, liftid::Int=1) where {L, T}
        if ntodim(n) != length(h)
            error("Number of variables and dimension does not match")
        end
        if liftid < 1
            error("liftid must be positive")
        end
        new{L, T, typeof(h)}(n, h, liftid)
    end

end

Polyhedra.fulldim(h::DualEntropy) = length(h.h)

DualEntropy{L, T}(n::Int, liftid::Int=1) where {L, T} = DualEntropy{L, T}(n, Vector{T}(ntodim(n)), liftid)
DualEntropy{L}(n::Int, h::AbstractVector{T}, liftid::Int=1) where {L, T} = DualEntropy{L, T}(n, h, liftid)
DualEntropy{L}(h::AbstractVector, liftid::Int=1) where {L} = DualEntropy{L}(dimton(length(h)), h, liftid)

#Base.convert{T}(::Type{HRepresentation{T}}, h::DualEntropy{T}) = Base.convert(HRepresentation{T}, [h])
#Doesn't work

# L <-> linearity/equality
mutable struct DualEntropyLift{L, T<:Real} <: AbstractDualEntropy{L, T}
    n::Vector{Int}
    h::AbstractVector{T}

    function DualEntropyLift{L, T}(n::Vector{Int}, h::AbstractVector{T}=spzeros(T, sum(ntodim(n)))) where {L, T}
        if sum(ntodim(n)) != length(h)
            error("Dimension should be equal to the length of h")
        end
        new{L, T}(n, h)
    end
end

Polyhedra.fulldim(h::DualEntropyLift) = length(h.h)

function DualEntropyLift(h::DualEntropy{L, T}, m) where {L, T}
    N = Polyhedra.fulldim(h)
    hlift = spzeros(T, m*N)
    offset = (h.liftid-1)*N
    hlift[(offset+1):(offset+N)] = h.h
    DualEntropyLift{L, T}(h.n*ones(Int, m), hlift)
end

DualEntropyLift{L}(n::Vector{Int}, h::AbstractVector{T}) where {L, T} = DualEntropyLift{L, T}(n, h)

setequality(h::DualEntropy{false}) = DualEntropy{true}(h.n, h.h, h.liftid)
setequality(h::DualEntropyLift{false}) = DualEntropyLift{true}(h.n, h.h)

Polyhedra.HRepElement(h::Union{DualEntropy{true, T}, DualEntropyLift{true, T}}) where {T}  = HyperPlane(h.h, zero(T))
Polyhedra.HRepElement(h::Union{DualEntropy{false, T}, DualEntropyLift{false, T}}) where {T} = HalfSpace(-h.h, zero(T))
Polyhedra.hrep(hs::Vector{<:DualEntropy}) = hrep(HRepElement.(hs))
Polyhedra.hrep(hs::DualEntropy) = hrep([hs])
#function Base.convert(::Type{HyperPlane{T, AT}}, h::DualEntropy{true}) where {T, AT}
#    HyperPlane{T, AT}(h.h, zero(T))
#end
#function Base.convert(::Type{HalfSpace{T, AT}}, h::DualEntropy{false}) where {T, AT}
#    HalfSpace{T, AT}(-h.h, zero(T))
#end

#function Polyhedra.hrep(hs::Vector{DualEntropy{false, T, AT}}) where {T, AT}
#    hrep(HalfSpace{T, AT}.(hs))
#end

abstract type AbstractPrimalEntropy{T<:Real} <: EntropyVector{T} end

mutable struct PrimalEntropy{T<:Real} <: AbstractPrimalEntropy{T}
    n::Int
    h::AbstractVector{T}
    liftid::Int # 1 by default: the first cone of the lift

    function PrimalEntropy{T}(n::Int, h::AbstractVector{T}, liftid::Int=1) where {T}
        if ntodim(n) != length(h)
            error("Number of variables and dimension does not match")
        end
        new{T}(n, h, liftid)
    end
end

Polyhedra.fulldim(h::PrimalEntropy) = length(h.h)

PrimalEntropy{T}(n::Int, liftid::Int=1) where {T} = PrimalEntropy{T}(n, Vector{T}(undef, ntodim(n)), liftid)
PrimalEntropy(n::Int, h::AbstractVector{T}, liftid::Int=1) where {T} = PrimalEntropy{T}(n, h, liftid)
PrimalEntropy(h::AbstractVector, liftid::Int=1) = PrimalEntropy(dimton(length(h)), h, liftid)

mutable struct PrimalEntropyLift{T<:Real} <: AbstractPrimalEntropy{T}
    n::Vector{Int}
    h::AbstractVector{T}
    liftid::Vector{Int}

    #function PrimalEntropyLift(n::Array{Int,1}, liftid::Array{Int,1})
    #  new(n, sum(ntodim(n)), liftid)
    #end

    function PrimalEntropyLift{T}(n::Vector{Int}, h::AbstractVector{T}, liftid::Vector{Int}) where {T}
        if sum(ntodim(n)) != length(h) # TODO Not usre
            error("Number of variables and dimension does not match")
        end
        new{T}(n, h, liftid)
    end

end

Polyhedra.fulldim(h::PrimalEntropyLift) = convert(Int, sum(ntodim(n)))

PrimalEntropyLift(n::Vector{Int}, h::AbstractVector{T}, liftid::Vector{Int}) where {T<:Real} = PrimalEntropyLift{T}(n, h, liftid)

function Base.:(*)(h1::AbstractPrimalEntropy{T}, h2::AbstractPrimalEntropy{T}) where {T<:Real}
    if length(h1.liftid) + length(h2.liftid) != length(union(BitSet(h1.liftid), BitSet(h2.liftid)))
        error("liftids must differ")
    end
    PrimalEntropyLift([h1.n; h2.n], [h1.h; h2.h], [h1.liftid; h2.liftid])
end

Base.convert(::Type{PrimalEntropy{T}}, h::PrimalEntropy{S}) where {T<:Real,S<:Real} = PrimalEntropy(Array{T}(h.h))

function subpdf(p::Array{Float64,n}, S::EntropyIndex) where n
    cpy = copy(p)
    for j = 1:n
        if !myin(j, S)
            cpy = reduce(+, cpy, dims=j)
        end
    end
    cpy
end

subpdf(p::Array{Float64,n}, s::Signed) where {n} = subpdf(p, set(s))

function entropyfrompdf(p::Array{Float64,n}) where n
    h = PrimalEntropy{Float64}(n, 1)
    for i = indset(n)
        h[i] = -sum(map(xlogx, subpdf(p, i)))
    end
    h
end

LinearAlgebra.dot(hd::DualEntropy, hp::PrimalEntropy) = dot(hp, hd)
function LinearAlgebra.dot(hp::PrimalEntropy, hd::DualEntropy{L}) where {L}
    @assert hp.liftid == hd.liftid
    dot(hp.h, hd.h)
end
Base.:(-)(h::PrimalEntropy{T}) where {T<:Real}      = PrimalEntropy{T}(-h.h, h.liftid)
Base.:(-)(h::DualEntropy{L, T}) where {L, T<:Real}  =   DualEntropy{L, T}(-h.h, h.liftid)
Base.:(-)(h::PrimalEntropyLift{T}) where {T<:Real}  = PrimalEntropyLift{T}(h.n, -h.h, h.liftid)
Base.:(-)(h::DualEntropyLift{L, T}) where {L, T<:Real} =   DualEntropyLift{L, T}(h.n, -h.h)
Base.:(*)(h::DualEntropy{true}, α::Number)  = DualEntropy{true}(h.n, h.h*α, h.liftid)
Base.:(*)(α::Number, h::DualEntropy{true})  = DualEntropy{true}(h.n, α*h.h, h.liftid)
Base.:(*)(h::DualEntropy{false}, α::Number) = (@assert α >= 0; DualEntropy{false}(h.n, h.h*α, h.liftid))
Base.:(*)(α::Number, h::DualEntropy{false}) = (@assert α >= 0; DualEntropy{false}(h.n, α*h.h, h.liftid))
Base.:(*)(h::PrimalEntropy, α::Number) = PrimalEntropy(h.n, h.h*α, h.liftid)
Base.:(*)(α::Number, h::PrimalEntropy) = PrimalEntropy(h.n, h.h*α, h.liftid)
Base.:(==)(h1::PrimalEntropy, h2::PrimalEntropy) = h1.liftid == h2.liftid && h1.h == h2.h
Base.:(==)(h1::DualEntropy, h2::DualEntropy) = h1.liftid == h2.liftid && h1.h == h2.h
function Base.:(+)(h1::PrimalEntropy, h2::PrimalEntropy)
    @assert h1.liftid == h2.liftid
    PrimalEntropy(h1.n, h1.h + h2.h, h1.liftid)
end
function Base.:(+)(h1::DualEntropy{L}, h2::DualEntropy{L}) where L
    @assert h1.liftid == h2.liftid
    DualEntropy{L}(h1.n, h1.h + h2.h, h1.liftid)
end
function Base.:(+)(h1::PrimalEntropyLift, h2::PrimalEntropyLift)
    @assert h1.n == h2.n
    PrimalEntropyLift(h1.n, h1.h + h2.h, h1.liftid)
end
function Base.:(+)(h1::DualEntropyLift{L}, h2::DualEntropyLift{L}) where L
    @assert h1.n == h2.n
    DualEntropyLift{L}(h1.n, h1.h + h2.h, h1.liftid)
end
function Base.:(-)(h1::PrimalEntropy, h2::PrimalEntropy)
    @assert h1.liftid == h2.liftid
    PrimalEntropy(h1.n, h1.h - h2.h, h1.liftid)
end
function Base.:(-)(h1::DualEntropy{L}, h2::DualEntropy{L}) where L
    @assert h1.liftid == h2.liftid
    DualEntropy{L}(h1.n, h1.h - h2.h, h1.liftid)
end
function Base.:(-)(h1::PrimalEntropyLift, h2::PrimalEntropyLift)
    @assert h1.n == h2.n
    PrimalEntropyLift(h1.n, h1.h - h2.h, h1.liftid)
end
function Base.:(-)(h1::DualEntropyLift{L}, h2::DualEntropyLift{L}) where L
    @assert h1.n == h2.n
    DualEntropyLift{L}(h1.n, h1.h - h2.h)
end

function constprimalentropy(n, x::T) where T<:Real
    PrimalEntropy(x * ones(T, ntodim(n)))
end
function constdualentropy(n, x::T) where T<:Real
    if x == 0
        DualEntropy{false}(spzeros(T, ntodim(n)))
    else
        DualEntropy{false}(x * ones(T, ntodim(n)))
    end
end

function Base.one(h::PrimalEntropy{T}) where T<:Real
    constprimalentropy(h.n, one(T))
end
function Base.one(h::DualEntropy{L, T}) where {L, T}
    constdualentropy(h.n, one(T))
end

# #Base.similar(h::EntropyVector) = EntropyVector(h.n)
# # Used by e.g. hcat
# Base.similar(h::PrimalEntropy, ::Type{T}, dims::Dims) where {T} = length(dims) == 1 ? PrimalEntropy{dims[1], T}(dimton(dims[1]), h.liftid) : Array{T}(dims...)
# Base.similar(h::DualEntropy, ::Type{T}, dims::Dims) where {T} = length(dims) == 1 ? DualEntropy{dims[1], T}(dimton(dims[1]), h.equality, h.liftid) : Array{T}(dims...)
# # Cheating here, I cannot deduce n just from dims so I use copy(h.n)
# Base.similar(h::PrimalEntropyLift, ::Type{T}, dims::Dims) where {T} = length(dims) == 1 ? PrimalEntropyLift{dims[1], T}(copy(h.n), h.liftid) : Array{T}(dims...)
# Base.similar(h::DualEntropyLift, ::Type{T}, dims::Dims) where {T} = length(dims) == 1 ? DualEntropyLift{dims[1], T}(copy(h.n), h.equality) : Array{T}(dims...)
# #Base.similar{T}(h::EntropyVector, ::Type{T}) = EntropyVector(h.n)

# Classical Entropy Inequalities
function dualentropywith(n, pos, neg)
    ret = constdualentropy(n, 0)
    # I use -= and += in case some I is in pos and neg
    for I in pos
        if I != emptyset()
            ret[I] += 1
        end
    end
    for I in neg
        if I != emptyset()
            ret[I] -= 1
        end
    end
    ret
end

function nonnegative(n, S::EntropyIndex)
    dualentropywith(n, [S], [])
end
function nonnegative(n, s::Signed)
    nonnegative(n, set(s))
end

function nondecreasing(n, S::EntropyIndex, T::EntropyIndex)
    T = union(S, T)
    x = dualentropywith(n, [T], [S]) # fix of weird julia bug
    print("")
    x
end
function nondecreasing(n, s::Signed, t::Signed)
    nondecreasing(n, set(s), set(t))
end

function submodular(n, S::EntropyIndex, T::EntropyIndex, I::EntropyIndex)
    S = union(S, I)
    T = union(T, I)
    U = union(S, T)
    h = dualentropywith(n, [S, T], [U, I])
    h
end
submodular(n, S::EntropyIndex, T::EntropyIndex) = submodular(n, S, T, S ∩ T)
submodular(n, s::Signed, t::Signed, i::Signed) = submodular(n, set(s), set(t), set(i))
submodular(n, s::Signed, t::Signed) = submodular(n, set(s), set(t))

submodulareq(n, S::EntropyIndex, T::EntropyIndex, I::EntropyIndex) = setequality(submodular(n, S, T, I))
submodulareq(n, S::EntropyIndex, T::EntropyIndex) = submodulareq(n, S, T, S ∩ T)
submodulareq(n, s::Signed, t::Signed, i::Signed) = submodulareq(n, set(s), set(t), set(i))
submodulareq(n, s::Signed, t::Signed) = submodulareq(n, set(s), set(t))

function ingleton(n, i, j, k, l)
    pos = []
    I = singleton(i)
    J = singleton(j)
    K = singleton(k)
    L = singleton(l)
    ij = union(I, J)
    kl = union(K, L)
    ijkl = union(ij, kl)
    for s in indset(n)
        if issubset(s, ijkl) && card(s) == 2 && s != ij
            pos = [pos; s]
        end
    end
    ikl = union(I, kl)
    jkl = union(J, kl)
    dualentropywith(n, pos, [ij, K, L, ikl, jkl])
end

function getkl(i, j)
    x = 1:4
    kl = x[(x .!= i) .& (x .!= j)]
    (kl[1], kl[2])
end

function ingleton(i, j)
    ingleton(4, i, j, getkl(i, j)...)
end

# Classical Entropy Vectors
function cardminusentropy(n, I::EntropyIndex)
    h = constprimalentropy(n, 0)
    for J in indset(n)
        h[J] = card(setdiff(J, I))
    end
    return h
end
cardminusentropy(n, i::Signed) = cardminusentropy(n, set(i))

function cardentropy(n)
    return cardminusentropy(n, emptyset())
end

#min(h1, h2) gives Array{Any,1} instead of EntropyVector :(
function _min(h1::PrimalEntropy{T}, h2::PrimalEntropy{T}) where {T<:Real} # FIXME cannot make it work with min :(
    PrimalEntropy(min.(h1.h, h2.h))
end

function invalidfentropy(S::EntropyIndex)
    n = 4
    #ret = min(constentropy(n, 4), 2 * cardentropy(n)) #can't make it work
    h = _min(constprimalentropy(n, 4), cardentropy(n) * 2)
    for i in indset(n)
        if i != S && card(i) == 2
            h[i] = 3
        end
    end
    return h
end

function invalidfentropy(s::Signed)
    return invalidfentropy(set(s))
end

function matusrentropy(t, S::EntropyIndex)
    n = 4
    h = _min(constprimalentropy(n, t), cardminusentropy(n, S))
    return h
end

function matusrentropy(t, s::Signed)
    return matusrentropy(t, set(s))
end

# Manipulation

function toTikz(h::EntropyVector)
    dens = [el.den for el in h.h]
    hlcm = reduce(lcm, 1, dens)
    x = h.h * hlcm
    println(join(Vector{Int}(x), " "))
end

function toTikz(h::EntropyVector{Rational{T}}) where T
    x = Vector{Real}(h.h)
    i = h.h .== round(h.h)
    x[i] = Vector{Int}(h.h[i])
    println(join(x, " "))
end

function Base.show(io::IO, h::EntropyVector)
    offset = 0
    for i in eachindex(collect(h.n))
        for l in h.n[i]:-1:1
            for j in indset(h, i)
                if card(j) == l
                    bitmap = bitstring(j)[end-h.n[i]+1:end]
                    val = h.h[offset+j]
                    if val == 0
                        print(io, " $(bitmap):$(val)")
                    else
                        printstyled(io, " $(bitmap):$(val)", color=:blue)
                    end
                end
            end
            if i != length(h.n) || l != 1
                println(io)
            end
        end
        offset += ntodim(h.n[i])
    end
end


text:
using Plots

# Nominal trajectories
x̄_nominal, ū_nominal = unpack(z̄_nominal, prob_nominal)
x̄_friction, ū_friction = unpack(z̄_friction, prob_friction)

# Plots results
s_friction_nominal = [ū_friction[t][7] for t = 1:T-1]
@assert norm(s_friction_nominal, Inf) < 1.0e-4

t_nominal = range(0, stop = h * (T - 1), length = T)

# Control
plt = plot(t_nominal[1:T-1], hcat(ū_nominal...)[1:1, :]',
    color = :magenta, width=2.0,
    title = "Cartpole",
	xlabel = "time (s)",
	ylabel = "control",
	label = "no friction",
    legend = :topright,
	linetype = :steppost)
plt = plot!(t_nominal[1:T-1], hcat(ū_friction...)[1:1, :]',
	color = :cyan,
    width = 2.0, label = "friction",
	linetype = :steppost)

# States
plt = plot(t_nominal, hcat(x̄_nominal...)[1:4, :]',
    color = :magenta, width = 2.0, xlabel = "time (s)",
    ylabel = "state", label = "", title = "Cartpole",
	legend = :topright)

plt = plot!(t_nominal, hcat(x̄_friction...)[1:4,:]',
    color = :cyan, width = 2.0, label = "")

include(joinpath(pwd(), "models/visualize.jl"))
vis = Visualizer()
render(vis)
# open(vis)
visualize!(vis, model_nominal, x̄_nominal, Δt = h)

# DPO
x_dpo, u_dpo = unpack(z[prob_dpo.prob.idx.nom], prob_dpo.prob.prob.nom)
s_dpo = [u_dpo[t][7] for t = 1:T-1]
@assert norm(s_dpo, Inf) < 1.0e-4

# Control
plt = plot(t_nominal[1:T-1], hcat(u_dpo...)[1:1, :]',
    color = :orange, width = 2.0,
    title = "Cartpole",
	xlabel = "time (s)", ylabel = "control", label = "dpo",
    legend = :topright, linetype = :steppost)

# States
plt = plot(t_nominal, hcat(x_dpo...)[1:4, :]',
    color = :orange, width = 2.0, xlabel = "time (s)",
    ylabel = "state", label = "dpo", title = "Cartpole",
	legend = :topright)

# Plots
plt_lqr_nom = plot(t_nominal, hcat(x̄_nominal...)[1:2, :]',
	legend = :topleft, color = :magenta,
    label = ["reference (no friction)" ""],
    width = 2.0, xlabel = "time",
	title = "Cartpole", ylabel = "state", ylims = (-1, 5))
plt_lqr_nom = plot!(t_sim, hcat(z_lqr...)[1:2,:]', color = :black,
    label=["lqr" ""], width = 1.0)

plt_lqr_friction = plot(t_nominal, hcat(x̄_friction...)[1:2,:]',
    color = :cyan, label = ["reference (friction)" ""],
    width=2.0,xlabel="time",title="Cartpole",ylabel="state",legend=:topleft)
plt_tvlqr_friction = plot!(t_sim, hcat(z_lqr_fr...)[1:2,:]',
    color = :black, label = ["lqr" ""], width = 1.0)

plt_dpo = plot(t_nominal, hcat(x_dpo...)[1:2,:]',
	legend=:topleft, color = :orange,
    label=["reference" ""],
    width = 2.0, xlabel = "time", title = "Cartpole", ylabel = "state")
plt_dpo = plot!(t_sim, hcat(z_dpo...)[1:2,:]', color=:black,
    label=["dpo" ""], width = 1.0)

using PGFPlots
const PGF = PGFPlots

# nominal trajectory
psx_nom = PGF.Plots.Linear(t_nominal,hcat(x̄_nominal...)[1,:],mark="none",
	style="color=magenta, line width=2pt")
psθ_nom = PGF.Plots.Linear(t_nominal,hcat(x̄_nominal...)[2,:],mark="none",
	style="color=magenta, line width=2pt")

psx_nom_sim_lqr = PGF.Plots.Linear(t_sim,hcat(z_lqr...)[1,:],mark="none",
	style="color=black, line width=1pt, densely dotted")
psθ_nom_sim_lqr = PGF.Plots.Linear(t_sim,hcat(z_lqr...)[2,:],mark="none",
	style="color=black, line width=1pt, dashed")

a_lqr_no_friction = Axis([psx_nom;psθ_nom;psx_nom_sim_lqr;psθ_nom_sim_lqr],
    xmin=0., ymin=-1, xmax=tf, ymax=3.5,
    axisEqualImage=false,
    hideAxis=false,
	ylabel="state",
	xlabel="time",
	legendStyle="{at={(0.01,0.99)},anchor=north west}")

# Save to tikz format
dir = joinpath(pwd(),"examples/direct_policy_optimization/figures")
PGF.save(joinpath(dir,"cartpole_lqr_no_friction.tikz"), a_lqr_no_friction, include_preamble=false)

# nominal trajectory
psx_nom_friction = PGF.Plots.Linear(t_nominal,hcat(x̄_friction...)[1,:],mark="none",
	style="color=cyan, line width=2pt")
psθ_nom_friction = PGF.Plots.Linear(t_nominal,hcat(x̄_friction...)[2,:],mark="none",
	style="color=cyan, line width=2pt")

psx_nom_sim_lqr_friction = PGF.Plots.Linear(t_sim,hcat(z_lqr_fr...)[1,:],mark="none",
	style="color=black, line width=1pt, densely dotted",legendentry="pos.")
psθ_nom_sim_lqr_friction = PGF.Plots.Linear(t_sim,hcat(z_lqr_fr...)[2,:],mark="none",
	style="color=black, line width=1pt, dashed",legendentry="ang.")

a_lqr_friction = Axis([psx_nom_friction;psθ_nom_friction;
    psx_nom_sim_lqr_friction;psθ_nom_sim_lqr_friction],
    xmin=0., ymin=-1, xmax=tf, ymax=3.5,
    axisEqualImage=false,
    hideAxis=false,
	ylabel="state",
	xlabel="time",
	legendStyle="{at={(0.01,0.99)},anchor=north west}")

# Save to tikz format
dir = joinpath(pwd(),"examples/direct_policy_optimization/figures")
PGF.save(joinpath(dir,"cartpole_lqr_friction.tikz"), a_tvlqr_friction, include_preamble=false)

# nominal trajectory
psx_dpo = PGF.Plots.Linear(t_nominal,hcat(x̄_dpo...)[1,:],mark="none",
	style="color=orange, line width=2pt")
psθ_dpo = PGF.Plots.Linear(t_nominal,hcat(x̄_dpo...)[2,:],mark="none",
	style="color=orange, line width=2pt")

psx_sim_dpo = PGF.Plots.Linear(t_sim,hcat(z_dpo...)[1,:],mark="none",
	style="color=black, line width=1pt, densely dotted",legendentry="pos.")
psθ_sim_dpo = PGF.Plots.Linear(t_sim,hcat(z_dpo...)[2,:],mark="none",
	style="color=black, line width=1pt, dashed",legendentry="ang.")

a_dpo = Axis([psx_dpo;psθ_dpo;psx_sim_dpo;
    psθ_sim_dpo],
    xmin=0., ymin=-1, xmax=tf, ymax=3.5,
    axisEqualImage=false,
    hideAxis=false,
	ylabel="state",
	xlabel="time",
	legendStyle="{at={(0.01,0.99)},anchor=north west}",
	)

# Save to tikz format
dir = joinpath(pwd(),"examples/direct_policy_optimization/figures")
PGF.save(joinpath(dir,"cartpole_dpo.tikz"), a_dpo, include_preamble=false)


text:
"""
    LeisenReimer()

[Leisen-Reimer binomial model](http://www.goddardconsulting.ca/option-pricing-binomial-alts.html#lr).
"""
struct LeisenReimer <: Model end

function h(z::T, n::Int64) where {T<:Number}
    return 0.5 + sign(z) * 0.5 * sqrt(1 - exp(-((z / (n + 1.0 / 3.0))^2.0) * (n + 1.0 / 6.0)))
end

"""
    evaluate(O, LeisenReimer(), N = 1001)

Evaluate option `O` using `LeisenReimer` binomial model.

# Arguments
- `N`: number of paths to simulate, must be odd
"""
function evaluate(O::Option, m::LeisenReimer, N::Int64 = 1001)
    Δt = 0.01 # O.t / N
    R = exp(O.r * Δt)
    d1 = (log(O.s / O.k) + (O.r + O.σ * O.σ / 2) * O.t) / (O.σ * √O.t)
    d2 = d1 - O.σ * √O.t
    p = h(d2, N)
    q = 1 - p

    if O.call == -1
        Z = [max(0, O.k - O.s * exp((2 * i - N) * O.σ * √Δt)) for i = 0:N]
    elseif O.call == 1
        Z = [max(0, O.s * exp((2 * i - N) * O.σ * √Δt) - O.k) for i = 0:N]
    end

    for n = N-1:-1:0, i = 0:n
        if O.call == -1
            x = O.k - O.s * exp((2 * i - n) * O.σ * √Δt)
        elseif O.call == 1
            x = O.s * exp((2 * i - n) * O.σ * √Δt) - O.k
        end
        y = (q * Z[i+1] + p * Z[i+2]) / exp(O.r * Δt)
        Z[i+1] = max(x, y)
    end

    return Z[1]
end


text:
### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ b2937383-dcbe-4863-9267-eca115a7b666
begin
	using Pkg
	Pkg.activate(mktempdir())
	Pkg.Registry.update()
	Pkg.add("Yao")
	Pkg.add("YaoPlots")
end

# ╔═╡ 57209cca-1ede-11eb-283b-196a50e6f1eb
using Yao,YaoPlots

# ╔═╡ 76794562-1ed0-11eb-18ec-ad4020344ce6
md"# Arithmetic using qubits"

# ╔═╡ ad61776c-1ed9-11eb-20af-7d41b6e281c3
md"## Binary addition"

# ╔═╡ 949416ee-1ed0-11eb-2a86-dd52c9b1d5f9
md"Suppose we've to add two numbers in binary form! Say... `` 5 `` and `` 7 ``."

# ╔═╡ aad0de2e-1ed0-11eb-3efc-95c0072fff76
md"In binary form, `` 5 `` can be represented by `` 101 `` and `` 7 `` can be represented by `` 111 ``. Remember when adding two numbers in binary form,

- `` 0 + 0 = 0 ``
- `` 0 + 1 = 1 ``
- `` 1 + 0 = 1 ``
- `` 1 + 1 = 0 `` with a carry of `` 1 ``

So adding `` 5 `` and `` 7 `` in binary form looks somewhat like this.
- `` \;\;\;\; ₁ \;₁\;\;₁\;₀ ``  ← **Carry**
- ``  \;\;\;\;\;\;1\; 0\; 1 ``
- `` \;\; `` + `` \; 1\; 1\; 1 ``
- `` \;\; `` _________
- `` \;\;\;1\;1\;0\;0 ``

Starting from right, and moving to left

- `` 1 + 1 = 0 `` with a carry of `` 1 ``
- `` 0 + 1 = 1 `` which is added to the carried value, which is `` 1 ``, so `` 1 + 1 = 0 `` with a carry of `` 1 ``, again
- `` 1 + 1 = 0 `` with a carry of `` 1 ``. The result is added again to the carried value, which was `` 1 ``, so `` 0 + 1 = 1 ``
- The remaining carried value, which is `` 1 ``, is put as it is.
"

# ╔═╡ c731fcb8-1ed7-11eb-2800-4fda7d5ae5e7
md"The result is `` \; [( 1 × 2^3 ) + ( 1 × 2^2 ) + ( 0 × 2^1 ) + ( 0 × 2^0 )] = 12. ``"

# ╔═╡ b914c0b4-1ed9-11eb-37e9-8f1a60dbfcb2
md"#### Quantum addition circuit for one pair of qubits"

# ╔═╡ ec622eb0-1eda-11eb-0ca2-1bc4b4f1a730
md"We'll try making the adder circuit for addition of two numbers  `` (0-7) ``. We'll need `` 3 `` qubits to represent each of the two numbers, which means a total of `` 6 `` qubits."

# ╔═╡ 30953b42-1ede-11eb-32ee-af55afa9221f
begin
	FirstNumber = ArrayReg(bit"111") #The first number is 7
	SecondNumber = ArrayReg(bit"101") #The second number is 5
end

# ╔═╡ 2ae7c4ec-1ee0-11eb-1c46-59d4d3084b39
md"For adding each pair of qubits, we'll need two more qubits to hold the carried in value and carried out value. Since the carry out of the 1st pair acts as the carry in of second pair, the carry out of the second pair acts as the carry in of the 3rd pair, we need 4 more qubits for carry in and carry out, with

- 1 qubit for the carry-in of first pair
- 1 qubit for the carry-out of first pair and the carry-in of the second pair
- 1 qubit for the carry-out of second pair and the carry-in of the third pair
- 1 qubit for the carry-out of third pair

In total, we require a total of `` 10 `` qubits to add two numbers in the range `` \; 0-7 ``."

# ╔═╡ 1ac1e8c0-1f2d-11eb-1cc5-5986d94a86c2
md"So lets try making the quantum circuit for adding two qubits. We need a qubit for carry-in, which will be zero for the rightmost pair, and a qubit for carry-out, which will act as the carry in for the next pair"

# ╔═╡ 1d8ce7a2-1f2e-11eb-1b74-0d601958a2c9
md"Consider this circuit"

# ╔═╡ 802cf394-1f2d-11eb-23ad-95c475382371
begin
	OneqbQFA = chain(4, control(2:3, 4=>X), control(2, 3=>X), control([1 3], 4=>X), control(1, 3=>X))
	plot(OneqbQFA)
end

# ╔═╡ 36613c90-1f2e-11eb-1f5e-d3e079a8deda
md"
- The top qubit holds the carry-in value.
- The bottom qubit has the state `` |0〉 ``.
- The 2nd and the 3rd qubits hold the pair to be added.
After passing throught the circuit,
- The top qubit holds the carry-in value.
- The bottom qubit holds the carry-out value.
- The 2nd qubit is as it was, while the 3rd qubit holds the addition of the 2nd and 3rd qubit.
"

# ╔═╡ 88d976e2-1f2e-11eb-0b7c-ad9557d124e6
md"### The use of CX gate in Arithmetics"

# ╔═╡ ade4a628-1f2e-11eb-32d0-c16546a16e44
md"Consider this circuit"

# ╔═╡ b307d638-1f2f-11eb-19ec-5fb58b49ff0f
begin
	a = chain(2, control(1, 2=>X))
	plot(a)
end

# ╔═╡ 31a37e02-1f30-11eb-3c16-494e05427f33
md"Lets try passing different values through this circuit"

# ╔═╡ 41c223ec-1f30-11eb-038b-efd7279954e6
measure((ArrayReg(bit"00") |> a)) #The output is 00, when we passed 00

# ╔═╡ 6ba13306-1f30-11eb-04c7-9f1c51a49915
measure(ArrayReg(bit"01") |> a) #The output is 11, when we passed 01
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |1〉, the 2nd qubit gets flipped to |1〉 too.

# ╔═╡ 95dbc5be-1f30-11eb-232f-e72c7bfbda61
measure(ArrayReg(bit"10") |> a) #The output is 10, when we passed 10
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |0〉, the 2nd qubit is left untouched.

# ╔═╡ d33faf76-1f30-11eb-0dd7-3d1331633c0a
measure(ArrayReg(bit"11") |> a) #The output is 01, when we passed 11
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |1〉, the 2nd qubit is flipped to |0〉.

# ╔═╡ b3823ea8-1f31-11eb-0651-d55953af774a
md"Lets analyze the output
- When the top qubit is `` |0〉 `` and the bottom qubit is `` |0〉 ``, the bottom qubit is left untouched.
- When the top qubit is `` |1〉 `` and the bottom qubit is `` |0〉 ``, the bottom qubit is flipped to `` |1〉 ``.
- When the top qubit is `` |0〉 `` and the bottom qubit is `` |1〉 ``, the bottom qubit is left untouched.
- When the top qubit is `` |1〉 `` and the bottom qubit is `` |1〉 ``, the bottom qubit is flipped to `` |0〉 ``.
Compare this with,
- `` 0 + 0 = 0 ``
- `` 0 + 1 = 1 ``
- `` 1 + 0 = 1 ``
- `` 1 + 1 = 0 ``
In all the cases, the bottom qubit holds the added value.

How about adding 3 qubits?
"

# ╔═╡ c7b7511c-1f34-11eb-37ba-f71851aceb94
begin
	b = chain(3, control(2, 3=>X), control(1, 3=>X))
	plot(b)
end

# ╔═╡ fce93e40-1f34-11eb-3ea6-a11d129d7af3
measure.([(ArrayReg(bit"000") |> b)
		(ArrayReg(bit"001") |> b)
		(ArrayReg(bit"010") |> b)
		(ArrayReg(bit"011") |> b)
		(ArrayReg(bit"110") |> b)
		(ArrayReg(bit"101") |> b)
		(ArrayReg(bit"100") |> b)
		(ArrayReg(bit"111") |> b)])
#If the output is not visible, click on the output to expand it.

# ╔═╡ 1604f776-1f36-11eb-03fc-c5575a528776
md"The toffoli gate acts as the carry-out. If both input qubits are `` |1〉 ``, the 3rd qubit is flipped."

# ╔═╡ ba41d664-1f3a-11eb-2334-83fa55bc3bb6
md"### The Quantum Adder Circuit"

# ╔═╡ f4db1d44-1f3a-11eb-0b8c-4589d745d435
md"We already made the circuit for adding two qubits. Remember that the carry-out for the first pair, becomes the carry in for the second pair. With this, here's the circuit for a quantum adder for 2 numbers between `` 0-7 ``."

# ╔═╡ 92b2e1b4-4457-11eb-22c0-7b5173d6bd97
plot(chain(10, put(1:4=>label(chain(4), "Full adder\nfor a pair\n of qubits")), put(4:7=>label(chain(4), "Full adder\nfor a pair\n of qubits")), put(7:10=>label(chain(4), "Full adder\nfor a pair\n of qubits"))))

# ╔═╡ 71d66a6e-4458-11eb-0a56-69919a6dbdca
md"Below is a complete circuit for quantum adder for 2 numbers between `` 0 - 7 ``."

# ╔═╡ 412203a2-1f3b-11eb-3de7-45f7eccd697e
begin
	nqbQFA = chain(10, put(1:4=>OneqbQFA), put(4:7=>OneqbQFA), put(7:10=>OneqbQFA))
	plot(nqbQFA)
end

# ╔═╡ c2afc0f8-1f3b-11eb-0626-c5120a452e38
md"The input to the circuit being - "

# ╔═╡ e4b012f0-1f3b-11eb-1336-af9ba42a98c8
begin
	input = join(zero_state(4), FirstNumber, SecondNumber)
	# 10 9 8 7 6 5 4  3 2 1
	# 1  0 1 1 1 1 0  0 0 0
	# 2  5 8 3 6 9 1  4 7 10
	reorder!(input, [2 5 8 3 6 9 1 4 7 10]) #Re-ordering the qubits, for being the input to the above circuit
end

# ╔═╡ 1abc9e96-4b70-11eb-1661-7f37fdea9130
measure(input)

# ╔═╡ 04c694e4-1f3c-11eb-1faf-0d08d391ce64
results = input |> nqbQFA |> r->measure(r, nshots=1024)

# ╔═╡ 1083740a-1f3c-11eb-2950-c56e4f20087c
stringres = reverse(string(Int(results[1]), base=2, pad=10)) #To convert it to string

# ╔═╡ 23e71498-1f3c-11eb-1c3e-2dac1694afeb
md"Remember, the 3, 6 and 9th qubits hold the added values. The 10 qubit holds the final carry-out, if any."

# ╔═╡ 4e89800a-1f3c-11eb-239f-f92807c81881
output = parse(Int64, reverse(stringres[3] * stringres[6] * stringres[9] * stringres[10]), base=2)

# ╔═╡ 270619d0-1f50-11eb-3a89-1187e12bd4f9
md"## Quantum subtractor"

# ╔═╡ 976e00e8-1f50-11eb-0d93-77e2ba841a09
md"Binary subtraction has a borrow instead of carry. The rules of binary subtraction look somewhat like this.
- `` 0 - 0 = 0 ``
- `` 0 - 1 = 1 `` with a borrow of `` 1 ``
- `` 1 - 0 = 1 ``
- `` 1 - 1 = 0 ``
"

# ╔═╡ 346af906-1f50-11eb-3286-dbf3014c8fab
begin
	OneqbSubtractor = chain(4, control(1, 3=>X), put(2=>X), control(2:3, 4=>X), control(1, 3=>X), control([1 3], 4=>X), put(2=>X), control(2, 3=>X), control(1, 3=>X))
	plot(OneqbSubtractor)
end

# ╔═╡ e2659b46-1f51-11eb-34db-dbc917b977dd
begin
	nqbQFS = chain(10, put(1:4=>OneqbSubtractor), put(4:7=>OneqbSubtractor), put(7:10=>OneqbSubtractor))
	plot(nqbQFS)
end

# ╔═╡ adeee882-1f55-11eb-27a4-d79963776e20
measure(join(zero_state(1), ArrayReg(bit"10"), zero_state(1)) |> OneqbSubtractor)

# ╔═╡ 2ff67ae4-4b4c-11eb-17a6-45e64c916c48
begin
	p = ArrayReg(bit"111")
	q = ArrayReg(bit"101")
	inp = join(zero_state(4), p, q)
	reorder!(inp, [3 6 9 2 5 8 1 4 7 10])
end

# ╔═╡ 31eb23f6-4b5e-11eb-2aa3-39ef184edb69
measure(inp)

# ╔═╡ 1f828098-1f52-11eb-1d00-27f10270a814
result = inp |> nqbQFS |> r->measure(r, nshots=1024)

# ╔═╡ 3cbd0f52-1f52-11eb-06f4-ffe29cccde57
stringsub = reverse(string(Int(result[1]), base=2, pad=10)) #To convert it to string

# ╔═╡ 658eec52-1f52-11eb-39aa-bbf1fcfb3517
out =  parse(Int64, reverse(stringsub[3] * stringsub[6] * stringsub[9] * stringsub[10]), base=2)

# ╔═╡ ffd40492-1f56-11eb-210d-ef6bf293244e
md"This circuit only subtracts numbers if the answer is expected to be positive. It can't solve for calculations like `` 5 - 6 = -1 ``."

# ╔═╡ Cell order:
# ╠═b2937383-dcbe-4863-9267-eca115a7b666
# ╟─76794562-1ed0-11eb-18ec-ad4020344ce6
# ╟─ad61776c-1ed9-11eb-20af-7d41b6e281c3
# ╟─949416ee-1ed0-11eb-2a86-dd52c9b1d5f9
# ╟─aad0de2e-1ed0-11eb-3efc-95c0072fff76
# ╟─c731fcb8-1ed7-11eb-2800-4fda7d5ae5e7
# ╟─b914c0b4-1ed9-11eb-37e9-8f1a60dbfcb2
# ╟─ec622eb0-1eda-11eb-0ca2-1bc4b4f1a730
# ╠═57209cca-1ede-11eb-283b-196a50e6f1eb
# ╠═30953b42-1ede-11eb-32ee-af55afa9221f
# ╟─2ae7c4ec-1ee0-11eb-1c46-59d4d3084b39
# ╟─1ac1e8c0-1f2d-11eb-1cc5-5986d94a86c2
# ╟─1d8ce7a2-1f2e-11eb-1b74-0d601958a2c9
# ╠═802cf394-1f2d-11eb-23ad-95c475382371
# ╟─36613c90-1f2e-11eb-1f5e-d3e079a8deda
# ╟─88d976e2-1f2e-11eb-0b7c-ad9557d124e6
# ╟─ade4a628-1f2e-11eb-32d0-c16546a16e44
# ╠═b307d638-1f2f-11eb-19ec-5fb58b49ff0f
# ╟─31a37e02-1f30-11eb-3c16-494e05427f33
# ╠═41c223ec-1f30-11eb-038b-efd7279954e6
# ╠═6ba13306-1f30-11eb-04c7-9f1c51a49915
# ╠═95dbc5be-1f30-11eb-232f-e72c7bfbda61
# ╠═d33faf76-1f30-11eb-0dd7-3d1331633c0a
# ╟─b3823ea8-1f31-11eb-0651-d55953af774a
# ╠═c7b7511c-1f34-11eb-37ba-f71851aceb94
# ╠═fce93e40-1f34-11eb-3ea6-a11d129d7af3
# ╟─1604f776-1f36-11eb-03fc-c5575a528776
# ╟─ba41d664-1f3a-11eb-2334-83fa55bc3bb6
# ╟─f4db1d44-1f3a-11eb-0b8c-4589d745d435
# ╟─92b2e1b4-4457-11eb-22c0-7b5173d6bd97
# ╟─71d66a6e-4458-11eb-0a56-69919a6dbdca
# ╠═412203a2-1f3b-11eb-3de7-45f7eccd697e
# ╟─c2afc0f8-1f3b-11eb-0626-c5120a452e38
# ╠═e4b012f0-1f3b-11eb-1336-af9ba42a98c8
# ╠═1abc9e96-4b70-11eb-1661-7f37fdea9130
# ╠═04c694e4-1f3c-11eb-1faf-0d08d391ce64
# ╠═1083740a-1f3c-11eb-2950-c56e4f20087c
# ╟─23e71498-1f3c-11eb-1c3e-2dac1694afeb
# ╠═4e89800a-1f3c-11eb-239f-f92807c81881
# ╟─270619d0-1f50-11eb-3a89-1187e12bd4f9
# ╟─976e00e8-1f50-11eb-0d93-77e2ba841a09
# ╠═346af906-1f50-11eb-3286-dbf3014c8fab
# ╠═e2659b46-1f51-11eb-34db-dbc917b977dd
# ╠═adeee882-1f55-11eb-27a4-d79963776e20
# ╠═2ff67ae4-4b4c-11eb-17a6-45e64c916c48
# ╠═31eb23f6-4b5e-11eb-2aa3-39ef184edb69
# ╠═1f828098-1f52-11eb-1d00-27f10270a814
# ╠═3cbd0f52-1f52-11eb-06f4-ffe29cccde57
# ╠═658eec52-1f52-11eb-39aa-bbf1fcfb3517
# ╟─ffd40492-1f56-11eb-210d-ef6bf293244e


text:
using MomentClosure
using MomentClosure: define_M, define_μ
using Test
using Catalyst

@parameters t, k_on, k_off, k_p, γ_p, b
@variables p(t), g(t)

vars = [g, p]
ps = [k_on, k_off, k_p, γ_p, b]
S = [1 -1 0 0;
     0 0 b -1]

as = [k_on*(1-g),    # 0 -> g
      k_off*g*(p^2), # g -> 0
      k_p*g,         # 0 -> mP, m ~ Geometric(mean=b)
      γ_p*p]         # p -> 0

binary_vars = [1]

rn = ReactionSystemMod(t, vars, ps, as, S)

μ = define_μ(2,4)
M = define_M(2,4)
sys = generate_central_moment_eqs(rn, 2)

closed_eqs = moment_closure(sys, "zero", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,4]], 0)

closed_eqs = moment_closure(sys, "normal", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,4]], 3*M[0,2]^2)

closed_eqs= moment_closure(sys, "log-normal", binary_vars)
expr1 = closed_eqs.closure[M[0,3]]
expr2 = μ[0,1]^3*(1+M[0,2]*μ[0,1]^-2)^3 - μ[0,1]^3 - 3*M[0,2]*μ[0,1]
@test length(closed_eqs.odes.states) == 4 && isequal(expand(expr1), expr2)

closed_eqs = moment_closure(sys, "poisson", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,3]], μ[0,1])

closed_eqs = moment_closure(sys, "gamma", binary_vars)
expr1 = closed_eqs.closure[M[0,3]]
expr2 = 2*M[0,2]^2*μ[0,1]^-1
@test length(closed_eqs.odes.states) == 4 && isequal(expr1, expr2)

closed_eqs = moment_closure(sys, "derivative matching", binary_vars)
expr1 = closed_eqs.closure[sys.M[0,3]]
expr2 = μ[0,1]^-3 * (M[0,2]+μ[0,1]^2)^3 - μ[0,1]^3 - 3*M[0,2]*μ[0,1]
@test length(closed_eqs.odes.states) == 4 && isequal(expr1, expr2)

sys = generate_raw_moment_eqs(rn, 2)

closed_eqs = moment_closure(sys, "zero", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "normal", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "log-normal", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "poisson", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "gamma", binary_vars)
@test isequal(expr1, expr2)

closed_eqs = moment_closure(sys, "derivative matching", binary_vars)
@test isequal(expr1, expr2)


text:
using Distributed
using ProgressMeter
addprocs(46)
@everywhere begin
using Random
using Printf
using LinearAlgebra
using Distributions
using StatsBase
using ProgressMeter
using StaticArrays
using POMDPs
using AutoViz
using AutomotiveDrivingModels
using AutomotivePOMDPs
using POMDPSimulators
using BeliefUpdaters
using ProgressMeter
using POMDPPolicies
using DeepQLearning
using Flux
using RLInterface
using POMDPModelTools
using TensorBoardLogger
using MCTS
include("environment.jl")
include("generative_mdp.jl")
include("masking.jl")
include("cooperative_IDM.jl")
include("belief_updater.jl")
include("belief_mdp.jl")

rng = MersenneTwister(1)

mdp = GenerativeMergingMDP(random_n_cars=true, min_cars=10,max_cars=14, traffic_speed = :mixed, driver_type=:random,
                           observe_cooperation=false)
for i=2:mdp.max_cars+1
    mdp.driver_models[i] = CooperativeIDM()
end

function POMDPs.reward(mdp::GenerativeMergingMDP, s::AugScene, a::Int64, sp::AugScene)
    ego = get_by_id(s.scene, EGO_ID)
    egop = get_by_id(sp.scene, EGO_ID)
    r = 0.0
    if ego.state.posG.x < egop.state.posG.x
      r += 0.1
    end
    if reachgoal(mdp, egop)
       r += mdp.goal_reward
    elseif is_crash(sp.scene)
        r += mdp.collision_cost
    end
    if caused_hard_brake(mdp, sp.scene)
        r += mdp.hard_brake_cost
    end
    return r
end
# pomdp = FullyObservablePOMDP(mdp)
# up = MergingUpdater(mdp)
# bmdp = GenerativeBeliefMDP{typeof(pomdp), MergingUpdater, MergingBelief,Int64}(pomdp, up)

#s0 = initialstate(mdp, rng)

#BSON.@load "log12/policy_true.bson" policy
#rlpolicy = NNPolicy(mdp, policy.qnetwork, policy.action_map, policy.n_input_dims)

solver = DPWSolver(depth = 1,
                   exploration_constant = 1.0,
                   n_iterations = 10,
                   k_state  = 2.0,
                   alpha_state = 0.2,
                   keep_tree = true,
                   enable_action_pw = false,
                   rng = rng,
                    tree_in_info = true,
                  estimate_value = RolloutEstimator(RandomPolicy(mdp, rng=rng))
                   )

if mdp.observe_cooperation
  policy = solve(solver, mdp)
else
  @show keys(mdp.driver_models)
  mdp_noobs = deepcopy(mdp)
  @show keys(mdp_noobs.driver_models)
  #BSON.@load "rl13-backup/policy_false.bson" policy
  #rlpolicy = NNPolicy(mdp_noobs, policy.qnetwork, policy.action_map, policy.n_input_dims)
  #solver.estimate_value = RolloutEstimator(rlpolicy)
  for i=EGO_ID+1:EGO_ID+mdp_noobs.max_cars
    mdp_noobs.driver_models[i].c = 0.5
  end
  policy = solve(solver, mdp_noobs)
end

# s0 = initialstate(mdp, rng)
# hr = HistoryRecorder(rng = rng, max_steps=100)
# @time hist = simulate(hr, mdp, policy, s0);
# @show n_steps(hist)

# include("visualizer.jl");

# make_gif(hist, mdp)

# ## run many simulations

res = @showprogress pmap(x -> quick_evaluation(mdp, policy, x, 1), [MersenneTwister(i) for i=1:100])

function quick_evaluation(mdp::GenerativeMergingMDP, policy::Policy, rng::AbstractRNG, n_eval=1000)
    avg_r, avg_dr, c_rate, avg_steps, t_out = 0.0, 0.0, 0.0, 0.0, 0.0
    for i=1:n_eval
        s0 = initialstate(mdp, rng)
        hr = HistoryRecorder(rng = rng, max_steps=100)
        hist = simulate(hr, mdp, policy, s0)
        avg_r += undiscounted_reward(hist)
        avg_dr += discounted_reward(hist)
        c_rate += hist.reward_hist[end] < 0.0
        t_out += n_steps(hist) >= hr.max_steps ? 1.0 : 0.0
        avg_steps += n_steps(hist)
    end
    avg_r /= n_eval
    avg_dr /= n_eval
    c_rate /= n_eval
    avg_steps /= n_eval
    t_out /= n_eval
    return avg_r, avg_dr, c_rate, avg_steps, t_out
end

end #@everywhere

res = @showprogress pmap(x -> quick_evaluation(mdp, policy, MersenneTwister(x), 1), 1:1000)

c_rate = mean(x[3] for x in res)
avg_steps = mean(x[4] for x in res)
t_out = mean(x[5] for x in res)

# avg_r, avg_dr, c_rate, avg_steps, t_out = quick_evaluation(mdp, policy, rng, 100)

println("Collisions ", c_rate*100)
println("Avg steps ", avg_steps)
println("Time outs ", t_out*100)
# println("avg disc reward ", avg_dr)
# println("avg reward ", avg_r)

#simlist = [Sim(mdp, policy,
#rng=MersenneTwister(i), max_steps=100) for i=1:100];

#res = run_parallel(simlist) do sim, hist
#    return [:steps=>n_steps(hist), :dreward=>discounted_reward(hist), :reward=>undiscounted_reward(hist)]
#end

#n_collisions = sum(res[:reward] .< 0.0)
#avg_steps = mean(res[:steps])
#avg_dreward = mean(res[:dreward])
#avg_reward = mean(res[:reward])

#println("Collisions ", n_collisions)
#println("Avg steps ", avg_steps)
#println("avg disc reward ", avg_dreward)
#println("avg reward ", avg_reward)