text:
# This file is a part of Julia. License is MIT: http://julialang.org/license

"""Mirror callback function

Function sets `+refs/*:refs/*` refspecs and `mirror` flag for remote reference.
"""
function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void},
                         name::Cstring, url::Cstring, payload::Ptr{Void})
    # Create the remote with a mirroring url
    fetch_spec = "+refs/*:refs/*"
    err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint,
                (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Cstring),
                remote, repo_ptr, name, url, fetch_spec)
    err != 0 && return Cint(err)

    # And set the configuration option to true for the push command
    config = GitConfig(GitRepo(repo_ptr,false))
    name_str = unsafe_string(name)
    err= try set!(config, "remote.$name_str.mirror", true)
         catch -1
         finally close(config)
         end
    err != 0 && return Cint(err)
    return Cint(0)
end

function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}},
        username_ptr, schema, host)
    isusedcreds = checkused!(creds)

    errcls, errmsg = Error.last_error()
    if errcls != Error.None
        # Check if we used ssh-agent
        if creds.usesshagent == "U"
            creds.usesshagent = "E" # reported ssh-agent error, disables ssh agent use for the future
        end
    end

    # first try ssh-agent if credentials support its usage
    if creds.usesshagent == "Y" || creds.usesshagent == "U"
        err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint,
                     (Ptr{Ptr{Void}}, Cstring), libgit2credptr, username_ptr)
        creds.usesshagent = "U" # used ssh-agent only one time
        err == 0 && return Cint(0)
    end

    if creds.prompt_if_incorrect
        # if username is not provided, then prompt for it
        username = if username_ptr == Cstring(C_NULL)
            uname = creds.user # check if credentials were already used
            !isusedcreds ? uname : prompt("Username for '$schema$host'", default=uname)
        else
            unsafe_string(username_ptr)
        end
        isempty(username) && return Cint(Error.EAUTH)

        # For SSH we need a private key location
        privatekey = if haskey(ENV,"SSH_KEY_PATH")
            ENV["SSH_KEY_PATH"]
        else
            keydefpath = creds.prvkey # check if credentials were already used
            if isempty(keydefpath) || isusedcreds
                defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa")
                if isempty(keydefpath) && isfile(defaultkeydefpath)
                    keydefpath = defaultkeydefpath
                else
                    keydefpath =
                        prompt("Private key location for '$schema$username@$host'", default=keydefpath)
                end
            end
            keydefpath
        end

        # If the private key changed, invalidate the cached public key
        (privatekey != creds.prvkey) &&
            (creds.pubkey = "")

        # For SSH we need a public key location, look for environment vars SSH_* as well
        publickey = if haskey(ENV,"SSH_PUB_KEY_PATH")
            ENV["SSH_PUB_KEY_PATH"]
        else
            keydefpath = creds.pubkey # check if credentials were already used
            if isempty(keydefpath) || isusedcreds
                if isempty(keydefpath)
                    keydefpath = privatekey*".pub"
                end
                if !isfile(keydefpath)
                    prompt("Public key location for '$schema$username@$host'", default=keydefpath)
                end
            end
            keydefpath
        end

        passphrase_required = true
        if !isfile(privatekey)
            warn("Private key not found")
        else
            # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED"
            open(privatekey) do f
                readline(f)
                passphrase_required = readline(f) == "Proc-Type: 4,ENCRYPTED"
            end
        end

        passphrase = if haskey(ENV,"SSH_KEY_PASS")
            ENV["SSH_KEY_PASS"]
        else
            passdef = creds.pass # check if credentials were already used
            if passphrase_required && (isempty(passdef) || isusedcreds)
                if is_windows()
                    passdef = Base.winprompt(
                        "Your SSH Key requires a password, please enter it now:",
                        "Passphrase required", privatekey; prompt_username = false)
                    isnull(passdef) && return Cint(Error.EAUTH)
                    passdef = Base.get(passdef)[2]
                else
                    passdef = prompt("Passphrase for $privatekey", password=true)
                end
            end
            passdef
        end
        ((creds.user != username) || (creds.pass != passphrase) ||
            (creds.prvkey != privatekey) || (creds.pubkey != publickey)) && reset!(creds)

        creds.user = username # save credentials
        creds.prvkey = privatekey # save credentials
        creds.pubkey = publickey # save credentials
        creds.pass = passphrase
    else
        isusedcreds && return Cint(Error.EAUTH)
    end

    return ccall((:git_cred_ssh_key_new, :libgit2), Cint,
                 (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring),
                 libgit2credptr, creds.user, creds.pubkey, creds.prvkey, creds.pass)
end

function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::Ptr{Ptr{Void}},
        schema, host, urlusername)
    isusedcreds = checkused!(creds)

    if creds.prompt_if_incorrect
        username = creds.user
        userpass = creds.pass
        if is_windows()
            if isempty(username) || isempty(userpass) || isusedcreds
                res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required",
                        isempty(username) ? urlusername : username; prompt_username = true)
                isnull(res) && return Cint(Error.EAUTH)
                username, userpass = Base.get(res)
            end
        elseif isusedcreds
            username = prompt("Username for '$schema$host'",
                default=isempty(username) ? urlusername : username)
            userpass = prompt("Password for '$schema$username@$host'", password=true)
        end
        ((creds.user != username) || (creds.pass != userpass)) && reset!(creds)
        creds.user = username # save credentials
        creds.pass = userpass # save credentials

        isempty(username) && isempty(userpass) && return Cint(Error.EAUTH)
    else
        isusedcreds && return Cint(Error.EAUTH)
    end

    return ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint,
                 (Ptr{Ptr{Void}}, Cstring, Cstring),
                 libgit2credptr, creds.user, creds.pass)
end

"""Credentials callback function

Function provides different credential acquisition functionality w.r.t. a connection protocol.
If a payload is provided then `payload_ptr` should contain a `LibGit2.AbstractCredentials` object.

For `LibGit2.Consts.CREDTYPE_USERPASS_PLAINTEXT` type, if the payload contains fields:
`user` & `pass`, they are used to create authentication credentials.
Empty `user` name and `pass`word trigger an authentication error.

For `LibGit2.Consts.CREDTYPE_SSH_KEY` type, if the payload contains fields:
`user`, `prvkey`, `pubkey` & `pass`, they are used to create authentication credentials.
Empty `user` name triggers an authentication error.

Credentials are checked in the following order (if supported):
- ssh key pair (`ssh-agent` if specified in payload's `usesshagent` field)
- plain text

**Note**: Due to the specifics of the `libgit2` authentication procedure, when
authentication fails, this function is called again without any indication whether
authentication was successful or not. To avoid an infinite loop from repeatedly
using the same faulty credentials, the `checkused!` function can be called. This
function returns `true` if the credentials were used.
Using credentials triggers a user prompt for (re)entering required information.
`UserPasswordCredentials` and `CachedCredentials` are implemented using a call
counting strategy that prevents repeated usage of faulty credentials.
"""
function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring,
                              username_ptr::Cstring,
                              allowed_types::Cuint, payload_ptr::Ptr{Void})
    err = Cint(0)
    url = unsafe_string(url_ptr)

    # parse url for schema and host
    urlparts = match(URL_REGEX, url)
    schema = urlparts[:scheme] === nothing ? "" : urlparts[:scheme] * "://"
    urlusername = urlparts[:user] === nothing ? "" : urlparts[:user]
    host = urlparts[:host]

    # get credentials object from payload pointer
    @assert payload_ptr != C_NULL
    creds = unsafe_pointer_to_objref(payload_ptr)
    explicit = !isnull(creds[]) && !isa(Base.get(creds[]), CachedCredentials)
    # use ssh key or ssh-agent
    if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY))
        sshcreds = get_creds!(creds, "ssh://$host", reset!(SSHCredentials(true), -1))
        if isa(sshcreds, SSHCredentials)
            err = authenticate_ssh(sshcreds, libgit2credptr, username_ptr, schema, host)
            err == 0 && return err
        end
    end

    if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT))
        defaultcreds = reset!(UserPasswordCredentials(true), -1)
        credid = "$schema$host"
        upcreds = get_creds!(creds, credid, defaultcreds)
        # If there were stored SSH credentials, but we ended up here that must
        # mean that something went wrong. Replace the SSH credentials by user/pass
        # credentials
        if !isa(upcreds, UserPasswordCredentials)
            upcreds = defaultcreds
            isa(Base.get(creds[]), CachedCredentials) && (Base.get(creds[]).creds[credid] = upcreds)
        end
        return authenticate_userpass(upcreds, libgit2credptr, schema, host, urlusername)
    end

    # No authentication method we support succeeded. The most likely cause is
    # that explicit credentials were passed in, but said credentials are incompatible
    # with the remote host.
    if err == 0
        if explicit
            warn("The explicitly provided credentials were incompatible with " *
                 "the server's supported authentication methods")
        end
        err = Cint(Error.EAUTH)
    end
    return err
end

function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring,
                        oid_ptr::Ptr{GitHash}, is_merge::Cuint, payload::Ptr{Void})
    fhead_vec = unsafe_pointer_to_objref(payload)::Vector{FetchHead}
    push!(fhead_vec, FetchHead(unsafe_string(ref_name), unsafe_string(remote_url),
        unsafe_load(oid_ptr), is_merge == 1))
    return Cint(0)
end

"C function pointer for `mirror_callback`"
mirror_cb() = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void}))
"C function pointer for `credentials_callback`"
credentials_cb() = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void}))
"C function pointer for `fetchhead_foreach_callback`"
fetchhead_foreach_cb() = cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{GitHash}, Cuint, Ptr{Void}))

max_stars_repo_name: liveinjs/julia

max_issues_repo_name: liveinjs/julia

max_forks_repo_name: liveinjs/julia


text:
"""
throttled(f::Function, timeout::Real)

Return a function that when invoked, will only be triggered at most once
during `timeout` seconds.
The throttled function will run as much as it can, without ever
going more than once per `wait` duration.

This throttle is 'leading' and has some other properties that are specifically designed for our use in Pluto, see the tests.

Inspired by FluxML
See: https://github.com/FluxML/Flux.jl/blob/8afedcd6723112ff611555e350a8c84f4e1ad686/src/utils.jl#L662
"""
function throttled(f::Function, timeout::Real)
    tlock = ReentrantLock()
    iscoolnow = Ref(false)
    run_later = Ref(false)

    function flush()
        lock(tlock) do
            run_later[] = false
            f()
        end
    end

    function schedule()
        @async begin
            sleep(timeout)
            if run_later[]
                flush()
            end
            iscoolnow[] = true
        end
    end
    schedule()

    function throttled_f()
        if iscoolnow[]
            iscoolnow[] = false
            flush()
            schedule()
        else
            run_later[] = true
        end
    end

    return throttled_f, flush
end

ext: jl


text:
module PkgREPL

import ..CondaPkg
import Pkg
import Markdown

### parsing

function parse_pkg(x::String)
    m = match(r"^\s*(([^:]+)::)?([-_.A-Za-z0-9]+)\s*([<>=!0-9].*)?$", x)
    m === nothing && error("invalid conda package: $x")
    channel = m.captures[2]
    name = m.captures[3]
    version = m.captures[4]
    if version === nothing
        version = ""
    end
    if channel === nothing
        channel = ""
    end
    CondaPkg.PkgSpec(name, version=version, channel=channel)
end

function parse_pip_pkg(x::String)
    m = match(r"^\s*([-_.A-Za-z0-9]+)\s*([~!<>=@].*)?$", x)
    m === nothing && error("invalid pip package: $x")
    name = m.captures[1]
    version = m.captures[2]
    if version === nothing
        version = ""
    end
    CondaPkg.PipPkgSpec(name, version=version)
end

function parse_channel(x::String)
    CondaPkg.ChannelSpec(x)
end

### options

const force_opt = Pkg.REPLMode.OptionDeclaration([
    :name => "force",
    :short_name => "f",
    :api => :force => true,
])

### status

function status()
    CondaPkg.status()
end

const status_help = Markdown.parse("""
```
conda st|status
```

Display information about the Conda environment.
""")

const status_spec = Pkg.REPLMode.CommandSpec(
    name = "status",
    short_name = "st",
    api = status,
    help = status_help,
    description = "information about the Conda environment",
)

### resolve

function resolve(; force=false)
    CondaPkg.resolve(force=force, interactive=true)
end

const resolve_help = Markdown.parse("""
```
conda [-f|--force] resolve
```

Ensure all Conda dependencies are installed into the environment.
""")

const resolve_spec = Pkg.REPLMode.CommandSpec(
    name = "resolve",
    api = resolve,
    help = resolve_help,
    description = "ensure all Conda dependencies are installed",
    option_spec = [force_opt],
)

### add

function add(args)
    CondaPkg.add(parse_pkg.(args))
end

const add_help = Markdown.parse("""
```
conda add pkg ...
```

Add packages to the environment.

**Examples**

```
pkg> conda add python
pkg> conda add python>=3.5,<4
pkg> conda add conda-forge::numpy
```
""")

const add_spec = Pkg.REPLMode.CommandSpec(
    name = "add",
    api = add,
    should_splat = false,
    help = add_help,
    description = "add Conda packages",
    arg_count = 0 => Inf,
)

### channel_add

function channel_add(args)
    CondaPkg.add(parse_channel.(args))
end

const channel_add_help = Markdown.parse("""
```
conda channel_add channel ...
```

Add channels to the environment.

**Examples**

```
pkg> conda channel_add conda-forge
```
""")

const channel_add_spec = Pkg.REPLMode.CommandSpec(
    name = "channel_add",
    api = channel_add,
    should_splat = false,
    help = channel_add_help,
    description = "add Conda channels",
    arg_count = 0 => Inf,
)

### pip_add

function pip_add(args)
    CondaPkg.add(parse_pip_pkg.(args))
end

const pip_add_help = Markdown.parse("""
```
conda pip_add pkg ...
```

Add Pip packages to the environment.

**Examples**

```
pkg> conda pip_add build~=0.7
```
""")

const pip_add_spec = Pkg.REPLMode.CommandSpec(
    name = "pip add",
    api = pip_add,
    should_splat = false,
    help = pip_add_help,
    description = "add Pip packages",
    arg_count = 0 => Inf,
)

### rm

function rm(args)
    CondaPkg.rm(parse_pkg.(args))
end

const rm_help = Markdown.parse("""
```
conda rm|remove pkg ...
```

Remove packages from the environment.

**Examples**

```
pkg> conda rm python
```
""")

const rm_spec = Pkg.REPLMode.CommandSpec(
    name = "remove",
    short_name = "rm",
    api = rm,
    should_splat = false,
    help = rm_help,
    description = "remove Conda packages",
    arg_count = 0 => Inf,
)

### channel_rm

function channel_rm(args)
    CondaPkg.rm(parse_channel.(args))
end

const channel_rm_help = Markdown.parse("""
```
conda channel_rm|channel_remove channel ...
```

Remove channels from the environment.

**Examples**

```
pkg> conda channel_rm conda-forge
```
""")

const channel_rm_spec = Pkg.REPLMode.CommandSpec(
    name = "channel_remove",
    short_name = "channel_rm",
    api = channel_rm,
    should_splat = false,
    help = channel_rm_help,
    description = "remove Conda channels",
    arg_count = 0 => Inf,
)

### pip_rm

function pip_rm(args)
    CondaPkg.rm(parse_pip_pkg.(args))
end

const pip_rm_help = Markdown.parse("""
```
conda pip_rm|pip_remove pkg ...
```

Remove Pip packages from the environment.

**Examples**

```
pkg> conda pip_rm build
```
""")

const pip_rm_spec = Pkg.REPLMode.CommandSpec(
    name = "pip_remove",
    short_name = "pip_rm",
    api = pip_rm,
    should_splat = false,
    help = pip_rm_help,
    description = "remove Pip packages",
    arg_count = 0 => Inf,
)

### gc

function gc()
    CondaPkg.gc()
end

const gc_help = Markdown.parse("""
```
conda gc
```

Delete any files no longer used by Conda.
""")

const gc_spec = Pkg.REPLMode.CommandSpec(
    name = "gc",
    api = gc,
    help = gc_help,
    description = "delete files no longer used by Conda",
)

## run

function run(args)
    CondaPkg.withenv() do
        Base.run(Cmd(args))
    end
end

const run_help = Markdown.parse("""
```
conda run cmd ...
```

Run the given command in the Conda environment.
""")

const run_spec = Pkg.REPLMode.CommandSpec(
    name = "run",
    api = run,
    should_splat = false,
    help = run_help,
    arg_count = 1 => Inf,
    description = "run a command in the Conda environment",
)

### all specs

const SPECS = Dict(
    "st" => status_spec,
    "status" => status_spec,
    "resolve" => resolve_spec,
    "add" => add_spec,
    "remove" => rm_spec,
    "rm" => rm_spec,
    "channel_add" => channel_add_spec,
    "channel_remove" => channel_rm_spec,
    "channel_rm" => channel_rm_spec,
    "pip_add" => pip_add_spec,
    "pip_remove" => pip_rm_spec,
    "pip_rm" => pip_rm_spec,
    "gc" => gc_spec,
    "run" => run_spec,
)

function __init__()
    # add the commands to the repl
    Pkg.REPLMode.SPECS["conda"] = SPECS
    # update the help with the new commands
    copy!(Pkg.REPLMode.help.content, Pkg.REPLMode.gen_help().content)
end

end


text:
# All bindings of RTLSDR exported by Clang
module LibRtlsdr
# --- Library call
using librtlsdr_jll
# --- Dependencies
using CEnum

# ----------------------------------------------------
# --- Enumeration
# ----------------------------------------------------
@cenum rtlsdr_tuner::UInt32 begin
    RTLSDR_TUNER_UNKNOWN = 0
    RTLSDR_TUNER_E4000 = 1
    RTLSDR_TUNER_FC0012 = 2
    RTLSDR_TUNER_FC0013 = 3
    RTLSDR_TUNER_FC2580 = 4
    RTLSDR_TUNER_R820T = 5
    RTLSDR_TUNER_R828D = 6
end

# ----------------------------------------------------
# --- Runtime structures
# ----------------------------------------------------
mutable struct rtlsdr_dev end
const rtlsdr_dev_t = rtlsdr_dev

# typedef void ( * rtlsdr_read_async_cb_t ) ( unsigned char * buf , uint32_t len , void * ctx )
const rtlsdr_read_async_cb_t = Ptr{Cvoid}

# ----------------------------------------------------
# --- Functions
# ----------------------------------------------------
function rtlsdr_get_device_count()
    ccall((:rtlsdr_get_device_count, librtlsdr), UInt32, ())
end

function rtlsdr_get_device_name(index)
    ccall((:rtlsdr_get_device_name, librtlsdr), Ptr{Cchar}, (UInt32,), index)
end

function rtlsdr_get_device_usb_strings(index, manufact, product, serial)
    ccall((:rtlsdr_get_device_usb_strings, librtlsdr), Cint, (UInt32, Ptr{Cchar}, Ptr{Cchar}, Ptr{Cchar}), index, manufact, product, serial)
end

function rtlsdr_get_index_by_serial(serial)
    ccall((:rtlsdr_get_index_by_serial, librtlsdr), Cint, (Ptr{Cchar},), serial)
end

function rtlsdr_open(dev, index)
    ccall((:rtlsdr_open, librtlsdr), Cint, (Ptr{Ptr{rtlsdr_dev_t}}, UInt32), dev, index)
end

function rtlsdr_close(dev)
    ccall((:rtlsdr_close, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_xtal_freq(dev, rtl_freq, tuner_freq)
    ccall((:rtlsdr_set_xtal_freq, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, UInt32, UInt32), dev, rtl_freq, tuner_freq)
end

function rtlsdr_get_xtal_freq(dev, rtl_freq, tuner_freq)
    ccall((:rtlsdr_get_xtal_freq, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{UInt32}, Ptr{UInt32}), dev, rtl_freq, tuner_freq)
end

function rtlsdr_get_usb_strings(dev, manufact, product, serial)
    ccall((:rtlsdr_get_usb_strings, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{Cchar}, Ptr{Cchar}, Ptr{Cchar}), dev, manufact, product, serial)
end

function rtlsdr_write_eeprom(dev, data, offset, len)
    ccall((:rtlsdr_write_eeprom, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{UInt8}, UInt8, UInt16), dev, data, offset, len)
end

function rtlsdr_read_eeprom(dev, data, offset, len)
    ccall((:rtlsdr_read_eeprom, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{UInt8}, UInt8, UInt16), dev, data, offset, len)
end

function rtlsdr_set_center_freq(dev, freq)
    ccall((:rtlsdr_set_center_freq, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, UInt32), dev, freq)
end

function rtlsdr_get_center_freq(dev)
    ccall((:rtlsdr_get_center_freq, librtlsdr), UInt32, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_freq_correction(dev, ppm)
    ccall((:rtlsdr_set_freq_correction, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, ppm)
end

function rtlsdr_get_freq_correction(dev)
    ccall((:rtlsdr_get_freq_correction, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_get_tuner_type(dev)
    ccall((:rtlsdr_get_tuner_type, librtlsdr), rtlsdr_tuner, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_get_tuner_gains(dev, gains)
    ccall((:rtlsdr_get_tuner_gains, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{Cint}), dev, gains)
end

function rtlsdr_set_tuner_gain(dev, gain)
    ccall((:rtlsdr_set_tuner_gain, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, gain)
end

function rtlsdr_set_tuner_bandwidth(dev, bw)
    ccall((:rtlsdr_set_tuner_bandwidth, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, UInt32), dev, bw)
end

function rtlsdr_get_tuner_gain(dev)
    ccall((:rtlsdr_get_tuner_gain, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_tuner_if_gain(dev, stage, gain)
    ccall((:rtlsdr_set_tuner_if_gain, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint, Cint), dev, stage, gain)
end

function rtlsdr_set_tuner_gain_mode(dev, manual)
    ccall((:rtlsdr_set_tuner_gain_mode, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, manual)
end

function rtlsdr_set_sample_rate(dev, rate)
    ccall((:rtlsdr_set_sample_rate, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, UInt32), dev, rate)
end

function rtlsdr_get_sample_rate(dev)
    ccall((:rtlsdr_get_sample_rate, librtlsdr), UInt32, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_testmode(dev, on)
    ccall((:rtlsdr_set_testmode, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, on)
end

function rtlsdr_set_agc_mode(dev, on)
    ccall((:rtlsdr_set_agc_mode, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, on)
end

function rtlsdr_set_direct_sampling(dev, on)
    ccall((:rtlsdr_set_direct_sampling, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, on)
end

function rtlsdr_get_direct_sampling(dev)
    ccall((:rtlsdr_get_direct_sampling, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_offset_tuning(dev, on)
    ccall((:rtlsdr_set_offset_tuning, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, on)
end

function rtlsdr_get_offset_tuning(dev)
    ccall((:rtlsdr_get_offset_tuning, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_reset_buffer(dev)
    ccall((:rtlsdr_reset_buffer, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_read_sync(dev, buf, len, n_read)
    ccall((:rtlsdr_read_sync, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Ptr{Cvoid}, Cint, Ptr{Cint}), dev, buf, len, n_read)
end

function rtlsdr_wait_async(dev, cb, ctx)
    ccall((:rtlsdr_wait_async, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, rtlsdr_read_async_cb_t, Ptr{Cvoid}), dev, cb, ctx)
end

function rtlsdr_read_async(dev, cb, ctx, buf_num, buf_len)
    ccall((:rtlsdr_read_async, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, rtlsdr_read_async_cb_t, Ptr{Cvoid}, UInt32, UInt32), dev, cb, ctx, buf_num, buf_len)
end

function rtlsdr_cancel_async(dev)
    ccall((:rtlsdr_cancel_async, librtlsdr), Cint, (Ptr{rtlsdr_dev_t},), dev)
end

function rtlsdr_set_bias_tee(dev, on)
    ccall((:rtlsdr_set_bias_tee, librtlsdr), Cint, (Ptr{rtlsdr_dev_t}, Cint), dev, on)
end

# exports
const PREFIXES = ["rtlsdr_"]
for name in names(@__MODULE__; all=true), prefix in PREFIXES
    if startswith(string(name), prefix)
        @eval export $name
    end
end

end


text:
using HPAT

function session_split(A::Vector{Tuple{Int64,Int64}}, timeout)
    out = Vector{Int64}[]
    if length(A)==0 return out end

    sort!(A)
    start_time = A[1][1]

    session = Int64[]
    for (a,b) in A
        if b-start_time>timeout
            push!(out, session)
            start_time = b
            session = Int64[]
        end
        push!(session,a)
    end
    push!(out, session)
    return out
end

function removeItem(S::Vector{Vector{Int64}}, a)
    out = Vector{Int64}[]
    for arr in S
        while findfirst(arr, a)!=0
            deletat!(arr, findfirst(arr, a))
        end
        push!(out, arr)
    end
    return out
end

@acc hpat function q02(my_item, timeout, num_top_items, file_name)

    web_clickstreams = DataSource(DataTable{:wcs_user_sk=Int64,:wcs_item_sk=Int64,wcs_click_time_sk=Int64,wcs_click_date_sk=Int64}, HDF5, file_name)
    web_clickstreams[:tstamp_inSec] = web_clickstreams[:wcs_click_date_sk]*24*60*60 .+ web_clickstreams[:wcs_click_time_sk]

    user_clicks = aggregate(web_clickstreams, :wcs_user_sk, collect(:tstamp_inSec),collect(:wcs_item_sk))
    map!(sort!, user_clicks)
    session_clicks = map(x->session_split(x,timeout), user_clicks)
    map!(session-> filter!(x->(my_item in x), session), session_clicks)
    map!(x->removeItem(my_item, x), session_clicks)

    all_items = flatten(session_clicks)
    item_counts = collect(countmap(all_items))
    sort!(item_counts, by=a->a[2])
    return item_counts[1:num_top_items]
end

println(q02(11, 60*60, 30, "data.hdf5"))


text:
# Implement number base conversion

function all_your_base(digits, input_base, output_base)
    # error checking behavior
    if input_base < 2
        throw(DomainError(input_base, "input base must be >= 2"))
    end

    if output_base < 2
        throw(DomainError(output_base, "output base must be >= 2"))
    end

    # short-circuit for [0] input; doesn't matter the base
    if isempty(digits)
        return [0]
    end

    # encode the list to a single number
    total = 0
    for i in digits
        if i >= input_base || i < 0
            throw(DomainError(input_base, "all digits must satisfy 0 <= d < input base"))
        end
        total = total * input_base + i
    end

    # decode to desired base
    output = Int[]
    while total > 0
        (total, rem) = divrem(total, output_base)
        push!(output, rem)
    end

    if isempty(output)
        return [0]
    end

    return reverse!(output)
end


text:
using Test
using LinearAlgebra
import GPLinearODEMaker
GLOM = GPLinearODEMaker

println("Testing...")

@testset "cholesky factorizations" begin
    A = [4. 12 -16; 12 37 -43; -16 -43 98]
    chol_A = GLOM.ridge_chol(A)
    @test chol_A==cholesky(A)==GLOM.symmetric_A(A, chol=true)
    @test chol_A.L==LowerTriangular([2. 0 0;6 1 0;-8 5 3])
    A = [4. 12 -16; 12 37 -43; -16 -43 88]
    @test_logs (:warn, "added a ridge")  GLOM.ridge_chol(A)
    println()
end

@testset "nlogL derivatives" begin
    kernel, n_kern_hyper = include("../src/kernels/se_kernel.jl")

    n = 100
    xs = 20 .* sort(rand(n))
    y1 = sin.(xs) .+ 0.1 .* randn(n)
    noise1 = 0.1 .* ones(n)
    y2 = cos.(xs) .+ 0.2 .* randn(n)
    noise2 = 0.2 .* ones(n)

    ys = collect(Iterators.flatten(zip(y1, y2)))
    noise = collect(Iterators.flatten(zip(noise1, noise2)))

    prob_def = GLOM.GLO(kernel, n_kern_hyper, 2, 2, xs, ys; noise = noise, a0=[[1. 0.1];[0.1 1]])

    @test GLOM.est_dΣdθ(prob_def, 1 .+ rand(n_kern_hyper); return_bool=true, print_stuff=false)
    @test GLOM.test_∇nlogL_GLOM(prob_def, 1 .+ rand(n_kern_hyper), print_stuff=false)
    @test GLOM.test_∇∇nlogL_GLOM(prob_def, 1 .+ rand(n_kern_hyper), print_stuff=false)
    println()
end


text:
using Polyopt
using Base.Test

# Simple convex quadratic problem with two variables in different cliques
let
    println("BSOS test 1")
    x = variables(["x1", "x2"])
    f = x[1] - x[2]
    g = [ 1.0-x[1]^2, 1-x[2]^2 ]

    I = Array{Int,1}[ [1], [2] ]
    prob = bsosprob_chordal(1, 1, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# P4_2 from Weisser's paper
let
    println("BSOS test 2")
    x = variables(["x1", "x2", "x3", "x4"])
    f = x[1]^2 - x[2]^2 + x[3]^2 - x[4]^2 + x[1] - x[2]
    g = [ 2*x[1]^2 + 3*x[2]^2 + 2*x[1]*x[2] + 2*x[3]^2 + 3*x[4]^2 + 2*x[3]*x[4],
          3*x[1]^2 + 2*x[2]^2 - 4*x[1]*x[2] + 3*x[3]^2 + 2*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 6*x[2]^2 - 4*x[1]*x[2] + x[3]^2 + 6*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 4*x[2]^2 - 3*x[1]*x[2] + x[3]^2 + 4*x[4]^2 - 3*x[3]*x[4],
          2*x[1]^2 + 5*x[2]^2 + 3*x[1]*x[2] + 2*x[3]^2 + 5*x[4]^2 + 3*x[3]*x[4],
          x[1],x[2],x[3],x[4]]

    I = Array{Int,1}[ [1,2,3,4] ]
    prob = bsosprob_chordal(1, 1, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# P4_4 from Weisser's paper (we cannot reproduce bounds)
let
    println("BSOS test 3")
    x = variables(["x1", "x2", "x3", "x4"])
    f = x[1]^4 - x[2]^4 + x[3]^4 - x[4]^4
    g = [ 2*x[1]^4 + 3*x[2]^2 + 2*x[1]*x[2] + 2*x[3]^4 + 3*x[4]^2 + 2*x[3]*x[4],
          3*x[1]^2 + 2*x[2]^2 - 4*x[1]*x[2] + 3*x[3]^2 + 2*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 6*x[2]^2 - 4*x[1]*x[2] + x[3]^2 + 6*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 4*x[2]^4 - 3*x[1]*x[2] + x[3]^2 + 4*x[4]^4 - 3*x[3]*x[4],
          2*x[1]^2 + 5*x[2]^2 + 3*x[1]*x[2] + 2*x[3]^2 + 5*x[4]^2 + 3*x[3]*x[4],
          x[1], x[2], x[3], x[4] ]

    I = Array{Int,1}[ [1,2,3,4] ]
    prob = bsosprob_chordal(2, 2, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# P4_6 from Weisser's paper
let
    println("BSOS test 4")
    x = variables(["x1", "x2", "x3", "x4"])
    f = x[1]^4*x[2]^2 + x[1]^2*x[2]^4 - x[1]^2*x[2]^2 + x[3]^4*x[4]^2 + x[3]^2*x[4]^4 - x[3]^2*x[4]^2
    g = [ x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2,
          3*x[1]^2 + 2*x[2]^2 - 4*x[1]*x[2] + 3*x[3]^2 + 2*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 6*x[2]^4 - 8*x[1]*x[2] + x[3]^2 + 6*x[4]^4 - 8*x[3]*x[4] + 2.5,
          x[1]^4 + 3*x[2]^4 + x[3]^4 + 3*x[4]^4,
          x[1]^2 + x[2]^3 + x[3]^2 + x[4]^4,
          x[1], x[2], x[3], x[4] ]

    I = Array{Int,1}[ [1,2,3,4] ]
    prob = bsosprob_chordal(3, 3, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# P4_8 from Weisser's paper
let
    println("BSOS test 5")
    x = variables(["x1", "x2", "x3", "x4"])
    f = x[1]^4*x[2]^2 + x[1]^2*x[2]^6 - x[1]^2*x[2]^2 + x[3]^4*x[4]^2 + x[3]^2*x[4]^6 - x[3]^2*x[4]^2
    g = [ x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2,
          3*x[1]^2 + 2*x[2]^2 - 4*x[1]*x[2] + 3*x[3]^2 + 2*x[4]^2 - 4*x[3]*x[4],
          x[1]^2 + 6*x[2]^4 - 8*x[1]*x[2] + x[3]^2 + 6*x[4]^4 - 8*x[3]*x[4] + 2.5,
          x[1]^4 + 3*x[2]^4 + x[3]^4 + 3*x[4]^4,
          x[1]^2 + x[2]^3 + x[3]^2 + x[4]^4,
          x[1], x[2], x[3], x[4] ]

    I = Array{Int,1}[ [1,2,3,4] ]
    prob = bsosprob_chordal(3, 4, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# Haverly1 from Marandi's paper
let
    println("BSOS test 6")
    x = variables(["x1", "x2", "x3", "x4", "x5"])
    f = -200*x[2]*(15*x[1]-12) - 200*x[3]*(15*x[1]-6) + 200*x[4] - 1000*x[5]
    g = [-3//4*(x[1]-1)*(x[2]+x[3]),
          1//4*(3*x[1]-1)*(x[2]+x[3]),
          1 - 2*(x[2]+x[4]),
          1 - (x[3]+x[5]),
          1//2*(x[4]+x[2]) - 2//5*x[4] - 3//5*x[1]*x[2],
          1//2*(x[5]+x[3]) - 2//3*x[5] - x[1]*x[3],
          x[1], x[2], x[3], x[4], x[5] ]

    #I = Array{Int,1}[ [1,2,3,4,5] ]
    I = Polyopt.chordal_embedding(Polyopt.correlative_sparsity(f,g))

    prob = bsosprob_chordal(3, 2, I, f, g)
    X, t, l, y, solsta = solve_mosek(prob)
end

# Generalized Rosenbrock function
let
    println("BSOS test 7")
    n = 100
    x = variables("x", n)

    I = Array{Int,1}[]
    for i=1:n-1
        push!(I, [i, i+1])
    end

    f = sum([ 100*(x[i]-x[i-1]^2)^2 + (1-x[i])^2 for i=2:n ])
    g = vcat(Polyopt.Poly{Int}[x[i] for i=1:n], Polyopt.Poly{Int}[2 - sum([xk^2 for xk=x[Ik]]) for Ik in I])

    prob = bsosprob_chordal(3, 2, I, f, g);
    X, t, l, y, solsta = solve_mosek(prob);
    t
end


text:
export PDB,
    PDBParseError,
    downloadpdb,
    parseatomrecord,
    spaceatomname,
    pdbline,
    writepdb,
    writepdblines

"Protein Data Bank (PDB) file format."
immutable PDB <: FileFormat end

"Error arising from parsing a PDB file."
type PDBParseError <: Exception
    message::String
    line_number::Int
    line::String
end

Base.showerror(io::IO, e::PDBParseError) = println(io, e.message, " at line ", e.line_number, " of file:\n", e.line)

"""
Download a PDB file or biological assembly from the RCSB PDB. By default
downloads the PDB file; if the keyword argument `ba_number` is set the
biological assembly with that number will be downloaded.
"""
function downloadpdb(pdbid::AbstractString, out_filepath::AbstractString="$pdbid.pdb"; ba_number::Integer=0)
    # Check PDB ID is 4 characters long and only consits of alphanumeric characters
    @assert length(pdbid) == 4 && !ismatch(r"[^a-zA-Z0-9]", pdbid) "Not a valid PDB ID: \"$pdbid\""
    if ba_number == 0
        download("http://www.rcsb.org/pdb/files/$pdbid.pdb", out_filepath)
    else
        # Currently will download error page if ba_number is too high
        download("http://www.rcsb.org/pdb/files/$pdbid.pdb$ba_number", out_filepath)
    end
end

function Base.read(input::IO,
            ::Type{PDB},
            selector_functions::Function...;
            structure_name::AbstractString="",
            remove_disorder::Bool=false,
            read_std_atoms::Bool=true,
            read_het_atoms::Bool=true)
    # Dictionary of model numbers and raw atom lists
    atom_lists = Dict(1 => Atom[])
    # Entries outside of a MODEL/ENDMDL block are added to model 1
    curr_model = 1
    line_number = 0
    for line in eachline(input)
        line_number += 1
        # Read ATOM and HETATM records as required
        if (read_std_atoms && startswith(line, "ATOM  ")) || (read_het_atoms && startswith(line, "HETATM"))
            push!(atom_lists[curr_model], parseatomrecord(rstrip(line, '\n'), line_number))
        # Read MODEL record
        elseif startswith(line, "MODEL ")
            try
                curr_model = parse(Int, line[11:14])
            catch
                throw(PDBParseError("Could not read model serial number", line_number, line))
            end
            # Create model if required
            if !haskey(atom_lists, curr_model)
                atom_lists[curr_model] = AbstractAtom[]
            end
        # Read ENDMDL record
        elseif startswith(line, "ENDMDL")
            curr_model = 1
        end
    end
    # Remove model 1 atom list if it was not added to
    length(atom_lists[1]) == 0 ? delete!(atom_lists, 1) : nothing
    # Form disordered atom containers or remove atoms depending on remove_disorder
    # Apply selectors at this point (so e.g. disorderselector work correctly)
    # Form structure by organising each atom list into a model
    return organisestructure(
        [organisemodel(applyselectors(formatomlist(atom_list; remove_disorder=remove_disorder), selector_functions...); model_number=model_number) for (model_number, atom_list) in atom_lists]; structure_name=structure_name
    )
end

function Base.read(filepath::AbstractString,
            ::Type{PDB},
            selector_functions::Function...;
            structure_name::AbstractString=splitdir(filepath)[2],
            kwargs...)
    open(filepath, "r") do input
        read(input, PDB, selector_functions...; structure_name=structure_name, kwargs...)
    end
end

"Parse a PDB ATOM or HETATM record and return an `Atom`."
function parseatomrecord(line::String, line_number::Integer=1)
    @assert startswith(line, "ATOM  ") || startswith(line, "HETATM") "Line does not appear to be an ATOM/HETATM record: \"$line\""
    return Atom(
        line[1:6] == "HETATM",
        parsestrict(line, (7,11), Int, "Could not read atom serial number", line_number),
        strip(parsestrict(line, (13,16), String, "Could not read atom name", line_number)),
        parsestrict(line, (17,17), Char, "Could not read alt loc identifier", line_number),
        strip(parsestrict(line, (18,20), String, "Could not read residue name", line_number)),
        parsestrict(line, (22,22), Char, "Could not read chain ID", line_number),
        parsestrict(line, (23,26), Int, "Could not read residue number", line_number),
        parsestrict(line, (27,27), Char, "Could not read insertion code", line_number),
        [
            parsestrict(line, (31,38), Float64, "Could not read x coordinate", line_number),
            parsestrict(line, (39,46), Float64, "Could not read y coordinate", line_number),
            parsestrict(line, (47,54), Float64, "Could not read z coordinate", line_number)
        ],
        parselenient(line, (55,60), Float64, 1.0),
        parselenient(line, (61,66), Float64, 0.0),
        strip(parselenient(line, (77,78), String, "")),
        strip(parselenient(line, (79,80), String, ""))
    )
end

"Parse columns from a line and return the value or throw a `PDBParseError`."
function parsestrict(line::String,
                    cols::Tuple{Integer, Integer},
                    out_type::Type,
                    error_message::AbstractString,
                    line_number::Integer)
    try
        return parsevalue(line, cols, out_type)
    catch
        throw(PDBParseError(error_message, line_number, line))
    end
end

"Parse columns from a line and return the value or a default value."
function parselenient(line::String,
                    cols::Tuple{Integer, Integer},
                    out_type::Type,
                    default)
    try
        return parsevalue(line, cols, out_type)
    catch
        return default
    end
end

"Parse columns from a line."
function parsevalue(line::String, cols::Tuple{Integer, Integer}, out_type::Type)
    try
        if out_type == Int
            return parse(Int, line[cols[1]:cols[2]])
        elseif out_type == Float64
            return parse(Float64, line[cols[1]:cols[2]])
        elseif out_type == String
            return line[cols[1]:cols[2]]
        elseif out_type == Char
            return line[cols[1]]
        else
            error()
        end
    catch
        error("Could not parse to desired type")
    end
end

"""
Form a string of a certain length from a value by adding spaces to the left.
Throws an error if the value is too long.
"""
function spacestring(val_in, new_length::Integer)
    string_out = string(val_in)
    @assert length(string_out) <= new_length "Cannot fit value \"$string_out\" into $new_length space(s)"
    return lpad(string_out, new_length)
end

"""
Space an `Atom` name such that the last element letter (generally) appears in
the second column. If the `element` property of the `Atom` is set it is used to
get the element, otherwise the name starts from the second column where
possible.
"""
function spaceatomname(atom::Atom)
    atom_name = atomname(atom)
    chars = length(atom_name)
    @assert chars <= 4 "Atom name is greater than four characters: \"$atom_name\""
    # In the absence of the element, the first index goes in column two
    if element(atom) == "" || findfirst(atom_name, element(atom)[1]) == 0
        cent_ind = 1
    # The last letter of the element goes in column two where possible
    else
        cent_ind = findfirst(atom_name, element(atom)[1]) + length(element(atom)) - 1
    end
    @assert cent_ind <= 2 "Atom name is too long to space correctly: \"$atom_name\""
    if cent_ind == 1 && chars < 4
        out_string = " $atom_name"
    else
        out_string = "$atom_name"
    end
    return rpad(out_string, 4)
end

"Form a Protein Data Bank (PDB) format ATOM or HETATM record from an `Atom`."
pdbline(atom::Atom) = String[
        ishetatom(atom) ? "HETATM" : "ATOM  ",
        spacestring(serial(atom), 5),
        " ",
        spaceatomname(atom),
        string(altlocid(atom)),
        spacestring(resname(atom), 3),
        " ",
        string(chainid(atom)),
        spacestring(resnumber(atom), 4),
        string(inscode(atom)),
        "   ",
        # This will throw an error for large coordinate values, e.g. -1000.123
        spacestring(round(x(atom), 3), 8),
        spacestring(round(y(atom), 3), 8),
        spacestring(round(z(atom), 3), 8),
        spacestring(round(occupancy(atom), 2), 6),
        # This will throw an error for large temp facs, e.g. 1000.12
        spacestring(round(tempfac(atom), 2), 6),
        "          ",
        spacestring(element(atom), 2),
        spacestring(charge(atom), 2),
    ]

"""
Write a `StructuralElementOrList` to a PDB format file. Only ATOM, HETATM, MODEL
and ENDMDL records are written - there is no header and no TER records.
Additional arguments are `selector_functions...` - only atoms that satisfy the
selector functions are written.
"""
function writepdb(output::IO, element::Union{ProteinStructure, Vector{Model}}, selector_functions::Function...)
    # If there are multiple models, write out MODEL/ENDMDL lines
    if length(element) > 1
        for model in element
            println(output, "MODEL     ", spacestring(modelnumber(model), 4), repeat(" ", 66))
            writepdblines(output, model, selector_functions...)
            println(output, "ENDMDL$(repeat(" ", 74))")
        end
    # If there is only one model, do not write out MODEL/ENDMDL lines
    else
        writepdblines(output, element, selector_functions...)
    end
end

writepdb(output::IO, element::StructuralElementOrList, selector_functions::Function...) = writepdblines(output, element, selector_functions...)

function writepdb(filepath::AbstractString, element::StructuralElementOrList, selector_functions::Function...)
    open(filepath, "w") do output
        writepdb(output, element, selector_functions...)
    end
end

"""
Write a `StructuralElementOrList` to an output as lines in PDB format.
Additional arguments are `selector_functions...` - only atoms that satisfy the
selector functions are written.
"""
function writepdblines(output::IO, element::StructuralElementOrList, selector_functions::Function...)
    # Collect residues then expand out disordered residues and atoms
    for res in collectresidues(element)
        if isa(res, Residue)
            for atom in collectatoms(res, selector_functions...), atom_record in atom
                println(output, pdbline(atom_record)...)
            end
        else
            for res_name in resnames(res), atom in collectatoms(disorderedres(res, res_name), selector_functions...), atom_record in atom
                println(output, pdbline(atom_record)...)
            end
        end
    end
end


text:
using KNITRO, Compat.Test

 ## Solve test problem 1 (Synthesis of processing system) in
 #  M. Duran & I.E. Grossmann, "An outer approximation algorithm for
 #  a class of mixed integer nonlinear programs", Mathematical
 #  Programming 36, pp. 307-339, 1986.  The problem also appears as
 #  problem synthes1 in the MacMINLP test set.
 #
 #  min   5 x4 + 6 x5 + 8 x6 + 10 x1 - 7 x3 -18 math.log(x2 + 1)
 #       - 19.2 math.log(x1 - x2 + 1) + 10
 #  s.t.  0.8 math.log(x2 + 1) + 0.96 math.log(x1 - x2 + 1) - 0.8 x3 >= 0
 #        math.log(x2 + 1) + 1.2 math.log(x1 - x2 + 1) - x3 - 2 x6 >= -2
 #        x2 - x1 <= 0
 #        x2 - 2 x4 <= 0
 #        x1 - x2 - 2 x5 <= 0
 #        x4 + x5 <= 1
 #        0 <= x1 <= 2
 #        0 <= x2 <= 2
 #        0 <= x3 <= 1
 #        x1, x2, x3 continuous
 #        x4, x5, x6 binary
 #
 #
 #  The solution is (1.30098, 0, 1, 0, 1, 0).
 ##

function eval_f(x::Vector{Float64})
    tmp1 = x[1] - x[2] + 1.0
    tmp2 = x[2] + 1.0
    linear_terms = 10.0 + 10.0*x[1] - 7.0*x[3] + 5.0*x[4] + 6.0*x[5] + 8.0*x[6]
    nonlinear_terms = - 18.0*log(tmp2) - 19.2*log(tmp1)
    linear_terms + nonlinear_terms
end

function eval_g(x::Vector{Float64}, cons::Vector{Float64})
    tmp1 = x[1] - x[2] + 1.0
    tmp2 = x[2] + 1.0
    cons[1] = 0.8*log(tmp2) + 0.96*log(tmp1) - 0.8*x[3]
    cons[2] = log(tmp2) + 1.2*log(tmp1) - x[3] - 2*x[6]
    cons[3] = x[2] - x[1]
    cons[4] = x[2] - 2*x[4]
    cons[5] = x[1] - x[2] - 2*x[5]
    cons[6] = x[4] + x[5]
end

function eval_grad_f(x::Vector{Float64}, grad::Vector{Float64})
    tmp1 = x[1] - x[2] + 1.0
    tmp2 = x[2] + 1.0
    grad[1] = 10.0 - (19.2 / tmp1)
    grad[2] = (-18.0 / tmp2) + (19.2 / tmp1)
    grad[3] = -7.0
    grad[4] = 5.0
    grad[5] = 6.0
    grad[6] = 8.0
end

function eval_jac_g(x::Vector{Float64}, jac::Vector{Float64})
    tmp1 = x[1] - x[2] + 1.0
    tmp2 = x[2] + 1.0
    #---- GRADIENT OF CONSTRAINT 0.
    jac[1] = 0.96 / tmp1
    jac[2] = (-0.96 / tmp1) + (0.8 / tmp2)
    jac[3] = -0.8
    #---- GRADIENT OF CONSTRAINT 1.
    jac[4] = 1.2 / tmp1
    jac[5] = (-1.2 / tmp1) + (1.0 / tmp2)
    jac[6] = -1.0
    jac[7] = -2.0
    #---- GRADIENT OF CONSTRAINT 2.
    jac[8] = -1.0
    jac[9] = 1.0
    #---- GRADIENT OF CONSTRAINT 3.
    jac[10] = 1.0
    jac[11] = -2.0
    #---- GRADIENT OF CONSTRAINT 4.
    jac[12] = 1.0
    jac[13] = -1.0
    jac[14] = -2.0
    #---- GRADIENT OF CONSTRAINT 5.
    jac[15] = 1.0
    jac[16] = 1.0
end

function eval_h(x::Vector{Float64}, lambda::Vector{Float64},
                sigma::Float64, hess::Vector{Float64})
    tmp1 = x[1] - x[2] + 1.0
    tmp2 = x[2] + 1.0
    hess[1] = sigma*(19.2 / (tmp1*tmp1)) + lambda[1]*(-0.96 / (tmp1*tmp1)) + lambda[2]*(-1.2 / (tmp1*tmp1))
    hess[2] = sigma*(-19.2 / (tmp1*tmp1)) + lambda[1]*(0.96 / (tmp1*tmp1)) + lambda[2]*(1.2 / (tmp1*tmp1))
    hess[3] = sigma*((19.2 / (tmp1*tmp1)) + (18.0 / (tmp2*tmp2))) + lambda[1]*((-0.96 / (tmp1*tmp1)) - (0.8 / (tmp2*tmp2))) + lambda[2]*((-1.2 / (tmp1*tmp1)) - (1.0 / (tmp2*tmp2)))
end

function eval_hv(x::Vector{Float64}, lambda::Vector{Float64},
                 sigma::Float64, hv::Vector{Float64})
    tmp1 = (x[1] - x[2] + 1.0)^2
    tmp2 = x[2] + 1.0

    hv1 = (sigma*(19.2/tmp1)+lambda[1]*(-0.96/tmp1)+lambda[2]*(-1.2/tmp1))*hv[1]
    hv1 = hv1 + (sigma*(-19.2/tmp1)+lambda[1]*(0.96/tmp1)+lambda[2]*(1.2/tmp1))*hv[2]
    hv2 = (sigma*(-19.2/tmp1)+lambda[1]*(0.96/tmp1)+lambda[2]*(1.2/tmp1))*hv[1]
    hv2 = hv2 + (sigma*((19.2/tmp1)+(18.0/tmp2))+lambda[1]*((-0.96/tmp1)-(0.8/tmp2))+lambda[2]*((-1.2/tmp1)-(1.0/tmp2)))*hv[2]
    hv[1] = hv1
    hv[2] = hv2
    hv[3] = 0.0
    hv[4] = 0.0
    hv[5] = 0.0
    hv[6] = 0.0
end

function eval_mip_node(kp::KnitroProblem, obj::Float64)
    # Print information about the status of the MIP solution
    println("callbackProcessNode:")
    println("    Node number    = ", get_mip_num_nodes(kp))
    println("    Node objective = ", obj)
    println("    Current relaxation bound = ", get_mip_relaxation_bnd(kp))
    bound = get_mip_incumbent_obj(kp)
    if abs(bound) >= KTR_INFBOUND
        println("    No integer feasible point found yet.")
    else
        println("    Current incumbent bound  = ", bound)
        println("    Absolute integrality gap = ", get_mip_abs_gap(kp))
        println("    Relative integrality gap = ", get_mip_rel_gap(kp))
    end
end

objType = KTR_OBJTYPE_GENERAL
objGoal = KTR_OBJGOAL_MINIMIZE
objFnType = KTR_FNTYPE_CONVEX

n = 6
x_L = zeros(n)
x_U = [2.0,2.0,1.0,1.0,1.0,1.0]

m = 6
c_Type = [KTR_CONTYPE_GENERAL,
          KTR_CONTYPE_GENERAL,
          KTR_CONTYPE_LINEAR,
          KTR_CONTYPE_LINEAR,
          KTR_CONTYPE_LINEAR,
          KTR_CONTYPE_LINEAR]
c_FnType = [KTR_FNTYPE_CONVEX,
           KTR_FNTYPE_CONVEX,
           KTR_FNTYPE_CONVEX,
           KTR_FNTYPE_CONVEX,
           KTR_FNTYPE_CONVEX,
           KTR_FNTYPE_CONVEX]
c_L = [0.0,-2.0,
       -KTR_INFBOUND,
       -KTR_INFBOUND,
       -KTR_INFBOUND,
       -KTR_INFBOUND]
c_U = [KTR_INFBOUND,
       KTR_INFBOUND,
       0.0,0.0,0.0,1.0]

jac_con = Int32[0,0,0,1,1,1,1,2,2,3,3,4,4,4,5,5]
jac_var = Int32[0,1,2,0,1,2,5,0,1,1,3,0,1,4,3,4]
hess_row = Int32[0,0,1]
hess_col = Int32[0,1,1]

x_Type = [KTR_VARTYPE_CONTINUOUS,
          KTR_VARTYPE_CONTINUOUS,
          KTR_VARTYPE_CONTINUOUS,
          KTR_VARTYPE_BINARY,
          KTR_VARTYPE_BINARY,
          KTR_VARTYPE_BINARY]

kp = createProblem()
@test applicationReturnStatus(kp) == :Uninitialized

# ------ Illustrate how to override default options ------
# --- (options must be set before calling init_problem) ---
setOption(kp, "mip_method", KTR_MIP_METHOD_BB)
setOption(kp, "algorithm", KTR_ALG_ACT_CG)
setOption(kp, "outmode", KTR_OUTMODE_SCREEN)
setOption(kp, KTR_PARAM_OUTLEV, KTR_OUTLEV_ALL)
setOption(kp, KTR_PARAM_MIP_OUTINTERVAL, 1)
setOption(kp, KTR_PARAM_MIP_MAXNODES, 10000)
# specify that user is able to provide evaluations of the
# hessian matrix without the objective component
# (turned off by default, but should be enabled if possible)
setOption(kp, KTR_PARAM_HESSIAN_NO_F, KTR_HESSIAN_NO_F_ALLOW)
@test applicationReturnStatus(kp) == :Uninitialized

# --- set callback functions ---
setCallbacks(kp, eval_f, eval_g, eval_grad_f, eval_jac_g, eval_h, eval_hv)
@test applicationReturnStatus(kp) == :Uninitialized
setMIPCallback(kp, eval_mip_node)
@test applicationReturnStatus(kp) == :Uninitialized

initializeProblem(kp, objGoal, objType, objFnType,
                  x_Type, x_L, x_U, c_Type, c_FnType, c_L, c_U,
                  jac_var, jac_con, hess_row, hess_col)
@test applicationReturnStatus(kp) == :Initialized
solveProblem(kp)

# --- test optimal solutions ---
@testset "Test optimal solutions" begin
    @test applicationReturnStatus(kp) == :Optimal
    @test isapprox(kp.x, [1.30098, 0.0, 1.0, 0.0, 1.0, 0.0], atol=1e-5)
end

freeProblem(kp)


text:
module Repeaters

using ReduxImGui.Redux
using ReduxImGui.CImGui
using ReduxImGui.ArrowButtons

# actions
abstract type AbstractRepeaterAction <: AbstractSyncAction end

"""
    Increment(id)
Increment counter.
"""
struct Increment <: AbstractRepeaterAction
    id::String
end

"""
    Decrement(id)
Decrement counter.
"""
struct Decrement <: AbstractRepeaterAction
    id::String
end

"""
    ChangeArrowToUpDown(id)
Change arrow direction to up-down.
"""
struct ChangeArrowToUpDown <: AbstractRepeaterAction
    id::String
end

# state
struct State <: AbstractImmutableState
    id::String
    counter::Int
    button1::ArrowButtons.State
    button2::ArrowButtons.State
end
State(id::AbstractString, counter) = State(
    id,
    counter,
    ArrowButtons.State("##left", CImGui.ImGuiDir_Left),
    ArrowButtons.State("##right", CImGui.ImGuiDir_Right),
)
State(id::AbstractString) = State(id, 0)

# reducers
repeater(state::AbstractState, action::AbstractAction) = state
repeater(state::Vector{<:AbstractState}, action::AbstractAction) = state
repeater(state::Dict{String,<:AbstractState}, action::AbstractAction) = state

function repeater(state::Dict{String,State}, action::AbstractRepeaterAction)
    s = Dict{String,State}()
    for (k, v) in state
        s[k] = v.id == action.id ? repeater(v, action) : v
    end
    return s
end

repeater(s::State, a::Increment) = State(s.id, s.counter + 1, s.button1, s.button2)
repeater(s::State, a::Decrement) = State(s.id, s.counter - 1, s.button1, s.button2)
repeater(s::State, a::ChangeArrowToUpDown) = State(
    s.id,
    s.counter,
    ArrowButtons.State("##up", CImGui.ImGuiDir_Up),
    ArrowButtons.State("##down", CImGui.ImGuiDir_Down),
)

# helper
"""
    Repeater(store::AbstractStore, get_state=s->Redux.get_state(s)) -> Bool
Return `true` when triggered.
`get_state` is a router function which tells how to find the target
state from `store`.
"""
function Repeater(store::AbstractStore, get_state=s->Redux.get_state(s))
    is_triggered = false
    spacing = CImGui.GetStyle().ItemInnerSpacing.x
    CImGui.PushButtonRepeat(true)
    if ArrowButtons.ArrowButton(store, s->get_state(s).button1)
        dispatch!(store, Decrement(get_state(store).id))
        is_triggered = true
    end
    CImGui.SameLine(0.0, spacing)
    if ArrowButtons.ArrowButton(store, s->get_state(s).button2)
        dispatch!(store, Increment(get_state(store).id))
        is_triggered = true
    end
    CImGui.PopButtonRepeat()
    CImGui.SameLine()
    CImGui.Text("$(get_state(store).counter)")
    return is_triggered
end

end


text:
export Client,
    me,
    enable_cache!,
    disable_cache!,
    start,
    start!,
    init!,
    add_handler!

include("limiter.jl")
include("state.jl")

# Messages are created regularly, and lose relevance quickly.
const DEFAULT_STRATEGIES = Dict{DataType, CacheStrategy}(
    Guild          => CacheForever(),
    DiscordChannel => CacheForever(),
    User           => CacheForever(),
    Member         => CacheForever(),
    Presence       => CacheForever(),
    Message        => CacheTTL(Hour(6)),
)

# A versioned WebSocket connection.
mutable struct Conn
    io
    v::Int
end

"""
    Client(
        token::String
        application_id::Snowflake
        intents::Int;
        presence::Union{Dict, NamedTuple}=Dict(),
        strategies::Dict{DataType, <:CacheStrategy}=Dict(),
        version::Int=$API_VERSION,
    ) -> Client

A Discord bot. `Client`s can connect to the gateway, respond to events, and make REST API
calls to perform actions such as sending/deleting messages, kicking/banning users, etc.

### Bot Token
A bot token can be acquired by creating a new application
[here](https://discordapp.com/developers/applications). Make sure not to hardcode the token
into your Julia code! Use an environment variable or configuration file instead.

### Application ID
The application id for your bot can be found [here](https://discordapp.com/developers/applications).
Make sure not to hardcode the application id into your Julia code!
Use an environment variable or configuration file instead.

### Intents
Integer representing intents. More information [here](https://discord.com/developers/docs/topics/gateway#gateway-intents).

### Presence
The `presence` keyword sets the bot's presence upon connection. It also sets defaults
for future calls to [`set_game`](@ref). The schema
[here](https://discordapp.com/developers/docs/topics/gateway#update-status-gateway-status-update-structure)
must be followed.

### Cache Control
By default, most data that comes from Discord is cached for later use. However, to avoid
memory leakage, not all of it is kept forever. The default setings are to keep everything
but [`Message`](@ref)s, which are deleted after 6 hours, forever. Although the default
settings are sufficient for most workloads, you can specify your own strategies per type
with the `strategies` keyword. Keys can be any of the following:

- [`Guild`](@ref)
- [`DiscordChannel`](@ref)
- [`Message`](@ref)
- [`User`](@ref)
- [`Member`](@ref)
- [`Presence`](@ref)

For potential values, see [`CacheStrategy`](@ref).

The cache can also be disabled/enabled permanently and temporarily as a whole with
[`enable_cache!`](@ref) and [`disable_cache!`](@ref).

### API Version
The `version` keyword chooses the Version of the Discord API to use. Using anything but
`$API_VERSION` is not officially supported by the Ekztazy.jl developers.

### Sharding
Sharding is handled automatically. The number of available processes is the number of
shards that are created. See the
[sharding example](https://github.com/Xh4H/Discord.jl/blob/master/examples/sharding.jl)
for more details.
"""
mutable struct Client
    token::AbstractString                                        # Bot token
    application_id::Snowflake                                    # Application ID for Interactions
    commands::Vector{ApplicationCommand}                         # Commands
    guild_commands::Dict{Snowflake, Vector{ApplicationCommand}}  # Guild commands
    intents::Int                                                 # Intents value
    handlers::Dict{Symbol, Vector{Handler}}                      # Handlers for each event
    hb_interval::Int                                             # Milliseconds between heartbeats.
    hb_seq::Nullable{Int}                                        # Sequence value sent by Discord for resuming.
    last_hb::DateTime                                            # Last heartbeat send.
    last_ack::DateTime                                           # Last heartbeat ack.
    version::Int                                                 # Discord API version.
    state::State                                                 # Client state, cached data, etc.
    shards::Int                                                  # Number of shards in use.
    shard::Int                                                   # Client's shard index.
    limiter::Limiter                                             # Rate limiter.
    ready::Bool                                                  # Client is connected and authenticated.
    use_cache::Bool                                              # Whether or not to use the cache.
    presence::Dict                                               # Default presence options.
    conn::Conn                                                   # WebSocket connection.
    p_guilds::Dict{Snowflake, String}                            # Command prefix overrides.
    no_auto_ack::Vector{String}                                  # No auto ack custom ids.
    auto_update_ack::Vector{String}                              # Auto ack for deferred message update custom ids.

    function Client(
        token::String,
        application_id::Snowflake,
        intents::Int;
        # kwargs
        presence::Union{Dict, NamedTuple}=Dict(),
        strategies::Dict{DataType, <:CacheStrategy}=Dict{DataType, CacheStrategy}(),
        version::Int=API_VERSION,
    )
        token = startswith(token, "Bot ") ? token : "Bot $token"
        state = State(merge(DEFAULT_STRATEGIES, strategies))
        app_id = application_id
        conn = Conn(nothing, 0)
        presence = merge(Dict(
            "since" => nothing,
            "game" => nothing,
            "status" => "online", # Should make a const out of statuses
            "afk" => false,
        ), Dict(string(k) => v for (k, v) in Dict(pairs(presence))))

        c = new(
            token,        # token
            app_id,       # application_id
            Vector(),     # Commands
            Dict(),       # Guild commands
            intents,      # intents
            Dict(),       # handlers
            0,            # hb_interval
            nothing,      # hb_seq
            DateTime(0),  # last_hb
            DateTime(0),  # last_ack
            version,      # version
            state,        # state
            nprocs(),     # shards
            myid() - 1,   # shard
            Limiter(),    # limiter
            false,        # ready
            true,         # use_cache
            presence,     # presence
            conn,         # conn
            Dict(),       # p_guilds
            [],           # no_auto_ack
            []            # auto_update_ack
        )

        return c
    end
end

Client(token::String, appid::Int, intents::Int, args...) = Client(token, UInt(appid), intents, args...)
Client(in::Int) = Client(
    ENV["DISCORD_TOKEN"],
    ENV["APPLICATION_ID"] isa Number ? ENV["APPLICATION_ID"] : parse(UInt, ENV["APPLICATION_ID"]),
    in
)
Client() = Client(intents(GUILDS, GUILD_MESSAGES))

add!(d::Dict{U, Union{Vector{T}}}, k::U, v::T) where T where U = haskey(d, k) ? push!(d[k], v) : d[k] = [v]

"""
    add_handler!(c::Client, handler::Handler)

Adds a handler to the client.
"""
add_handler!(c::Client, handler::Handler) = add!(c.handlers, handlerkind(handler), handler)
"""
    add_handler!(c::Client, handler::Handler)

Bulk adds handlers to the client.
"""
add_handler!(c::Client, args...) = map(h -> add_handler!(c, h), args)

add_command!(c::Client; kwargs...) = add_command!(c, ApplicationCommand(; application_id=c.application_id, kwargs...))
add_command!(c::Client, cmd::ApplicationCommand) = push!(c.commands, cmd)
add_command!(c::Client, args...) = map(cmd -> add_command!(c, cmd), args)
add_command!(c::Client, g::Snowflake; kwargs...) = add_command!(c, g, ApplicationCommand(; application_id=c.application_id, kwargs...))
add_command!(c::Client, g::Snowflake, cmd::ApplicationCommand) = add!(c.guild_commands, g, cmd)
add_command!(c::Client, g::Snowflake, args...) = map(cmd -> add_command!(c, g, cmd), args)

mock(::Type{Client}; kwargs...) = Client("token")

function Base.show(io::IO, c::Client)
    print(io, "Discord.Client(shard=$(c.shard + 1)/$(c.shards), api=$(c.version), ")
    isopen(c) || print(io, "not ")
    print(io, "logged in)")
end

"""
    me(c::Client) -> Nullable{User}

Get the [`Client`](@ref)'s bot user.
"""
me(c::Client) = c.state.user

"""
    enable_cache!(c::Client)
    enable_cache!(f::Function c::Client)

Enable the cache. `do` syntax is also accepted.
"""
enable_cache!(c::Client) = c.use_cache = true
enable_cache!(f::Function, c::Client) = set_cache(f, c, true)

"""
    disable_cache!(c::Client)
    disable_cache!(f::Function, c::Client)

Disable the cache. `do` syntax is also accepted.
"""
disable_cache!(c::Client) = c.use_cache = false
disable_cache!(f::Function, c::Client) = set_cache(f, c, false)

# Compute some contextual keywords for log messages.
function logkws(c::Client; kwargs...)
    kws = Pair[:time => now()]
    c.shards > 1 && push!(kws, :shard => c.shard)
    c.conn.io === nothing || push!(kws, :conn => c.conn.v)

    for kw in kwargs
        if kw.second === undef  # Delete any keys overridden with undef.
            filter!(p -> p.first !== kw.first, kws)
        else
            # Replace any overridden keys.
            idx = findfirst(p -> p.first === kw.first, kws)
            if idx === nothing
                push!(kws, kw)
            else
                kws[idx] = kw
            end
        end
    end

    return kws
end

# Parse some data (usually a Dict from JSON), or return the thrown error.
function Base.tryparse(c::Client, T::Type, data)
    return try
        T <: Vector ? eltype(T).(data) : T(data), nothing
    catch e
        kws = logkws(c; T=T, exception=(e, catch_backtrace()))
        @error "Parsing failed" kws...
        nothing, e
    end
end

# Run some function with the cache enabled or disabled.
function set_cache(f::Function, c::Client, use_cache::Bool)
    old = c.use_cache
    c.use_cache = use_cache
    try
        f()
    finally
        # Usually the above function is going to be calling REST endpoints. The cache flag
        # is checked asynchronously, so by the time it happens there's a good chance we've
        # already returned and set the cache flag back to its original value.
        sleep(Millisecond(1))
        c.use_cache = old
    end
end

"""
    start!(c::Client)

Calls [`init`](@ref) on the client
Calls `open` then `wait` on the [`Client`](@ref).
"""
function start!(c::Client, duration=Minute(0))
    open(c)
    duration != Second(0) && auto_shutdown(c, duration, "SHUTDOWN")
    wait(c)
end

"""
    auto_shutdown(run_duration::TimePeriod)

Run a background process to track the program's run time and exit
the program when it has exceeded the specified `run_duration` or
when a file exists at `trigger_path`.
"""
function auto_shutdown(c::Client, run_duration::TimePeriod, trigger_path::AbstractString="SHUTDOWN", s=5)
    start_time = now()
    @async while true
        if now() > start_time + run_duration
            @info "Times up! The bot is shutting down automatically."
            shutdown_gracefully(c)
            break
        end
        if length(trigger_path) > 0 && isfile(trigger_path)
            @info "The bot is shutting down via trigger path `$trigger_path`."
            rm(trigger_path)
            shutdown_gracefully(c)
            break
        end
        sleep(s)
    end
end

function shutdown_gracefully(c::Client)
    try
        close(c)
    catch ex
        @warn "Unable to close client connection" ex
    end
end

"""
    init!(c::Client)

Creates a handler to generate [`ApplicationCommand`](@ref)s.\n
Creates handlers for GuildCreate events.\n
"""
function init!(c::Client)
    hcreate = OnGuildCreate() do (ctx)
        put!(c.state, ctx.guild)
    end
    hupdate = OnGuildUpdate() do (ctx)
        put!(c.state, ctx.guild)
    end
    hready = OnReady() do (ctx)
        c.state.user = ctx.user
        for (g, cmds) in c.guild_commands
            create(c, Vector{ApplicationCommand}, g, cmds)
        end
        create(c, Vector{ApplicationCommand}, c.commands)
    end
    add_handler!(c, hcreate, hupdate, hready)
end

const start = start! # backwards compat


text:
using DataFrames
using Dates
using Gadfly
# using VegaLite
using FileTools, DataFrameTools
using Colors, ColorSchemes

using MAT
using CSV

using Revise
using JuliaToolsForMagTIP
dir = raw"D:\GeoMag (main)\GeoMag_1"
allpaths = filelistall(r"\.mat",dir);
dtstr = gettag.("dt", allpaths)
dt = Date.(dtstr, dateformat"yyyymmdd")
stn = gettag.("stn", allpaths)
typ = gettag.("type", allpaths)

df = DataFrame(:filename => basename.(allpaths), :date => dt, :fullpath=>allpaths, :station=>stn, :color => 1, :type => typ)
filter!(:type => x -> occursin(r"(full|Full)", x), df)

sort!(df, :date)
unique!(df, :filename)

fulldt = df.date[1]:Day(1):df.date[end] |> collect
gd = groupby(df, :station)

dfs = []
for dfi in gd
    fulldf = DataFrame(:date=> fulldt);
    push!(dfs, outerjoin(dfi, fulldf, on=:date))
end

df2 = vcat(dfs...)
dropmissing!(df2, :station)
if !isequal(sort(df2,[:date,:station]), df)
    error("They should be identical")
end

select!(df2, Not(:color), :color =>ByRow(x -> ismissing(x) ? 0 : 1); renamecols=false)

transform!(df2, :date => ByRow(x -> floor(x, Dates.Month)) => :month)
gd_stnmon = groupby(df2, [:station, :month])
daycount_inmonth_stn = combine(gd_stnmon, nrow => :counts)

rt0t1 = floor.(extrema(fulldt), Dates.Month)

mycs = ColorScheme(range(colorant"white", stop=colorant"lightgreen", length=100)) # range(HSV(0,1,1), stop=HSV(250,1,1), length=100)
# platettef = Scale.lab_gradient(...)

set_default_plot_size(9inch,2.5inch)
p = plot(daycount_inmonth_stn, x=:month, y=:station, color=:counts,
Geom.rectbin,
# Coord.cartesian(xmin=rt0t1[1], xmax=rt0t1[2]), # this makes xtick label protrude
Guide.xticks(ticks=DateTime.(collect(rt0t1[1]:Dates.Month(3):rt0t1[2]))),
# Scale.x_continuous(;minticks=10, maxticks=50),
Scale.color_continuous(colormap=p->get(mycs, p),minvalue=0, maxvalue=30),
)
# A vector of `Date` needs to be converted to `DateTime` to avoid a conversion error (trying to get the `hour` in Showoff.jl).

# df2 |>
# @vlplot(
#     :rect, x={:date, bin={maxbins=100}}, y=:station, color=:color,
#     width=400, height=200
# )

df3 = filter(:date => x -> (Date(2017,10,1) > x > Date(2010,9,30)) ,df2)
gd_stn = groupby(df3, [:station])
targetfolder = raw"D:\GeoMag (main)\GeoMag4HMM"
extrema(df3.date) # (Date("2010-10-01"), Date("2017-09-30"))
for df_i in gd_stn
    stnm = df_i.station[1]
    fname = "D_$(stnm)_ti1.csv"
    fpath = joinpath(targetfolder, fname);
    if isfile(fpath)
        continue
    end
    sort!(df_i, :date)
    try
        for path in df_i.fullpath
            file = matopen(path)
            M = read(file, "M")
            dfm = DataFrame(M[:,1:7],
                    [:year, :month,:day,:hour,:minute,:second,:data])

            filter!(AsTable(:) => (nt -> isgeomagdatavalid(nt)) , dfm)

            select!(dfm,
            [:year, :month,:day,:hour,:minute,:second] => ByRow((yyyy,mm,dd,hh,MM,ss) -> toordinal(yyyy,mm,dd,hh,MM,ss)) => :ordinaldate, :data)

            CSV.write(fpath, dfm; writeheader=false, append=true,compress=true)
            close(file)
        end
    catch e
        rm(fpath);
        @warn "Error occurred while loading $path, writing to $fpath. $fpath has been deleted."
        rethrow(e)
    end
end


text:
include("../imports.jl")

@testset ExtendedTestSet "`CustomCallback`" begin
    cb = CustomCallback(Events.StepEnd, TrainingPhase) do learner
        throw(CancelFittingException("test"))
    end
    learner = testlearner(Recorder(), cb, coeff = 3)
    @test_throws CancelFittingException epoch!(learner, TrainingPhase())
    @test learner.cbstate.history[TrainingPhase()].epochs == 0
    @test learner.cbstate.history[TrainingPhase()].steps == 0
end


text:
#-----------------------------------------------------------------------# GradientAlgorithm
function gradient!(a::GradientAlgorithm, o::SModel)
    mul!(a.nvec, o.x, o.β)          # nvec ← x * β
    deriv!(a.nvec, o.loss, o.y, a.nvec) # nvec ← deriv(L, y, x * β)
    multiply_by_weights!(a.nvec, o.w)   # nvec .*= w ./ n
    mul!(a.pvec, o.x', a.nvec)      # pvec ← x'nvec
end
multiply_by_weights!(nvec, w::Nothing) = rmul!(nvec, 1 / length(nvec))
function multiply_by_weights!(nvec, w)
    wt = inv(length(nvec))
    for i in eachindex(nvec)
        @inbounds nvec[i] *= w[i] * wt
    end
end

# TODO: do something smarter than this
stepsize(o::SModel) = 1.0
stepsize(o::SModel{LinearRegression}) = 1.0

#-----------------------------------------------------------------------# LineSearch
"""
    LineSearch(algorithm)

Use a line search in the `update!` of `algorithm`.  Currently, [`ProxGrad`](@ref),
[`Fista`](@ref), and [`GradientDescent`](@ref) are supported.

# Example

    x, y, β = SparseRegression.fakedata(L2DistLoss(), 1000, 10)
    s = SModel(x, y, L2DistLoss())
    strat = strategy(MaxIter(50), LineSearch(ProxGrad(s)))
    learn!(s, strat)
"""
struct LineSearch{A <: Algorithm} <: Algorithm
    alg::A
    divisor::Float64
    β::Vector{Float64}
end
function LineSearch(a::GradientAlgorithm, divisor::Float64 = 1.5)
    LineSearch(a, divisor, zeros(length(a.pvec)))
end
Base.show(io::IO, a::LineSearch) = print(io, "LineSearch: $(a.alg)")
function update!(o::SModel, a::LineSearch{<: GradientAlgorithm})
    l1 = value(o)
    l2 = Inf
    copyto!(a.β, o.β)
    firststep = a.alg.step
    while l2 > l1
        a.alg.step /= a.divisor
        copyto!(o.β, a.β)
        update!(o, a.alg)
        l2 = value(o)
    end
    a.alg.step = firststep
end

#-----------------------------------------------------------------------# Adaptive ProxGrad
"""
    AdaptiveProxGrad(s, divisor = 1.5, init = 1.0)

Proximal gradient method with adaptive step sizes.  AdaptiveProxGrad uses element-wise
learning rates.  Every time the sign of a coefficient switches, the step size for that
coefficient is divided by `divisor`.
"""
struct AdaptiveProxGrad <: GradientAlgorithm
    nvec::Vector{Float64}
    pvec::Vector{Float64}
    steps::Vector{Float64}
    divisor::Float64
end
function AdaptiveProxGrad(o::SModel, divisor::Float64 = 1.5, init::Float64 = 1.0)
    n, p = size(o.x)
    AdaptiveProxGrad(zeros(n), zeros(p), fill(init, p), divisor)
end
Base.show(io::IO, a::AdaptiveProxGrad) = print(io, "AdaptiveProxGrad")
function update!(o::SModel, a::AdaptiveProxGrad)
    gradient!(a, o)
    ∇ = a.pvec
    for j in eachindex(o.β)
        s = a.steps[j]
        sign_old = sign(o.β[j])
        o.β[j] = prox(o.penalty, o.β[j] - s * ∇[j], s * o.λ[j])
        if sign_old != sign(o.β[j])
            a.steps[j] /= a.divisor
        end
    end
end

#-----------------------------------------------------------------------# ProxGrad
"""
    ProxGrad(model, step = 1.0)

Proximal gradient method with step size `step`.  Works for any loss and any penalty with a `prox` method.

# Example

    x, y, β = SparseRegression.fakedata(L2DistLoss(), 1000, 10)
    s = SModel(x, y, L2DistLoss())
    strat = strategy(MaxIter(50), ProxGrad(s))
    learn!(s, strat)
"""
mutable struct ProxGrad <: GradientAlgorithm
    nvec::Vector{Float64}
    pvec::Vector{Float64}
    step::Float64
end
ProxGrad(o::SModel, s::Float64 = stepsize(o)) = ProxGrad(zeros(size(o.x, 1)), zeros(size(o.x, 2)), s)
Base.show(io::IO, a::ProxGrad) = print(io, "ProxGrad(step = $(a.step))")
function update!(o::SModel, a::ProxGrad)
    gradient!(a, o)
    ∇ = a.pvec
    s = a.step
    for j in eachindex(o.β)
        o.β[j] = prox(o.penalty, o.β[j] - s * ∇[j], s * o.λ[j])
    end
end

#
# #-----------------------------------------------------------------------# Fista
"""
    Fista(model, step = 1.0)

Accelerated proximal gradient method.  Works for any loss and any penalty with a `prox` method.
"""
mutable struct Fista <: GradientAlgorithm
    step::Float64
    nvec::Vector{Float64}
    pvec::Vector{Float64}
    β1::Vector{Float64}
    β2::Vector{Float64}
    t::Int
end
function Fista(o::SModel, step::Float64 = stepsize(o))
    n, p = size(o.x)
    Fista(step, zeros(n), zeros(p), zeros(p), zeros(p), 0)
end
setup!(a::Fista, o::SModel) = (a.t = 0)
function update!(o::SModel, a::Fista)
    copyto!(a.β2, a.β1)
    copyto!(a.β1, o.β)
    a.t += 1
    if a.t > 2
        γ = (a.t - 2) / (a.t + 1)
        for j in eachindex(o.β)
            @inbounds o.β[j] = a.β1[j] + γ * (a.β1[j] - a.β2[j])
        end
    end
    gradient!(a, o)
    ∇ = a.pvec
    s = a.step
    for j in eachindex(o.β)
        @inbounds o.β[j] = prox(o.penalty, o.β[j] - s * ∇[j], s * o.λ[j])
    end
end

#-----------------------------------------------------------------------# GradientDescent
"""
    GradientDescent(model, step = 1.0)

Gradient Descent.  Works for any loss and any penalty.

# Example

    x, y, β = SparseRegression.fakedata(L2DistLoss(), 1000, 10)
    s = SModel(x, y, L2DistLoss())
    strat = strategy(MaxIter(50), GradientDescent(s))
    learn!(s, strat)
"""
mutable struct GradientDescent <: GradientAlgorithm
    step::Float64
    nvec::Vector{Float64}
    pvec::Vector{Float64}
end
function GradientDescent(o::SModel, step::Float64 = stepsize(o))
    n, p = size(o.x)
    GradientDescent(step, zeros(n), zeros(p))
end
Base.show(io::IO, a::GradientDescent) = print(io, "GradientDescent")
function update!(o::SModel, a::GradientDescent)
    gradient!(a, o)
    ∇ = a.pvec
    s = a.step
    for j in eachindex(o.β)
        @inbounds o.β[j] -= s * (∇[j] + o.λ[j] * deriv(o.penalty, o.β[j]))
    end
    o
end

#-----------------------------------------------------------------------# Sweep
"""
    Sweep(model)

Linear/ridge regression via sweep operator.  Works for (scaled) L2DistLoss
with NoPenalty or L2Penalty.  The `Sweep` algorithm has a closed form solution and is
complete after one iteration.  It therefore doesn't need additional learning strategies
such as `MaxIter`, `Converged`, etc.

# Example

    x, y, β = SparseRegression.fakedata(L2DistLoss(), 1000, 10)
    s = SModel(x, y, L2DistLoss())
    learn!(s, Sweep(s))
"""
struct Sweep <: OneIterAlgorithm
    A::Matrix{Float64}  # [x y]'[x y]
    S::Matrix{Float64}  # sweep!(A, 1:p)
end
Sweep(o::SModel) = (A = make_A(o); Sweep(A, similar(A)))
Base.show(io::IO, a::Sweep) = print(io, "Sweep")
function make_A(o::SModel{L,P,X,Y,W}) where {L,P,X,Y,W<:AbstractWeights}
    n, p = size(o.x)
    a = zeros(p + 1, p + 1)
    # b = zeros(p + 1, p + 1)
    @views BLAS.syrk!('U', 'T', 1 / n, Diagonal(sqrt.(o.w)) * o.x, 0.0, a[1:p, 1:p]) # x'wx
    @views BLAS.gemv!('T', 1 / n, Diagonal(o.w) * o.x, o.y, 0.0, a[1:p, end])        # x'wy
    a[end, end] = dot(o.y, Diagonal(o.w) * o.y) / n                                  # y'wy
    a
end
function make_A(o::SModel{L,P,X,Y,Nothing}) where {L,P,X,Y}
    n, p = size(o.x)
    a = zeros(p + 1, p + 1)
    @views BLAS.syrk!('U', 'T', 1 / n, o.x, 0.0, a[1:p, 1:p])    # x'x
    @views BLAS.gemv!('T', 1 / n, o.x, o.y, 0.0, a[1:p, end])   # x'y
    a[end, end] = dot(o.y, o.y) / n                             # y'y
    a
end
function update!(o::SModel, a::Sweep)
    n, p = size(o.x)
    copyto!(a.S, a.A)
    add_ridge!(o, a)
    sweep!(a.S, 1:p)
    for j in eachindex(o.β)
        @inbounds o.β[j] = a.S[j, end]
    end
end
function add_ridge!(o::SModel{L, NoPenalty}, a::Algorithm) where {L} end
function add_ridge!(o::SModel{L, L2Penalty}, a::Algorithm) where {L}
    for i in eachindex(o.β)
        @inbounds a.S[i, i] += o.λ[i]
    end
end

#-----------------------------------------------------------------------# LinRegCholesky
"""
    LinRegCholesky(model)

Linear/ridge regression via cholesky decomposition.  Works for (scaled) L2DistLoss
with NoPenalty or L2Penalty.  The `LinRegCholesky` algorithm has a closed form solution
and is complete after one iteration.  It therefore doesn't need additional learning
strategies such as `MaxIter`, `Converged`, etc.

# Example

    x, y, β = SparseRegression.fakedata(L2DistLoss(), 1000, 10)
    s = SModel(x, y, L2DistLoss())
    learn!(s, Sweep(s))
"""
struct LinRegCholesky<: OneIterAlgorithm
    A::Matrix{Float64}
    S::Matrix{Float64}
end
LinRegCholesky(o::SModel) = (A = make_A(o); LinRegCholesky(A, similar(A)))
Base.show(io::IO, a::LinRegCholesky) = print(io, "LinRegCholesky")
function update!(o::SModel, a::LinRegCholesky)
    copyto!(a.S, a.A)
    add_ridge!(o, a)
    cholesky!(Symmetric(a.S))
    @views o.β[:] = UpperTriangular(a.S[1:end-1, 1:end-1]) \ a.S[1:end-1, end]
end


text:
# This file was generated, do not modify it. # hide
datac = coerce(data, autotype(data));


text:
# figure out at what byte position the header row(s) start and at what byte position the data starts
function detectheaderdatapos(buf, pos, len, oq, eq, cq, cmt, ignoreemptylines, header, datarow)
    headerpos = 0
    datapos = 1
    if header isa Integer
        if header <= 0
            # no header row in dataset; skip to data
            datapos = skiptorow(buf, pos, len, oq, eq, cq, 1, datarow)
        else
            headerpos = skiptorow(buf, pos, len, oq, eq, cq, 1, header)
            headerpos = checkcommentandemptyline(buf, headerpos, len, cmt, ignoreemptylines)
            datapos = skiptorow(buf, headerpos, len, oq, eq, cq, header, datarow)
        end
    elseif header isa AbstractVector{<:Integer}
        headerpos = skiptorow(buf, pos, len, oq, eq, cq, 1, header[1])
        headerpos = checkcommentandemptyline(buf, headerpos, len, cmt, ignoreemptylines)
        datapos = skiptorow(buf, headerpos, len, oq, eq, cq, header[1], datarow)
    elseif header isa Union{AbstractVector{Symbol}, AbstractVector{String}}
        datapos = skiptorow(buf, pos, len, oq, eq, cq, 1, datarow)
    else
        throw(ArgumentError("unsupported header argument: $header"))
    end
    datapos = max(1, checkcommentandemptyline(buf, datapos, len, cmt, ignoreemptylines))
    return headerpos, datapos
end

# this function scans a few rows and tracks the # of bytes and characters encountered
# it tries to guess a file's delimiter by which character showed up w/ the same frequency
# over all rows scanned; we use the average # of bytes per row w/ total length of the file
# to guess the total # of rows in the file
function detectdelimandguessrows(buf, headerpos, datapos, len, oq, eq, cq, delim, cmt, ignoreemptylines)
    nbytes = 0
    lastbytenewline = false
    parsedanylines = false
    nlines = 0
    headerbvc = ByteValueCounter()
    bvc = ByteValueCounter()
    b = 0x00
    # don't parse header row if there isn't one: #508
    if headerpos > 0
        pos = headerpos
        # parsing our header row is useful for delimiter
        # detection, but we don't track nbytes here
        # because the header row size doesn't necessarily
        # correlate w/ data row size
        while pos <= len
            parsedanylines = true
            @inbounds b = buf[pos]
            pos += 1
            if b == oq
                while pos <= len
                    @inbounds b = buf[pos]
                    pos += 1
                    if b == eq
                        if pos > len
                            break
                        elseif eq == cq && buf[pos] != cq
                            break
                        end
                        @inbounds b = buf[pos]
                        pos += 1
                    elseif b == cq
                        break
                    end
                end
            elseif b == UInt8('\n')
                nlines += 1
                lastbytenewline = true
                break
            elseif b == UInt8('\r')
                pos <= len && buf[pos] == UInt8('\n') && (pos += 1)
                nlines += 1
                lastbytenewline = true
                break
            else
                lastbytenewline = false
                incr!(headerbvc, b)
                incr!(bvc, b)
            end
        end
    end
    pos = max(1, checkcommentandemptyline(buf, datapos, len, cmt, ignoreemptylines))
    while pos <= len && nlines < 10
        parsedanylines = true
        @inbounds b = buf[pos]
        pos += 1
        nbytes += 1
        if b == oq
            while pos <= len
                @inbounds b = buf[pos]
                pos += 1
                nbytes += 1
                if b == eq
                    if pos > len
                        break
                    elseif eq == cq && buf[pos] != cq
                        break
                    end
                    @inbounds b = buf[pos]
                    pos += 1
                    nbytes += 1
                elseif b == cq
                    break
                end
            end
        elseif b == UInt8('\n')
            pos = checkcommentandemptyline(buf, pos, len, cmt, ignoreemptylines)
            nlines += 1
            lastbytenewline = true
        elseif b == UInt8('\r')
            pos <= len && buf[pos] == UInt8('\n') && (pos += 1)
            pos = checkcommentandemptyline(buf, pos, len, cmt, ignoreemptylines)
            nlines += 1
            lastbytenewline = true
        else
            lastbytenewline = false
            incr!(bvc, b)
        end
    end
    nlines += parsedanylines && !lastbytenewline
    if delim == UInt8('\n')
        if nlines > 0
            d = UInt8('\n')
            for attempted_delim in (UInt8(','), UInt8('\t'), UInt8(' '), UInt8('|'), UInt8(';'), UInt8(':'))
                cnt = bvc.counts[Int(attempted_delim)]
                # @show Char(attempted_delim), cnt, nlines
                if cnt > 0 && cnt % nlines == 0
                    d = attempted_delim
                    break
                end
            end
            if d == UInt8('\n')
                maxcnt = 0
                for attempted_delim in (UInt8(','), UInt8('\t'), UInt8(' '), UInt8('|'), UInt8(';'), UInt8(':'))
                    cnt = headerbvc.counts[Int(attempted_delim)]
                    if cnt > maxcnt
                        d = attempted_delim
                        maxcnt = cnt
                    end
                end
            end
            if d == UInt8('\n')
                d = UInt8(',')
            end
        else
            d = UInt8(',')
        end
    else # delim explicitly provided
        d = delim
    end
    guess = ((len - datapos) / (nbytes / nlines))
    rowsguess = isfinite(guess) ? ceil(Int, guess) : 0
    return d, max(1, rowsguess)
end

struct ByteValueCounter
    counts::Vector{Int64}
    ByteValueCounter() = new(zeros(Int64, 256))
end

function incr!(c::ByteValueCounter, b::UInt8)
    @inbounds c.counts[b] += 1
    return
end

# given the various header and normalization options, figure out column names for a file
function detectcolumnnames(buf, headerpos, datapos, len, options, header, normalizenames)
    if header isa Union{AbstractVector{Symbol}, AbstractVector{String}}
        fields, pos = readsplitline(buf, datapos, len, options)
        if isempty(header)
            return [Symbol(:Column, i) for i = 1:length(fields)]
        elseif headerpos > 0
            length(header) == length(fields) || throw(ArgumentError("The length of provided header ($(length(header))) doesn't match the number of columns in data ($(length(fields)))"))
        end
        names = header
    elseif headerpos == 0
        fields, pos = readsplitline(buf, datapos, len, options)
        # generate column names
        return [Symbol(:Column, i) for i = 1:length(fields)]
    elseif header isa Integer
        names, pos = readsplitline(buf, headerpos, len, options)
    elseif header isa AbstractVector{<:Integer}
        names, pos = readsplitline(buf, headerpos, len, options)
        for row = 2:length(header)
            pos = skiptorow(buf, pos, len, options.oq, options.e, options.cq, 1, header[row] - header[row - 1])
            fields, pos = readsplitline(buf, pos, len, options)
            for (i, x) in enumerate(fields)
                names[i] *= "_" * x
            end
        end
    end
    return makeunique([normalizenames ? normalizename(x) : Symbol(x) for x in names])
end

# efficiently skip from `cur` to `dest` row
function skiptorow(buf, pos, len, oq, eq, cq, cur, dest)
    cur >= dest && return pos
    for _ = 1:(dest - cur)
        while pos <= len
            @inbounds b = buf[pos]
            pos += 1
            if b == oq
                while pos <= len
                    @inbounds b = buf[pos]
                    pos += 1
                    if b == eq
                        if pos > len
                            break
                        elseif eq == cq && buf[pos] != cq
                            break
                        end
                        @inbounds b = buf[pos]
                        pos += 1
                    elseif b == cq
                        break
                    end
                end
            elseif b == UInt8('\n')
                typeof(buf) == ReversedBuf && pos <= len && buf[pos] == UInt8('\r') && (pos += 1)
                break
            elseif b == UInt8('\r')
                pos <= len && buf[pos] == UInt8('\n') && (pos += 1)
                break
            end
        end
    end
    return pos
end

# read a single row, splitting cells on delimiters; used for parsing column names from header row(s)
function readsplitline(buf, pos, len, options::Parsers.Options{ignorerepeated}) where {ignorerepeated}
    vals = String[]
    (pos > len || pos == 0) && return vals, pos
    col = 1
    while true
        _, code, vpos, vlen, tlen = Parsers.xparse(String, buf, pos, len, options)
        push!(vals, columnname(buf, vpos, vlen, code, options, col))
        pos += tlen
        col += 1
        ignorerepeated && Parsers.newline(code) && break
        Parsers.delimited(code) && continue
        (Parsers.newline(code) || pos > len) && break
    end
    return vals, pos
end

function columnname(buf, vpos, vlen, code, options, i)
    if Parsers.sentinel(code) || vlen == 0
        return "Column$i"
    elseif Parsers.escapedstring(code)
        return unescape(PointerString(pointer(buf, vpos), vlen), options.e)
    else
        return unsafe_string(pointer(buf, vpos), vlen)
    end
end

@inline function skipemptyrow(buf, pos, len)
    @inbounds b = buf[pos]
    if b == UInt8('\n')
        return pos + 1
    elseif b == UInt8('\r')
        if pos + 1 < len && buf[pos + 1] == UInt8('\n')
            return pos + 2
        else
            return pos + 1
        end
    end
    return pos
end

function checkcommentandemptyline(buf, pos, len, cmt, ignoreemptylines)
    cmtptr, cmtlen = cmt === nothing ? (C_NULL, 0) : cmt
    ptr = pointer(buf, pos)
    while pos <= len
        skipped = matched = false
        if ignoreemptylines
            newpos = skipemptyrow(buf, pos, len)
            if newpos > pos
                pos = newpos
                skipped = true
            end
        end
        if cmtlen > 0 && (pos + cmtlen - 1) <= len
            matched = Parsers.memcmp(ptr, cmtptr, cmtlen)
            if matched
                pos += cmtlen
                pos > len && break
                @inbounds b = buf[pos]
                while b != UInt8('\n') && b != UInt8('\r')
                    pos += 1
                    pos > len && break
                    @inbounds b = buf[pos]
                end
                b == UInt8('\r') && pos <= len && buf[pos + 1] == UInt8('\n') && (pos += 1)
                pos += 1
            end
        end
        (skipped | matched) || break
        ptr = pointer(buf, pos)
    end
    return pos
end

# here we try to "chunk" up a file; given the equally spaced out byte positions in `ranges`, we start at each
# byte position and start parsing until we find the start of the next row; if the next rows all verify w/ the
# right # of expected columns then we move on to the next file chunk byte position. If we fail, we start over
# at the byte position, assuming we were in the a quoted field (and encountered a newline inside the quoted
# field the first time through)
function findrowstarts!(buf, len, options::Parsers.Options{ignorerepeated}, ranges, ncols, types, flags) where {ignorerepeated}
    totalbytes = 0
    totalrows = 0
    totalmissing = 0
    detectedtypes = copy(types)
    for i = 2:(length(ranges) - 1)
        pos = ranges[i]
        while pos <= len
            startpos = pos
            code = Parsers.ReturnCode(0)
            # assume not in quoted field; start parsing, count ncols + newline and if things match, return
            while pos <= len
                _, code, _, _, tlen = Parsers.xparse(String, buf, pos, len, options)
                pos += tlen
                if Parsers.newline(code)
                    # assume we found the correct start of the next row
                    ranges[i] = pos
                    break
                end
            end
            # now we read the next 5 rows and see if we get the right # of columns
            rowstartpos = pos
            correct = true
            for j = 1:5
                for col = 1:ncols
                    _, code, vpos, vlen, tlen = Parsers.xparse(String, buf, pos, len, options)
                    if !typedetected(flags[col])
                        D = detectedtypes[col]
                        det = detect(buf, vpos, vpos + vlen - 1, options)
                        T = typeof(something(det, ""))
                        if T === Missing
                            totalmissing += 1
                        elseif T !== String
                            detectedtypes[col] = promote_types(D, T)
                            # if detectedtypes[col] !== D
                            #     println("promoted column type from $D to $(detectedtypes[col]) for task chunk = $i")
                            # end
                        end
                    end
                    pos += tlen
                    pos > len && break
                end
                correct &= Parsers.newline(code)
            end
            if correct
                # boom, we read a whole row and got correct # of columns
                totalbytes += pos - rowstartpos
                totalrows += 5
                for col = 1:ncols
                    types[col] = detectedtypes[col]
                end
                break
            end
            # else, assume we were inside a quoted field:
            pos = startpos
            # if first byte is quotechar, need to check previous char for escapechar and if so, skip forward
            if buf[pos] == options.cq && buf[pos - 1] == options.e
                pos += 1
            end
            # start parsing until we find quotechar (ignoring escaped quote chars)
            cq, eq = options.cq, options.e
            while pos <= len
                b = buf[pos]
                pos += 1
                if b == eq
                    if pos > len
                        break
                    elseif eq == cq && buf[pos] != cq
                        break
                    end
                    b = buf[pos]
                    pos += 1
                elseif b == cq
                    break
                end
            end
            col = 1
            while pos <= len
                _, code, _, _, tlen = Parsers.xparse(String, buf, pos, len, options)
                if !typedetected(flags[col])
                    T = typeof(something(detect(buf, pos, len, options), ""))
                    if T === Missing
                        totalmissing += 1
                    elseif T !== String
                        types[col] = promote_types(types[col], T)
                    end
                end
                pos += tlen
                col += 1
                if Parsers.newline(code)
                    # assume we found the correct start of the next row
                    ranges[i] = pos
                    break
                end
            end
            # in the worse case, we read to the end of the file; this shouldn't happen
            # because we're only identifying the starting byte positions of rows
            # in the middle of the file; if we hit this, it's most likely a corrupt file
            # with unquoted delimiters in string cells, or misquoted cells.
            # but if there's an actual bug here somehow, let's ask the user to tell us about it for now
            if pos > len
                error("$i; something went wrong trying to determine row positions for multithreading; often it's a mismatch in the # of columns expected in the file vs. actual; pass `threaded=false` to avoid using multithreaded parsing; if you think you're seeing this error incorrectly, it'd be very helpful if you could open an issue at https://github.com/JuliaData/CSV.jl/issues so package authors can investigate")
            end
        end
    end
    for col = 1:ncols
        if types[col] !== Union{} && types[col] !== Missing && types[col] !== String && types[col] !== Union{Missing, String}
            flags[col] |= TYPEDETECTED
        end
    end
    return totalbytes / totalrows
end

function detecttranspose(buf, pos, len, options, header, datarow, normalizenames)
    if isa(header, Integer) && header > 0
        # skip to header column to read column names
        row, pos = skiptofield!(buf, pos, len, options, 1, header)
        # io now at start of 1st header cell
        _, code, vpos, vlen, tlen = Parsers.xparse(String, buf, pos, len, options)
        columnnames = [columnname(buf, vpos, vlen, code, options, 1)]
        pos += tlen
        row, pos = skiptofield!(buf, pos, len, options, header+1, datarow)
        columnpositions = Int64[pos]
        datapos = pos
        rows, pos = countfields(buf, pos, len, options)

        # we're now done w/ column 1, if EOF we're done, otherwise, parse column 2's column name
        cols = 1
        while pos <= len
            # skip to header column to read column names
            row, pos = skiptofield!(buf, pos, len, options, 1, header)
            cols += 1
            _, code, vpos, vlen, tlen = Parsers.xparse(String, buf, pos, len, options)
            push!(columnnames, columnname(buf, vpos, vlen, code, options, cols))
            pos += tlen
            row, pos = skiptofield!(buf, pos, len, options, header+1, datarow)
            push!(columnpositions, pos)
            _, pos = countfields(buf, pos, len, options)
        end
    elseif isa(header, AbstractRange)
        # column names span several columns
        columnpositions = Int64[]
        columnnames = String[]
        throw(ArgumentError("not implemented for transposed csv files"))
    elseif pos > len
        # emtpy file, use column names if provided
        datapos = pos
        columnpositions = Int64[]
        columnnames = header isa Vector && !isempty(header) ? String[string(x) for x in header] : []
        rows = 0
    else
        # column names provided explicitly or should be generated, they don't exist in data
        # skip to datarow
        row, pos = skiptofield!(buf, pos, len, options, 1, datarow)
        # io now at start of 1st data cell
        columnnames = [isa(header, Integer) || isempty(header) ? "Column1" : string(header[1])]
        columnpositions = Int64[pos]
        datapos = pos
        rows, pos = countfields(buf, pos, len, options)
        # we're now done w/ column 1, if EOF we're done, otherwise, parse column 2's column name
        cols = 1
        while pos <= len
            # skip to datarow column
            row, pos = skiptofield!(buf, pos, len, options, 1, datarow)
            cols += 1
            push!(columnnames, isa(header, Integer) || isempty(header) ? "Column$cols" : string(header[cols]))
            push!(columnpositions, pos)
            _, pos = countfields(buf, pos, len, options)
        end
    end
    return rows, makeunique(map(x->normalizenames ? normalizename(x) : Symbol(x), columnnames))::Vector{Symbol}, columnpositions
end

function skiptofield!(buf, pos, len, options, row, header)
    while row < header
        while pos <= len
            _, code, _, _, tlen = Parsers.xparse(String, buf, pos, len, options)
            pos += tlen
            Parsers.delimited(code) && break
        end
        row += 1
    end
    return row, pos
end

function countfields(buf, pos, len, options)
    rows = 0
    while pos <= len
        _, code, _, _, tlen = Parsers.xparse(String, buf, pos, len, options)
        pos += tlen
        rows += 1
        Parsers.delimited(code) && continue
        (Parsers.newline(code) || pos > len) && break
    end
    return rows, pos
end