text:
# =========================================================================
#    BASIC DATA GENERATION
# =========================================================================

const vertex_data = Ref{Any}(nothing)
V() = sort!(collect(keys(vertex_data[])))
V(i) = vertex_data[][i]

const distance_data = Ref{Any}(nothing)
dist() = distance_data[]
dist(i,j) = distance_data[][i,j]

const vehicle_data = Ref{Any}(nothing)
K() = sort!(collect(keys(vehicle_data[])))
K(k) = vehicle_data[][k]

const period_data = Ref{Any}(nothing)
T() = period_data[]
T(t) = period_data[][t]

const demand_data = Ref{Any}(nothing)
d() = demand_data[]
d(i,t) = demand_data[][i,t]

function extract!(path::String;f::String) #extract from excel
    xf = XLSX.readxlsx(path) #READ WORKSHEET
    data = Dict{Symbol,DataFrame}() #DATAFRAME DICT

    for sheets in XLSX.sheetnames(xf) #TURN SHEETS INTO DATAFRAME
        df = DataFrame(XLSX.gettable(xf[sheets])...) #TRANSFORM TABLE INTO DATAFRAME
        data[Symbol(sheets)] = df #DEFINE THE NAME FROM THE WORKSHEET
    end

    V = Dict{Int64,vtx}() #INITIATE VERTICES
    for v in eachrow(data[:vertices]) #ITERATE OVER DATA
        V[v.id] = vtx(
            v.name, v.type,
            v.x, v.y, v.MAX, v.MIN, v.START,
            v.h
        )
    end

    dist = JuMP.Containers.DenseAxisArray{Float64}(undef, keys(V), keys(V))
    dist .= 999999999
    for i in keys(V), j in keys(V)
        if i != j
            if f == "haversine"
                dist[i,j] = haversine([V[i].x,V[i].y],[V[j].x,V[j].y],6378.137)
            else #if f == "euclidean"
                dist[i,j] = euclidean([V[i].x,V[i].y],[V[j].x,V[j].y])
            end
        end
    end

    K = Dict{Int64,veh}() #INITIATE VEHICLES
    for k in eachrow(data[:vehicles]) #ITERATE OVER DATA
        K[k.id] = veh(
            k.name, k.type,
            parse.(Int64,split(k.cover)),
            Dict(parse.(Int64,split(k.cover)) .=>  parse.(Int64,split(k.BP))), k.Q,
            k.vx, k.vl, k.fp, k.fd
        )
    end

    T = collect( #range from starting month for duration
        range(
            last(data[:periods].start),
            length = last(data[:periods].T),
            step = 1
        )
    )

    d = JuMP.Containers.DenseAxisArray(
        Array{Float64}(data[:demands][:,string.(T)]), #dataset
        Array{Int64}(data[:demands].point), #dims 1
        T #dims 2
    )

    vertex_data[] = V
    distance_data[] = dist
    vehicle_data[] = K
    period_data[] = T
    demand_data[] = d

    return V,dist,K,T,d
end

function initStab()
    slackCoeff = sl_C()
    surpCoeff = su_C()

    slackLim = Dict{Tuple,Int64}()
    surpLim = Dict{Tuple,Int64}()
    for k in K(), t in T()
        slackLim[(k,t)] = sum(K(k).BP[i] for i in K(k).cover) * K(k).Q
        surpLim[(k,t)] = sum(K(k).BP[i] for i in K(k).cover) * K(k).Q
    end

    return stabilizer(slackCoeff,surpCoeff,slackLim,surpLim)
end

function root()
    root = node(
        uuid1(), uuid1(),
        Vector{bound}(),Vector{col}(),
        initStab(),
        ["UNVISITED"]
    )

    #=R = Dict{Int64,col}()

    for k in K()
        o = JuMP.Containers.DenseAxisArray{Float64}(undef,K(k).cover)
        u = JuMP.Containers.DenseAxisArray{Float64}(undef,K(k).cover)
        v = JuMP.Containers.DenseAxisArray{Float64}(undef,K(k).cover)
        x = JuMP.Containers.DenseAxisArray{Float64}(undef,K(k).cover,K(k).cover)
        l = JuMP.Containers.DenseAxisArray{Float64}(undef,K(k).cover,K(k).cover)

        o .= 0
        u .= 0
        v .= 0
        l .= 0
        x .= 0

        R[k] = col(u,v,l,o,x)
    end

    push!(root.columns,R)=#

    return root
end


text:
# build a random mask se with extra symmetricity constraint
function rand_se_mask(r, N; symmetric=false)
    se = centered(rand(Bool, ntuple(_ -> r, N)))
    symmetric || return se
    dual_se = copy(se)
    for i in CartesianIndices(dual_se)
        dual_se[i] = se[-i]
    end

    out = @. (Int(dual_se) + Int(se)) ÷ 2
    out[OffsetArrays.center(out)...] = 1
    return Bool.(out)
end

ext: jl


text:
using PatModules
@once module example_once
    include("subpackage/subpackage.jl")
    include("utils.jl")
    import .subpackage
    import .utils

    function greetings()
        utils.do_nothing()
        subpackage.say_hi()
        # Can descend into subsubpackages
        subpackage.two.say_bye()
    end
end


text:
function aspect_ratio(g::OSMGraph)
    max_y, min_y = extrema(first, g.node_coordinates)
    mid_y = (max_y + min_y)/2
    return 1/cos(mid_y * pi/180)
end
aspect_ratio(sg::SimplifiedOSMGraph) = aspect_ratio(sg.parent)

RecipesBase.@recipe function f(g::AbstractOSMGraph)
    color --> :black
    aspect_ratio --> aspect_ratio(g)
    MultiLineString(GeoInterface.coordinates.(highway_gdf(g).geom))
end


text:
"""
    ground_state_τeo(g::AGraph, J::Vector{Vector{Int}}
        τ::Float64 = 1.2,
        maxiters::Int = typemax(Int),
        infotime::Int = typemax(Int))

Computes the ground state of an Ising model using the τ-EO heuristic.

Returns a configuration `σ` (a vector with ±1 components) and its energy `E`.
"""
function ground_state_τeo(g::AGraph, J::Vector{Vector{Int}};
        τ::Float64 = 1.2,
        maxiters::Int = typemax(Int),
        infotime::Int = typemax(Int))

    N = nv(g)
    σ = fill(1, N)
    hmax = maximum(v->sum(abs,v), J)
    sets = [Set{Int}() for s=1:2hmax+1]
    for i=1:N
        f = frust(g, σ, J[i], i)
        idx = ftoidx(f, hmax)
        push!(sets[idx], i)
    end
    distr = DiscreteDistribution(Float64[(τ-1)/(1-N^(1-τ)) / l^τ for l=1:N])
    E = energy(g,σ, zeros(Int, N), J)
    Emin = E
    σmin = deepcopy(σ)

    niter_tranche = 100
    itmultiplier = 4

    niters = 0
    foundnewmin = true

    while foundnewmin && niters <= maxiters
        foundnewmin = false
        for it=1:niter_tranche
            niters += 1
            niters > maxiters && break
            for _=1:N
                l = rand(distr)
                idx = searchsortedfirst(cumsum(length.(sets)), l)
                i = rand(sets[idx])

                delete!(sets[idx], i)
                for j in neighbors(g, i)
                    fj = frust(g, σ, J[j], j)
                    idxj = ftoidx(fj, hmax)
                    delete!(sets[idxj], j)
                end

                σ[i] *= -1
                fi = frust(g, σ, J[i], i)
                idx = ftoidx(fi, hmax)
                push!(sets[idx], i)
                for j in neighbors(g, i)
                    fj = frust(g, σ, J[j], j)
                    idxj = ftoidx(fj, hmax)
                    push!(sets[idxj], j)
                end

                E += 2fi
                if E < Emin && niters > 5 # avoids too much copying in the very first steps
                    foundnewmin = true
                    Emin = E
                    σmin = deepcopy(σ)
                end
            end
            if niters % infotime == 0
                println("it=$niters E/N=$(E/N)  Emin/N=$(Emin/N)")
            end
        end

        foundnewmin && (niter_tranche *= itmultiplier)
    end
    return σmin, Emin
end

ftoidx(f::Int, hmax::Int) = hmax - f + 1

function frust(g::AGraph, σ::Vector, J::Vector{Int}, i)
    H = 0
    for (k,j) in enumerate(neighbors(g, i))
        H += J[k] * σ[j]
    end
    return -H*σ[i]
end

max_stars_repo_name: JuliaTagBot/LittleScienceTools

max_issues_repo_name: JuliaTagBot/LittleScienceTools

max_forks_repo_name: JuliaTagBot/LittleScienceTools


text:
"""
Notes:
- Usually we want to retain more than the intersection of columns of the input tables, but less than the union of columns.
  E.g., A table may contain name and DOB, another table may contain name, address and social security number,
  and both tables may also contain other event-level data such as timestamps and service locations.
  We want to retain the entity-level data (name, DOB, address and social security number) and not the event-level data.
  Therefore we require a schema for the spine so that the desired columns are explicitly specified.
"""
module runlinkage

export run_linkage

using CSV
using DataFrames
using Dates
using Logging
using Schemata

using ..TableIndexes
using ..distances
using ..config

function run_linkage(configfile::String)
    @info "$(now()) Configuring linkage"
    cfg = LinkageConfig(configfile)
    run_linkage(cfg, configfile)
end

function run_linkage(cfg::LinkageConfig, configfile::String="")
    @info "$(now()) Initialising output directory: $(cfg.output_directory)"
    d = cfg.output_directory
    mkdir(d)
    mkdir(joinpath(d, "input"))
    mkdir(joinpath(d, "output"))
    if isempty(configfile)
        write_config(joinpath(d, "input", "config.yaml"), cfg)  # Write cfg to config.yaml
    else
        cp(configfile, joinpath(d, "input", basename(configfile)))  # Copy config file to d/input
    end
    software_versions = construct_software_versions_table()
    CSV.write(joinpath(d, "input", "SoftwareVersions.csv"), software_versions; delim=',')  # Write software_versions to d/input
    criteria = construct_criteria_table(cfg)
    CSV.write(joinpath(d, "output", "criteria.tsv"), criteria; delim='\t')  # Write criteria to d/output

    # Init spine
    if isnothing(cfg.spine.datafile)
        @info "$(now()) Initialising spine"
        colnames = cfg.spine.schema.columnorder
        isnothing(findfirst(==(:EntityId), colnames)) && error("The spine has no EntityId column.")
        spine = DataFrame()
        for colname in colnames
            spine[!, colname] = colname == :EntityId ? UInt[] : Union{Missing, String}[]
        end
    else
        @info "$(now()) Importing spine"
        spine = DataFrame(CSV.File(cfg.spine.datafile; types=Union{Missing, String}))  # For performance only Strings are compared (avoids parsing values)
        spine[!, :EntityId] = [parse(UInt, x) for x in spine[!, :EntityId]]
    end

    # Init Links table
    links         = DataFrame(TableName=String[], EventId=UInt[], EntityId=UInt[], CriteriaId=Int[])
    nlinks        = 0  # Number of rows in the links table stored on disk
    links_outfile = joinpath(cfg.output_directory, "output", "links.tsv")
    CSV.write(links_outfile, links; delim='\t')
    n     = 1_000_000
    links = DataFrame(TableName=Vector{String}(undef, n), EventId=Vector{UInt}(undef, n), EntityId=Vector{UInt}(undef, n), CriteriaId=Vector{Int}(undef, n))

    # Do the linkage for each events table
    construct_entityid_from = cfg.construct_entityid_from
    for tablecriteria in cfg.criteria
        # Create an empty output file for the events table and store it in the output directory
        tablename = tablecriteria[1].tablename
        @info "$(now()) Starting linkage for table $(tablename)"
        events_schema  = cfg.tables[tablename].schema
        events         = init_events(events_schema, 0)  # Columns are [:EventId, primarykey_columns...]
        events_outfile = joinpath(cfg.output_directory, "output", "$(tablename)_primarykey_and_eventid.tsv")
        CSV.write(events_outfile, events; delim='\t')

        # Run the data through each linkage iteration
        events_infile = cfg.tables[tablename].datafile
        nlinks = link_table_to_events!(links, nlinks, links_outfile,
                                       spine, construct_entityid_from, cfg.append_to_spine,
                                       events_infile, events_outfile, events_schema, tablecriteria)
    end

    @info "$(now()) Writing spine to the output directory ($(format_number(size(spine, 1))) rows)"
    spine_outfile = joinpath(cfg.output_directory, "output", "spine.tsv")
    CSV.write(spine_outfile, spine; delim='\t')

    @info "$(now()) Finished linkage"
    cfg.output_directory
end

################################################################################
# Unexported

"Returns a DataFrame with n rows and columns [:EventId, events_schema.primarykey...]."
function init_events(events_schema::TableSchema, n::Int)
    result = DataFrame(EventId = Vector{Union{Missing, UInt}}(undef, n))
    for colname in events_schema.primarykey
        result[!, colname] = Vector{Union{Missing, String}}(undef, n)
    end
    result
end

"""
Modified: links, spine.

For each eventrow in the events table:
1. If the eventrow concerns an entity in the spine, append a row to links.
2. Else if append_to_spine is true, append a row to the spine and link it to the eventrow.
"""
function link_table_to_events!(links::DataFrame, nlinks::Int, links_outfile::String,
                               spine::DataFrame, construct_entityid_from::Vector{Symbol}, append_to_spine::Bool,
                               events_infile::String, events_outfile::String, events_schema::TableSchema, tablecriteria::Vector{LinkageCriteria})
    nlinks0    = nlinks  # Number of links due to other tables
    events     = init_events(events_schema, 1_000_000)  # Process the events in batches of 1_000_000 rows
    eventids   = Set{UInt}()  # Enables avoidance of duplicated EventIds
    buffer     = IOBuffer()   # For building EventIds
    i_events   = 0  # Number of this table's rows stored in-memory
    nevents    = 0  # Number of this table's rows stored on disk
    i_links    = 0  # Number of this table's rows stored in the in-memory links table
    tablename  = String(events_schema.name)
    n_criteria = length(tablecriteria)
    events_primarykey = events_schema.primarykey
    criteriaid2index  = construct_table_indexes(tablecriteria, spine)  # criteria.id => TableIndex(spine, colnames, index)
    criteriaid2key    = Dict(id => fill("", length(tableindex.colnames)) for (id, tableindex) in criteriaid2index)  # Place-holder for lookup keys
    for eventrow in CSV.Rows(events_infile; reusebuffer=true)
        # Store event primary key and construct eventid at the same time
        i_events += 1
        primarykey_is_incomplete = false
        print(buffer, tablename)  # Include the tablename in the eventid
        for colname in events_primarykey
            val = getproperty(eventrow, colname)
            if ismissing(val)
                primarykey_is_incomplete = true
                break
            end
            print(buffer, val)
            events[i_events, colname] = val
        end
        eventid = hash(String(take!(buffer)))

        # Check whether an EntityId can be constructed (requires non-missing values for columns in construct_entityid_from)
        cannot_construct_entityid = false
        if append_to_spine
            for colname in construct_entityid_from
                cannot_construct_entityid = ismissing(getproperty(eventrow, colname))
                cannot_construct_entityid && break
            end
        end

        # Roll back if primary key is incomplete, cannot construct EntityId or if EventId is a duplicate
        if primarykey_is_incomplete || cannot_construct_entityid || in(eventid, eventids)
            i_events -= 1  # Unstore the row
            continue
        end

        # Store EventId
        push!(eventids, eventid)
        events[i_events, :EventId] = eventid

        # Link the eventrow to the spine using the first LinkageCriteria that are satisfied (if any)
        # n_hasmissing = Number of criteria for which eventrow has missing data
        # If n_hasmissing == n_criteria then the entity in the eventrow cannot be appended to the spine because no criteria can be satisfied
        i_links, n_hasmissing, islinked = link_event_to_spine!(eventrow, eventid, spine, links, i_links, tablecriteria, criteriaid2index, criteriaid2key, tablename)

        # If eventrow is unlinked, append it to the spine, update the TableIndexes and link
        if append_to_spine && n_hasmissing < n_criteria && !islinked
            append_row_to_spine!(eventrow, spine, construct_entityid_from)  # Create a new spine record
            for linkagecriteria in tablecriteria
                tableindex = criteriaid2index[linkagecriteria.id]
                update!(tableindex, spine, size(spine, 1))  # Update the tableindex
            end
            i_links, n_hasmissing, islinked = link_event_to_spine!(eventrow, eventid, spine, links, i_links, tablecriteria, criteriaid2index, criteriaid2key, tablename)
        end

        # If events table is full, write to disk
        if i_events == 1_000_000
            CSV.write(events_outfile, events; delim='\t', append=true)
            nevents += i_events
            @info "$(now()) Exported $(div(nevents, 1_000_000))M rows of $(tablename)"
            i_events = 0  # Reset the eventrow number
        end

        # If links table is full, write to disk
        if i_links == 1_000_000
            CSV.write(links_outfile, links; delim='\t', append=true)
            nlinks += i_links
            @info "$(now()) Exported $(div(nlinks - nlinks0, 1_000_000))M links between $(tablename) and the spine"
            i_links = 0  # Reset the row number
        end
    end
    if i_events != 0  # Write remaining events if they exist
        CSV.write(events_outfile, view(events, 1:i_events, :); append=true, delim='\t')
        nevents += i_events
        @info "$(now()) Exported $(format_number(nevents)) rows of $(tablename)"
    end
    if i_links != 0  # Write remaining links if they exist
        CSV.write(links_outfile, view(links, 1:i_links, :); append=true, delim='\t')
        nlinks += i_links
        @info "$(now()) Exported $(format_number(nlinks - nlinks0)) links between $(tablename) and the spine"
    end
    nlinks
end

"""
Modified: events[i_events, :]

If possible, link the eventrow to the spine using the first LinkageCriteria that are satisfied (if any).
Record the linkage in the events table.
"""
function link_event_to_spine!(eventrow, eventid::UInt, spine, links::DataFrame, i_links::Int, tablecriteria::Vector{LinkageCriteria}, criteriaid2index, criteriaid2key, tablename)
    n_hasmissing = 0  # Number of criteria for which row has missing data
    islinked     = false
    for linkagecriteria in tablecriteria
        # Identify the spine records that match the row on linkagecriteria.exactmatch
        criteriaid = linkagecriteria.id
        tableindex = criteriaid2index[criteriaid]
        hasmissing = constructkey!(criteriaid2key[criteriaid], eventrow, tableindex.colnames)
        if hasmissing
            n_hasmissing += 1
            continue  # datarow[tableindex.colnames] includes a missing value
        end
        k = Tuple(criteriaid2key[criteriaid])
        !haskey(tableindex.index, k) && continue  # eventrow doesn't match any spine records on linkagecriteria.exactmatch
        candidate_indices = tableindex.index[k]

        # Identify the spine record that best matches the eventrow (if it exists)
        entityid, i_spine = select_best_candidate(spine, candidate_indices, eventrow, linkagecriteria.approxmatch)
        entityid == 0 && continue  # None of the candidates satisfy the approxmatch criteria

        # Create a link between the spine and the events data
        islinked = true
        i_links += 1
        links[i_links, :TableName]  = tablename
        links[i_links, :EventId]    = eventid
        links[i_links, :EntityId]   = entityid
        links[i_links, :CriteriaId] = criteriaid
        break  # eventrow has been linked, no need to link on other criteria
    end
    i_links, n_hasmissing, islinked
end

function select_best_candidate(spine, candidate_indices::Vector{Int}, row, approxmatches::Vector{ApproxMatch})
    if isempty(approxmatches)
        if length(candidate_indices) == 1
            i_spine = candidate_indices[1]
            return spine[i_spine, :EntityId], i_spine  # There is 1 candidate and no further selection criteria to be met
        else
            return 0, 0  # There is more than 1 candidate and no way to select the best one
        end
    end
    result = 0, 0
    min_distance = 1.0
    for i_spine in candidate_indices
        dist = 0.0
        ok   = true  # Each ApproxMatch criterion is satisfied
        for approxmatch in approxmatches
            dataval  = getproperty(row, approxmatch.datacolumn)
            spineval = spine[i_spine, approxmatch.spinecolumn]
            d        = distance(approxmatch.distancemetric, dataval, spineval)
            if d <= approxmatch.threshold
                dist += d
            else
                ok = false
                break
            end
        end
        !ok && continue  # Not every ApproxMatch criterion is satisfied
        dist >= min_distance && continue  # distance(row, candidate) is not minimal among candidates tested so far
        min_distance = dist
        result = spine[i_spine, :EntityId], i_spine
    end
    result
end

"""
Modified: spine.

Append the row to the spine and return the EntityId of the new row.
"""
function append_row_to_spine!(row, spine, construct_entityid_from::Vector{Symbol})
    push!(spine, (EntityId=UInt(0),), cols=:subset)  # Append row containing a dummy EntityId and missing values for all other columns
    i = size(spine, 1)
    spinecols = names(spine)
    for colname_str in spinecols
        colname = Symbol(colname_str)
        colname == :EntityId       && continue
        !hasproperty(row, colname) && continue
        spine[i, colname] = getproperty(row, colname)
    end
    spine[i, :EntityId] = hash(spine[i, construct_entityid_from])
end

################################################################################
# Utils

function construct_software_versions_table()
    pkg_version = get_package_version()
    DataFrame(software=["Julia", "SpineBasedRecordLinkage.jl"], version=[VERSION, pkg_version])
end

function get_package_version()
    pkg_version = "unknown"
    pkgdir = dirname(@__DIR__)
    f = open(joinpath(pkgdir, "Project.toml"))
    i = 0
    for line in eachline(f)
        i += 1
        if i == 4
            v = split(line, "=")  # line: version = "0.1.0"
            pkg_version = replace(strip(v[2]), "\"" => "")  # v0.1.0
            close(f)
            return pkg_version
        end
    end
end

"Convert integer to string and insert commas for pretty printing."
function format_number(x::Int)
    s      = string(x)
    n      = length(s)
    lead   = rem(n, 3)  # Number of digits before the 1st comma
    lead   = lead == 0 ? 3 : lead
    result = s[1:lead]
    n_done = lead
    for i= 1:100  # Append remaining digits in groups of 3
        n_done == n && break
        s2 = ",$(s[(n_done + 1):(n_done + 3)])"
        result = result * s2
        n_done += 3
    end
    result
end

function construct_criteria_table(cfg::LinkageConfig)
    colnames = (:CriteriaId, :TableName, :ExactMatches, :ApproxMatches)
    coltypes = Tuple{Int, String, Dict{Symbol, Symbol}, Union{Missing, Vector{ApproxMatch}}}
    result   = NamedTuple{colnames, coltypes}[]
    for v in cfg.criteria
        for x in v
            am = isempty(x.approxmatch) ? missing : x.approxmatch
            r  = (CriteriaId=x.id, TableName=x.tablename, ExactMatches=x.exactmatch, ApproxMatches=am)
            push!(result, r)
        end
    end
    DataFrame(result)
end

"Returns: Dict(CriteriaId => TableIndex(spine, colnames))"
function construct_table_indexes(criteria::Vector{LinkageCriteria}, spine)
    # Create TableIndexes
    tmp = Dict{Int, TableIndex}()
    for linkagecriteria in criteria
        colnames = [spine_colname for (data_colname, spine_colname) in linkagecriteria.exactmatch]
        tmp[linkagecriteria.id] = TableIndex(spine, colnames)
    end

    # Replace spine colnames with data colnames
    # A hack to avoid converting from spine colnames to data colnames on each lookup
    result = Dict{Int, TableIndex}()
    for linkagecriteria in criteria
        data_colnames = [data_colname for (data_colname, spine_colname) in linkagecriteria.exactmatch]
        tableindex    = tmp[linkagecriteria.id]
        result[linkagecriteria.id] = TableIndex(spine, data_colnames, tableindex.index)
    end
    result
end

"Update the TableIndex with table[i, :]"
function update!(tableindex::TableIndex, table, i::Int)
    k = Tuple(table[i, tableindex.colnames])
    TableIndexes.update!(tableindex.index, k, i)
end

"Returns: true if row[colnames] includes a missing value."
function constructkey!(result::Vector{String}, row, colnames::Vector{Symbol})
    for (j, colname) in enumerate(colnames)  # Populate result with the row's values of tableindex.colnames (iteration.exactmatches)
        val = getproperty(row, colname)
        ismissing(val) && return true
        result[j] = val
    end
    false
end

end


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

include("model_ApplicationGateway.jl")
include("model_ApplicationGatewayAuthenticationCertificate.jl")
include("model_ApplicationGatewayAuthenticationCertificatePropertiesFormat.jl")
include("model_ApplicationGatewayAutoscaleConfiguration.jl")
include("model_ApplicationGatewayAvailableRequestHeadersResult.jl")
include("model_ApplicationGatewayAvailableResponseHeadersResult.jl")
include("model_ApplicationGatewayAvailableServerVariablesResult.jl")
include("model_ApplicationGatewayAvailableSslOptions.jl")
include("model_ApplicationGatewayAvailableSslOptionsPropertiesFormat.jl")
include("model_ApplicationGatewayAvailableSslPredefinedPolicies.jl")
include("model_ApplicationGatewayAvailableWafRuleSetsResult.jl")
include("model_ApplicationGatewayBackendAddress.jl")
include("model_ApplicationGatewayBackendAddress2.jl")
include("model_ApplicationGatewayBackendAddressPool.jl")
include("model_ApplicationGatewayBackendAddressPool2.jl")
include("model_ApplicationGatewayBackendAddressPoolPropertiesFormat.jl")
include("model_ApplicationGatewayBackendAddressPoolPropertiesFormat2.jl")
include("model_ApplicationGatewayBackendHealth.jl")
include("model_ApplicationGatewayBackendHealthHttpSettings.jl")
include("model_ApplicationGatewayBackendHealthOnDemand.jl")
include("model_ApplicationGatewayBackendHealthPool.jl")
include("model_ApplicationGatewayBackendHealthServer.jl")
include("model_ApplicationGatewayBackendHttpSettings.jl")
include("model_ApplicationGatewayBackendHttpSettingsPropertiesFormat.jl")
include("model_ApplicationGatewayConnectionDraining.jl")
include("model_ApplicationGatewayCustomError.jl")
include("model_ApplicationGatewayFirewallDisabledRuleGroup.jl")
include("model_ApplicationGatewayFirewallExclusion.jl")
include("model_ApplicationGatewayFirewallRule.jl")
include("model_ApplicationGatewayFirewallRuleGroup.jl")
include("model_ApplicationGatewayFirewallRuleSet.jl")
include("model_ApplicationGatewayFirewallRuleSetPropertiesFormat.jl")
include("model_ApplicationGatewayFrontendIPConfiguration.jl")
include("model_ApplicationGatewayFrontendIPConfigurationPropertiesFormat.jl")
include("model_ApplicationGatewayFrontendPort.jl")
include("model_ApplicationGatewayFrontendPortPropertiesFormat.jl")
include("model_ApplicationGatewayHeaderConfiguration.jl")
include("model_ApplicationGatewayHttpListener.jl")
include("model_ApplicationGatewayHttpListenerPropertiesFormat.jl")
include("model_ApplicationGatewayIPConfiguration.jl")
include("model_ApplicationGatewayIPConfigurationPropertiesFormat.jl")
include("model_ApplicationGatewayListResult.jl")
include("model_ApplicationGatewayOnDemandProbe.jl")
include("model_ApplicationGatewayPathRule.jl")
include("model_ApplicationGatewayPathRulePropertiesFormat.jl")
include("model_ApplicationGatewayPrivateEndpointConnection.jl")
include("model_ApplicationGatewayPrivateEndpointConnectionListResult.jl")
include("model_ApplicationGatewayPrivateEndpointConnectionProperties.jl")
include("model_ApplicationGatewayPrivateLinkConfiguration.jl")
include("model_ApplicationGatewayPrivateLinkConfigurationProperties.jl")
include("model_ApplicationGatewayPrivateLinkIpConfiguration.jl")
include("model_ApplicationGatewayPrivateLinkIpConfigurationProperties.jl")
include("model_ApplicationGatewayPrivateLinkResource.jl")
include("model_ApplicationGatewayPrivateLinkResourceListResult.jl")
include("model_ApplicationGatewayPrivateLinkResourceProperties.jl")
include("model_ApplicationGatewayProbe.jl")
include("model_ApplicationGatewayProbeHealthResponseMatch.jl")
include("model_ApplicationGatewayProbePropertiesFormat.jl")
include("model_ApplicationGatewayPropertiesFormat.jl")
include("model_ApplicationGatewayProtocol.jl")
include("model_ApplicationGatewayRedirectConfiguration.jl")
include("model_ApplicationGatewayRedirectConfigurationPropertiesFormat.jl")
include("model_ApplicationGatewayRequestRoutingRule.jl")
include("model_ApplicationGatewayRequestRoutingRulePropertiesFormat.jl")
include("model_ApplicationGatewayRewriteRule.jl")
include("model_ApplicationGatewayRewriteRuleActionSet.jl")
include("model_ApplicationGatewayRewriteRuleCondition.jl")
include("model_ApplicationGatewayRewriteRuleSet.jl")
include("model_ApplicationGatewayRewriteRuleSetPropertiesFormat.jl")
include("model_ApplicationGatewaySku.jl")
include("model_ApplicationGatewaySslCertificate.jl")
include("model_ApplicationGatewaySslCertificatePropertiesFormat.jl")
include("model_ApplicationGatewaySslPolicy.jl")
include("model_ApplicationGatewaySslPredefinedPolicy.jl")
include("model_ApplicationGatewaySslPredefinedPolicyPropertiesFormat.jl")
include("model_ApplicationGatewayTrustedRootCertificate.jl")
include("model_ApplicationGatewayTrustedRootCertificatePropertiesFormat.jl")
include("model_ApplicationGatewayUrlConfiguration.jl")
include("model_ApplicationGatewayUrlPathMap.jl")
include("model_ApplicationGatewayUrlPathMapPropertiesFormat.jl")
include("model_ApplicationGatewayWebApplicationFirewallConfiguration.jl")
include("model_ApplicationSecurityGroup.jl")
include("model_ApplicationSecurityGroupPropertiesFormat.jl")
include("model_BackendAddressPool.jl")
include("model_BackendAddressPoolPropertiesFormat.jl")
include("model_CipherSuitesEnum.jl")
include("model_CloudError.jl")
include("model_CloudErrorBody.jl")
include("model_CustomDnsConfigPropertiesFormat.jl")
include("model_DdosSettings.jl")
include("model_Delegation.jl")
include("model_Error.jl")
include("model_ErrorDetails.jl")
include("model_FlowLog.jl")
include("model_FlowLogFormatParameters.jl")
include("model_FlowLogPropertiesFormat.jl")
include("model_FrontendIPConfiguration.jl")
include("model_FrontendIPConfigurationPropertiesFormat.jl")
include("model_IPAllocationMethod.jl")
include("model_IPConfiguration.jl")
include("model_IPConfigurationProfile.jl")
include("model_IPConfigurationProfilePropertiesFormat.jl")
include("model_IPConfigurationPropertiesFormat.jl")
include("model_IPVersion.jl")
include("model_InboundNatRule.jl")
include("model_InboundNatRulePropertiesFormat.jl")
include("model_IpTag.jl")
include("model_LoadBalancerBackendAddress.jl")
include("model_LoadBalancerBackendAddressPropertiesFormat.jl")
include("model_NetworkInterface.jl")
include("model_NetworkInterfaceDnsSettings.jl")
include("model_NetworkInterfaceIPConfiguration.jl")
include("model_NetworkInterfaceIPConfigurationPrivateLinkConnectionProperties.jl")
include("model_NetworkInterfaceIPConfigurationPropertiesFormat.jl")
include("model_NetworkInterfacePropertiesFormat.jl")
include("model_NetworkInterfaceTapConfiguration.jl")
include("model_NetworkInterfaceTapConfigurationPropertiesFormat.jl")
include("model_NetworkSecurityGroup.jl")
include("model_NetworkSecurityGroupPropertiesFormat.jl")
include("model_PolicyNameEnum.jl")
include("model_PrivateEndpoint.jl")
include("model_PrivateEndpointProperties.jl")
include("model_PrivateLinkServiceConnection.jl")
include("model_PrivateLinkServiceConnectionProperties.jl")
include("model_PrivateLinkServiceConnectionState.jl")
include("model_ProtocolsEnum.jl")
include("model_ProvisioningState.jl")
include("model_PublicIPAddress.jl")
include("model_PublicIPAddressDnsSettings.jl")
include("model_PublicIPAddressPropertiesFormat.jl")
include("model_PublicIPAddressSku.jl")
include("model_RedirectTypeEnum.jl")
include("model_Resource.jl")
include("model_ResourceNavigationLink.jl")
include("model_ResourceNavigationLinkFormat.jl")
include("model_RetentionPolicyParameters.jl")
include("model_Route.jl")
include("model_RouteNextHopType.jl")
include("model_RoutePropertiesFormat.jl")
include("model_RouteTable.jl")
include("model_RouteTablePropertiesFormat.jl")
include("model_SecurityRule.jl")
include("model_SecurityRuleAccess.jl")
include("model_SecurityRuleDirection.jl")
include("model_SecurityRulePropertiesFormat.jl")
include("model_ServiceAssociationLink.jl")
include("model_ServiceAssociationLinkPropertiesFormat.jl")
include("model_ServiceDelegationPropertiesFormat.jl")
include("model_ServiceEndpointPolicy.jl")
include("model_ServiceEndpointPolicyDefinition.jl")
include("model_ServiceEndpointPolicyDefinitionPropertiesFormat.jl")
include("model_ServiceEndpointPolicyPropertiesFormat.jl")
include("model_ServiceEndpointPropertiesFormat.jl")
include("model_SubResource.jl")
include("model_Subnet.jl")
include("model_SubnetPropertiesFormat.jl")
include("model_TagsObject.jl")
include("model_TrafficAnalyticsConfigurationProperties.jl")
include("model_TrafficAnalyticsProperties.jl")
include("model_TransportProtocol.jl")
include("model_VirtualNetworkTap.jl")
include("model_VirtualNetworkTapPropertiesFormat.jl")


max_stars_repo_name: egmreynolds/MANA

max_issues_repo_name: egmreynolds/MANA

max_forks_repo_name: egmreynolds/MANA


text:
#==============================================================================#
# AWSS3.jl
#
# S3 API. See http://docs.aws.amazon.com/AmazonS3/latest/API/APIRest.html
#
# Copyright OC Technology Pty Ltd 2014 - All rights reserved
#==============================================================================#

module AWSS3

export S3Path,
    s3_arn,
    s3_put,
    s3_get,
    s3_get_file,
    s3_exists,
    s3_delete,
    s3_copy,
    s3_create_bucket,
    s3_put_cors,
    s3_enable_versioning,
    s3_delete_bucket,
    s3_list_buckets,
    s3_list_objects,
    s3_list_keys,
    s3_list_versions,
    s3_get_meta,
    s3_directory_stat,
    s3_purge_versions,
    s3_sign_url,
    s3_begin_multipart_upload,
    s3_upload_part,
    s3_complete_multipart_upload,
    s3_multipart_upload,
    s3_get_tags,
    s3_put_tags,
    s3_delete_tags

using AWS
using AWS.AWSServices: s3
using FilePathsBase
using FilePathsBase: /, join
using HTTP
using OrderedCollections: OrderedDict, LittleDict
using SymDict
using Retry
using XMLDict
using EzXML
using Dates
using Base64
using UUIDs
using URIs
using Compat: @something

@service S3

const SSDict = Dict{String,String}
const AbstractS3Version = Union{AbstractString,Nothing}
const AbstractS3PathConfig = Union{AbstractAWSConfig,Nothing}

__init__() = FilePathsBase.register(S3Path)

"""
    s3_arn(resource)
    s3_arn(bucket,path)

[Amazon Resource Name](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
for S3 `resource` or `bucket` and `path`.
"""
s3_arn(resource) = "arn:aws:s3:::$resource"
s3_arn(bucket, path) = s3_arn("$bucket/$path")

"""
    s3_get([::AbstractAWSConfig], bucket, path; <keyword arguments>)

[Get Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html)
from `path` in `bucket`.

# Optional Arguments
- `version=`: version of object to get.
- `retry=true`: try again on "NoSuchBucket", "NoSuchKey"
                (common if object was recently created).
- `raw=false`:  return response as `Vector{UInt8}`
                (by default return type depends on `Content-Type` header).
- `byte_range=nothing`:  given an iterator of `(start_byte, end_byte)` gets only
    the range of bytes of the object from `start_byte` to `end_byte`.  For example,
    `byte_range=1:4` gets bytes 1 to 4 inclusive.  Arguments should use the Julia convention
    of 1-based indexing.
- `header::Dict{String,String}`: pass in an HTTP header to the request.

As an example of how to set custom HTTP headers, the below is equivalent to
`s3_get(aws, bucket, path; byte_range=range)`:

```julia
s3_get(aws, bucket, path; headers=Dict{String,String}("Range" => "bytes=\$(first(range)-1)-\$(last(range)-1)"))
```
"""
function s3_get(
    aws::AbstractAWSConfig,
    bucket,
    path;
    version::AbstractS3Version=nothing,
    retry::Bool=true,
    byte_range::Union{Nothing,AbstractVector}=nothing,
    raw::Bool=false,
    headers::AbstractDict{<:AbstractString,<:Any}=Dict{String,Any}(),
    return_stream::Bool=false,
    kwargs...,
)
    @repeat 4 try
        params = Dict{String,Any}("return_raw" => raw, "return_stream" => return_stream)
        if version !== nothing
            params["versionId"] = version
        end

        if byte_range !== nothing
            headers = copy(headers)  # make sure we don't mutate existing object
            # we make sure we stick to the Julia convention of 1-based indexing
            a, b = (first(byte_range) - 1), (last(byte_range) - 1)
            headers["Range"] = "bytes=$a-$b"
        end

        if !isempty(headers)
            params["headers"] = headers
        end

        return S3.get_object(bucket, path, params; aws_config=aws, kwargs...)
    catch e
        #! format: off
        # https://github.com/domluna/JuliaFormatter.jl/issues/459
        @delay_retry if retry && ecode(e) in ["NoSuchBucket", "NoSuchKey"] end
        #! format: on
    end
end

s3_get(a...; b...) = s3_get(global_aws_config(), a...; b...)

"""
    s3_get_file([::AbstractAWSConfig], bucket, path, filename; [version=], kwargs...)

Like `s3_get` but streams result directly to `filename`.  Keyword arguments accept are
the same as those for `s3_get`.
"""
function s3_get_file(
    aws::AbstractAWSConfig,
    bucket,
    path,
    filename;
    version::AbstractS3Version=nothing,
    kwargs...,
)
    stream = s3_get(aws, bucket, path; version=version, return_stream=true, kwargs...)

    open(filename, "w") do file
        while !eof(stream)
            write(file, readavailable(stream))
        end
    end
end

s3_get_file(a...; b...) = s3_get_file(global_aws_config(), a...; b...)

function s3_get_file(
    aws::AbstractAWSConfig,
    buckets::Vector,
    path,
    filename;
    version::AbstractS3Version=nothing,
    kwargs...,
)
    i = start(buckets)

    @repeat length(buckets) try
        bucket, i = next(buckets, i)
        s3_get_file(aws, bucket, path, filename; version=version, kwargs...)
    catch e
        #! format: off
        @retry if ecode(e) in ["NoSuchKey", "AccessDenied"] end
        #! format: on
    end
end

"""
   s3_get_meta([::AbstractAWSConfig], bucket, path; [version=], kwargs...)

[HEAD Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html)

Retrieves metadata from an object without returning the object itself.
"""
function s3_get_meta(
    aws::AbstractAWSConfig, bucket, path; version::AbstractS3Version=nothing, kwargs...
)
    params = Dict{String,Any}()
    if version !== nothing
        params["versionId"] = version
    end

    return S3.head_object(bucket, path, params; aws_config=aws, kwargs...)
end

s3_get_meta(a...; b...) = s3_get_meta(global_aws_config(), a...; b...)

function _s3_exists_file(aws::AbstractAWSConfig, bucket, path)
    q = Dict("prefix" => path, "delimiter" => "", "max-keys" => 1)
    l = S3.list_objects_v2(bucket, q; aws_config=aws)
    c = get(l, "Contents", nothing)
    c === nothing && return false
    return get(c, "Key", "") == path
end

"""
    _s3_exists_dir(aws::AbstractAWSConfig, bucket, path)

Internal, called by [`s3_exists`](@ref).

Checks whether the given directory exists.  This is a bit subtle because of how the
AWS API handles empty directories.  Empty directories are really just 0-byte nodes
which are named like directories, i.e. their name has a trailing `"/"`.

What this function does is, given a directory name `dir/`, check for all keys which
are lexographically greater than `dir.`.  The reason for this is that, if `dir/`
is a 0-byte node, checking for it directly will not reveal its existence due to
some rather peculiar design choices on the part of the S3 developers.

In all such cases, if the directory exists it will be seen in the *first* item
returned from `S3.list_objects_v2`: for empty directories this is because using
`start-after` explicitly excludes `dir.` itself and `dir/` is next; for directories
with actual keys, it is guaranteed that the first contained key will start with
the directory name.
"""
function _s3_exists_dir(aws::AbstractAWSConfig, bucket, path)
    a = chop(string(path)) * "."
    # note that you are not allowed to use *both* `prefix` and `start-after`
    q = Dict("delimiter" => "", "max-keys" => 1, "start-after" => a)
    l = S3.list_objects_v2(bucket, q; aws_config=aws)
    c = get(l, "Contents", nothing)
    c === nothing && return false
    return startswith(get(c, "Key", ""), path)
end

"""
    s3_exists_versioned([::AbstractAWSConfig], bucket, path, version)

Check if the version specified by `version` of the object in bucket `bucket` exists at `path`.

Note that this function relies on error catching and may be less performant than [`s3_exists_unversioned `](@ref)
which is preferred.  The reason for this is that support for versioning in the AWS API is very limited.
"""
function s3_exists_versioned(
    aws::AbstractAWSConfig, bucket, path, version::AbstractS3Version
)
    @repeat 2 try
        s3_get_meta(aws, bucket, path; version=version)
        return true
    catch e
        #! format: off
        @delay_retry if ecode(e) in ["NoSuchBucket", "404", "NoSuchKey", "AccessDenied"] end
        #! format: on

        @ignore if ecode(e) in ["404", "NoSuchKey", "AccessDenied"]
            return false
        end
    end
end

"""
    s3_exists_unversioned([::AbstractAWSConfig], bucket, path)

Returns a boolean whether an object exists at  `path` in `bucket`.

See [`s3_exists_versioned`](@ref) to check for specific versions.
"""
function s3_exists_unversioned(aws::AbstractAWSConfig, bucket, path)
    f = endswith(path, '/') ? _s3_exists_dir : _s3_exists_file
    return f(aws, bucket, path)
end

"""
    s3_exists([::AbstractAWSConfig], bucket, path; version=nothing)

Returns a boolean whether an object exists at `path` in `bucket`.

Note that the AWS API's support for object versioning is quite limited and this
check will involve `try` `catch` logic if `version` is not `nothing`.
"""
function s3_exists(aws::AbstractAWSConfig, bucket, path; version::AbstractS3Version=nothing)
    if version !== nothing
        s3_exists_versioned(aws, bucket, path, version)
    else
        s3_exists_unversioned(aws, bucket, path)
    end
end
s3_exists(a...; b...) = s3_exists(global_aws_config(), a...; b...)

"""
    s3_delete([::AbstractAWSConfig], bucket, path; [version=], kwargs...)

[DELETE Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html)
"""
function s3_delete(
    aws::AbstractAWSConfig, bucket, path; version::AbstractS3Version=nothing, kwargs...
)
    params = Dict{String,Any}()
    if version !== nothing
        params["versionId"] = version
    end

    return S3.delete_object(bucket, path, params; aws_config=aws, kwargs...)
end

s3_delete(a...; b...) = s3_delete(global_aws_config(), a...; b...)

"""
    s3_copy([::AbstractAWSConfig], bucket, path; to_bucket=bucket, to_path=path, kwargs...)

[PUT Object - Copy](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html)

# Optional Arguments
- `metadata::Dict=`; optional `x-amz-meta-` headers.
"""
function s3_copy(
    aws::AbstractAWSConfig,
    bucket,
    path;
    acl::AbstractString="",
    to_bucket=bucket,
    to_path=path,
    metadata::AbstractDict=SSDict(),
    kwargs...,
)
    headers = SSDict(
        "x-amz-metadata-directive" => "REPLACE",
        Pair["x-amz-meta-$k" => v for (k, v) in metadata]...,
    )

    if !isempty(acl)
        headers["x-amz-acl"] = acl
    end

    return S3.copy_object(
        to_bucket,
        to_path,
        "$bucket/$path",
        Dict("headers" => headers);
        aws_config=aws,
        kwargs...,
    )
end

s3_copy(a...; b...) = s3_copy(global_aws_config(), a...; b...)

"""
    s3_create_bucket([:AbstractAWSConfig], bucket; kwargs...)

[PUT Bucket](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html)
"""
function s3_create_bucket(aws::AbstractAWSConfig, bucket; kwargs...)
    @protected try
        if aws.region == "us-east-1"
            S3.create_bucket(bucket; aws_config=aws, kwargs...)
        else
            bucket_config = """
                                <CreateBucketConfiguration
                                            xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
                                    <LocationConstraint>$(aws.region)</LocationConstraint>
                                </CreateBucketConfiguration>
                            """

            S3.create_bucket(
                bucket,
                Dict("CreateBucketConfiguration" => bucket_config);
                aws_config=aws,
                kwargs...,
            )
        end
    catch e
        #! format: off
        @ignore if ecode(e) == "BucketAlreadyOwnedByYou" end
        #! format: on
    end
end

s3_create_bucket(a) = s3_create_bucket(global_aws_config(), a)

"""
    s3_put_cors([::AbstractAWSConfig], bucket, cors_config; kwargs...)

[PUT Bucket cors](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTcors.html)

```
s3_put_cors("my_bucket", \"\"\"
    <?xml version="1.0" encoding="UTF-8"?>
    <CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
        <CORSRule>
            <AllowedOrigin>http://my.domain.com</AllowedOrigin>
            <AllowedOrigin>http://my.other.domain.com</AllowedOrigin>
            <AllowedMethod>GET</AllowedMethod>
            <AllowedMethod>HEAD</AllowedMethod>
            <AllowedHeader>*</AllowedHeader>
            <ExposeHeader>Content-Range</ExposeHeader>
        </CORSRule>
    </CORSConfiguration>
\"\"\"
```
"""
function s3_put_cors(aws::AbstractAWSConfig, bucket, cors_config; kwargs...)
    return S3.put_bucket_cors(bucket, cors_config; aws_config=aws, kwargs...)
end

s3_put_cors(a...; b...) = s3_put_cors(AWS.global_aws_config(), a...; b...)

"""
    s3_enable_versioning([::AbstractAWSConfig], bucket; kwargs...)

[Enable versioning for `bucket`](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html).
"""
function s3_enable_versioning(aws::AbstractAWSConfig, bucket, status="Enabled"; kwargs...)
    versioning_config = """
        <VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
            <Status>$status</Status>
        </VersioningConfiguration>
        """

    return s3(
        "PUT",
        "/$(bucket)?versioning",
        Dict("body" => versioning_config);
        aws_config=aws,
        kwargs...,
    )
end

s3_enable_versioning(a; b...) = s3_enable_versioning(global_aws_config(), a; b...)

"""
    s3_put_tags([::AbstractAWSConfig], bucket, [path,] tags::Dict; kwargs...)

PUT `tags` on
[`bucket`](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTtagging.html)
or
[object (`path`)](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUTtagging.html).

See also `tags=` option on [`s3_put`](@ref).
"""
function s3_put_tags(aws::AbstractAWSConfig, bucket, tags::SSDict; kwargs...)
    return s3_put_tags(aws, bucket, "", tags; kwargs...)
end

function s3_put_tags(aws::AbstractAWSConfig, bucket, path, tags::SSDict; kwargs...)
    tag_set = Dict("Tag" => [Dict("Key" => k, "Value" => v) for (k, v) in tags])
    tags = Dict("Tagging" => Dict("TagSet" => tag_set))

    tags = XMLDict.node_xml(tags)

    if isempty(path)
        s3("PUT", "/$(bucket)?tagging", Dict("body" => tags); aws_config=aws, kwargs...)
    else
        s3(
            "PUT",
            "/$(bucket)/$(path)?tagging",
            Dict("body" => tags);
            aws_config=aws,
            kwargs...,
        )
    end
end

s3_put_tags(a...) = s3_put_tags(global_aws_config(), a...)

"""
    s3_get_tags([::AbstractAWSConfig], bucket, [path]; kwargs...)

Get tags from
[`bucket`](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETtagging.html)
or
[object (`path`)](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGETtagging.html).
"""
function s3_get_tags(aws::AbstractAWSConfig, bucket, path=""; kwargs...)
    @protected try
        tags = []

        if isempty(path)
            tags = S3.get_bucket_tagging(bucket; aws_config=aws, kwargs...)
        else
            tags = S3.get_object_tagging(bucket, path; aws_config=aws, kwargs...)
        end

        if isempty(tags["TagSet"])
            return SSDict()
        end

        tags = tags["TagSet"]
        tags = isa(tags["Tag"], Vector) ? tags["Tag"] : [tags["Tag"]]

        return SSDict(x["Key"] => x["Value"] for x in tags)
    catch e
        @ignore if ecode(e) == "NoSuchTagSet"
            return SSDict()
        end
    end
end

s3_get_tags(a...; b...) = s3_get_tags(global_aws_config(), a...; b...)

"""
    s3_delete_tags([::AbstractAWSConfig], bucket, [path])

Delete tags from
[`bucket`](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEtagging.html)
or
[object (`path`)](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETEtagging.html).
"""
function s3_delete_tags(aws::AbstractAWSConfig, bucket, path=""; kwargs...)
    if isempty(path)
        S3.delete_bucket_tagging(bucket; aws_config=aws, kwargs...)
    else
        S3.delete_object_tagging(bucket, path; aws_config=aws, kwargs...)
    end
end

s3_delete_tags(a...; b...) = s3_delete_tags(global_aws_config(), a...; b...)

"""
    s3_delete_bucket([::AbstractAWSConfig], "bucket"; kwargs...)

[DELETE Bucket](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETE.html).
"""
function s3_delete_bucket(aws::AbstractAWSConfig, bucket; kwargs...)
    return S3.delete_bucket(bucket; aws_config=aws, kwargs...)
end
s3_delete_bucket(a; b...) = s3_delete_bucket(global_aws_config(), a; b...)

"""
    s3_list_buckets([::AbstractAWSConfig]; kwargs...)

[List of all buckets owned by the sender of the request](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html).
"""
function s3_list_buckets(aws::AbstractAWSConfig=global_aws_config(); kwargs...)
    r = S3.list_buckets(; aws_config=aws, kwargs...)
    buckets = r["Buckets"]

    isempty(buckets) && return []

    buckets = buckets["Bucket"]
    return [b["Name"] for b in (isa(buckets, Vector) ? buckets : [buckets])]
end

"""
    s3_list_objects([::AbstractAWSConfig], bucket, [path_prefix]; delimiter="/", max_items=1000, kwargs...)

[List Objects](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html)
in `bucket` with optional `path_prefix`.

Returns an iterator of `Dict`s with keys `Key`, `LastModified`, `ETag`, `Size`,
`Owner`, `StorageClass`.
"""
function s3_list_objects(
    aws::AbstractAWSConfig,
    bucket,
    path_prefix="";
    delimiter="/",
    start_after="",
    max_items=nothing,
    kwargs...,
)
    return Channel(; ctype=LittleDict, csize=128) do chnl
        more = true
        num_objects = 0
        token = ""

        while more
            q = Dict{String,String}()
            for (name, v) in [
                ("prefix", path_prefix),
                ("delimiter", delimiter),
                ("start-after", start_after),
                ("continuation-token", token),
            ]
                isempty(v) || (q[name] = v)
            end
            if max_items !== nothing
                # Note: AWS seems to only return up to 1000 items
                q["max-keys"] = string(max_items - num_objects)
            end

            @repeat 4 try
                # Request objects
                r = S3.list_objects_v2(bucket, q; aws_config=aws, kwargs...)

                token = get(r, "NextContinuationToken", "")
                isempty(token) && (more = false)
                if haskey(r, "Contents")
                    l = isa(r["Contents"], Vector) ? r["Contents"] : [r["Contents"]]
                    for object in l
                        put!(chnl, object)
                        num_objects += 1
                    end
                end
            catch e
                #! format: off
                @delay_retry if ecode(e) in ["NoSuchBucket"] end
                #! format: on
            end
        end
    end
end
s3_list_objects(a...; kw...) = s3_list_objects(global_aws_config(), a...; kw...)

"""
    s3_directory_stat([::AbstractAWSConfig], bucket, path)

Determine the properties of an S3 "directory", size and time of last modification, that cannot be determined
directly with the standard AWS API.  This returns a tuple `(s, tmlast)` where `s` is the size in bytes, and
`tmlast` is the time of the latest modification to a file within that directory.
"""
function s3_directory_stat(
    aws::AbstractAWSConfig, bucket::AbstractString, path::AbstractString
)
    s = 0  # total size in bytes
    tmlast = typemin(DateTime)
    # setting delimiter is needed to get all objects within path,
    # additionally, we have to make sure the path ends with "/" or it will pick up extra stuff
    endswith(path, "/") || (path = path * "/")
    for obj in s3_list_objects(aws, bucket, path; delimiter="")
        s += parse(Int, get(obj, "Size", "0"))
        t = get(obj, "LastModified", nothing)
        t = t ≡ nothing ? tmlast : DateTime(t[1:(end - 4)])
        tmlast = max(tmlast, t)
    end
    return s, tmlast
end
s3_directory_stat(a...) = s3_directory_stat(global_aws_config(), a...)

"""
    s3_list_keys([::AbstractAWSConfig], bucket, [path_prefix]; kwargs...)

Like [`s3_list_objects`](@ref) but returns object keys as `Vector{String}`.
"""
function s3_list_keys(aws::AbstractAWSConfig, bucket, path_prefix=""; kwargs...)
    return (o["Key"] for o in s3_list_objects(aws, bucket, path_prefix; kwargs...))
end

s3_list_keys(a...; b...) = s3_list_keys(global_aws_config(), a...; b...)

"""
    s3_list_versions([::AbstractAWSConfig], bucket, [path_prefix]; kwargs...)

[List object versions](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html) in `bucket` with optional `path_prefix`.
"""
function s3_list_versions(aws::AbstractAWSConfig, bucket, path_prefix=""; kwargs...)
    more = true
    versions = []
    marker = ""

    while more
        query = Dict{String,Any}(
            "versions" => "", "prefix" => path_prefix, "return_raw" => true
        )

        if !isempty(marker)
            query["key-marker"] = marker
        end

        r = S3.list_object_versions(bucket, query; aws_config=aws, kwargs...)
        r = parse_xml(String(r))

        more = r["IsTruncated"] == "true"

        for e in eachelement(EzXML.root(r.x))
            if nodename(e) in ["Version", "DeleteMarker"]
                version = xml_dict(e)
                version["state"] = nodename(e)
                push!(versions, version)
                marker = version["Key"]
            end
        end
    end

    return versions
end

s3_list_versions(a...; b...) = s3_list_versions(global_aws_config(), a...; b...)

"""
    s3_purge_versions([::AbstractAWSConfig], bucket, [path [, pattern]]; kwargs...)

[DELETE](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html)
all object versions except for the latest version.
"""
function s3_purge_versions(aws::AbstractAWSConfig, bucket, path="", pattern=""; kwargs...)
    for v in s3_list_versions(aws, bucket, path; kwargs...)
        if pattern == "" || occursin(pattern, v["Key"])
            if v["IsLatest"] != "true"
                S3.delete_object(
                    bucket,
                    v["Key"],
                    Dict("versionId" => v["VersionId"]);
                    aws_config=aws,
                    kwargs...,
                )
            end
        end
    end
end

s3_purge_versions(a...; b...) = s3_purge_versions(global_aws_config(), a...; b...)

"""
    s3_put([::AbstractAWSConfig], bucket, path, data, data_type="", encoding=""; <keyword arguments>)

[PUT Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)
`data` at `path` in `bucket`.

# Optional Arguments
- `data_type=`; `Content-Type` header.
- `encoding=`; `Content-Encoding` header.
- `acl=`; `x-amz-acl` header for setting access permissions with canned config.
    See [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl).
- `metadata::Dict=`; `x-amz-meta-` headers.
- `tags::Dict=`; `x-amz-tagging-` headers
                 (see also [`s3_put_tags`](@ref) and [`s3_get_tags`](@ref)).
"""
function s3_put(
    aws::AbstractAWSConfig,
    bucket,
    path,
    data::Union{String,Vector{UInt8}},
    data_type="",
    encoding="";
    acl::AbstractString="",
    metadata::SSDict=SSDict(),
    tags::AbstractDict=SSDict(),
    kwargs...,
)
    headers = Dict{String,Any}(["x-amz-meta-$k" => v for (k, v) in metadata])

    if isempty(data_type)
        data_type = "application/octet-stream"
        ext = splitext(path)[2]
        for (e, t) in [
            (".html", "text/html"),
            (".js", "application/javascript"),
            (".pdf", "application/pdf"),
            (".csv", "text/csv"),
            (".txt", "text/plain"),
            (".log", "text/plain"),
            (".dat", "application/octet-stream"),
            (".gz", "application/octet-stream"),
            (".bz2", "application/octet-stream"),
        ]
            if ext == e
                data_type = t
                break
            end
        end
    end

    headers["Content-Type"] = data_type

    if !isempty(tags)
        headers["x-amz-tagging"] = HTTP.escapeuri(tags)
    end

    if !isempty(acl)
        headers["x-amz-acl"] = acl
    end

    if !isempty(encoding)
        headers["Content-Encoding"] = encoding
    end

    args = Dict("body" => data, "headers" => headers)

    return S3.put_object(bucket, path, args; aws_config=aws, kwargs...)
end

s3_put(a...; b...) = s3_put(global_aws_config(), a...; b...)

function s3_begin_multipart_upload(
    aws::AbstractAWSConfig,
    bucket,
    path,
    args=Dict{String,Any}();
    kwargs...,
    # format trick: using this comment to force use of multiple lines
)
    return S3.create_multipart_upload(bucket, path, args; aws_config=aws, kwargs...)
end

function s3_upload_part(
    aws::AbstractAWSConfig,
    upload,
    part_number,
    part_data;
    args=Dict{String,Any}(),
    kwargs...,
)
    args["body"] = part_data
    args["return_headers"] = true

    _, headers = S3.upload_part(
        upload["Bucket"],
        upload["Key"],
        part_number,
        upload["UploadId"],
        args;
        aws_config=aws,
        kwargs...,
    )

    return Dict(headers)["ETag"]
end

function s3_complete_multipart_upload(
    aws::AbstractAWSConfig,
    upload,
    parts::Vector{String},
    args=Dict{String,Any}();
    kwargs...,
)
    doc = XMLDocument()
    rootnode = setroot!(doc, ElementNode("CompleteMultipartUpload"))

    for (i, etag) in enumerate(parts)
        part = addelement!(rootnode, "Part")
        addelement!(part, "PartNumber", string(i))
        addelement!(part, "ETag", etag)
    end

    args["body"] = string(doc)

    response = S3.complete_multipart_upload(
        upload["Bucket"], upload["Key"], upload["UploadId"], args; aws_config=aws, kwargs...
    )

    return response
end

function s3_multipart_upload(
    aws::AbstractAWSConfig, bucket, path, io::IO, part_size_mb=50; kwargs...
)
    part_size = part_size_mb * 1024 * 1024

    upload = s3_begin_multipart_upload(aws, bucket, path)
    tags = Vector{String}()
    buf = Vector{UInt8}(undef, part_size)

    i = 0
    while (n = readbytes!(io, buf, part_size)) > 0
        if n < part_size
            resize!(buf, n)
        end

        push!(tags, s3_upload_part(aws, upload, (i += 1), buf; kwargs...))
    end

    return s3_complete_multipart_upload(aws, upload, tags; kwargs...)
end

using MbedTLS

function _s3_sign_url_v2(
    aws::AbstractAWSConfig,
    bucket,
    path,
    seconds=3600;
    verb="GET",
    content_type="application/octet-stream",
    protocol="http",
)
    path = HTTP.escapepath(path)

    expires = round(Int, Dates.datetime2unix(now(Dates.UTC)) + seconds)

    query = SSDict(
        "AWSAccessKeyId" => aws.credentials.access_key_id,
        "x-amz-security-token" => aws.credentials.token,
        "Expires" => string(expires),
        "response-content-disposition" => "attachment",
    )

    if verb != "PUT"
        content_type = ""
    end

    to_sign =
        "$verb\n\n$content_type\n$(query["Expires"])\n" *
        "x-amz-security-token:$(query["x-amz-security-token"])\n" *
        "/$bucket/$path?" *
        "response-content-disposition=attachment"

    key = aws.credentials.secret_key
    query["Signature"] = strip(base64encode(digest(MD_SHA1, to_sign, key)))

    endpoint = string(protocol, "://", bucket, ".s3.", aws.region, ".amazonaws.com")
    return "$endpoint/$path?$(HTTP.escapeuri(query))"
end

function _s3_sign_url_v4(
    aws::AbstractAWSConfig,
    bucket,
    path,
    seconds=3600;
    verb="GET",
    content_type="application/octet-stream",
    protocol="http",
)
    path = HTTP.escapepath("/$bucket/$path")

    now_datetime = now(Dates.UTC)
    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
    date_stamp = Dates.format(now_datetime, "YYYYmmdd")

    service = "s3"
    scheme = "AWS4"
    algorithm = "HMAC-SHA256"
    terminator = "aws4_request"

    scope = "$date_stamp/$(aws.region)/$service/$terminator"
    host = if aws.region == "us-east-1"
        "s3.amazonaws.com"
    else
        "s3-$(aws.region).amazonaws.com"
    end

    headers = OrderedDict{String,String}("Host" => host)
    sort!(headers; by=name -> lowercase(name))
    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")

    query = OrderedDict{String,String}(
        "X-Amz-Expires" => string(seconds),
        "X-Amz-Algorithm" => "$scheme-$algorithm",
        "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope",
        "X-Amz-Date" => datetime_stamp,
        "X-Amz-Security-Token" => aws.credentials.token,
        "X-Amz-SignedHeaders" => canonical_header_names,
    )

    if !isempty(aws.credentials.token)
        query["X-Amz-Security-Token"] = aws.credentials.token
    end

    sort!(query; by=name -> lowercase(name))

    canonical_headers = join(
        map(
            header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
            collect(headers),
        ),
    )

    canonical_request = string(
        "$verb\n",
        "$path\n",
        "$(HTTP.escapeuri(query))\n",
        "$canonical_headers\n",
        "$canonical_header_names\n",
        "UNSIGNED-PAYLOAD",
    )

    string_to_sign = string(
        "$scheme-$algorithm\n",
        "$datetime_stamp\n",
        "$scope\n",
        bytes2hex(digest(MD_SHA256, canonical_request)),
    )

    key_secret = string(scheme, aws.credentials.secret_key)
    key_date = digest(MD_SHA256, date_stamp, key_secret)
    key_region = digest(MD_SHA256, aws.region, key_date)
    key_service = digest(MD_SHA256, service, key_region)
    key_signing = digest(MD_SHA256, terminator, key_service)
    signature = digest(MD_SHA256, string_to_sign, key_signing)

    query["X-Amz-Signature"] = bytes2hex(signature)

    return string(protocol, "://", host, path, "?", HTTP.escapeuri(query))
end

"""
    s3_sign_url([::AbstractAWSConfig], bucket, path, [seconds=3600];
                [verb="GET"], [content_type="application/octet-stream"],
                [protocol="http"], [signature_version="v4"])

Create a
[pre-signed url](http://docs.aws.amazon.com/AmazonS3/latest/dev/ShareObjectPreSignedURL.html) for `bucket` and `path` (expires after for `seconds`).

To create an upload URL use `verb="PUT"` and set `content_type` to match
the type used in the `Content-Type` header of the PUT request.

For compatibility, the signature version 2 signing process can be used by setting
`signature_version="v2"` but it is [recommended](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) that the default version 4 is used.

```
url = s3_sign_url("my_bucket", "my_file.txt"; verb="PUT")
Requests.put(URI(url), "Hello!")
```
```
url = s3_sign_url("my_bucket", "my_file.txt";
                  verb="PUT", content_type="text/plain")

Requests.put(URI(url), "Hello!";
             headers=Dict("Content-Type" => "text/plain"))
```
"""
function s3_sign_url(
    aws::AbstractAWSConfig,
    bucket,
    path,
    seconds=3600;
    verb="GET",
    content_type="application/octet-stream",
    protocol="http",
    signature_version="v4",
)
    if signature_version == "v2"
        _s3_sign_url_v2(
            aws,
            bucket,
            path,
            seconds;
            verb=verb,
            content_type=content_type,
            protocol=protocol,
        )
    elseif signature_version == "v4"
        _s3_sign_url_v4(
            aws,
            bucket,
            path,
            seconds;
            verb=verb,
            content_type=content_type,
            protocol=protocol,
        )
    else
        throw(ArgumentError("Unknown signature version $signature_version"))
    end
end

s3_sign_url(a...; b...) = s3_sign_url(global_aws_config(), a...; b...)

"""
    s3_nuke_bucket(bucket_name)

This function is NOT exported on purpose. AWS does not officially support this type of action,
although it is a very nice utility one this is not exported just as a safe measure against
accidentally blowing up your bucket.

!!! warning

    It will delete all versions of objects in the given bucket and then the bucket itself.
"""
function s3_nuke_bucket(aws::AbstractAWSConfig, bucket_name)
    for v in s3_list_versions(aws, bucket_name)
        s3_delete(aws, bucket_name, v["Key"]; version=v["VersionId"])
    end

    return s3_delete_bucket(aws, bucket_name)
end

include("s3path.jl")

end #module AWSS3

#==============================================================================#
# End of file.
#==============================================================================#


text:
"""
    Gradient function
"""
struct GradFunc
    function_arg
    S_func
    dim
    N
    len_arg
end

function (self::GradFunc)(dt, X)
    return reshape(grad_func(dt,X, self.S_func), self.dim, self.N)  # Output gradient
end

"""
Arguement
---------
- `ODE`{Function}: ODE(du,u,p,t)
- `dim`{Integer}: Number of variables (`length of u`)
- `N` {Integer}: Sections of time intervals
"""
function gradient_gen(ODE!, p, dim, N)

    @variables X[1:dim, 1:N]
    @parameters dt

    S = 0

    D = DiffusionMatrix(dim)

    for k in 2:N
        dxdt = get_dxdt(ODE!, X[:,k-1], p)
        S = S + (1/4)*dt* (Δ(X,k,dt) - dxdt)' * (D)^(-1) * (Δ(X,k,dt) - dxdt )
    end

    vecX =  collect(Iterators.flatten(X))
    vec = collect([dt, vecX...])
    S_func = build_function(S,vec, expression=Val{false}) # S_func( [dt Xs])

    return GradFunc(vec, S_func, dim, N, dim*N)
end

grad_func(dt,Xs, S_func) = ForwardDiff.gradient(S_func, [dt,Xs...])[2:end] # Index 1 is df/dt

function get_dxdt(ODE!, x, p; t=nothing)
    dx = deepcopy(x)
    ODE!(dx, x, p, t)
    return dx
end


text:
using OrdinaryDiffEq
using Trixi

# define new structs inside a module to allow re-evaluating the file
module TrixiExtension

using Trixi

struct IndicatorVortex{Cache<:NamedTuple} <: Trixi.AbstractIndicator
  cache::Cache
end

function IndicatorVortex(semi)
  basis = semi.solver.basis
  alpha = Vector{real(basis)}()
  A = Array{real(basis), 2}
  indicator_threaded = [A(undef, nnodes(basis), nnodes(basis))
                        for _ in 1:Threads.nthreads()]
  cache = (; semi.mesh, alpha, indicator_threaded)

  return IndicatorVortex{typeof(cache)}(cache)
end

function (indicator_vortex::IndicatorVortex)(u::AbstractArray{<:Any,4},
                                             mesh, equations, dg, cache;
                                             t, kwargs...)
  mesh = indicator_vortex.cache.mesh
  alpha = indicator_vortex.cache.alpha
  indicator_threaded = indicator_vortex.cache.indicator_threaded
  resize!(alpha, nelements(dg, cache))

  # get analytical vortex center (based on assumption that center=[0.0,0.0]
  # at t=0.0 and that we stop after one period)
  domain_length = mesh.tree.length_level_0
  if t < 0.5 * domain_length
    center = (t, t)
  else
    center = (t-domain_length, t-domain_length)
  end

  Threads.@threads for element in eachelement(dg, cache)
    cell_id = cache.elements.cell_ids[element]
    coordinates = (mesh.tree.coordinates[1, cell_id], mesh.tree.coordinates[2, cell_id])
    # use the negative radius as indicator since the AMR controller increases
    # the level with increasing value of the indicator and we want to use
    # high levels near the vortex center
    alpha[element] = -periodic_distance_2d(coordinates, center, domain_length)
  end

  return alpha
end

function periodic_distance_2d(coordinates, center, domain_length)
  dx = @. abs(coordinates - center)
  dx_periodic = @. min(dx, domain_length - dx)
  return sqrt(sum(abs2, dx_periodic))
end

end # module TrixiExtension

import .TrixiExtension

###############################################################################
# semidiscretization of the compressible Euler equations

equations = CompressibleEulerEquations2D(1.4)

"""
    initial_condition_isentropic_vortex(x, t, equations::CompressibleEulerEquations2D)

The classical isentropic vortex test case of
- Chi-Wang Shu (1997)
  Essentially Non-Oscillatory and Weighted Essentially Non-Oscillatory
  Schemes for Hyperbolic Conservation Laws
  [NASA/CR-97-206253](https://ntrs.nasa.gov/citations/19980007543)
"""
function initial_condition_isentropic_vortex(x, t, equations::CompressibleEulerEquations2D)
  # needs appropriate mesh size, e.g. [-10,-10]x[10,10]
  # make sure that the inicenter does not exit the domain, e.g. T=10.0
  # initial center of the vortex
  inicenter = SVector(0.0, 0.0)
  # size and strength of the vortex
  iniamplitude = 0.2
  # base flow
  rho = 1.0
  v1 = 1.0
  v2 = 1.0
  vel = SVector(v1, v2)
  p = 10.0
  rt = p / rho                  # ideal gas equation
  cent = inicenter + vel*t      # advection of center
  cent = x - cent               # distance to centerpoint
  #cent=cross(iniaxis,cent)     # distance to axis, tangent vector, length r
  # cross product with iniaxis = [0,0,1]
  cent = SVector(-cent[2], cent[1])
  r2 = cent[1]^2 + cent[2]^2
  du = iniamplitude/(2*π)*exp(0.5*(1-r2)) # vel. perturbation
  dtemp = -(equations.gamma-1)/(2*equations.gamma*rt)*du^2            # isentrop
  rho = rho * (1+dtemp)^(1\(equations.gamma-1))
  vel = vel + du*cent
  v1, v2 = vel
  p = p * (1+dtemp)^(equations.gamma/(equations.gamma-1))
  prim = SVector(rho, v1, v2, p)
  return prim2cons(prim, equations)
end
initial_condition = initial_condition_isentropic_vortex
solver = DGSEM(polydeg=3, surface_flux=flux_lax_friedrichs)

coordinates_min = (-10.0, -10.0)
coordinates_max = ( 10.0,  10.0)
mesh = TreeMesh(coordinates_min, coordinates_max,
                initial_refinement_level=3,
                n_cells_max=10_000)

semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver)

###############################################################################
# ODE solvers, callbacks etc.

tspan = (0.0, 20.0)
ode = semidiscretize(semi, tspan)

summary_callback = SummaryCallback()

analysis_interval = 200

analysis_callback = AnalysisCallback(semi, interval=analysis_interval, save_analysis=true,
                                     extra_analysis_errors=(:conservation_error,),
                                     extra_analysis_integrals=(entropy, energy_total,
                                                               energy_kinetic, energy_internal))

alive_callback = AliveCallback(analysis_interval=analysis_interval)

save_solution = SaveSolutionCallback(interval=50,
                                     save_initial_solution=true,
                                     save_final_solution=true,
                                     solution_variables=cons2prim)

amr_controller = ControllerThreeLevel(semi, TrixiExtension.IndicatorVortex(semi),
                                      base_level=3,
                                      med_level=4, med_threshold=-3.0,
                                      max_level=5, max_threshold=-2.0)
amr_callback = AMRCallback(semi, amr_controller,
                           interval=5,
                           adapt_initial_condition=true,
                           adapt_initial_condition_only_refine=true)

stepsize_callback = StepsizeCallback(cfl=1.1)

callbacks = CallbackSet(summary_callback,
                        analysis_callback, alive_callback,
                        save_solution,
                        amr_callback, stepsize_callback)

###############################################################################
# run the simulation

sol = solve(ode, CarpenterKennedy2N54(williamson_condition=false),
            dt=1.0, # solve needs some value here but it will be overwritten by the stepsize_callback
            save_everystep=false, callback=callbacks);
summary_callback() # print the timer summary


text:
#=
    Provides different functions to check the debugTree
=#

"""
    traverse(entry,callback,params,results)

Traverse over the debug dict and call callback for every entry with the hash, step_obj
as well as parameters.
Returns the results array in the end.
"""
function traverse(entry,callback,params,results)
    push!(results,callback(entry[:hash],entry[:step_obj],params))
    if haskey(entry,:children)
        results = traverse(entry[:children][1], callback, params, results)
        results = traverse(entry[:children][2], callback, params, results)
    end
    return results
end

"""
    c_isstate(hash,entry,params)

A callback function for traverse which checks whether a given state (params[:state]) is present
"""
function c_isstate(hash,entry,params)
    return entry[:node][:state] == params[:state]
end

"""
    c_hashes(hash,entry,params)

A callback function for traverse which gets the hash
"""
function c_hashes(hash,entry,params)
    return hash
end

"""
    c_counter(hash,entry,params)

A callback function for traverse which gets the counter
"""
function c_counter(hash,entry,params)
    return haskey(entry,:counter) ? entry[:counter] : 0
end

"""
    getnstate(d,state)

Get number of how often a given state arised
"""
function getnstate(d,state)
    dictTree = d[:tree]
    params = Dict{Symbol,Symbol}()
    params[:state] = state
    results = Vector{Bool}()
    results = traverse(dictTree, c_isstate, params, results)
    return sum(results)
end

"""
    different_hashes(d)

Return if all hashes are different
"""
function different_hashes(d)
    dictTree = d[:tree]
    results = Vector{String}()
    results = traverse(dictTree, c_hashes, [], results)
    return length(Set(results)) == length(results)
end

"""
    counter_test(d,nbranches)

Check whether every counter is used exactly onces and that it is the same as the
number of branches.
"""
function counter_test(d, nbranches)
    dictTree = d[:tree]
    results = Vector{Int64}()
    results = traverse(dictTree, c_counter, [], results)
    sort!(results)
    results = results[something(findlast(isequal(0), results), 0)+1:end]
    @assert length(Set(results)) == length(results)
    @assert results[1] == 1
    @assert results[end] == length(results)
end


text:
using DataGridReader
using Test

@testset "DataGridReader.jl" begin

end


text:
using Distributions, KLDivergence

const div_func = [kl_divergence, skl_divergence, js_divergence]

for i in 1:length(div_func)
    try
        @assert div_func[i](Normal(0, 1), Normal(0, 1)) == 0.0
        @assert div_func[i](Normal(0, 1), Normal(1, 1)) >
                  div_func[i](Normal(0, 1), Normal(0, 1))
        @assert div_func[i](Normal(0, 1), Normal(0, 3)) >
                  div_func[i](Normal(0, 1), Normal(0, 1))
    catch e
        throw(string(e.msg, ", where div_func[i] == ", div_func[i]))
    end
end


text:
function SeisNMO(in;dt=0.001,offset=1000.,tnmo=0.,vnmo=1500.,max_stretch=1000)

	nt,nx = size(in)
	if length(offset) < size(in,2)
		offset = offset[1]*ones(in[1,:])
	end
	# interpolate tau,v pairs to match sampling of input data
	if (length(vnmo) == 1)
		tnmo = convert(Float64,tnmo);
		vnmo = convert(Float64,vnmo);
		ti = collect(0:1:nt-1)*dt
		vi = ones(1,nt)*vnmo
	else
		tnmo = convert(Array{Float64,1},vec(tnmo))
		vnmo = convert(Array{Float64,1},vec(vnmo))
		ti = collect(0:1:nt-1)*dt
		#g = InterpIrregular(tnmo, vnmo, BCnan, InterpLinear)
		g = interpolate(tnmo, vnmo, Gridded(Linear()))
		vi = g[ti]
	end
	out = zeros(size(in))
	M = zeros(nt,1)
	for it = 1:nt
		for ix = 1:nx
			time = sqrt(ti[it]^2 + (offset[ix]/vi[it]).^2)
			stretch = (time-ti[it])/(ti[it]+1e-10)
			if (stretch<max_stretch/100)
				its = round(Int,time/dt)+1
				it1 = round(Int,floor(time/dt))+1
				it2 = it1+1
				a = its-it1
				if (it2 <= nt)
					out[it,ix] = (1-a)*in[it1,ix]+a*in[it2,ix]
				end
			end
		end
	end
	return out
end

function SeisNMO(in,h::Array{Header,1};tnmo=0.,vnmo=1500.,max_stretch=1000)

	offset = Seismic.ExtractHeader(h,"h")
	dt = h[1].d1
	out = SeisNMO(in;dt=dt,offset=offset,tnmo=tnmo,vnmo=vnmo,max_stretch=max_stretch)
	return out,h

end

function SeisNMO(in::AbstractString,out::AbstractString;tnmo=0.,vnmo=1500.,max_stretch=1000)

	@compat parameters = Dict(:tnmo=>tnmo,:vnmo=>vnmo,:max_stretch=>max_stretch)
	SeisProcess(in,out,[SeisNMO],[parameters],key=["imx"])

end

max_stars_repo_name: fercarozzi/myseismicjulia

max_issues_repo_name: fercarozzi/myseismicjulia

max_forks_repo_name: fercarozzi/myseismicjulia


text:
"""

Package `AstronomicalDetectors` deals with the calibration files of
astronomical detectors like those of the VLT/Sphere instrument.

Example of usage:

    using AstronomicalDetectors, Glob
    list = scan_calibrations(glob("SPHER.2015-12-2*", dir))
    data = read(CalibrationData{Float64}, list; part=(501:580,601:650))
    calib = ReducedCalibration(data)

"""
module AstronomicalDetectors

export
    CalibrationCategory,
    CalibrationData,
    CalibrationInformation,
    scan_calibrations,
    ReadCalibrationFiles

using FITSIO, EasyFITS
using EasyFITS: FitsHeader

using SimpleExpressions

using ScientificDetectors
using ScientificDetectors: CalibrationCategory

include("ReadCalibration.jl")
import .YAMLCalibrationFiles: ReadCalibrationFiles

struct CalibrationInformation
    path::String             # FITS file
    dims::NTuple{3,Int}      # (width,height,nframes)
    Δt::Float64              # exposure time (seconds)
    cat::CalibrationCategory # calibration category
end

"""
    lst = scan_calibrations(args...; scanner=default_scanner, kwds...)

scans calibration data files specified as arguments (file and/or directory
names) and returns a list that can be used by `read(CalibrationData,...)` to
produce an instance of `CalibrationData`.

See [`AstronomicalDetectors.scan_calibration!](@ref) for a
description of available keywords.

Example:

    using AstronomicalDetectors, Glob
    lst = scan_calibrations(glob("SPHER.2015-12-2*", dir))

"""
function scan_calibrations(args::Union{AbstractString,
                                       AbstractVector{<:AbstractString}}...;
                           kdws...)
    # Auxiliary functions for channels that yield filenames.
    function yield_filenames(out::Channel, A::AbstractVector{<:AbstractString})
        for name in A
            yield_filenames(out, name)
        end
    end
    function yield_filenames(out::Channel, name::AbstractString)
        if isfile(name)
            put!(out, name)
        elseif isdir(name)
            for other in readdir(name; join=true, sort=false)
                isfile(other) && put!(out, other)
            end
        end
    end

    # Build a channel that yields all file names.
    chnl = Channel{String}() do out
        for arg in args
            yield_filenames(out, arg)
        end
    end

    # Collect all calibration information.
    list = CalibrationInformation[]
    for filename in chnl
        scan_calibration!(list, filename; kdws...)
    end
    return list
end

"""
    scan_calibration!(dst, filename; scanner=default_scanner, kwds...) -> dst

scans the calibration information if FITS file `filename` and pushes this
information in destination `dst` which is returned.

Keyword `scanner` may be used to specifiy your own method for scanning
calibration information.  The method is called as:

    scanner(filename; kwds...)

and shall return an instance of `CalibrationInformation` for the given file.

All other keywords `kwds...` specified in the call to `scan_calibration!` are
passed to the scanner.  The default scanner accept the following keywords:

- Keyword `exptime` can be set with the name of the FITS card which stores the
  exposure time (in seconds).  By default `exptime="ESO DET SEQ1 REALDIT"`.

- Keyword `category` can be set with the name of the FITS card which stores the
  calibration category.  By default `category = "ESO DPR TYPE"`.

"""
function scan_calibration!(dest::Vector{<:CalibrationInformation},
                           filename::AbstractString;
                           scanner = default_scanner,
                           kwds...)
    isfile(filename) || error("\"", filename, "\" is not a file")
    return push!(dest, scanner(filename; kwds...))
end

# Calibration categories.
#    OBJECT ≈ ESO DPR TYPE  except for science which yields "OBJECT"
#    ESO DPR CATG = SCIENCE or CALIB
#
function default_scanner(filename::AbstractString,
                         hdr::FitsHeader =  read(FitsHeader, filename);
                         exptime::AbstractString = "ESO DET SEQ1 REALDIT",
                         category::AbstractString = "ESO DPR TYPE")

    # Get dimensions.
    naxis = get(Int, hdr, "NAXIS")
    if !(2 ≤ naxis ≤ 3)
        error("other dimensions than 2D and 3D not implemented")
    end
    dims = (get(Int, hdr, "NAXIS1"),
            get(Int, hdr, "NAXIS2"),
            (naxis == 2 ? 1 : get(Int, hdr, "NAXIS3")))

    # Get exposure time.
    Δt = get(Float64, hdr, exptime)

    # Get category of calibration and provide the corresponding linear
    # combination of sources.
    cat = uppercase(strip(get(String, hdr, category)))
    if cat == "DARK"
        expr = :(dark)
    elseif cat == "DARK,BACKGROUND"
        expr = :(dark + background)
    elseif cat == "FLAT,LAMP"
        expr = :(dark + background + flat)
    elseif cat == "LAMP,WAVE"
        expr = :(dark + background + wave)
    elseif cat == "OBJECT"
        # Use object's name as source and category.
        cat =  uppercase(strip(get(String, hdr, "OBJECT")))
        object = Symbol(lowercase(cat))
        expr = :(dark + sky + $object)
    elseif cat == "SKY"
        expr = :(dark + sky)
    else
        @warn "unknown calibration category: \"", cat, "\" in file \"", filename, "\""
        expr = :(dark + $cat)
        return
    end
    return CalibrationInformation(filename, dims, Δt,CalibrationCategory(cat, expr))
end

"""
    read(CalibrationData{T}, lst; part=(:,:))

"""
function Base.read(::Type{CalibrationData},
                   list::AbstractVector{<:CalibrationInformation};
                   kwds...)
    return read(CalibrationData{Float64}, list; kwds...)
end

function Base.read(::Type{CalibrationData{T}},
                   list::AbstractVector{<:CalibrationInformation};
                   part::NTuple{2} = (:,:)) where {T<:AbstractFloat}
    # Get detector size.
    width, height = -1, -1
    first = true
    for item in list
        if first
            width, height = item.dims[1], item.dims[2]
            first = false
        else
            (width, height) == (item.dims[1], item.dims[2]) || error(
                "incompatible sizes")
        end
    end
    inds = (Base.OneTo(width)[part[1]],
            Base.OneTo(height)[part[2]])

    # Build a calibration data frame producer.
    N = 2 # number of dimensions
    roi = DetectorAxes(inds)
    producer = Channel{CalibrationDataFrame{T,N}}() do chn
        for item in list
            hdu = FITS(item.path)[1] :: ImageHDU
            naxis = ndims(hdu)
            width, height, nframes = item.dims
            if naxis == N && nframes == 1
                put!(chn, CalibrationDataFrame{T,N}(item.cat.name, item.Δt,
                                                    read(hdu, inds...);
                                                    roi = roi))
            elseif naxis == N+1
                for j in 1:nframes
                    put!(chn, CalibrationDataFrame{T,N}(item.cat.name, item.Δt,
                                                        read(hdu, inds..., j);
                                                        roi = roi))
                end
            else
                error("other dimensions than ", N, "-D and ",
                      N+1, "-D not implemented")
            end
        end
        close(chn)
    end

    # Load all calibration data.
    dat = CalibrationData{T}(roi, map(x -> x.cat, list))
    for frame in producer
        push!(dat, frame)
    end
    return dat
end

end # module


text:
struct Impulse <: IRMeasurement
    len::Int
    prepad::Int
    gain::Float64
end

prepadding(i::Impulse) = i.prepad

"""
    impulse(samples; options...)
    impulse(seconds::Time, samplerate; options...)
    impulse(seconds::Time, samplerate::Frequency; options...)

Create an impulse response measurement using a literal digital impulse (a single
sample). This is typically not used in practice because the limited energy in
the stimulus signal gives poor signal-to-noise in the measured impulse response.
It is included in this package mostly as a reference.

## Options
$optiondoc_gain
$optiondoc_prepad
"""
impulse(L; prepad=L, gain=1) = Impulse(L, prepad, gain)
impulse(t::Time, samplerate::Frequency; kwargs...) = impulse(Int(t*samplerate); kwargs...)
impulse(t::Time, samplerate; kwargs...) = impulse(Int(t/(1s) * samplerate); kwargs...)

stimulus(m::Impulse) = [zeros(m.prepad); m.gain; zeros(m.len-1)]
_analyze(m::Impulse, response::AbstractArray) =
        timeslice(response, m.prepad+1:size(response, 1)) ./ m.gain


text:
# This file was generated, do not modify it. # hide
coef_vals = [c[2] for c in coefs]
sum(coef_vals .≈ 0) / length(coefs)


text:
# Convert a string to a decimal, e.g. "0.01" -> Decimal(0, 1, -2)
function decimal(str::AbstractString)
    if 'e' in str
        return decimal(scinote(str))
    end
    c, q = parameters(('.' in str) ? split(str, '.') : str)
    norm(Decimal((str[1] == '-') ? 1 : 0, c, q))
end

# Convert a number to a decimal
decimal(x::Number) = decimal(string(x))

# If it's already a decimal, do nothing
decimal(x::Decimal) = x

# Convert an array to an array of decimals
decimal(x::Array) = map(decimal, x)

# Get Decimal constructor parameters from string
parameters(x::AbstractString) = (abs(parse(BigInt, x)), 0)

# Get Decimal constructor parameters from array
function parameters(x::Array)
    c = parse(BigInt, join(x))
    (abs(c), -length(x[2]))
end

# Get decimal() argument from scientific notation
function scinote(str::AbstractString)
    s = (str[1] == '-') ? "-" : ""
    n, expo = split(str, 'e')
    n = split(n, '.')
    if s == "-"
        n[1] = n[1][2:end]
    end
    if parse(Int64, expo) > 0
        shift = parse(Int64, expo) - ((length(n) == 2) ? length(n[2]) : 0)
        s * join(n) * repeat("0", shift)
    else
        shift = -parse(Int64, expo) - ((length(n) == 2) ? length(n[1]) : length(n))
        s * "0." * repeat("0", shift) * join(n)
    end
end

# Convert a decimal to a string
function Base.string(x::Decimal)
    c = string(x.c)
    if x.q > 0
        for i in 1:x.q
            c *= "0"
        end
    elseif x.q < 0
        shift = x.q + length(c)
        if shift > 0
            c = c[1:shift] * "." * c[(shift+1):end]
        else
            c = "0." * repeat("0", -shift) * c
        end
    end
    ((x.s == 1) ? "-" : "") * c
end

# Zero value
Base.zero(::Type{Decimals.Decimal}) = Decimal(0,0,0)

# Convert a decimal to a float
Base.float(x::Decimal) = float(string(x))
Base.float(x::Array{Decimal}) = map(float, x)

# Convert a decimal to an integer if possible, a float if not
function number(x::Decimal)
    ix = (str = string(x) ; fx = float(str); round(Int64, fx))
    (ix == fx) ? ix : fx
end
number(x::Array{Decimal}) = map(number, x)

# Check if integer
isint(x::Integer) = (length(string(x)) < length(string(typemax(Int))))
isint(x::AbstractFloat) = ((round(x) == x) && isint(round(x)))
isint(x::AbstractString) = isint(float(x))


text:
import Base: convert, ==

# import JuLIP: energy, virial, site_energies, forces
# import JuLIP.Potentials: evaluate

import JuLIP.FIO: read_dict, write_dict
using JuLIP: JVec, JMat, chemical_symbols

export OneBody

"""
`mutable struct OneBody{T}  <: NBodyFunction{1}`

this should not normally be constructed by a user, but instead E0 should be
passed to the relevant lsq functions, which will construct it.
This structure deals with multi-species configurations.
"""
mutable struct OneBody{T <: AbstractFloat} <: AbstractCalculator
   E0::Dict{Symbol, T}
end

@pot OneBody

OneBody(args...) = OneBody(Dict(args...))

evaluate(V::OneBody, sp) = V.E0[sp]

function site_energies(V::OneBody, at::AbstractAtoms{T}) where {T}
   E = zeros(T, length(at))
   for i in 1:length(at)
      E[i] = V(chemical_symbols(at)[i])
   end
   return E
end

energy(V::OneBody, at::AbstractAtoms) = sum(site_energies(V, at))

forces(V::OneBody, at::AbstractAtoms{T}) where {T} = zeros(JVec{T}, length(at))

virial(V::OneBody, at::AbstractAtoms{T}) where {T} = zero(JMat{T})

write_dict(V::OneBody) =
         Dict("__id__" => "JuLIP_OneBody",
              "E0" => Dict([String(key) => val for (key, val) in V.E0]...))

# convert the E0 Dict{String} read from JSON into a Dict{Symbol}
OneBody(D::Dict{String}, T=Float64) =
   OneBody([Symbol(key) => T(val) for (key, val) in D]...)

read_dict(::Val{:JuLIP_OneBody}, D::Dict) = OneBody(D["E0"])

==(V1::OneBody, V2::OneBody) = (V1.E0 == V2.E0)