text:
using ReplicaExchange

# FIXME, and rename
rescale(l, t) = exp(l, t)

function custom_proposal(rng = MersenneTwister(0),  nreplicas = 4)
  m2 = MixtureModel(
	  [MvNormal([0.9, 0.0], [0.3, 0.2]),
	   MvNormal([2.0, 3.0], [0.3, 0.4]),
	   MvNormal([-2.0, -1.0], [0.25, 0.33])],
	  [0.5, 0.3, 0.2]);
  # nreplicas initial start states
  ωinits = [((x_, ϵ_) = rand(2); (x = x_, ϵ = ϵ_, y = x_ + ϵ_)) for _ = 1:nreplicas]

  # Create one temperature for each replica
  logtemps(n, k) = exp.(k * range(-2.0, stop = 1.0, length = n))
  ctxs = logtemps(nreplicas)

  function somefunc(alg, temp, ωinit)
    let ℓ(ω) = logenergy(ω, temp)
      mh!(rng, ℓ, y, 10000, ωinit, CustomProp(prop))
    end
  end

end

max_stars_repo_name: zenna/expect

max_issues_repo_name: zenna/expect

max_forks_repo_name: zenna/expect


text:
file = open("hello.md")

test1 = eachline(file)

chunk = []
all = []

for i in test1
  #println(i)
  ismatched = ismatch(r"^##", i)
  println(ismatched ? i : false)
end

ext: jl

max_stars_repo_name: yuffiy/RabbitWannaADocumentTool

max_issues_repo_name: yuffiy/RabbitWannaADocumentTool

max_forks_repo_name: yuffiy/RabbitWannaADocumentTool


text:
#!/usr/bin/env julia

using Gasp
using Base.Threads

cpu_hz = 0.0

@inline ntputs(tid, s...) = ccall(:puts, Cint, (Ptr{Int8},), string("[$(grank())]<$tid> ", s...))

function threadfun(dt, ni, ci, li, ilock, rundt, dura)
    tid = threadid()
    if rundt && tid == 1
        ntputs(tid, "running tree")
        while runtree(dt)
            Gasp.cpu_pause()
        end
    else
        ntputs(tid, string("begin, ", ni[], " items, ", length(dura), " available delays"))
        while ni[] > 0
            lock(ilock)
            if li[] == 0
                ntputs(tid, string("out of work"))
                unlock(ilock)
                break
            end
            if ci[] == li[]
                ntputs(tid, string("work consumed (last was ", li[], "); requesting more"))
                ni[], (ci[], li[]) = getwork(dt)
                ntputs(tid, string("got ", ni[], " work items (", ci[], " to ", li[], ")"))
                unlock(ilock)
                continue
            end
            item = ci[]
            ci[] = ci[] + 1
            unlock(ilock)

            # wait dura[item] seconds
            global cpu_hz
            ticks = dura[item] * cpu_hz
            #ntputs(tid, "item $item: $(dura[item]) secs, $ticks ticks")
            startts = Gasp.rdtsc()
            while Gasp.rdtsc() - startts < ticks
                Gasp.cpu_pause()
            end
        end
    end
end

function bench(nwi, meani, stddevi, first_distrib, rest_distrib, min_distrib, fan_out = 1024)
    # get CPU speed
    global cpu_hz
    t = Gasp.rdtsc()
    sleep(1)
    cpu_hz = Gasp.rdtsc() - t

    # create the tree
    dt, is_parent = Dtree(fan_out, nwi, true, 1.0, first_distrib, rest_distrib, min_distrib)

    # ---
    if grank() == 1
        println("dtreebench -- $(ngranks()) ranks")
        println("  system clock speed is $(cpu_hz/1e9) GHz")
    end

    # roughly how many work items will each rank will handle?
    each, r = divrem(nwi, ngranks())
    if r > 0
        each = each + 1
    end

    # ---
    if grank() == 1
        println("  ", nwi, " work items, ~", each, " per rank")
    end

    # generate random numbers for work item durations
    dura = Float64[]
    mn = repmat([meani-0.5*stddevi, meani+0.5*stddevi], ceil(Int, ngranks()/2))
    mt = MersenneTwister(7777777)
    for i = 1:ngranks()
        r = randn(mt)*stddevi*0.25+mn[i]
        append!(dura, max.(randn(mt, each)*stddevi+r, zero(Float64)))
    end

    # ---
    if grank() == 1
        println("  initializing...")
    end

    # get the initial allocation
    ilock = SpinLock()
    ni, (ci, li) = initwork(dt)

    # ---
    if grank() == 1
        println("  ...done.")
    end

    # start threads and run
    tfargs = Core.svec(threadfun, dt, Ref(ni), Ref(ci), Ref(li), ilock, runtree(dt), dura)
    ccall(:jl_threading_run, Void, (Any,), tfargs)

    # ---
    if grank() == 1
        println("complete")
    end
    tic()
    finalize(dt)
    wait_done = toq()
    ntputs(1, "wait for done: $wait_done secs")
end

bench(500, 0.5, 0.125, 0.5, 0.5, nthreads())


text:
push!(LOAD_PATH,"../src/")
using MicrobiomeAnalysis, Documenter
ENV["GKSwstype"] = "100"

generated_path = joinpath(@__DIR__, "src")
base_url = "https://github.com/JuliaTurkuDataScience/MicrobiomeAnalysis.jl/blob/main/"
isdir(generated_path) || mkdir(generated_path)

open(joinpath(generated_path, "readme.md"), "w") do io
    # Point to source readme file
    println(
        io,
        """
        ```@meta
        EditURL = "$(base_url)README.md"
        ```
        """,
    )
    # Write the contents out below the meta block
    for line in eachline(joinpath(dirname(@__DIR__), "README.md"))
        println(io, line)
    end
end

makedocs(format = Documenter.HTML(),
         authors = "Giulio Benedetti, Leo Lahti",
         sitename = "MicrobiomeAnalysis.jl",
         modules = [MicrobiomeAnalysis],
         pages=[
             "Home" => "readme.md",
             "Manual" => "index.md",
             "Tutorial" => Any[
                "SE construction and analysis" => "example1.md",
                "MAE construction and manipulation" => "example2.md",
                "SE and MAE retrieval" => "example3.md",
                "CommunityProfile conversion" => "example4.md",
                "Methods for real data" => "example5.md"
                ]
         ])

deploydocs(repo="github.com/JuliaTurkuDataScience/MicrobiomeAnalysis.jl", push_preview=true)


text:
#
# Negatives Tests
#

@testset "negatives" begin
    Factor(:X, [-2016, 4])

    h = set_negative_mode(NegativeError)
    @test_throws ArgumentError log(Factor(:X, [1E-2, 1//4]))
    @test h == NegativeIgnore
    set_negative_mode(h)
end


text:
##############################   Born-Mayer  ###################################
struct BornMayer <: EmpiricalPotential
    A
    ρ
    rcutoff
    species :: AbstractVector
end

get_parameters(bm::BornMayer) = Parameter{ (:A,:ρ) }((bm.A, bm.ρ))
set_parameters(p::Parameter{(:A, :ρ)}, bm::BornMayer) = BornMayer(p.A, p.ρ, bm.rcutoff, bm.species)

deserialize_parameters(p::Parameter{(:A, :ρ)}, bm::BornMayer) = [p.A, p.ρ]
serialize_parameters(p::Vector, bm::BornMayer) = Parameter{(:A, :ρ)}( (p[1], p[2]) )

get_hyperparameters(bm::BornMayer) = Parameter{(:rcutoff,)}( (bm.rcutoff,) )
set_hyperparameters(p::Parameter{(:rcutoff,)}, bm::BornMayer) = BornMayer(bm.A, bm.ρ, p.rcutoff, bm.species)

deserialize_hyperparameters(p::Parameter{(:rcutoff,)}, bm::BornMayer) = [p.rcutoff]
serialize_hyperparameters(p::Vector, bm::BornMayer) = Parameter{(:rcutoff, )}( (p[1],) )

# ##############################   Energy  ###################################

function potential_energy(R::AbstractFloat, bm::BornMayer)
    bm.A * exp(-R / bm.ρ)
end

# ##############################   Force   ###################################

function force(R::AbstractFloat, r::SVector{3,<:AbstractFloat}, bm::BornMayer)
    SVector(bm.A /bm.ρ * exp(-R / bm.ρ ) .* r ./ R)
end

# ##############################   Gradients  ###################################

function grad_potential_energy(r::Vector{<:Real}, bm::BornMayer)
    d = norm(r)
    return (dpdA = exp(-d / bm.ρ),
            dpdρ = bm.A * d * exp(-d/bm.ρ) / bm.ρ^2 )
end

function grad_force(r::Vector{<:Real}, bm::BornMayer)
    d = norm(r)
    return (dfdA = 1.0 /bm.ρ * exp(-d / bm.ρ ) .* r ./ d,
            dfdρ = bm.A / bm.ρ^3 * exp(-d / bm.ρ )*(d - bm.ρ)  .* r ./ d)
end

function grad_virial(r::Vector{<:Real}, bm::BornMayer)
    df = grad_force(r, bm)
    dfdA = df[:dfdA]
    dfdrho = df[:dfdρ]
    return (dvdA = dfdA[1]*r[1] + dfdA[2]*r[2] + dfdA[3]*r[3],
            dvdρ = dfdrho[1]*r[1] + dfdrho[2]*r[2] + dfdrho[3]*r[3])
end


text:
"""
Bag-of-random-embedding-projections (BOREP) structure for
document embedding using word vectors.

# References
  * [Wieting, Kiela ICLR 2019, "No training required: Exploring random encoders
     for sentence classification"](https://arxiv.org/abs/1901.10444)
"""
struct BOREPEmbedder{T,S} <: WordVectorsEmbedder{T,S}
    embeddings::EmbeddingsLibrary{T,S}
    R::Matrix{T}
    fpool::Function
    config::NamedTuple
end

function BOREPEmbedder(embeddings::EmbeddingsLibrary{T,S},
                       config;
                       dim=DEFAULT_BOREP_DIMENSION,
                       pooling_function=DEFAULT_BOREP_POOLING_FUNCTION,
                       initialization::Symbol=:heuristic,
                       kwargs...
                      ) where {T<:AbstractFloat, S<:AbstractString}
    # Check initialization option and generate random matrix
    d = size(embeddings)[1]  # number of vector components
    if initialization == :heuristic
        R = rand(T[-1/sqrt(d), 1/sqrt(d)], dim, d)
    elseif initialization == :uniform
        R = rand(T[-0.1, 0.1], dim, d)
    elseif initialization == :normal
        R = randn(T, dim, d)
    end

    # Check pooling function option anf generate pooling function
    if pooling_function == :sum
        fpool = x->vec(sum(x, dims=2))
    elseif pooling_function == :max
        fpool = x->vec(maximum(x, dims=2))
    end

    return BOREPEmbedder(embeddings, R, fpool, config)
end

# Dimensionality function
function dimensionality(embedder::BOREPEmbedder)
    # The final number of dimensions is given by
    # the dimension of the random space (i.e. number
    # of rows of the projection matrix)
    return size(embedder.R, 1)
end

# Sentence embedding function
function sentences2vec(embedder::BOREPEmbedder,
                       document_embedding::Vector{Matrix{T}};
                       kwargs...) where {T,S}
    n = length(document_embedding)
    X = zeros(T, dimensionality(embedder), n)
    @inbounds @simd for i in 1:n
        X[:,i] = embedder.fpool(embedder.R * document_embedding[i])
    end
    return X
end


text:
import REPL:ends_with_semicolon
import .Configuration
import .ExpressionExplorer: FunctionNameSignaturePair, is_joined_funcname, UsingsImports, external_package_names
import .WorkspaceManager: macroexpand_in_workspace

Base.push!(x::Set{Cell}) = x

"Run given cells and all the cells that depend on them, based on the topology information before and after the changes."
function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; deletion_hook::Function=WorkspaceManager.move_vars, persist_js_state::Bool=false, already_in_run::Bool=false)::TopologicalOrder
	if !already_in_run
		# make sure that we're the only `run_reactive!` being executed - like a semaphor
		take!(notebook.executetoken)

		old_workspace_name, new_workspace_name = WorkspaceManager.bump_workspace_module((session, notebook))

		if !is_resolved(new_topology)
			unresolved_topology = new_topology
			new_topology = notebook.topology = resolve_topology(session, notebook, unresolved_topology, old_workspace_name)

			# update cache and save notebook because the dependencies might have changed after expanding macros
			update_dependency_cache!(notebook)
			session.options.server.disable_writing_notebook_files || save_notebook(notebook)
		end
	else
		workspace = WorkspaceManager.get_workspace((session, notebook))
		old_workspace_name = new_workspace_name = workspace.module_name
	end

	removed_cells = setdiff(keys(old_topology.nodes), keys(new_topology.nodes))
	roots = Cell[roots..., removed_cells...]

	# by setting the reactive node and expression caches of deleted cells to "empty", we are essentially pretending that those cells still exist, but now have empty code. this makes our algorithm simpler.
	new_topology = NotebookTopology(
		nodes=merge(
			new_topology.nodes,
			Dict(cell => ReactiveNode() for cell in removed_cells),
		),
		codes=merge(
			new_topology.codes,
			Dict(cell => ExprAnalysisCache() for cell in removed_cells)
		),
		unresolved_cells=new_topology.unresolved_cells,
	)

	# save the old topological order - we'll delete variables assigned from it and re-evalutate its cells
	old_order = topological_order(notebook, old_topology, roots)

	old_runnable = old_order.runnable
	to_delete_vars = union!(Set{Symbol}(), defined_variables(old_topology, old_runnable)...)
	to_delete_funcs = union!(Set{Tuple{UUID,FunctionName}}(), defined_functions(old_topology, old_runnable)...)

	# get the new topological order
	new_order = topological_order(notebook, new_topology, union(roots, keys(old_order.errable)))
	to_run_raw = setdiff(union(new_order.runnable, old_order.runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters

	# find (indirectly) deactivated cells and update their status
	deactivated = filter(c -> c.running_disabled, notebook.cells)
	indirectly_deactivated = collect(topological_order(notebook, new_topology, deactivated))
	for cell in indirectly_deactivated
		cell.running = false
		cell.queued = false
		cell.depends_on_disabled_cells = true
	end

    to_run = setdiff(to_run_raw, indirectly_deactivated)

	# change the bar on the sides of cells to "queued"
	for cell in to_run
		cell.queued = true
		cell.depends_on_disabled_cells = false
	end
	for (cell, error) in new_order.errable
		cell.running = false
		cell.queued = false
		relay_reactivity_error!(cell, error)
	end

	# Send intermediate updates to the clients at most 20 times / second during a reactive run. (The effective speed of a slider is still unbounded, because the last update is not throttled.)
	# flush_send_notebook_changes_throttled,
	send_notebook_changes_throttled, flush_notebook_changes = throttled(1.0 / 20) do
		send_notebook_changes!(ClientRequest(session=session, notebook=notebook))
	end
	send_notebook_changes_throttled()

	# delete new variables that will be defined by a cell
	new_runnable = new_order.runnable
	to_delete_vars = union!(to_delete_vars, defined_variables(new_topology, new_runnable)...)
	to_delete_funcs = union!(to_delete_funcs, defined_functions(new_topology, new_runnable)...)

	# delete new variables in case a cell errors (then the later cells show an UndefVarError)
	new_errable = keys(new_order.errable)
	to_delete_vars = union!(to_delete_vars, defined_variables(new_topology, new_errable)...)
	to_delete_funcs = union!(to_delete_funcs, defined_functions(new_topology, new_errable)...)

	to_reimport = union!(Set{Expr}(), map(c -> new_topology.codes[c].module_usings_imports.usings, setdiff(notebook.cells, to_run))...)
	deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs, to_reimport; to_run=to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars`

	delete!.([notebook.bonds], to_delete_vars)

	local any_interrupted = false
	for (i, cell) in enumerate(to_run)

		cell.queued = false
		cell.running = true
		send_notebook_changes_throttled()

		if any_interrupted || notebook.wants_to_interrupt
			relay_reactivity_error!(cell, InterruptException())
		else
			run = run_single!(
				(session, notebook), cell,
				new_topology.nodes[cell], new_topology.codes[cell];
				persist_js_state=(persist_js_state || cell ∉ roots)
			)
			any_interrupted |= run.interrupted
		end

		cell.running = false

		if !is_resolved(new_topology) && can_help_resolve_cells(new_topology, cell)
			notebook.topology = new_new_topology = resolve_topology(session, notebook, new_topology, old_workspace_name; skip_reimports=true)

			return run_reactive!(session, notebook, new_topology, new_new_topology, to_run[i+1:end]; deletion_hook=deletion_hook, persist_js_state=persist_js_state, already_in_run=true)
		end
	end

	notebook.wants_to_interrupt = false
	flush_notebook_changes()
	# allow other `run_reactive!` calls to be executed
	put!(notebook.executetoken)
	return new_order
end

run_reactive_async!(session::ServerSession, notebook::Notebook, to_run::Vector{Cell}; kwargs...) = run_reactive_async!(session, notebook, notebook.topology, notebook.topology, to_run; kwargs...)

function run_reactive_async!(session::ServerSession, notebook::Notebook, old::NotebookTopology, new::NotebookTopology, to_run::Vector{Cell}; run_async::Bool=true, kwargs...)::Union{Task,TopologicalOrder}
	maybe_async(run_async) do
		run_reactive!(session, notebook, old, new, to_run; kwargs...)
	end
end

function maybe_async(f::Function, async::Bool)
	run_task = @asynclog f()
	if async
		run_task
	else
		fetch(run_task)
	end
end

const lazymap = Base.Generator

function defined_variables(topology::NotebookTopology, cells)
	lazymap(cells) do cell
		topology.nodes[cell].definitions
	end
end

function defined_functions(topology::NotebookTopology, cells)
	lazymap(cells) do cell
		((cell.cell_id, namesig.name) for namesig in topology.nodes[cell].funcdefs_with_signatures)
	end
end

"Run a single cell non-reactively, set its output, return run information."
function run_single!(session_notebook::Union{Tuple{ServerSession,Notebook},WorkspaceManager.Workspace}, cell::Cell, reactive_node::ReactiveNode, expr_cache::ExprAnalysisCache; persist_js_state::Bool=false)
	run = WorkspaceManager.eval_format_fetch_in_workspace(
		session_notebook,
		expr_cache.parsedcode,
		cell.cell_id,
		ends_with_semicolon(cell.code),
		expr_cache.function_wrapped ? (filter(!is_joined_funcname, reactive_node.references), reactive_node.definitions) : nothing,
		cell.cell_dependencies.contains_user_defined_macrocalls,
	)
	set_output!(cell, run, expr_cache; persist_js_state=persist_js_state)
	if session_notebook isa Tuple && run.process_exited
		session_notebook[2].process_status = ProcessStatus.no_process
	end
	return run
end

function set_output!(cell::Cell, run, expr_cache::ExprAnalysisCache; persist_js_state::Bool=false)
	cell.output = CellOutput(
		body=run.output_formatted[1],
		mime=run.output_formatted[2],
		rootassignee=ends_with_semicolon(expr_cache.code) ? nothing : ExpressionExplorer.get_rootassignee(expr_cache.parsedcode),
		last_run_timestamp=time(),
		persist_js_state=persist_js_state,
	)
	cell.published_objects = run.published_objects
	cell.runtime = run.runtime
	cell.errored = run.errored
	cell.running = cell.queued = false
end

will_run_code(notebook::Notebook) = notebook.process_status != ProcessStatus.no_process && notebook.process_status != ProcessStatus.waiting_to_restart

is_macro_identifier(symbol::Symbol) = startswith(string(symbol), "@")

"Tells whether or not a cell can 'unlock' the resolution of other cells"
function can_help_resolve_cells(topology::NotebookTopology, cell::Cell)
    cell_code = topology.codes[cell]
    cell_node = topology.nodes[cell]
    !isempty(cell_code.module_usings_imports.imports) ||
	!isempty(cell_code.module_usings_imports.usings) ||
	any(is_macro_identifier, cell_node.funcdefs_without_signatures)
end

"We still have 'unresolved' macrocalls, use the pre-created workspace to do macro-expansions"
function resolve_topology(session::ServerSession, notebook::Notebook, unresolved_topology::NotebookTopology, old_workspace_name::Symbol; skip_reimports::Bool=false)
	sn = (session, notebook)

	if !skip_reimports
	    to_reimport = union!(Set{Expr}(), map(c -> unresolved_topology.codes[c].module_usings_imports.usings, notebook.cells)...)
	    WorkspaceManager.do_reimports(sn, to_reimport)
	end

	function macroexpand_cell(cell)
		try_macroexpand(module_name::Union{Nothing,Symbol}=nothing) =
			macroexpand_in_workspace(sn, unresolved_topology.codes[cell].parsedcode, cell.cell_id, module_name)
		# Several trying steps
		# 1. Try in the new module with moved imports
		# 2. Try in the previous module
		# 3. Move imports and re-try in the new module
		res = try_macroexpand() # 1.
		if (res isa LoadError && res.error isa UndefVarError) || res isa UndefVarError
			# We have not found the macro in the new workspace after reimports
			# this most likely means that the macro is user defined, we try to expand it
			# in the old workspace to see whether or not it is defined there

			res = try_macroexpand(old_workspace_name) # 2.
			# It was not defined previously, we try searching modules in our own batch
			if (res isa LoadError && res.error isa UndefVarError) || res isa UndefVarError
				to_import_from_batch = mapreduce(union, unresolved_topology.codes) do (_, cache)
				    union(cache.module_usings_imports.imports,
					  cache.module_usings_imports.usings)
				end
				WorkspaceManager.do_reimports(sn, to_import_from_batch)
				# Last try and we leave
				res = try_macroexpand() # 3.
			end
		end
		res
	end

	function analyze_macrocell(cell::Cell, current_symstate)
		if unresolved_topology.nodes[cell].macrocalls ⊆ ExpressionExplorer.can_macroexpand
			return current_symstate, true
		end

		result = macroexpand_cell(cell)
		if result isa Exception
		    # if expansion failed, we use the "shallow" symbols state
		    err = result
		    @debug "Expansion failed" err
		    current_symstate, false
		else # otherwise, we use the expanded expression + the list of macrocalls
		    expanded_symbols_state = ExpressionExplorer.try_compute_symbolreferences(result)
		    expanded_symbols_state, true
		end
	end

	# create new node & new codes for macrocalled cells
	new_nodes = Dict{Cell,ReactiveNode}()
	still_unresolved_nodes = Dict{Cell,SymbolsState}()
	for (cell, current_symstate) in unresolved_topology.unresolved_cells
			(new_symstate, succeeded) = analyze_macrocell(cell, current_symstate)
			if succeeded
				new_node = ReactiveNode(new_symstate)
				union!(new_node.macrocalls, unresolved_topology.nodes[cell].macrocalls)
				union!(new_node.references, new_node.macrocalls)
				new_nodes[cell] = new_node
			else
				still_unresolved_nodes[cell] = current_symstate
			end
	end

	all_nodes = merge(unresolved_topology.nodes, new_nodes)

	NotebookTopology(nodes=all_nodes, codes=unresolved_topology.codes, unresolved_cells=still_unresolved_nodes)
end

function static_macroexpand(topology::NotebookTopology, cell::Cell, old_symstate)
	new_symstate = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |>
		ExpressionExplorer.try_compute_symbolreferences
	union!(new_symstate.macrocalls, old_symstate.macrocalls)

	ReactiveNode(new_symstate)
end

"The same as `resolve_topology` but does not require custom code execution, only works with a few `Base` & `PlutoRunner` macros"
function static_resolve_topology(topology::NotebookTopology)
	new_nodes = Dict{Cell,ReactiveNode}(cell => static_macroexpand(topology, cell, symstate) for (cell, symstate) in topology.unresolved_cells)
	all_nodes = merge(topology.nodes, new_nodes)

	NotebookTopology(nodes=all_nodes, codes=topology.codes, unresolved_cells=topology.unresolved_cells)
end

###
# CONVENIENCE FUNCTIONS
###

"Do all the things!"
function update_save_run!(session::ServerSession, notebook::Notebook, cells::Array{Cell,1}; save::Bool=true, run_async::Bool=false, prerender_text::Bool=false, kwargs...)
	old = notebook.topology
	new = notebook.topology = updated_topology(old, notebook, cells) # macros are not yet resolved

	update_dependency_cache!(notebook)
	session.options.server.disable_writing_notebook_files || (save && save_notebook(notebook))

	# _assume `prerender_text == false` if you want to skip some details_
	to_run_online = if !prerender_text
		cells
	else
		# this code block will run cells that only contain text offline, i.e. on the server process, before doing anything else
		# this makes the notebook load a lot faster - the front-end does not have to wait for each output, and perform costly reflows whenever one updates
		# "A Workspace on the main process, used to prerender markdown before starting a notebook process for speedy UI."
		original_pwd = pwd()
		offline_workspace = WorkspaceManager.make_workspace(
			(
				ServerSession(),
				notebook,
			),
			force_offline=true,
		)

		new = notebook.topology = static_resolve_topology(new)

		to_run_offline = filter(c -> !c.running && is_just_text(new, c) && is_just_text(old, c), cells)
		for cell in to_run_offline
			run_single!(offline_workspace, cell, new.nodes[cell], new.codes[cell])
		end

		cd(original_pwd)
		setdiff(cells, to_run_offline)
	end

	maybe_async(run_async) do
		sync_nbpkg(session, notebook; save=(save && !session.options.server.disable_writing_notebook_files))
		if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook)
			# not async because that would be double async
			run_reactive_async!(session, notebook, old, new, to_run_online; run_async=false, kwargs...)
			# run_reactive_async!(session, notebook, old, new, to_run_online; deletion_hook=deletion_hook, run_async=false, kwargs...)
		end
	end
end

update_save_run!(session::ServerSession, notebook::Notebook, cell::Cell; kwargs...) = update_save_run!(session, notebook, [cell]; kwargs...)
update_run!(args...) = update_save_run!(args...; save=false)

function update_from_file(session::ServerSession, notebook::Notebook)
	just_loaded = try
		sleep(1.2) ## There seems to be a synchronization issue if your OS is VERYFAST
		load_notebook_nobackup(notebook.path)
	catch e
		@error "Skipping hot reload because loading the file went wrong" exception=(e,catch_backtrace())
		return
	end

	old_codes = Dict(
		id => c.code
		for (id,c) in notebook.cells_dict
	)
	new_codes = Dict(
		id => c.code
		for (id,c) in just_loaded.cells_dict
	)

	added = setdiff(keys(new_codes), keys(old_codes))
	removed = setdiff(keys(old_codes), keys(new_codes))
	changed = let
		remained = keys(old_codes) ∩ keys(new_codes)
		filter(id -> old_codes[id] != new_codes[id], remained)
	end

	# @show added removed changed

	for c in added
		notebook.cells_dict[c] = just_loaded.cells_dict[c]
	end
	for c in removed
		delete!(notebook.cells_dict, c)
	end
	for c in changed
		notebook.cells_dict[c].code = new_codes[c]
	end

	notebook.cell_order = just_loaded.cell_order
	update_save_run!(session, notebook, Cell[notebook.cells_dict[c] for c in union(added, changed)])
end

"""
	throttled(f::Function, timeout::Real)

Return a function that when invoked, will only be triggered at most once
during `timeout` seconds.
The throttled function will run as much as it can, without ever
going more than once per `wait` duration.
Inspired by FluxML
See: https://github.com/FluxML/Flux.jl/blob/8afedcd6723112ff611555e350a8c84f4e1ad686/src/utils.jl#L662
"""
function throttled(f::Function, timeout::Real)
	tlock = ReentrantLock()
	iscoolnow = false
	run_later = false

	function flush()
		lock(tlock) do
			run_later = false
			f()
		end
	end

	function schedule()
		@async begin
			sleep(timeout)
			if run_later
				flush()
			end
			iscoolnow = true
		end
	end
	schedule()

	function throttled_f()
		if iscoolnow
			iscoolnow = false
			flush()
			schedule()
		else
			run_later = true
		end
	end

	return throttled_f, flush
end


text:
# Function traits to support delayed evaluation

type TFun{S} end
type TCallSig{S, N} end

abstract FunKind

type EWiseOp <: FunKind end
type ReducOp <: FunKind end

get_op_kind{S,N}(::TCallSig{S,N}) = nothing

##########################################################################
#
#   op-assignment
#
#########################################################################

is_opassign(f::TFun) = false
is_opassign(::TFun{:+=}) = true
is_opassign(::TFun{:-=}) = true
is_opassign(::TFun{:.*=}) = true
is_opassign(::TFun{:./=}) = true

is_opassign(f::Symbol) = is_opassign(TFun{f}())

extract_assign_op(::TFun{:+=}) = (:+)
extract_assign_op(::TFun{:-=}) = (:-)
extract_assign_op(::TFun{:.*=}) = (:.*)
extract_assign_op(::TFun{:./=}) = (:./)

##########################################################################
#
#   general devices to facilitate function registration
#
#########################################################################

function register_ewise_mathop(sym::Symbol, nargs::Integer)
    # the function to generate the codes to register a ewise function

    tc = TCallSig{sym, nargs}
    s1 = :( get_op_kind(::$(Meta.quot(tc))) = EWiseOp() )

    tf = TFun{sym}

    if nargs == 1
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type) = T1 )
    elseif nargs == 2
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type, T2::Type) = promote_type(T1, T2) )
    elseif nargs == 3
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type, T2::Type, T3::Type) = promote_type(promote_type(T1, T2), T3) )
    else
        error("register_ewise_mathop supports up to three arguments.")
    end

    :( $s1; $s2 )
end

function register_ewise_pred(sym::Symbol, nargs::Integer)
    # the function to generate the codes to register a ewise predicate

    tc = TCallSig{sym, nargs}
    s1 = :( get_op_kind(::$(Meta.quot(tc))) = EWiseOp() )

    tf = TFun{sym}

    if nargs == 1
        s2 = :( result_type(::$(Meta.quot(tf)), ::Type) = Bool )
    elseif nargs == 2
        s2 = :( result_type(::$(Meta.quot(tf)), ::Type, ::Type) = Bool )
    elseif nargs == 3
        s2 = :( result_type(::$(Meta.quot(tf)), ::Type, ::Type, ::Type) = Bool )
    else
        error("register_ewise_pred supports up to three arguments.")
    end

    :( $s1; $s2 )
end

function register_reductor(sym::Symbol, nargs::Integer)
    # the function to generate the codes to register a reduction function

    tc = TCallSig{sym, nargs}
    s1 = :( get_op_kind(::$(Meta.quot(tc))) = ReducOp() )

    tf = TFun{sym}

    if nargs == 1
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type) = T1 )
    elseif nargs == 2
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type, T2::Type) = promote_type(T1, T2) )
    elseif nargs == 3
        s2 = :( result_type(::$(Meta.quot(tf)),
            T1::Type, T2::Type, T3::Type) = promote_type(promote_type(T1, T2), T3) )
    else
        error("register_reductor supports up to three arguments.")
    end

    :( $s1; $s2 )
end

##########################################################################
#
#   registration of known operators and functions
#
#########################################################################

# arithmetics

for s in [:+, :-]
    @eval $(register_ewise_mathop(s, 1))
end

for s in [:+, :-, :.+, :.-, :.*, :./, :.^, :max, :min]
    @eval $(register_ewise_mathop(s, 2))
end

@eval $(register_ewise_mathop(:+, 3))
@eval $(register_ewise_mathop(:clamp, 3))

# comparison & logical

for s in [:.==, :.!=, :.<, :.>, :.<=, :.>= ]
    @eval $(register_ewise_pred(s, 2))
end

get_op_kind(::TCallSig{:&, 2}) = EWiseOp()
get_op_kind(::TCallSig{:|, 2}) = EWiseOp()

result_type(::TFun{:&}, ::Type{Bool}, ::Type{Bool}) = Bool
result_type(::TFun{:|}, ::Type{Bool}, ::Type{Bool}) = Bool

# math functions

sqr(x::Number) = x * x
sqr{T<:Number}(a::AbstractArray{T}) = a .* a

rcp(x::AbstractFloat) = one(x) / x
rcp{T<:AbstractFloat}(a::AbstractArray{T}) =  one(eltype(a)) ./ a

for s in [
    :sqrt, :cbrt, :abs, :sqr, :rcp,
    :floor, :ceil, :round, :trunc, :sign,
    :exp, :log, :log10, :exp2, :log2, :expm1, :log1p,
    :sin, :cos, :tan, :asin, :acos, :atan,
    :sinh, :cosh, :tanh, :asinh, :acosh, :atanh,
    :erf, :erfc, :gamma, :lgamma, :digamma ]

    @eval $(register_ewise_mathop(s, 1))
end

for s in [:mod, :hypot, :atan2]
    @eval $(register_ewise_mathop(s, 2))
end

# blending

blend{T1, T2}(c::Bool, x::T1, y::T2) = (c ? x : y)

get_op_kind(::TCallSig{:blend, 3}) = EWiseOp()
result_type(::TFun{:blend}, ::Type{Bool}, T1::Type, T2::Type) = promote_type(T1, T2)

# reduction functions

for s in [ :sum, :maximum, :minimum, :mean ]
    @eval $(register_reductor(s, 1))
end

for s in [ :dot ]
    @eval $(register_reductor(s, 2))
end

# reduction function traits

reduc_result_getter(f::TFun, s::Symbol, n::Symbol) = s

reduc_initializer(f::TFun{:sum}, ty::Symbol) = :( zero($ty) )
reduc_emptyval(f::TFun{:sum}, ty::Symbol) = :( zero($ty) )
reduc_updater(f::TFun{:sum}, s::Symbol, x::Symbol) = :( $s += $x )

reduc_initializer(f::TFun{:maximum}, ty::Symbol) = :( typemin($ty) )
reduc_emptyval(f::TFun{:maximum}, ty::Symbol) = :( typemin($ty) )
reduc_updater(f::TFun{:maximum}, s::Symbol, x::Symbol) = :( $s = max($s, $x) )

reduc_initializer(f::TFun{:minimum}, ty::Symbol) = :( typemax($ty) )
reduc_emptyval(f::TFun{:minimum}, ty::Symbol) = :( typemax($ty) )
reduc_updater(f::TFun{:minimum}, s::Symbol, x::Symbol) = :( $s = min($s, $x) )

reduc_initializer(f::TFun{:mean}, ty::Symbol) = :( zero($ty) )
reduc_emptyval(f::TFun{:mean}, ty::Symbol) = :( nan($ty) )
reduc_updater(f::TFun{:mean}, s::Symbol, x::Symbol) = :( $s += $x )
reduc_result_getter(f::TFun{:mean}, s::Symbol, n::Symbol) = :( $s / $n )

reduc_initializer(f::TFun{:dot}, ty::Symbol) = :( zero($ty) )
reduc_emptyval(f::TFun{:dot}, ty::Symbol) = :( zero($ty) )
reduc_updater(f::TFun{:dot}, s::Symbol, x::Symbol, y::Symbol) = :( $s += $x * $y )


text:
export SimpleHRepresentation, SimpleVRepresentation

# H-Representation

# No copy since I do not modify anything and a copy is done when building a polyhedron
type SimpleHRepresentation{N, T} <: HRepresentation{N, T}
  # Ax <= b
  A::AbstractMatrix{T}
  b::AbstractVector{T}
  linset::IntSet

  function SimpleHRepresentation(A::AbstractMatrix{T}, b::AbstractVector{T}, linset::IntSet=IntSet())
    if size(A, 1) != length(b)
      error("The length of b must be equal to the number of rows of A")
    end
    if ~isempty(linset) && last(linset) > length(b)
      error("The elements of linset should be between 1 and the number of rows of A/length of b")
    end
    if size(A, 2) != N
      error("dimension does not match")
    end
    new(A, b, linset)
  end
end

decomposedfast(rep::SimpleHRepresentation) = false

function SimpleHRepresentation{S <: Real, T <: Real}(A::AbstractMatrix{S}, b::AbstractVector{T}, linset::IntSet=IntSet())
  U = promote_type(S, T)
  SimpleHRepresentation{size(A,2),U}(AbstractMatrix{U}(A), AbstractVector{U}(b), linset)
end
SimpleHRepresentation{T <: Real}(A::AbstractMatrix{T}, b::AbstractVector{T}, linset::IntSet=IntSet()) = SimpleHRepresentation{size(A,2),T}(A, b, linset)

function SimpleHRepresentation{N, T}(it::HRepIterator{N, T})
  A = Matrix{T}(length(it), N)
  b = Vector{T}(length(it))
  linset = IntSet()
  for (i, h) in enumerate(it)
    A[i,:] = h.a
    b[i] = h.β
    if islin(h)
      push!(linset, i)
    end
  end
  new(A, b, linset)
end
function SimpleHRepresentation{N, T}(;eqs::Nullable{EqIterator{N, T}}=nothing, ineqs::Nullable{IneqIterator{N, T}}=nothing)
  neq = isnull(eqs) ? 0 : length(eqs)
  nineq = isnull(ineqs) ? 0 : length(ineqs)
  nhrep = neq + nineq
  A = Matrix{T}(nhrep, N)
  b = Vector{T}(nhrep)
  linset = IntSet(1:neq)
  if !(eqs === nothing)
    for (i, h) in enumerate(get(eqs))
      A[i,:] = h.a
      b[i] = h.β
    end
  end
  if !(ineqs === nothing)
    for (i, h) in enumerate(get(ineqs))
      A[neq+i,:] = h.a
      b[neq+i] = h.β
    end
  end
  new(A, b, linset)
end

Base.length(ine::SimpleHRepresentation) = size(ine.A, 1)

Base.copy{N,T}(ine::SimpleHRepresentation{N,T}) = SimpleHRepresentation{N,T}(copy(ine.A), copy(ine.b), copy(ine.linset))

starthrep(ine::SimpleHRepresentation) = 1
donehrep(ine::SimpleHRepresentation, state) = state > length(ine)
nexthrep(ine::SimpleHRepresentation, state) = (state in ine.linset ? HyperPlane(ine.A[state,:], ine.b[state]) : HalfSpace(ine.A[state,:], ine.b[state]), state+1)

neqs(ine::SimpleHRepresentation) = length(ine.linset)
starteq(ine::SimpleHRepresentation) = start(ine.linset)
doneeq(ine::SimpleHRepresentation, state) = done(ine.linset)
function nexteq(ine::SimpleHRepresentation, state)
  (i, nextstate) = next(ine.linset)
  (HyperPlane(ine.A[i,:], ine.b[i]), nextstate)
end

function nextz(is::IntSet, i)
  while i in is
    i += 1
  end
  i
end
nineqs(ine::SimpleHRepresentation) = length(ine) - neqs(ine)
startineq(ine::SimpleHRepresentation) = nextz(ine.linset, 1)
doneineq(ine::SimpleHRepresentation, state) = state > length(ine)
nextineq(ine::SimpleHRepresentation, state) = (HalfSpace(ine.A[state,:], ine.b[state]), nextz(state+1))

# V-Representation

type SimpleVRepresentation{N,T} <: VRepresentation{N,T}
  V::AbstractMatrix{T} # each row is a vertex
  R::AbstractMatrix{T} # each row is a ray
  Vlinset::IntSet
  Rlinset::IntSet

  function SimpleVRepresentation(V::AbstractMatrix{T}, R::AbstractMatrix{T}, Vlinset::IntSet=IntSet(), Rlinset::IntSet=IntSet())
    if length(R) > 0 && size(R, 2) != N
      error("dimension does not match")
    end
    if length(V) > 0 && size(V, 2) != N
      error("dimension does not match")
    end
    if ~isempty(Vlinset) && last(Vlinset) > size(V, 1)
      error("The elements of Vlinset should be between 1 and the number of rows of V")
    end
    if ~isempty(Rlinset) && last(Rlinset) > size(R, 1)
      error("The elements of Rlinset should be between 1 and the number of rows of R")
    end
    new(V, R, Vlinset, Rlinset)
  end
end

SimpleVRepresentation{T <: Real}(V::AbstractMatrix{T}, R::AbstractMatrix{T}, Vlinset::IntSet=IntSet(), Rlinset::IntSet=IntSet()) = SimpleVRepresentation{size(V,2),T}(V, R, Vlinset, Rlinset)

SimpleVRepresentation{T <: Real}(V::AbstractMatrix{T}, linset::IntSet=IntSet()) = SimpleVRepresentation{size(V, 2),T}(V, similar(V, 0, size(V, 2)), linset, IntSet())

Base.copy{N,T}(ext::SimpleVRepresentation{N,T}) = SimpleVRepresentation{N,T}(copy(ext.V), copy(ext.R), copy(ext.Vlinset), copy(ext.Rlinset))


text:
# ---
# title: 260. Single Number III
# id: problem260
# author: Indigo
# date: 2021-06-05
# difficulty: Medium
# categories: Bit Manipulation
# link: <https://leetcode.com/problems/single-number-iii/description/>
# hidden: true
# ---
#
# Given an integer array `nums`, in which exactly two elements appear only once
# and all the other elements appear exactly twice. Find the two elements that
# appear only once. You can return the answer in **any order**.
#
# **Follow up:  **Your algorithm should run in linear runtime complexity. Could
# you implement it using only constant space complexity?
#
#
#
# **Example 1:**
#
#
#
#     Input: nums = [1,2,1,3,2,5]
#     Output: [3,5]
#     Explanation: [5, 3] is also a valid answer.
#
#
# **Example 2:**
#
#
#
#     Input: nums = [-1,0]
#     Output: [-1,0]
#
#
# **Example 3:**
#
#
#
#     Input: nums = [0,1]
#     Output: [1,0]
#
#
#
#
# **Constraints:**
#
#   * `1 <= nums.length <= 30000`
#   *  Each integer in `nums` will appear twice, only two integers will appear once.
#
#
## @lc code=start
using LeetCode

function single_number_iii(nums::Vector{T}) where T <: Integer
    res = reduce(⊻, nums; init = zero(T))
    div = one(T)
    while (div & res) == 0
        div <<= one(T)
    end
    a = b = 0
    for n in nums
        (div & n == 0) ? a ⊻= n : b ⊻= n
    end
    return a, b
end
## @lc code=end


text: module TestModuleB end


text:
# # Car travel in 1D
#
# How a car travels back and forth between two locations.

using Plots

# On my computer GR, the default backend crashes Julia, thus this:
try
    pyplot() # run with PyPlot.jl backend, if installed
catch
    nothing # run with GR.jl backend
end

if !@isdefined plotyes
    plotyes = true
end

"""
    update_position(X, V, dir, dt, L)

Return updated position of car and new travel direction;
given, old postion `X`, speed `V`, direction `dir`, time step `dt`
and domain size `L`.

## Example
```jldoctest
julia> update_position(0.0, 10, 1, 1, 200)
(10.0, 1)
```
"""
function update_position(X, V, dir, dt, L)
    X = X + dir*V*dt
    if X > L
        dir = -1      # if beyond L, go back (left)
    elseif X < 0
        dir = 1       # if beyond 0, go back (right)
    end
    return X, dir
end

"""
    car_travel_1D()

Run the model.
"""
function car_travel_1D()
    ## Physical parameters
    V     = 43.0           # speed, km/h
    L     = 200.0          # length of segment, km
    dir   = 1              # switch 1 = go right, -1 = go left
    ttot  = 16.0           # total time, h
    ## Numerical parameters
    dt    = 0.1            # time step, h
    nt    = Int(cld(ttot, dt))  # number of time steps
    ## Array initialisation
    T     = zeros(nt)
    X     = zeros(nt)
    ## Time loop
    for it = 2:nt
        T[it] = T[it-1] + dt
        X[it], dir = update_position(X[it-1], V, dir, dt, L)
    end
    ## Visualisation
    return T, X
end

#-

## Only run this in an interactive session:
if plotyes
    println("Running model")
    T, X = car_travel_1D()
    plot(T, X, markersize=5, xlabel="time, hrs", ylabel="distance, km", framestyle=:box, legend=:none)
end
# Figure showing how the car bounces back and forth, nice!


text:
# Autogenerated wrapper script for ICU_jll for powerpc64le-linux-gnu-cxx11
export libicudata, libicui18n, libicuio, libicutest, libicutu, libicuuc

JLLWrappers.@generate_wrapper_header("ICU")
JLLWrappers.@declare_library_product(libicudata, "libicudata.so.69")
JLLWrappers.@declare_library_product(libicui18n, "libicui18n.so.69")
JLLWrappers.@declare_library_product(libicuio, "libicuio.so.69")
JLLWrappers.@declare_library_product(libicutest, "libicutest.so.69")
JLLWrappers.@declare_library_product(libicutu, "libicutu.so.69")
JLLWrappers.@declare_library_product(libicuuc, "libicuuc.so.69")
function __init__()
    JLLWrappers.@generate_init_header()
    JLLWrappers.@init_library_product(
        libicudata,
        "lib/libicudata.so",
        nothing,
    )

    JLLWrappers.@init_library_product(
        libicui18n,
        "lib/libicui18n.so",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libicuio,
        "lib/libicuio.so",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libicutest,
        "lib/libicutest.so",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libicutu,
        "lib/libicutu.so",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libicuuc,
        "lib/libicuuc.so",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
using Documenter, Subspaces

DocMeta.setdocmeta!(Subspaces, :DocTestSetup, :( using Subspaces; using LinearAlgebra ); recursive=true)

makedocs(
    sitename="Subspaces.jl",
    modules=[Subspaces],
    format = Documenter.HTML(
        prettyurls = get(ENV, "CI", nothing) == "true"
    ),
    pages = [
        "Home" => "index.md",
        "Usage" => "usage.md",
        "Reference" => "reference.md",
    ],
)

deploydocs(
    repo = "github.com/dstahlke/Subspaces.jl.git",
    devbranch = "main",
    branch = "gh-pages",
)


text:
module STables

using DataFrames
using Dates
import DelimitedFiles

include("grisu.jl")
include("types.jl")
include("schema.jl")
include("table.jl")
include("tablerow.jl")
include("schema_inference.jl")
include("csv.jl")

end # module STables


text:
mutable struct CircuitParsed{T<:AbstractString}
  circuitfilearray :: Array{T,1}
  parameternames :: Array{T,1}
  parametervalues :: ParameterValuesArray{Float64,1}
  parametermultiplier :: Array{Float64,1}
  parameterindex :: Array{Int,1}
  measurementnames :: Array{T,1}
  stepnames :: Array{T,1}
  circuitfileencoding
  cardlist
  circuitpath::T
  workingcircuitpath::T
  executablepath::T
  includesearchpath::Array{T,1}
  librarysearchpath::Array{T,1}
  CircuitParsed{T}() where T =
    new{T}([],[],[],[],[],[],[],nothing,standardcardlist,"","","",[],[])
end

# LTspice allows multiple directives (cards) in a single block
# in the GUI, ctrl-M is used to create a new line.
# this puts a backslash n in the file, NOT a newline character.
#
# EachCard is an iterator that separates the lines around the backslash n

#=
Base.start(::EachCard) = 1
function Base.next(ec::EachCard, state)
    p = first(something(findnext("\\n", ec.line, state), 0:-1))
    if p!=0 && p!=lastindex(ec.line)
        card = ec.line[state:p+1]
        state = p+2
    else
        card = ec.line[state:end]
        state = lastindex(ec.line)
    end
    return (card, state)
end
Base.done(ec::EachCard, state) = state>=lastindex(ec.line)
=#
struct EachCard{T<:AbstractString}
    line :: T
end
function Base.iterate(ec::EachCard, state=1)
    if state>=lastindex(ec.line)
        return nothing
    end
    p = first(something(findnext("\\n", ec.line, state), 0:-1))
    if p!=0 && p!=lastindex(ec.line)
        card = ec.line[state:p+1]
        state = p+2
    else
        card = ec.line[state:end]
        state = lastindex(ec.line)
    end
    return (card, state)
end

abstract type Card end
struct Parameter<:Card end
struct Measure<:Card end
struct Step<:Card end
struct Other<:Card end
struct Include<:Card end
struct Library<:Card end
const standardcardlist = [Parameter(), Measure(), Step(), Other()]
const tempdircardlist = [Parameter(), Measure(), Step(),
                         Include(), Library(), Other()]

const parameterregex = r"[.](?:parameter|param)[ ]+()"ix
const parametercaptureregex =
          r"""
          ([^\d ][^ =]*)
          ([ ]*={0,1}[ ]*)
          ([-+]{0,1}[0-9.]+e{0,1}[-+0-9]*)(k|meg|g|t|m|u|μ|n|p|f){0,1}
          (
          [^-+*/ ]*
          (?:\s|\\n|\r|$)
          (?![-+*/])
          )()
          """ix
#=
parametercaptureregex offsets
1 - name
2 - after name
3 - value
4 - unit
5 - everything to the end
6 - index after match
=#
function parsecard!(cp::CircuitParsed, ::Parameter, card::AbstractString)
  m = match(parameterregex, card)
  m == nothing && return false
  currentposition = m.offsets[1]
  push!(cp.circuitfilearray,card[1:currentposition-1])
  while currentposition<lastindex(card) && m!=nothing
#    println("      parameter: ",card[currentposition:end])
    m = match(parametercaptureregex,card,currentposition)
    if m!=nothing
      pushnamevalue!(cp,m,currentposition,card)
      currentposition = m.offsets[6]
    else
      push!(cp.circuitfilearray,card[currentposition:end])
    end
  end
  return true
end
function pushnamevalue!(cp::CircuitParsed,
                        m::RegexMatch,
                        currentposition::Int,
                        card::AbstractString)
  name = m.captures[1]
  after_name = m.captures[2]
  value = m.captures[3]
  unit = m.captures[4]
  to_end = m.captures[5]
  push!(cp.circuitfilearray,card[currentposition:m.offsets[1]-1])
  push!(cp.circuitfilearray,name)
  push!(cp.circuitfilearray,after_name)
  push!(cp.circuitfilearray,value)
  if haskey(units,unit)
      multiplier = units[unit]
  else
      multiplier = 1.0
  end
  valuenounit = parse(Float64,value)
  push!(cp.parameternames, name)
  index_parameternames = length(cp.parameternames)
  push!(cp.parametervalues.values, valuenounit * multiplier)
  push!(cp.parametermultiplier, multiplier)
  index_circuitfilearray = length(cp.circuitfilearray)
  push!(cp.parameterindex, index_circuitfilearray)
  unit != nothing && push!(cp.circuitfilearray,unit)
  push!(cp.circuitfilearray,to_end)
end

const measureregex = r"[.](?:measure|meas)[ ]+(?:ac |dc |op |tran |tf |noise ){0,1}[ ]*([^\d ][^ =]*)[ ]+"ix
function parsecard!(cp::CircuitParsed, ::Measure, card::AbstractString)
    m = match(measureregex, card)
    if m == nothing # exit if not a measure card
        return false
    end
    name = m.captures[1]
    push!(cp.measurementnames, name)
    push!(cp.circuitfilearray, card)
    return true
end
const step1regex = r"[.](?:step)[ ]+(?:oct |param ){0,1}[ ]*([^\d ][^ =]*)[ ]+(?:list ){0,1}[ ]*[0-9.e+-]+[a-z]*[ ]+"ix
const step2regex = r"[.](?:step)[ ]+(?:\w+)[ ]+(\w+[(]\w+[)])[ ]+"ix
function parsecard!(cp::CircuitParsed, ::Step, card::AbstractString)
    m = match(step1regex, card)
    if m == nothing
        m = match(step2regex, card)
        if m == nothing
            return false # exit if not a step card
        end
    end
    name = m.captures[1] #lowercase(m.captures[1])
    push!(cp.stepnames, name)
    push!(cp.circuitfilearray, card)
    return true
end
const includeregex = r"""(?:.include|.inc)[ ]+
                          [\"]{0,1}(.*?)[\"]{0,1}(?:\\n|\r|$)"""ix
function parsecard!(cp::CircuitParsed, ::Include, card::AbstractString)
  m = match(includeregex, card)
  m == nothing && return false
  incabspath = absolutepath(cp.includesearchpath,m.captures[1])
  push!(cp.circuitfilearray,replace(card,m.captures[1] => incabspath))
  return true
end
const libraryregex = r"""(?:.lib)[ ]+
                          [\"]{0,1}(.*?)[\"]{0,1}(?:\\n|\r|$)"""ix
function parsecard!(cp::CircuitParsed, ::Library, card::AbstractString)
  m = match(libraryregex, card)
  m == nothing && return false
  libabspath = absolutepath(cp.librarysearchpath,m.captures[1])
  push!(cp.circuitfilearray,replace(card,m.captures[1] => libabspath))
  return true
end
function absolutepath(patharray, file)
  for path in patharray
    testfile = joinpath(path,file)
    isfile(testfile) && return testfile
  end
end
function parsecard!(cp::CircuitParsed, ::Other, card::AbstractString)
    push!(cp.circuitfilearray, card)
    return true
end
function parsecard!(cp::CircuitParsed, card::AbstractString)
    for cardtype in cp.cardlist
        if parsecard!(cp,cardtype,card)
          break
        end
    end
end

iscomment(line::AbstractString) = occursin(r"^TEXT .* ;",line)
function parsecircuitfile(circuitpath::AbstractString,
                          workingcircuitpath::AbstractString,
                          executablepath::AbstractString,
                          librarysearhpaths)
  cp = CircuitParsed{String}()
  cp.circuitpath = abspath(circuitpath)
  cp.workingcircuitpath = abspath(workingcircuitpath)
  cp.executablepath = abspath(executablepath)
  if abspath(circuitpath) != abspath(workingcircuitpath)
    cp.cardlist = tempdircardlist
    executabledir = abspath(dirname(executablepath))
    originalcircuitdir = abspath(dirname(circuitpath))
    cp.includesearchpath = [joinpath(executabledir,"lib\\sub"),
                            originalcircuitdir]
    cp.librarysearchpath = [joinpath(executabledir,"lib\\cmp"),
                        joinpath(executabledir,"lib\\sub"),
                        originalcircuitdir]
    append!(cp.librarysearchpath,librarysearhpaths)
    cp.parametervalues.ismodified = true
  else
    cp.parametervalues.ismodified = false
  end
  cp.circuitfileencoding = circuitfileencoding(circuitpath)
#  println(circuitpath,"  ",cp.circuitfileencoding)
  open(circuitpath,cp.circuitfileencoding) do io
    for line in eachline(io, keep=true)
#      print("line: ",line)
      if iscomment(line)
        push!(cp.circuitfilearray, line)
      else
        for card in EachCard(line) # might be multi-line directive(s) created with Ctrl-M
#          println("  card: ",card)
          parsecard!(cp, card)
        end
      end
    end
  end
  return cp
end

function circuitfileencoding(path::AbstractString)
  function checkencoding(i)
    open(path,encodings[i]) do io
      if occursin(r"^Version",readline(io, keep=true))
        correct_i = i
      end
    end
  end
  firstwordshouldbe = "Version"
  encodings = [enc"windows-1252",enc"UTF-16LE",enc"UTF-8"] #enc"windows-1252",
  correct_i = 0
  for i in eachindex(encodings)
    try checkencoding(i) catch nothing end
    correct_i !=0 && break
  end
  correct_i == 0 && error("invalid LTspice circuit file")
#  println(path," ",encodings[correct_i])
  return encodings[correct_i]
end

# units as defined in LTspice
const units = Dict("K" => 1.0e3,
                   "k" => 1.0e3,
                   "MEG" => 1.0e6,
                   "meg" => 1.0e6,
                   "G" => 1.0e9,
                   "g" => 1.0e9,
                   "T" => 1.0e12,
                   "t" => 1.0e12,
                   "M" => 1.0e-3,
                   "m" => 1.0e-3,
                   "U" => 1.0e-6,
                   "u" => 1.0e-6,
                   "μ" => 1.0e-6,
                   "N" => 1.0e-9,
                   "n" => 1.0e-9,
                   "P" => 1.0e-12,
                   "p" => 1.0e-12,
                   "F" => 1.0e-15,
                   "f" => 1.0e-15,
                   )


text:
struct POWNodeBelief{S,A,O,P}
    model::P
    a::A # may be needed in push_weighted! and since a is constant for a node, we store it
    o::O
    dist::CategoricalVector{Tuple{S,Float64}}

    POWNodeBelief{S,A,O,P}(m,a,o,d) where {S,A,O,P} = new(m,a,o,d)
    function POWNodeBelief{S, A, O, P}(m::P, s::S, a::A, sp::S, o::O, r) where {S, A, O, P}
        cv = CategoricalVector{Tuple{S,Float64}}((sp, convert(Float64, r)),
                                                 1.0)
        new(m, a, o, cv)#initial

    end
end

function POWNodeBelief(model::POMDP{S,A,O}, s::S, a::A, sp::S, o::O, r) where {S,A,O}
    POWNodeBelief{S,A,O,typeof(model)}(model, s, a, sp, o, r)
end

rand(rng::AbstractRNG, b::POWNodeBelief) = rand(rng, b.dist)
state_mean(b::POWNodeBelief) = first_mean(b.dist)
POMDPs.currentobs(b::POWNodeBelief) = b.o
POMDPs.history(b::POWNodeBelief) = tuple((a=b.a, o=b.o))

struct POWNodeFilter end

belief_type(::Type{POWNodeFilter}, ::Type{P}) where {P<:POMDP} = POWNodeBelief{statetype(P), actiontype(P), obstype(P), P}

init_node_sr_belief(::POWNodeFilter, p::POMDP, s, a, sp, o, r) = POWNodeBelief(p, s, a, sp, o, r)

function push_weighted!(b::POWNodeBelief, ::POWNodeFilter, s, sp, r)
    w = 1.0
    insert!(b.dist, (sp, convert(Float64, r)), w)
end
"""
原来的r是在o下执行a得到的r，因为我们现在需要把s'加入到其他o分支中，所以这个r也应该变成在其他o下执行a得到的r，但是目前来说，我不知道应该如何实现，所以暂且不实现
"""
struct StateBelief{SRB<:POWNodeBelief}
    sr_belief::SRB
end

rand(rng::AbstractRNG, b::StateBelief) = first(rand(rng, b.sr_belief))
mean(b::StateBelief) = state_mean(b.sr_belief)
POMDPs.currentobs(b::StateBelief) = currentobs(b.sr_belief)
POMDPs.history(b::StateBelief) = history(b.sr_belief)


text:
# https://adventofcode.com/2020/day/5

# Instead of zones or groups, this airline uses binary space partitioning to seat people.
# A seat might be specified like FBFBBFFRLR, where F means "front", B means "back", L
# means "left", and R means "right".
#
# The first 7 characters will either be F or B; these specify exactly one of the 128 rows
# on the plane (numbered 0 through 127). Each letter tells you which half of a region the
# given seat is in. Start with the whole list of rows; the first letter indicates whether
# the seat is in the front (0 through 63) or the back (64 through 127). The next letter
# indicates which half of that region the seat is in, and so on until you're left with
# exactly one row.
#
# For example, consider just the first seven characters of FBFBBFFRLR:
#
#   Start by considering the whole range, rows 0 through 127.
#   F means to take the lower half, keeping rows 0 through 63.
#   B means to take the upper half, keeping rows 32 through 63.
#   F means to take the lower half, keeping rows 32 through 47.
#   B means to take the upper half, keeping rows 40 through 47.
#   B keeps rows 44 through 47.
#   F keeps rows 44 through 45.
#   The final F keeps the lower of the two, row 44.

function code_to_int(code)
    code = replace(code, r"F|L" => '0')
    code = replace(code, r"B|R" => '1')
    return parse(Int, code, base=2)
end

@assert code_to_int("BFFFBBFRRR") == 567
@assert code_to_int("FFFBBBFRRR") == 119
@assert code_to_int("BBFFBBFRLL") == 820
@assert code_to_int("FFFFFFFFLL") == 0
@assert code_to_int("BBBBBBBBRR") == 1023

function part1(codes)
    highest = 0
    for code in split(codes, '\n', keepempty=false)
        highest = max(highest, code_to_int(code))
    end
    return highest
end

example = "
FFFBBFFFLL
BFFBBFBBRR
BBBBBBBBRR
FFFFFFFFLL
"

@assert part1(example) == 1023

function find_missing(numbers)
    return findfirst(diff(numbers) .> 1)
end

@assert find_missing([1, 2, 3, 4, 6, 7, 8]) == 4

function part2(codes)
    seat_numbers = sort([code_to_int(code) for code in split(codes, '\n', keepempty=false)])
    idx = find_missing(seat_numbers)
    return seat_numbers[idx] + 1
end

test = read("data/day-05.txt", String)
println("Part 1: $(result1 = part1(test))")
println("Part 2: $(result2 = part2(test))")

@assert result1 == 913
@assert result2 == 717

max_stars_repo_name: twolodzko/advent-of-code

max_issues_repo_name: twolodzko/advent-of-code

max_forks_repo_name: twolodzko/advent-of-code


text:
using DataStructures

function slurp_map(filename)
    result = DefaultDict(Set)
    for (from, to) in split.(readlines(filename), "-")
        push!(result[from], to)
        push!(result[to], from)
    end
    for (_, reachable) in result
        delete!(reachable, "start")
    end
    result
end

slurp_map("12eg1.txt")

function islower(s)
    if s == "start"
        false
    elseif s == "end"
        false
    else
        islowercase(s[1])
    end
end

function count_paths(current::AbstractString, links::AbstractDict)
    if current == "end"
        1
    elseif isempty(links[current])
        0
    else
        path_count = 0
        new_links = deepcopy(links)
        if islower(current)
            for (from, tos) in new_links
                delete!(tos, current)
            end
        end
        for next in links[current]
            path_count += count_paths(next, deepcopy(new_links))
        end
        path_count
    end
end

count_paths(links::AbstractDict) = count_paths("start", links)

count_paths(slurp_map("12eg1.txt")) == 10
count_paths(slurp_map("12eg2.txt")) == 19
count_paths(slurp_map("12eg3.txt")) == 226
count_paths(slurp_map("12.txt"))

function count_paths2(current::AbstractString, links::AbstractDict, small_visited::AbstractDict)
    links = deepcopy(links)
    small_visited = deepcopy(small_visited)
    if current == "end"
        1
    elseif isempty(links[current])
        0
    else
        path_count = 0
        if islower(current)
            small_visited[current] += 1
        end
        for next in links[current]
            if islower(next) && any(>=(2), values(small_visited))
                if small_visited[next] == 0
                    path_count += count_paths2(next, links, small_visited)
                end
            else
                path_count += count_paths2(next, links, small_visited)
            end
        end
        path_count
    end
end

count_paths2(links::AbstractDict) = count_paths2("start", links, DefaultDict(0))

count_paths2(slurp_map("12eg1.txt"))
count_paths2(slurp_map("12eg2.txt"))
count_paths2(slurp_map("12eg3.txt")) == 3509

count_paths2(slurp_map("12.txt"))


text:
using Distributed
using Dates
using DelimitedFiles
using PyPlot

if (nprocs() < 2 && isdefined(Main, :usePrecaching) && usePrecaching)
    println("Adding process for file precaching!")
    addprocs(1)
end

scriptspath =  pwd()
if ! isdefined(Main, :FunctionsLoaded)
    #load includes
    include("includes/BaselineFunctions.jl")
    include("includes/InterpolationFunctions.jl")
    include("includes/DatasetPreloader.jl")
    include("includes/MasslistFunctions.jl")
    include("includes/PeakshapeFunctions.jl")
    include("includes/TofFunctions.jl")
    include("includes/MultipeakFunctions.jl")
    include("includes/ResultFileFunctions.jl")
    #misc module leftovers
    include("includes/TofTracer.jl")

    #Load Personal MassLibrary Definitions
    include("$(scriptspath)/manualMassLibrary.jl")

    #Load workflows
    include("$(scriptspath)/workflows/combinedMassScaleAndExtractSumSpec.jl")
    include("$(scriptspath)/workflows/peakShape.jl")
    include("$(scriptspath)/workflows/deconvolutionMatrix.jl")
    FunctionsLoaded = true
end


text:
using DataDeps

register(DataDep(
  "qsar-biodegradation",
  "http://archive.ics.uci.edu/ml/datasets/QSAR+biodegradation",
  "http://archive.ics.uci.edu/ml/machine-learning-databases/00254/biodeg.csv",
  "c66ac1a23aba11fb166fd5054fea14ac97b8e4f7e4940ae679dc863d1cc4b3c9",
  post_fetch_method=(path -> begin
    UCIData.process_dataset(path,
        target_index=42,
        feature_indices=1:41,
        delim=';',
    )
  end),
))

max_stars_repo_name: JackDunnNZ/uci-data

max_issues_repo_name: JackDunnNZ/uci-data

max_forks_repo_name: JackDunnNZ/uci-data


text:
import LightXML
using Parameters
import HTTP

abstract type AbstractGeneSet end

@with_kw mutable struct GeneSet <: AbstractGeneSet
    std_name::String
    ids::Array
    sys_name::String = ""
    org::String = ""
    pmid::String = ""
    cat::String = ""
    subcat::String = ""
    geoid::String = ""
    source::String = ""
    url::String = ""
    description::String = ""
end

"""
    get_msigdb(version="v7.2")

This function automatically fetches msigdb gene sets from the Broad Institute and generate an array of GeneSet objects containing relevant information
Input:
version: The version of the database, starts with "v" (e.g. "v.7.2"). Currently defaulted to Version.
"""
function get_msigdb(version::String="v7.2")
    urlname = string("https://data.broadinstitute.org/gsea-msigdb/msigdb/release/7.2/msigdb_",version, ".xml")
    println(string("Currently at ", version, " of MSigDB"))
    r = HTTP.get(urlname);
    open("msigdb.xml", "w") do io
        write(io,String(r.body))
    end;
    xdoc = LightXML.parse_file("msigdb.xml");
    rm("msigdb.xml");
    xroot = LightXML.root(xdoc);
    sets = LightXML.get_elements_by_tagname(xroot, "GENESET");
    output = [];
    attributes = ["STANDARD_NAME", "MEMBERS", "SYSTEMATIC_NAME", "ORGANISM", "PMID", "CATEGORY_CODE",
                    "SUB_CATEGORY_CODE", "GEOID", "EXACT_SOURCE", "GENE_SET_LISTING_URL", "DESCRIPTION_BRIEF"]
    # loop through sets to retrieve elements
    for i in sets
        if LightXML.attribute(i, "CATEGORY_CODE") == "ARCHIVED"
            continue
        end
        if LightXML.attribute(i, "STANDARD_NAME") == Nothing() || LightXML.attribute(i, "MEMBERS") == Nothing()
            continue
        end
        args = []
        # members for gene sets should be automatically
        for j in attributes
            if j == "MEMBERS"
                value = String.(split(LightXML.attribute(i,j), ","))
            else
                value = LightXML.attribute(i, j)
                value = ifelse(value == Nothing(), "", value)
            end
            push!(args, value)
        end
        # TODO: Ask for substitute to do.call
        set = GeneSet(args[1], args[2], args[3], args[4], args[5], args[6], args[7],
                        args[8], args[9], args[10], args[11]);
        push!(output, set)
    end
    return(output)
end


text:
module Dilemma

end # module


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule XML2_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("XML2")
JLLWrappers.@generate_main_file("XML2", UUID("02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"))
end  # module XML2_jll


text:
mutable struct Point{X<:FR,Y<:FR} <: Object
    x::X
    y::Y
    opts::Option{LittleDict{Symbol,Any}}
end

point(x::FR, y::FR; kw...) = Point(x, y, dict(;kw...))
point(x::FR, y::FR, d::LittleDict{Symbol,Any}) = Point(x, y, d)
point(n::String, x, y; kw...) = point(x, y; name=n, kw...)

# ---------------------------------------------------------------------------

function str(p::Point, b::Board)
    xs, xss, xrp = strf(p.x, "FX", b)
    ys, yss, yrp = strf(p.y, "FY", b)
    opts = get_opts(p)
    jss = js".create('point', [$xss, $yss], $opts);"
    name = get(opts, :name, nothing)
    isnothing(name) && return xs * ys * b.name * replacefn(jss.s, xrp, yrp)
    return xs * ys * "var $name =" * b.name * replacefn(jss.s, xrp, yrp)
end


text:
function calc_n_bar(cur_reactor::AbstractReactor)
  cur_n = K_n(cur_reactor)

  cur_n *= safe_symbol(cur_reactor, :I_P)

  cur_n /= safe_symbol(cur_reactor, :R_0) ^ 2

  cur_n
end


text:
int main() {
    int x;
    double y;
    printInt(x);
    printDouble(y);
    return 0;
}

max_stars_repo_name: carlostome/Javalette-Compiler

max_issues_repo_name: carlostome/Javalette-Compiler

max_forks_repo_name: carlostome/Javalette-Compiler


text:
const GLASSCAT_ROOT = joinpath(@__DIR__, "..", "src", "GlassCat")
const GLASS_JL_FILE = "AGFGlassCat.jl"
const GLASS_JL_PATH = GLASS_JL_FILE
const DOWNLOAD_DIR = mktempdir()

include(joinpath(GLASSCAT_ROOT, "GlassTypes.jl"))
include("utils.jl")

glasspaths = [joinpath(GLASSCAT_ROOT, "OtherGlassCat.jl")]

# load all AGF files to dictionary and
# for each catalog create a module with structs containing the default values
# this is done as a string then written to a file which is then included
# download glassfiles that are easily accessed from the web so user will have some glasses to start with
downloadcatalogs(DOWNLOAD_DIR)

@info "Generating $GLASS_JL_PATH\n"
generate_cat_jl(load_glass_db(DOWNLOAD_DIR), GLASS_JL_PATH)
push!(glasspaths, GLASS_JL_PATH)

# write a deps/deps.jl file that points to the generated glass cat source files
open("deps.jl", "w") do io
    for glasspath in glasspaths
        println(io, "include(raw\"$(joinpath(pwd(), glasspath))\")")
    end
end


text:
@testset "Multi-threaded runs test, no init pop" begin

@info "Commencing test of multi-threaded parallel simulation runs. This test may take a few minutes, depending on the user's hardware and Julia settings."
tStart = now()
rm( "parallel/tmps", recursive=true, force=true )
rm( "parallel/simDB.sqlite", force=true )

mrs = MRS( mpSim )
setMRSruns!( mrs, 30 )
setMRSdatabaseName!( mrs, "parallel/simDB.sqlite" )
setMRSmaxThreads!( mrs, 10 )
@test verifySimulation!( mrs )
run( mrs )
tElapsed = ( now() - tStart ).value / 1000
@info string( "Running 30 simulations in parallel on ",
    min( 10, Threads.nthreads() ), " threads took ", tElapsed,
    " seconds. Thank you for your patience." )

db = SQLite.DB( "parallel/simDB.sqlite" )
@test length( SQLite.tables( db )[:name] ) == 92

println()

end  # @testset "Multi-threaded runs test, no init pop"


text:
import DataFrames: DataFrame

function rast2df(r::AbstractGeoArray)
    LON, LAT = st_coords(r)
    DataFrame(id=seq_along(LON), value=r.A[:], lon=LON[:], lat=LAT[:])
end

function rast2df(list::Vector{AbstractGeoArray})
    map(rast2df, list)
end


text:
module DGApplication

import LinearAlgebra

# Types
export Application
# Functions
export is_second_order, nstates
export flux_c!, flux_d!, flux_l!
export numerical_flux_c!, boundary_flux_c!
export numerical_flux_d!, boundary_flux_d!
export numerical_flux_l!, boundary_flux_l!
export source!

"""
    abstract type Application{N}
Supertype for applications (PDEs) that specific PDEs can extend. Generalizes
the equation `u_t + ∇⋅f_c(u) + ∇⋅f_d(u,∇u) = s(t,x) where u = u(t,x)`. N is the
dimension of the PDE, aka size of the vector x.
"""
abstract type Application{N} end

"""
    is_second_order(app::Application)
Returns whether the PDE is 2nd order. Default: false => f_d(u,∇u) = 0
"""
is_second_order(app::Application) = false
"""
    nstates(app::Application)
Returns the number of state variables for the given PDE. Default: 1
"""
nstates(app::Application) = 1
"""
    flux_c!(flux, app::Application, u)
Computes the 1st order, or convective, flux f_c(u) for the given application.
Default: f_c(u) = 0.0
"""
flux_c!(flux, app::Application{N}, u) where {N} = (flux .= zeros(typeof(u[1]),nstates(app),N))
"""
    flux_d!(flux, app::Application, u)
Computes the 2nd order, or diffusive, flux f_d(u,∇u) for the given application.
Default: f_d(u,∇u) = 0.0
"""
flux_d!(flux, app::Application{N}, u, Du) where {N} = (flux .= zeros(typeof(u[1]),nstates(app),N))
"""
    flux_l!(flux, app::Application, u)
Computes the local DG flux function for the given application. For CDG, f_l(u)=u
Default: f_l(u) = u
"""
flux_l!(flux, app::Application{N}, u) where {N} = (flux .= deepcopy(u))
"""
    numerical_flux_c!(flux, app::Application, uK, uN, normal_k)
Given the state vector at this element (uK) and its neighbor (uN), compute the
convection numerical flux function for this PDE dotted with the normal, and
store the results in flux. Default: f(uK,uN)*n = upwinding with positive speed
"""
function numerical_flux_c!(flux, app::Application{N}, uK, uN, normal_k) where {N}
    @assert N == length(normal_k)
    flux_k = Array{Float64,2}(undef, nstates(app), N)
    flux_c!(flux_k, app, uK)
    flux_n = Array{Float64,2}(undef, nstates(app), N)
    flux_c!(flux_n, app, uN)
    flux .= zeros(typeof(uK[1]), nstates(app))
    for l=1:N
        @views flux .+= (normal_k[l] > 0.0 ? flux_k[:,l] : flux_n[:,l]).*normal_k[l]
    end
end
"""
    boundary_flux_c!(flux, app::Application, u, bc, normal_k)
Given the state vector u, compute the boundary condition numerical flux function
for this PDE dotted with the normal, and store the results in flux.
Default: f(u)*n = 0.0 (Homogeneous Dirichlet BC)
"""
function boundary_flux_c!(flux, app::Application, u, bc, normal_k)
    flux .= zeros(typeof(u[1]),nstates(app))
end
"""
    function numerical_flux_l!(flux, app::Application, uK, uN, switch, normal_k)
Compute the numerical local DG flux function dotted with n for this PDE. Note
that there is a different local DG flux depending on the gradient directions.
Therefore, normal_k is actually only the normal in the direction we care about.
Default: f(u)*n = f(uK)*n or f(uN)*n based on mesh's Local DG switch
"""
function numerical_flux_l!(flux, app::Application{N}, uK, uN, switch, normal_k) where {N}
    flux_k = similar(uK)
    flux_l!(flux_k, app, uK)
    flux_n = similar(uN)
    flux_l!(flux_n, app, uN)
    flux .= (switch ? flux_k : flux_n)*normal_k
end
"""
    function boundary_flux_l!(flux, app::Application{N}, u, bc, normal_k)
Compute the numerical local DG flux function dotted with n for this PDE on the
boundary of the domain. Note that there is a different local DG flux depending
on the gradient directions. Therefore normal_k is actually only the normal in
the direction we care about. Must override function for different BCs than
bc==-1 => Homogeneous Dirichlet and bc==-2 => Homogeneous Neumann
Default: f(u)*n = given bc*n for Dirichlet BC and f(u)*n for Neumann BC
"""
function boundary_flux_l!(flux, app::Application{N}, u, bc, normal_k) where {N}
    flux_k = similar(u)
    if bc == -1 # Homogeneous Dirichlet BC
        fill!(flux_k, 0.0)
    elseif bc == -2 # Homogeneous Neumann BC
        flux_l!(flux_k, app, u)
    else
        flux_l!(flux_k, app, u)
        error("Unknown default BC - must override operator!")
    end
    flux .= flux_k*normal_k
end
"""
    function numerical_flux_d!(flux, app::Application, uK,uN, DuK,DuN, switch, normal_k)
Compute the numerical diffusive (viscous) flux dotted with n for this PDE.
Default: f(uK,uN,DuK,DuN) = -f_d(uK, !ldg_switch(DuK,DuN))*n
"""
function numerical_flux_d!(flux, app::Application{N}, uK,uN, DuK,DuN, switch, normal_k) where {N}
    fluxes = similar(DuK)
    if !switch # Negative ldg switch for choosing DuK or DuN
        flux_d!(fluxes, app, uK, DuK)
    else
        flux_d!(fluxes, app, uK, DuN)
    end
    # flux = -fluxes*n
    LinearAlgebra.mul!(flux, fluxes, normal_k, -1.0, 0.0)
end
"""
    function boundary_flux_d!(flux, app::Application{N}, u,Du,bc, normal_k)
Compute the numerical diffusive (viscous) flux dotted with n for this PDE at a
boundary.
Default: f(u,Du)*n = 0.0 (Homogeneous Neumann BC)
"""
function boundary_flux_d!(flux, app::Application{N}, u,Du,bc, normal_k) where {N}
    flux .= zeros(typeof(Du[1]),size(Du)...)
end
"""
    source!(s_val, app::Application, t=0.0, x=0.0)
Computes the source(t,x) of the PDE. Default: source(t,x) = 0.0
"""
source!(s_val, app::Application, t=0.0, x=0.0) = (s_val .= zeros(nstates(app)))

include("Applications\\Convection.jl")
include("Applications\\ConvectionDiffusion.jl")
include("Applications\\NavierStokes.jl")
include("Applications\\ElasticWave.jl")

end # Module DGApplication

max_stars_repo_name: NoseKnowsAll/DGToolkit

max_issues_repo_name: NoseKnowsAll/DGToolkit

max_forks_repo_name: NoseKnowsAll/DGToolkit


text:
using Pkg
Pkg.add(url="https://github.com/dojo-sim/Dojo.jl") # TODO: change to General repository
Pkg.add("PyCall")
Pkg.build("PyCall")

using Dojo
using PyCall

max_stars_repo_name: mcx/dojopy

max_issues_repo_name: mcx/dojopy

max_forks_repo_name: mcx/dojopy


text:
#NOTES:
#   Transposes give a GrB_DIMENSION_MISMATCH. So we do not provide a GBMatOrTrans arg.

@kwargs function apply!(C::GBVecOrMat, A::GBVecOrMat, op::UnaryUnion)
    op = getoperator(op, eltype(C))
    if C isa GBVector && A isa GBVector
        libgb.GrB_Vector_apply(C, mask, accum, op, A, desc)
    elseif C isa GBMatrix && A isa GBMatrix
        libgb.GrB_Matrix_apply(C, mask, accum, op, A, desc)
    end
end
@kwargs apply!(A::GBVecOrMat, op::UnaryUnion) = apply!(A, A, op; mask, accum, desc)
@kwargs function apply(A::GBVecOrMat, op::UnaryUnion)
    C = similar(A)
    apply!(C, A, op; mask, accum, desc)
    return C
end

@kwargs function apply!(C::GBVecOrMat, x, A::GBVecOrMat, op::BinaryUnion)
    op = getoperator(op, eltype(C))
    if C isa GBVector && A isa GBVector
        libgb.scalarvecapply1st[eltype(C)](C, mask, accum, op, x, A, desc)
    elseif C isa GBMatrix && A isa GBMatrix
        libgb.scalarmatapply1st[eltype(C)](C, mask, accum, op, x, A, desc)
    end
end

@kwargs function apply!(x, A::GBVecOrMat, op::BinaryUnion)
    apply!(A, x, A, op; mask, accum, desc)
end

@kwargs function apply(x, A::GBVecOrMat, op::BinaryUnion)
    C = similar(A)
    apply!(C, x, A, op; mask, accum, desc)
    return C
end

@kwargs function apply!(C::GBVecOrMat, A::GBVecOrMat, x, op::BinaryUnion)
    op = getoperator(op, eltype(C))
    if C isa GBVector && A isa GBVector
        libgb.scalarvecapply2nd[eltype(C)](C, mask, accum, op, A, x, desc)
    elseif C isa GBMatrix && A isa GBMatrix
        libgb.scalarmatapply2nd[eltype(C)](C, mask, accum, op, A, x, desc)
    end
end

@kwargs function apply!(A::GBVecOrMat, x, op::BinaryUnion)
    apply!(A, A, x, op; mask, accum, desc)
end

@kwargs function apply(A::GBVecOrMat, x, op::BinaryUnion)
    C = similar(A)
    apply!(C, A, x, op; mask, accum, desc)
    return C
end

@kwargs function Base.broadcasted(::typeof(+), u::GBVecOrMat, x::valid_union)
    apply(u, x, BinaryOps.PLUS; mask, accum, desc)
end
@kwargs function Base.broadcasted(::typeof(+), x::valid_union, u::GBVecOrMat)
    apply(x, u, BinaryOps.PLUS; mask, accum, desc)
end

@kwargs function Base.broadcasted(::typeof(*), u::GBVecOrMat, x::valid_union)
    apply(u, x, BinaryOps.TIMES; mask, accum, desc)
end
@kwargs function Base.broadcasted(::typeof(*), x::valid_union, u::GBVecOrMat)
    apply(x, u, BinaryOps.TIMES; mask, accum, desc)
end

#NOT WORKING
function Base.broadcasted(::typeof(^), u::GBVecOrMat, x::valid_union)
    apply(u, x, BinaryOps.POW)
end
@kwargs function Base.broadcasted(::typeof(^), x::valid_union, u::GBVecOrMat)
    apply(x, u, BinaryOps.POW; mask, accum, desc)
end


text:
# Transformations
# ---------------

function Base.push!{A}(seq::BioSequence{A}, x)
    bin = enc64(seq, x)
    resize!(seq, length(seq) + 1)
    encoded_setindex!(seq, bin, endof(seq))
    return seq
end

function Base.pop!(seq::BioSequence)
    if isempty(seq)
        throw(ArgumentError("sequence must be non-empty"))
    end
    x = seq[end]
    deleteat!(seq, endof(seq))
    return x
end

function Base.insert!{A}(seq::BioSequence{A}, i::Integer, x)
    checkbounds(seq, i)
    bin = enc64(seq, x)
    resize!(seq, length(seq) + 1)
    copy!(seq, i + 1, seq, i, endof(seq) - i)
    encoded_setindex!(seq, bin, i)
    return seq
end

function Base.deleteat!{A,T<:Integer}(seq::BioSequence{A}, range::UnitRange{T})
    checkbounds(seq, range)
    copy!(seq, range.start, seq, range.stop + 1, length(seq) - range.stop)
    resize!(seq, length(seq) - length(range))
    return seq
end

function Base.deleteat!(seq::BioSequence, i::Integer)
    checkbounds(seq, i)
    copy!(seq, i, seq, i + 1, length(seq) - i)
    resize!(seq, length(seq) - 1)
    return seq
end

function Base.append!{A}(seq::BioSequence{A}, other::BioSequence{A})
    resize!(seq, length(seq) + length(other))
    copy!(seq, endof(seq) - length(other) + 1, other, 1)
    return seq
end

function Base.shift!(seq::BioSequence)
    if isempty(seq)
        throw(ArgumentError("sequence must be non-empty"))
    end
    x = seq[1]
    deleteat!(seq, 1)
    return x
end

function Base.unshift!{A}(seq::BioSequence{A}, x)
    bin = enc64(seq, x)
    resize!(seq, length(seq) + 1)
    copy!(seq, 2, seq, 1, length(seq) - 1)
    encoded_setindex!(seq, bin, 1)
    return seq
end

"""
    resize!(seq, size)

Resize a biological sequence `seq`, to a given `size`.
"""
function Base.resize!{A}(seq::BioSequence{A}, size::Integer)
    if size < 0
        throw(ArgumentError("size must be non-negative"))
    end
    orphan!(seq, size)
    resize!(seq.data, seq_data_len(A, size + seq.part.start - 1))
    seq.part = seq.part.start:seq.part.start+size-1
    return seq
end

"""
    empty!(seq)

Completely empty a biological sequence `seq` of nucleotides.
"""
Base.empty!(seq::BioSequence) = resize!(seq, 0)

function Base.filter!{A}(f::Function, seq::BioSequence{A})
    orphan!(seq)

    len = 0
    next = bitindex(seq, 1)
    j = index(next)
    datum::UInt64 = 0
    for i in 1:endof(seq)
        x = inbounds_getindex(seq, i)
        if f(x)
            datum |= enc64(seq, x) << offset(next)
            len += 1
            next += bitsof(A)
            if index(next) != j
                seq.data[j] = datum
                datum = 0
                j = index(next)
            end
        end
    end
    if offset(next) > 0
        seq.data[j] = datum
    end
    resize!(seq, len)

    return seq
end

function Base.filter(f::Function, seq::BioSequence)
    return filter!(f, copy(seq))
end

function Base.map!(f::Function, seq::BioSequence)
    orphan!(seq)
    for i in 1:endof(seq)
        unsafe_setindex!(seq, f(inbounds_getindex(seq, i)), i)
    end
    return seq
end

function Base.map(f::Function, seq::BioSequence)
    return map!(f, copy(seq))
end

"""
    reverse!(seq)

Reverse a biological sequence `seq` in place.
"""
function Base.reverse!(seq::BioSequence)
    orphan!(seq)
    for i in 1:div(endof(seq), 2)
        x = inbounds_getindex(seq, i)
        unsafe_setindex!(seq, inbounds_getindex(seq, endof(seq) - i + 1), i)
        unsafe_setindex!(seq, x, endof(seq) - i + 1)
    end
    return seq
end

"""
    reverse(seq)

Create a sequence which is the reverse of the bioloigcal sequence `seq`.
"""
Base.reverse(seq::BioSequence) = reverse!(copy(seq))

@generated function Base.reverse{A<:Union{DNAAlphabet,RNAAlphabet}}(seq::BioSequence{A})
    n = bitsof(A)
    if n == 2
        nucrev = :nucrev2
    elseif n == 4
        nucrev = :nucrev4
    else
        error("n (= $n) ∉ (2, 4)")
    end

    quote
        data = Vector{UInt64}(seq_data_len(A, length(seq)))
        i = 1
        next = bitindex(seq, endof(seq))
        stop = bitindex(seq, 0)
        r = rem(offset(next) + $n, 64)
        if r == 0
            @inbounds while next - stop > 0
                x = seq.data[index(next)]
                data[i] = $nucrev(x)
                i += 1
                next -= 64
            end
        else
            @inbounds while next - stop > 64
                j = index(next)
                x = (seq.data[j] << (64 - r)) | (seq.data[j-1] >> r)
                data[i] = $nucrev(x)
                i += 1
                next -= 64
            end
            if next - stop > 0
                j = index(next)
                x = seq.data[j] << (64 - r)
                if r < next - stop
                    x |= seq.data[j-1] >> r
                end
                data[i] = $nucrev(x)
            end
        end
        return BioSequence{A}(data, 1:length(seq), false)
    end
end

@inline function nucrev2(x::UInt64)
     x = (x & 0x3333333333333333) <<  2 | (x & 0xCCCCCCCCCCCCCCCC) >>  2
     x = (x & 0x0F0F0F0F0F0F0F0F) <<  4 | (x & 0xF0F0F0F0F0F0F0F0) >>  4
     x = (x & 0x00FF00FF00FF00FF) <<  8 | (x & 0xFF00FF00FF00FF00) >>  8
     x = (x & 0x0000FFFF0000FFFF) << 16 | (x & 0xFFFF0000FFFF0000) >> 16
     x = (x & 0x00000000FFFFFFFF) << 32 | (x & 0xFFFFFFFF00000000) >> 32
     return x
end

@inline function nucrev4(x::UInt64)
     x = (x & 0x0F0F0F0F0F0F0F0F) <<  4 | (x & 0xF0F0F0F0F0F0F0F0) >>  4
     x = (x & 0x00FF00FF00FF00FF) <<  8 | (x & 0xFF00FF00FF00FF00) >>  8
     x = (x & 0x0000FFFF0000FFFF) << 16 | (x & 0xFFFF0000FFFF0000) >> 16
     x = (x & 0x00000000FFFFFFFF) << 32 | (x & 0xFFFFFFFF00000000) >> 32
     return x
end

"""
    complement!(seq)

Make a complement sequence of `seq` in place.
"""
function complement!{A<:Union{DNAAlphabet{2},RNAAlphabet{2}}}(seq::BioSequence{A})
    orphan!(seq)
    next = bitindex(seq, 1)
    stop = bitindex(seq, endof(seq) + 1)
    @inbounds while next < stop
        seq.data[index(next)] = ~seq.data[index(next)]
        next += 64
    end
    return seq
end

"""
    complement!(seq)

Transform `seq` into it's complement.
"""
function complement!{A<:Union{DNAAlphabet{4},RNAAlphabet{4}}}(seq::BioSequence{A})
    orphan!(seq)
    next = bitindex(seq, 1)
    stop = bitindex(seq, endof(seq) + 1)
    @inbounds while next < stop
        x = seq.data[index(next)]
        seq.data[index(next)] = (
            ((x & 0x1111111111111111) << 3) | ((x & 0x8888888888888888) >> 3) |
            ((x & 0x2222222222222222) << 1) | ((x & 0x4444444444444444) >> 1))
        next += 64
    end
    return seq
end

"""
    complement(seq)

Make a complement sequence of `seq`.
"""
function complement{A<:Union{DNAAlphabet,RNAAlphabet}}(seq::BioSequence{A})
    return complement!(copy(seq))
end

"""
    reverse_complement!(seq)

Make a reversed complement sequence of `seq` in place.

Ambiguous nucleotides are left as-is.
"""
function reverse_complement!{A<:Union{DNAAlphabet,RNAAlphabet}}(seq::BioSequence{A})
    return complement!(reverse!(seq))
end

"""
    reverse_complement(seq)

Make a reversed complement sequence of `seq`.

Ambiguous nucleotides are left as-is.
"""
function reverse_complement{A<:Union{DNAAlphabet,RNAAlphabet}}(seq::BioSequence{A})
    return complement!(reverse(seq))
end

# Shuffle
# -------

function Base.shuffle(seq::BioSequence)
    return shuffle!(copy(seq))
end

function Base.shuffle!(seq::BioSequence)
    orphan!(seq)
    # Fisher-Yates shuffle
    for i in 1:endof(seq)-1
        j = rand(i:endof(seq))
        seq[i], seq[j] = seq[j], seq[i]
    end
    return seq
end


text:
using .tensorboard: Summary_Audio

"""
    log_audios(logger::TBLogger, name::AbstractString, samples::AbstractArray, samplerate::Real; step=step(logger))

Logs multiple audio clips at step `step`
- samples:
    Array of audio clips which are Arrays of samples N*C where N = number of samples and C = number of channel
- samplerate: Sampling rate or Sampling frequency: a Real value same for all clips
"""
function log_audios(logger::TBLogger, name::AbstractString, samplesArray::AbstractArray, samplerate::Real; step=nothing)
    for (n, sample) in enumerate(samplesArray)
        log_audio(logger, name*"/$n", sample, samplerate, step = step)
    end
end
"""
    log_audio(logger::TBLogger, name::AbstractString, samples::AbstractArray, samplerate::Real; step=step(logger))

Logs an audio clip with name `name` at step `step`
- samples: Array of samples N*C where N = number of samples and C = number of channels
- samplerate: Sampling rate or Sampling frequency: a Real value
"""
function log_audio(logger::TBLogger, name::AbstractString, samples::AbstractArray, samplerate::Real; step=nothing)
    summ = SummaryCollection(audio_summary(name, samples, samplerate))
    write_event(logger.file, make_event(logger, summ, step=step))
end

function audio_summary(name::AbstractString, samples::AbstractArray, samplerate::Real)
    @assert ndims(samples) <= 2
    samples = samples./max(maximum(samples), 1)
    samples = Int16.(floor.(samples.*32767))
    io = IOBuffer()
    save(Stream(format"WAV", io), samples)
    eas = io.data
    audio = Summary_Audio(sample_rate = samplerate, num_channels = ndims(samples), length_frames = size(samples, 1), encoded_audio_string = eas, content_type = "audio/wav")
    Summary_Value(tag=name, audio=audio)
end


text:
using BinDeps
using Compat

@BinDeps.setup

libflac = library_dependency("libflac",aliases=["libFLAC"])
libogg = library_dependency("libogg")
libvorbis = library_dependency("libvorbis")

provides(AptGet,"libflac-dev",libflac)
provides(AptGet,"libogg-dev",libogg)
provides(AptGet,"libvorbis-dev",libvorbis)

@osx_only begin
    if Pkg.installed("Homebrew") === nothing
        error("Homebrew package not installed, please run Pkg.add(\"Homebrew\")")  end
    using Homebrew
    provides( Homebrew.HB, "flac", libflac, os = :Darwin )
    provides( Homebrew.HB, "libogg", libogg, os = :Darwin )
    provides( Homebrew.HB, "libvorbis", libvorbis, os = :Darwin )
end

@BinDeps.install Dict(:libflac => :libflac,:libogg => :libogg)


text:
using LEAPMacro
using Test

curr_working_dir = pwd()
cd(@__DIR__)

@testset "LEAPMacro.run" begin
	@test LEAPMacro.run("LEAPMacro_params.yml", dump_err_stack = true) == 0
end

cd(curr_working_dir)

max_stars_repo_name: sei-international/LEAPMacro

max_issues_repo_name: sei-international/LEAPMacro

max_forks_repo_name: sei-international/LEAPMacro


text:
# Stream-orderdered memory allocator

export CuMemoryPool, default_memory_pool, memory_pool, memory_pool!, trim, attribute, attribute!

@enum_without_prefix CUmemAllocationType CU_MEM_
@enum_without_prefix CUmemAllocationHandleType CU_MEM_

mutable struct CuMemoryPool
    handle::CUmemoryPool
    ctx::CuContext

    function CuMemoryPool(dev::CuDevice;
                          alloc_type::CUmemAllocationType=ALLOCATION_TYPE_PINNED,
                          handle_type::CUmemAllocationHandleType=HANDLE_TYPE_NONE)
        props = Ref(CUmemPoolProps(
            alloc_type,
            handle_type,
            CUmemLocation(
                CU_MEM_LOCATION_TYPE_DEVICE,
                deviceid(dev)
            ),
            C_NULL,
            ntuple(i->Cuchar(0), 64)
        ))
        handle_ref = Ref{CUmemoryPool}()
        cuMemPoolCreate(handle_ref, props)

        ctx = current_context()
        obj = new(handle_ref[], ctx)
        finalizer(unsafe_destroy!, obj)
        return obj
    end

    global function default_memory_pool(dev::CuDevice)
        handle_ref = Ref{CUmemoryPool}()
        cuDeviceGetDefaultMemPool(handle_ref, dev)

        ctx = current_context()
        new(handle_ref[], ctx)
    end

    global function memory_pool(dev::CuDevice)
        handle_ref = Ref{CUmemoryPool}()
        cuDeviceGetMemPool(handle_ref, dev)

        ctx = current_context()
        new(handle_ref[], ctx)
    end
end

function unsafe_destroy!(pool::CuMemoryPool)
    @finalize_in_ctx pool.ctx cuMemPoolDestroy(pool)
end

Base.unsafe_convert(::Type{CUmemoryPool}, pool::CuMemoryPool) = pool.handle

Base.:(==)(a::CuMemoryPool, b::CuMemoryPool) = a.handle == b.handle
Base.hash(pool::CuMemoryPool, h::UInt) = hash(pool.handle, h)

memory_pool!(dev::CuDevice, pool::CuMemoryPool) = cuDeviceSetMemPool(dev, pool)

trim(pool::CuMemoryPool, bytes_to_keep::Integer=0) = cuMemPoolTrimTo(pool, bytes_to_keep)

## pool attributes

@enum_without_prefix CUmemPool_attribute CU_

"""
    attribute(X, pool::CuMemoryPool, attr)

Returns attribute `attr` about `pool`. The type of the returned value depends on the
attribute, and as such must be passed as the `X` parameter.
"""
function attribute(X::Type, pool::CuMemoryPool, attr::CUmemPool_attribute) where {T}
    value = Ref{X}()
    cuMemPoolGetAttribute(pool, attr, value)
    return value[]
end

"""
    attribute!(ptr::Union{Ptr,CuPtr}, attr, val)

Sets attribute` attr` on a pointer `ptr` to `val`.
"""
function attribute!(pool::CuMemoryPool, attr::CUmemPool_attribute, value) where {T}
    cuMemPoolSetAttribute(pool, attr, Ref(value))
    return
end


text:
# julia1_4.3.jl Perceptron Part 1
# using UCI Machine learning laboratory data
# you need to know your training/test data
# it generally doesn't come in a ready-to-use form
# This sonar data is 60 magnitudes in a time sequence
#  followed by a letter M=mine and R =rock
# read in training data
# Create Sonar Data Set
using Random
numsonar=208     # total number of sonar sweeps
trainln=138      # training length after initial randomization
burstln=60       # sonar burst length
testln=numsonar-trainln   # 70 is test length
#
traindata=Array{Float64}(undef,trainln,burstln)
trainlabel=Array{Int32}(undef,trainln)
testdata=Array{Float64}(undef,testln,burstln)
testlabel=Array{Int32}(undef,testln)
#
function create_sonar_data_arrays()
    global traindata, trainlabel, testdata, testlabel
    sdata=Array{Float64}(undef,numsonar,burstln)
    stype=Array{Int32}(undef,numsonar)
    sonardata=zeros(Float64,burstln)
    f1=open("sonar-data.csv","r")
    for i=1:numsonar
        sonarlist=split(readline(f1),",")
        stype[i]=Char(sonarlist[burstln+1][1])
        sdata[i,:]=parse.(Float64,sonarlist[1:burstln])
    end
    # randonize and partition training from test data
    seq=randperm(numsonar) # create random index sequence
    for i=1:numsonar
        sonardata[:]=sdata[seq[i],:]
        sonartype=(stype[seq[i]]==Int32('M')) ? 1 : 0
        if i <= trainln
            traindata[i,:]=sonardata
            trainlabel[i]=sonartype
        else
            testdata[i-trainln,:]=sonardata
            testlabel[i-trainln]=sonartype
        end
    end
    close(f1)
end
create_sonar_data_arrays()


text:
module BSEParse

using PyCall
using HDF5
using JSON

bse = pyimport("basis_set_exchange")
const am_to_shell_mapping = Dict(
    0 => "S",
    1 => "P",
    2 => "D",
    3 => "F",
    4 => "G",
    5 => "H",
    6 => "I"
  )

#===========================================================#
#= perform parsing operation on single atom/basis set pair =#
#===========================================================#
function parse_individual(atom::Dict{String,Any}, atomid::String, basis::String,
  bsed::HDF5File)

  for (shell_num, shell) in enumerate(atom["electron_shells"])
    ang_mom = shell["angular_momentum"]
    shell_type_string = length(ang_mom) > 1 ? "L" : am_to_shell_mapping[ang_mom[1]]

    exponents::Array{Float64,1} = parse.(Float64,shell["exponents"])

    coeff::Array{Float64} = begin
      if shell_type_string == "L"
        s_coeff = parse.(Float64,shell["coefficients"][1])
        p_coeff = parse.(Float64,shell["coefficients"][2])
        vcat(s_coeff,p_coeff)
      else
        parse.(Float64,shell["coefficients"][1])
      end
    end

    #println("Writing to $atomid/$basis/$shell_num...")
    h5write("bsed.h5",
      "$atomid/$basis/$shell_num/Shell Type", shell_type_string)
    h5write("bsed.h5",
      "$atomid/$basis/$shell_num/Exponents", exponents)
    h5write("bsed.h5",
      "$atomid/$basis/$shell_num/Coefficients", coeff)

    #=
    println("Exponents:")
    display(h5read("bsed.h5",
      "$atomid/$basis/$shell_num/Exponents"))
    println("")
    println("Contraction Coefficients:")
    display(h5read("bsed.h5",
      "$atomid/$basis/$shell_num/Coefficients"))
    println("")
    =#
  end
end

#===========================================================#
#= perform parsing operation on single atom/basis set pair =#
#===========================================================#
function parse_individual_cc_new(atom::Dict{String,Any}, atomid::String, basis::String,
  bsed::HDF5File)

  #if atomid == "O"

  # loop over the shells
  shell_num = 1
  for shell in atom["electron_shells"]
    ang_mom = shell["angular_momentum"]
    shell_type_string = am_to_shell_mapping[ang_mom[1]]

    #println("#===============================================#")
    #println("#== CREATE INITIAL EXPONENT/COEFFICIENT LISTS ==#")
    #println("#===============================================#")
    #println()
    #println("#== EXPONENTS ==#")
    exponents::Vector{Float64} = parse.(Float64,shell["exponents"])
    #display(exponents); println()

    #println()
    #println("#== COEFFICIENTS ==#")
    coeffs::Vector{Vector{Float64}} = []
    coefs_01::Vector{Union{Nothing,Int64}} = []
    for icoef in shell["coefficients"]
      coeff::Vector{Float64} = parse.(Float64,icoef)
      #display(coeff); println()
      push!(coeffs, coeff)
      push!(coefs_01, findfirst(i->(i == 1.0),coeff))
    end

    #display(coefs_01); println()

    #println("#=====================================#")
    #println("#== TRIM EXPONENT/COEFFICIENT LISTS ==#")
    #println("#=====================================#")
    idx_to_remove = findall(x->typeof(x)==Int64, coefs_01)
    #display(idx_to_remove)

    #println()
    #println("#== EXPONENTS ==#")
    exp_cc::Vector{Vector{Float64}} = [ deepcopy(exponents) ]
    exps_to_remove::Vector{Vector{Float64}} = []
    for iexp_cc in exp_cc, idx in idx_to_remove
      push!(exps_to_remove, [ iexp_cc[coefs_01[idx]]])
      iexp_cc[coefs_01[idx]] = 0.0
    end
    for iexp in exps_to_remove
      push!(exp_cc, iexp)
    end
    for iexp_cc in exp_cc
      filter!(x->x!=0.0, iexp_cc)
    end
    filter!(!isempty, exp_cc)
    #display(exp_cc); println()

    #println()
    #println("#== COEFFICIENTS ==#")
    coeff_cc::Vector{Vector{Float64}} = [ deepcopy(icoef) for icoef in coeffs ]
    coeffs_to_remove::Vector{Vector{Float64}} = []
    for (coef_idx, icoef_cc) in enumerate(coeff_cc), idx in idx_to_remove
      if typeof(coefs_01[coef_idx]) == Nothing
        push!(coeffs_to_remove, [ icoef_cc[coefs_01[idx]] ])
        icoef_cc[coefs_01[idx]] = 0.0
      end
    end
    for icoeff in coeffs_to_remove
      push!(coeff_cc, icoeff)
    end
    for icoeff_cc in coeff_cc
      filter!(x->x!=0.0, icoeff_cc)
    end
    filter!(!isempty, coeff_cc)
    #display(coeff_cc); println()

    exp_id = typeof(coefs_01[1]) == Int64 ? 0 : 1
    for coef_id in 1:length(coefs_01)
      if typeof(coefs_01[coef_id]) == Int64
        exp_id += 1
      end

      h5write("bsed.h5",
        "$atomid/$basis/$shell_num/Shell Type", shell_type_string)
      h5write("bsed.h5",
        "$atomid/$basis/$shell_num/Exponents", exp_cc[exp_id])
      h5write("bsed.h5",
        "$atomid/$basis/$shell_num/Coefficients", coeff_cc[coef_id])

      shell_num += 1
    end #for icoef
  end #for shells

  #=
  println("#==========================================#")
  println("#== FINAL LIST OF EXPONENTS/COEFFICIENTS ==#")
  println("#==========================================#")
  for ishl in 1:(shell_num-1)
    println("Shell Type $atomid/$basis/$ishl:")
    display(h5read("bsed.h5",
      "$atomid/$basis/$ishl/Shell Type"))
    println()
    println("Exponents $atomid/$basis/$ishl:")
    display(h5read("bsed.h5",
      "$atomid/$basis/$ishl/Exponents"))
    println()
    println("Contraction Coefficients $atomid/$basis/$ishl:")
    display(h5read("bsed.h5",
      "$atomid/$basis/$ishl/Coefficients"))
    println()
  end
  =#
  #end # if
end #function

#===========================================#
#= parse all selected atom-basis set pairs =#
#===========================================#
function parse_all()
    #== open HDF5 file for writing ==#
    hdf5name::String = "bsed.h5"
    h5open(hdf5name, "w") do bsed
        flush(bsed)

        #== parse "sto" basis family ==#
        atoms::Array{String,1} = [
            "H", "He",
            "Li", "Be", "B", "C", "N", "O", "F", "Ne",
            "Na", "Mg", "Al", "Si", "P", "S", "Cl", "Ar",
            "K", "Ca", "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn",
            "Ga", "Ge", "As", "Se", "Br", "Kr",
            "Rb", "Sr", "Y", "Zr", "Nb", "Mo", "Tc", "Ru", "Rh", "Pd", "Ag", "Cd",
            "In", "Sn", "Sb", "Te", "I", "Xe",
            "Cs"
            ] #H-Cs
        #=
        atoms::Array{String,1} = [
            "1", "2",
            "3", "4", "5", "6", "7", "8", "9", "10",
            "11", "12", "13", "14", "15", "16", "17", "18",
            "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29",
            "30", "31", "32", "33", "34", "35", "36",
            "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48",
            "49", "50", "51", "52", "53", "54",
            "55"
            ] #H-Cs
        =#
        basis_sets::Array{String,1} = ["STO-2G", "STO-3G", "STO-4G", "STO-5G",
            "STO-6G" ] #STO family

        for basis::String in basis_sets
            println("Basis: $basis")
            bs_dict = bse.get_basis(basis,fmt="json", header=false)
            bs_dict_json = JSON.parse(bs_dict)

            for atom in bs_dict_json["elements"]
                parse_individual(atom.second, atoms[parse(Int64,atom.first)],
                  basis, bsed)
            end
        end

        #== parse "pople" basis family ==#
        basis_sets = ["3-21G"          , "4-31G"             , "5-21G"        , "6-21G"       ,
                      "6-31G"          , "6-31G*"            , "6-31G**"      ,
                      "6-31+G"         , "6-31+G*"           , "6-31+G**"     , "6-31+G*-J"   ,
                      "6-31++G"        , "6-31++G*"          , "6-31++G**"    , "6-31++G**-J" ,
                      "6-31G(2df,p)"   , "6-31G(3df,3pd)"    , "6-31G(d,p)"   , "6-31G-J"     ,
                      "6-311G"         , "6-311G*"           , "6-311G**"     ,
                      "6-311+G"        , "6-311+G*"          , "6-311+G**"    , "6-311+G*-J"  ,
                      "6-311++G"       , "6-311++G*"         , "6-311++G**"   , "6-311++G**-J",
                      "6-311G(d,p)"    , "6-311G(2df,2pd)"   , "6-311+G(2d,p)",
                      "6-311++G(2d,2p)", "6-311++G(3df,3pd)" ] #pople family

        for basis::String in basis_sets
            println("Basis: $basis")
            bs_dict = bse.get_basis(basis,fmt="json", header=false)
            bs_dict_json = JSON.parse(bs_dict)

            for atom in bs_dict_json["elements"]
                parse_individual(atom.second, atoms[parse(Int64,atom.first)],
                  basis, bsed)
            end
        end

        #== parse polarization-consistent basis family ==#
        basis_sets = ["PCSeg-0", "PCSeg-1", "PCSeg-2"] #polarization-consistent family
        for basis::String in basis_sets
            println("Basis: $basis")
            bs_dict = bse.get_basis(basis,fmt="json", header=false)
            bs_dict_json = JSON.parse(bs_dict)

            for atom in bs_dict_json["elements"]
                parse_individual(atom.second, atoms[parse(Int64,atom.first)],
                  basis, bsed)
            end
        end

        #== parse RI basis family ==#
        basis_sets = ["cc-pVDZ-RIFIT", "aug-cc-pVDZ-RIFIT", "cc-pVTZ-RIFIT", "aug-cc-pVTZ-RIFIT"] # RIFIT family
        for basis::String in basis_sets
            println("Basis: $basis")
            bs_dict = bse.get_basis(basis,fmt="json", header=false)
            bs_dict_json = JSON.parse(bs_dict)

            for atom in bs_dict_json["elements"]
                parse_individual(atom.second, atoms[parse(Int64,atom.first)],
                  basis, bsed)
            end
        end

        #== parse correlation-consistent basis family ==#
        basis_sets = ["cc-pVDZ", "aug-cc-pVDZ", "cc-pVTZ", "aug-cc-pVTZ"]

        for basis::String in basis_sets
            println("Basis: $basis")
            bs_dict = bse.get_basis(basis,fmt="json", header=false)
            bs_dict_json = JSON.parse(bs_dict)

            for atom in bs_dict_json["elements"]
              parse_individual_cc_new(atom.second, atoms[parse(Int64,atom.first)],
                basis, bsed)
            end
        end

    end
end

end

BSEParse.parse_all()