text:
abstract type PostProcessingMethod end
struct noPostProcessing <: PostProcessingMethod end

@with_kw struct KlauAlgo <: PostProcessingMethod
    k::Int = -1;
    a::Int = 1;
	b::Int = 1;
	stepm::Int = 25;
	rtype::Int = 1;
	maxiter::Int = 500;
	verbose::Bool = false;
	gamma::Float64 =.4;
end

abstract type SKUpdateRule end
struct constant_R <: SKUpdateRule end
struct linear_R <: SKUpdateRule end
struct fgap_R <: SKUpdateRule end
struct overlap_R <: SKUpdateRule end
struct edges_R <: SKUpdateRule end

@with_kw struct SuccessiveKlauAlgo <: PostProcessingMethod
    k::Int = -1;
	iterDelta::Int = 100;
	verbose::Bool = true;
    update_rule::SKUpdateRule = overlap_R();
    maximum_Klau_iters::Int=1000;
    successive_iter::Int=10;
    start_maxIter::Int = 100;
	first_run_klau::KlauAlgo = KlauAlgo();
end

@with_kw struct LocalSearch <: PostProcessingMethod
    k::Int = -1;
	iterations::Int=10;
	verbose::Bool = true;
end

"""-----------------------------------------------------------------------------
   Greedily creates a b matching from a rank 1 bipartite matching problem. Only
  adds pairs to the matchings if they have the same sign, and neither edge value
  is zero.

  Output:
  -------
  * Matchings - (2D Int Array):
    The ith row corresponds to the b-matchings of the ith vertex represented by
    the vector u to the vertices in v. unmatched vertices are set to -1.

-----------------------------------------------------------------------------"""
#TODO: This version is outdated, should be removed and replaced
#      with 'rank_one_bmatching' below
function b_matching(u::Array{Float64,1}, v::Array{Float64,1}, b::Int)

    @assert length(u) <= length(v) #make first vector shorter

    n = length(u)

    #get indices sorted by increasing order
    u_indices = sort(1:length(u),by=i->-u[i])
    v_indices = sort(1:length(v),by=i->-v[i])

    matching_degrees = zeros(length(v))

    Matchings = Array{Tuple{Int,Int,Float64},1}(undef,b*n)  #-1 indicates unset matching
    match_idx = 1
    starting_index = 1
    ending_index = b

    for i in 1:n

        if u[i] == 0.0
            continue
        else
            #check matching degree of starting index
            while matching_degrees[v_indices[starting_index]] >= b
                starting_index += 1
            end
            ending_index = minimum((starting_index + b,length(v)))
            #check that sign of ending vertex doesn't change
            while v[v_indices[ending_index]]*u[u_indices[i]] <= 0
                ending_index -= 1
            end

            @assert ending_index != starting_index  #make sure something's added

            for j in 1:(ending_index - starting_index)
                v_j = v_indices[starting_index + (j-1)]
                Matchings[match_idx] = (u_indices[i],v_j,v[v_j]*u[u_indices[i]])
                match_idx += 1
                matching_degrees[v_j] += 1
            end
        end
    end

    return Matchings[1:(match_idx-1)]

end

function rank_one_bmatching(u::Array{T,1},v::Array{T,1},b::Int) where T
	#current implementation will end up rounding b to be odd

    #get maximum matching by rearrangement theorem
    u_perm = sortperm(u,rev=true)
    v_perm = sortperm(v,rev=true)

    #matching_weight = zero(T )
    Match_candidates = Set()
    #for (i,j) in zip(u_perm,v_perm)
	for i = 1:length(u_perm)

		u_i = u_perm[i]
		#look at a size b window centered around
		for offset = ceil(-b/2):1:floor(b/2)

			j = i + Int(offset)
			#println(j == i)
			#println("i:$i  j:$j")
			#print(offset)
			if j >= 1 && j <= length(v_perm)
				v_j = v_perm[j]
				#only match positive to positive and negative to negative
				if (u[u_i] > 0 && v[v_j] > 0 ) || (u[u_i] < 0 && v[v_j] < 0)
					push!(Match_candidates,(u_i,v_j))
				end
			end
		end
    end
    return Match_candidates
end

"""-----------------------------------------------------------------------------
    Computes the cost of swapping the matching of (i,j) & (k,l) to (i,l) & (k,j)
  in terms of the number of triangles. Inputs are the triangles associated with
  each of the vertices

  Inputs:
  -------
  * T_{i,j,k,l} -(Set{Array{Int,1}}):
    All triangles incident with the vertices i,j,k,l .
  * original_matching - (Dict{Int,Int})
    The original matching from the vertices in graph A to the vertices in graph
    B.

-----------------------------------------------------------------------------"""
function compute_triangles_aligned(T_i::Set{Array{Int,1}},T_j::Set{Array{Int,1}},
  #                                  T_k::Set{Array{Int,1}},T_l::Set{Array{Int,1}},
                                    M::Dict{Int,Int})
    #compute the original number of aligned triangles
    matched_tris = 0
    for (v_i,v_j,v_k) in T_i
        if sort([get(M,v_i,-1),get(M,v_j,-1),get(M,v_k,-1)]) in T_j
            matched_tris += 1
        end
    end

    #=
    for (v_i,v_j,v_k) in T_k
        if sort([get(M,v_i,-1),get(M,v_j,-1),get(M,v_k,-1)]) in T_l
            matched_tris += 1
        end
    end

    #compute the new matched_triangles end
    new_matched_tris = 0
    for (v_i,v_j,v_k) in T_i
        if sort([get(M,v_i,-1),get(M,v_j,-1),get(M,v_k,-1)]) in T_l
            new_matched_tris += 1
        end
    end
    for (v_i,v_j,v_k) in T_k
        if sort([get(M,v_i,-1),get(M,v_j,-1),get(M,v_k,-1)]) in T_j
            new_matched_tris += 1
        end
    end
    =#
    return matched_tris#, new_matched_tris
end

"""-----------------------------------------------------------------------------
     Finds all the triangles associated with the vertices and preprocesses them
   into a list of Sets. Used to preprocess the searching for triangles
   associated with a vertex.

   Inputs:
   -------
   * triangles - (Array{Int,2}):
     The triangles contained within the graph. indices in each row are expected
     to be sorted in ascending order.
   * n - (Int):
     The number of vertices in the graph.

   Outputs:
   --------
   * incidence - (Array{Set{Array{Int,1}}},1):
     An array where the ith entry contains a set linking the vertex to the
     triangles it's contained within.
-----------------------------------------------------------------------------"""
function produce_triangle_incidence(triangles::Array{Int,2},n::Int)
    incidence = Array{Set{Array{Int,1}},1}(undef,n)
    for i in 1:n
        incidence[i] = Set{Array{Int,1}}()
    end

    for t in eachrow(triangles)
        for v in t
            push!(incidence[v],t)
        end
    end

    return incidence
end

"""-----------------------------------------------------------------------------
                    Klau's algorithm PostProcessing
-----------------------------------------------------------------------------"""

function netalignmr(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                    U::Matrix{T},V::Matrix{T},params::KlauAlgo;kwargs...) where T
    (_,_,matching,_)=  bipartite_matching_primal_dual(U*V';kwargs...)
    return netalignmr(A,B,knearest_sparsification(U,V,matching_array_to_dict(matching),2*min(size(U,2),size(V,2))),params)
end

function netalignmr(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                    U::Matrix{T},V::Matrix{T},k::Int,params::KlauAlgo;kwargs...) where T
    (_,_,matching,_)=  bipartite_matching_primal_dual(U*V';kwargs...)
    return netalignmr(A,B,knearest_sparsification(U,V,matching_array_to_dict(matching),k),params)
end

function netalignmr(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                    U::Matrix{T},V::Matrix{T},
                    matching::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},k::Int,params::KlauAlgo) where T
    return netalignmr(A,B,knearest_sparsification(U,V,matching,k),params)
end

function netalignmr(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                    U::Matrix{T},V::Matrix{T},
                    matching::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},params::KlauAlgo) where T
    return netalignmr(A,B,knearest_sparsification(U,V,matching,2*min(size(U,2),size(V,2))),params)
end

function netalignmr(A::SparseMatrixCSC{T1,Int},B::SparseMatrixCSC{T1,Int},L::SparseMatrixCSC{T2,Int},params::KlauAlgo) where {T1,T2}

	@assert size(L,1) == size(A,1) && size(L,2) == size(B,1)

	#L = sparse(X)
	S,w,li,lj = netalign_setup(A,B,L)

	# align networks
	(xbest,st,status,hist), t_netalignmr = @timed NetworkAlign.netalignmr(S,w,params.a,params.b,li,lj,params.gamma,params.stepm,
                                                                          params.rtype,params.maxiter, params.verbose)
    final_link_L = sparse(li,lj,xbest,size(A,1),size(B,1))
    dropzeros!(final_link_L)

    matching = MatrixNetworks.edge_list(bipartite_matching_local(final_link_L))

    matching = Dict(zip(matching...))

	return matching, t_netalignmr, nnz(L)/(size(A,1)*size(B,1)), status

end

function successive_netalignmr_profiled(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                                        U::Matrix{T},V::Matrix{T},
                                        m0::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},
                                        params::SuccessiveKlauAlgo) where T

    if typeof(m0) === Vector{NTuple{2,Int}}
        m0 = Dict(m0)
    end

    overlap_check = Int(ceil(.05*min(size(A,1),size(B,1))))
	edge_match = matching -> edges_matched(A,B,matching)[1]/2

	#  --  Build first link matrix  --  #
    T_U,t_u = @timed BallTree(U')
	T_V,t_v = @timed BallTree(V')
    kkn_rt = t_u + t_v

	L0,t  = @timed knearest_sparsification(T_U,T_V,U,V,m0,size(U,2))
    kkn_rt += t

	#run one call of Klau like normal
	(old_matching,_,_,status),rt = @timed netalignmr(A,B,L0,params.first_run_klau)

	matching_stats = Dict([
		("edges_matched",[edge_match(old_matching)]),
		("runtime",[rt]),
		("kkn L runtime",[kkn_rt]),
		("iters_used",[params.first_run_klau.maxiter]),
		("match_overlap",[-1]),
		("status",[status])
	])

	f_gap = matching_stats["status"][end][2] - matching_stats["status"][end][1]
    if params.verbose
	    println("iter 0:rt:$(matching_stats["runtime"][end]) -- match_overlap:$(matching_stats["match_overlap"][end]) -- edges_mapped:$(matching_stats["edges_matched"][end]) -- status:$(matching_stats["status"][end]) -- f_gap:$(f_gap)")
    end
    cur_maxiter = params.start_maxIter

	prev_check_val = 0
	nextTimeBreak=false

	best_edgeMatch = -1
	best_matching = Dict{Int,Int}()
	for i = 1:params.successive_iter

		L,t = @timed knearest_sparsification(T_U,T_V,U,V,old_matching,size(U,2))
		push!(matching_stats["kkn L runtime"],t)
		(klau_matching,_,_,status),rt = @timed netalignmr(A,B,L,KlauAlgo(maxiter=cur_maxiter,stepm=5))

		push!(matching_stats["runtime"],rt)
		push!(matching_stats["match_overlap"],length(intersect(Set(klau_matching),Set(old_matching))))
		push!(matching_stats["edges_matched"],edge_match(klau_matching))
		push!(matching_stats["iters_used"],cur_maxiter)
		push!(matching_stats["status"],status)

		fgap = matching_stats["status"][end][2] - matching_stats["status"][end][1]
        if params.verbose
    		println("iter $i: rt:$(matching_stats["runtime"][end]) -- match_overlap:$(matching_stats["match_overlap"][end]) -- edges_mapped:$(matching_stats["edges_matched"][end]) -- status:$(matching_stats["status"][end]) -- f_gap:$(fgap)")
        end

		if cur_maxiter < params.maximum_Klau_iters

			if params.update_rule === fgap_R()
				should_miter_increase = fgap > prev_check_val
			elseif params.update_rule === edges_R()
				should_miter_increase = matching_stats["edges_matched"][end] < prev_check_val
			elseif params.update_rule === overlap_R()
				should_miter_increase = ((matching_stats["match_overlap"][end] - prev_check_val) <= overlap_check)
			elseif params.update_rule === linear_R()
				should_miter_increase = true
			elseif params.update_rule == constant_R()
				should_miter_increase = false
			end

			if should_miter_increase
				cur_maxiter += params.iterDelta
                if params.verbose
				    println("upping Klau iterations to $cur_maxiter")
                end
			end
		end

		if matching_stats["edges_matched"][end] >= best_edgeMatch
			best_edgeMatch = matching_stats["edges_matched"][end]
			best_matching = copy(klau_matching)
		end

		if length(old_matching) == matching_stats["match_overlap"][end]
            if params.verbose
			    print("matching didn't change")
            end
            if nextTimeBreak
                if params.verbose
				    println(" twice in a row, breaking...")
                end
				break
			else
                if params.verbose
    				print("\n")
                end
				nextTimeBreak=true
			end
		else
			nextTimeBreak=false
		end

		if params.update_rule === fgap_R()
			prev_check_val = fgap
		elseif params.update_rule === edges_R()
			prev_check_val = matching_stats["edges_matched"][end]
		elseif params.update_rule === overlap_R()
			prev_check_val = matching_stats["match_overlap"][end]
		end

		old_matching = copy(klau_matching)
	end
	return best_matching, matching_stats

end

function successive_netalignmr(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
                               U::Matrix{T},V::Matrix{T},
                               m0::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},
                               params::SuccessiveKlauAlgo) where T

    if typeof(m0) === Vector{NTuple{2,Int}}
        m0 = Dict(m0)
    end

    overlap_check = Int(ceil(.05*min(size(A,1),size(B,1))))

	edge_match = matching -> edges_matched(A,B,matching)[1]/2

	#  --  Build first link matrix  --  #

	T_U = BallTree(U')
	T_V = BallTree(V')
	L0 = knearest_sparsification(T_U,T_V,U,V,m0,size(U,2))

	#run one call of Klau like normal
	klau_params = KlauAlgo()
	old_matching,_,_,status = netalignmr(A,B,L0,klau_params)

	f_gap = status[2] - status[1]
    if params.verbose
	    println("iter 0: match_overlap:n/a -- edges_mapped:$(edge_match(old_matching)) -- status:$status -- f_gap:$f_gap")
    end
    cur_maxiter = params.start_maxIter

	prev_check_val = 0
	nextTimeBreak=false

	best_edgeMatch = -1
	best_matching = Dict{Int,Int}()
	for i = 1:params.successive_iter

		L = knearest_sparsification(T_U,T_V,U,V,old_matching,2*size(U,2))

		klau_matching,_,_,status = netalignmr(A,B,L,KlauAlgo(maxiter=cur_maxiter))
        overlap = length(intersect(Set(klau_matching),Set(old_matching)))
        fgap = status[2] - status[1]
        edges_matched = edge_match(klau_matching)

        if params.verbose
            println("iter $i: match_overlap:$overlap -- edges_mapped:$edges_matched -- status:$status -- f_gap:$f_gap")
        end

		if cur_maxiter < params.maximum_Klau_iters

			if params.update_rule === fgap_R()
                fgap = status[2] - status[1]
				should_miter_increase = fgap > prev_check_val
			elseif params.update_rule === edges_R()
				should_miter_increase = edges_matched  < prev_check_val
			elseif params.update_rule === overlap_R()
                overlap = length(intersect(Set(klau_matching),Set(old_matching)))
				should_miter_increase = ((overlap - prev_check_val) <= overlap_check)
			elseif params.update_rule === linear_R()
				should_miter_increase = true
			elseif params.update_rule == constant_R()
				should_miter_increase = false
			end

			if should_miter_increase
				cur_maxiter += params.iterDelta
                if params.verbose
				    println("upping Klau iterations to $cur_maxiter")
                end
			end
		end

		if edges_matched >= best_edgeMatch
			best_edgeMatch = edges_matched
			best_matching = copy(klau_matching)
		end

		if length(old_matching) == overlap
            if params.verbose
			    print("matching didn't change")
            end
            if nextTimeBreak
                if params.verbose
				    println(" twice in a row, breaking...")
                end
				break
			else
                if params.verbose
    				print("\n")
                end
				nextTimeBreak=true
			end
		else
			nextTimeBreak=false
		end

		#TODO: check to see if this fails
		if params.update_rule === fgap_R()
			prev_check_val = fgap
		elseif params.update_rule === edges_R()
			prev_check_val = edges_matched
		elseif params.update_rule === overlap_R()
			prev_check_val = overlap
		end

		old_matching = copy(klau_matching)
	end
	return best_matching

end

function knearest_sparsification(U::Matrix{T},V::Matrix{T},matching::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},k) where T

    @assert maximum([i for (i,j) in matching]) <= size(U,1)
    @assert maximum([j for (i,j) in matching]) <= size(V,1)

	m = size(U,1)
	n = size(V,1)

	candidates = Set()
	T_U = BallTree(U')
	T_V = BallTree(V')

	for (i,j) in matching
		U_idxs = knn(T_U, U[i,:], minimum((k,m)))[1]
		V_idxs = knn(T_V, V[j,:], minimum((k,n)))[1]

		for ip in U_idxs
			push!(candidates,(ip,j))
		end

		for jp in V_idxs
			push!(candidates,(i,jp))
		end
	end

	sparse_L = spzeros(m,n)

	for (i,j) in candidates
		sparse_L[i,j] = U[i,:]'*V[j,:]
	end

	return sparse_L
end

function knearest_sparsification(T_U::BT,T_V::BT,U::Matrix{T},V::Matrix{T},matching::Union{Vector{NTuple{2,Int}},Dict{Int,Int}},k) where {BT <: BallTree,T}

    @assert maximum([i for (i,j) in matching]) <= size(U,1)
    @assert maximum([j for (i,j) in matching]) <= size(V,1)

	m = size(U,1)
	n = size(V,1)

	candidates = Set()

	for (i,j) in matching
		U_idxs = knn(T_U, U[i,:], minimum((k,m)))[1]
		V_idxs = knn(T_V, V[j,:], minimum((k,n)))[1]

		for ip in U_idxs
			push!(candidates,(ip,j))
		end

		for jp in V_idxs
			push!(candidates,(i,jp))
		end
	end

    li = []
    lj = []
    lv = []

	for (i,j) in candidates
        push!(li,i)
        push!(lj,j)
		push!(lv,U[i,:]'*V[j,:])
	end
	return sparse(li,lj,lv,m,n)
end

function findEmbeddingLookalikes(X::Matrix{S},idx,k) where S
	n = size(X,1)
	#all_nodes = vcat(U,V)
	T = BallTree(X')
	idxs = knn(T, X[idx,:]', minimum((k,n)))[1]
	# form the edges for sparse
	ei = Int[]
	ej = Int[]
	for i=1:length(idx)
	  for j=idxs[i]
		if i != j
		  push!(ei,i)
		  push!(ej,j)
		end
	  end
	end
	return ei,ej
end

#
#  Local Search Methods
#

#=
@with_kw struct LocalSearch <: PostProcessingMethod
    k::Int = -1;
	iterations::Int=10;
	verbose::Bool = true;
end
=#

#function local_search_profiled(A, B, A_ten, B_ten, U, V ,matching,max_iters,k,method="new")
function local_search_profiled(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
							  A_ten, B_ten, U::Matrix{T},V::Matrix{T},
							  m0::Dict{Int,Int},  params::LocalSearch) where T
	#k = 15

	profile_results = Dict{String,Union{Vector{Tuple{Int64,Int64}},Vector{Float64}}}([
		("runtimes",Vector{Float64}(undef,0)),
		("motif match",Vector{Tuple{Int64,Int64}}(undef,0)),
		("edge match",Vector{Tuple{Int64,Int64}}(undef,0)),
		("seq runtimes",Vector{Float64}(undef,0)),
		("top runtimes",Vector{Float64}(undef,0))
	])

	if params.k == -1
		k = 2*size(U,2)
	else
		k = params.k
	end

	if params.verbose
		edges_matched = LambdaTAME.edges_matched(A,B,m0)
		motifs_matched = LambdaTAME.TAME_score(A_ten,B_ten,m0)
		println("iter 0: edges_matched -- $edges_matched  motifs matched -- $motifs_matched")
	end
	#cur_matching = LT_res.matching
	cur_matching = copy(m0)
	for i =1:params.iterations

		(new_matching, seq_scoring_runtimes, top_scoring_runtimes),rt = @timed local_search_profiled(A,B,A_ten,B_ten,U,V,cur_matching,k)

		push!(profile_results["runtimes"],rt)
		push!(profile_results["seq runtimes"],seq_scoring_runtimes)
		push!(profile_results["top runtimes"],top_scoring_runtimes)

		push!(profile_results["motif match"],LambdaTAME.TAME_score(A_ten,B_ten,new_matching))
									#TAME_score wants most motifs in left arg
		push!(profile_results["edge match"],LambdaTAME.edges_matched(A,B,new_matching))
		cur_matching = copy(new_matching)

		if params.verbose
			println("iter $i: edges_matched -- $(profile_results["edge match"][end])  motifs matched -- $(profile_results["motif match"][end])  rt -- $rt seq_rt:$seq_scoring_runtimes, top_rt:$top_scoring_runtimes")
		end

	end

	return cur_matching, profile_results
end

function local_search(A::SparseMatrixCSC{T,Int},B::SparseMatrixCSC{T,Int},
							  A_ten, B_ten, U::Matrix{T},V::Matrix{T},
							  m0::Dict{Int,Int},  params::LocalSearch) where T

	if params.k == -1
		k = 2*size(U,2)
	else
		k = params.k
	end

	if params.verbose
		edges_matched = LambdaTAME.edges_matched(A,B,m0)
		motifs_matched = LambdaTAME.TAME_score(A_ten,B_ten,m0)
		println("iter 0: edges_matched -- $edges_matched  motifs matched -- $motifs_matched")
	end
	#cur_matching = LT_res.matching
	cur_matching = copy(m0)
	for i =1:params.iterations

		new_matching = local_search(A,B,A_ten,B_ten,U,V,cur_matching,k)

		if params.verbose
			edges_matched = LambdaTAME.edges_matched(A,B,new_matching)
			motifs_matched = LambdaTAME.TAME_score(A_ten,B_ten,new_matching)
			println("iter $i: edges_matched -- $edges_matched  motifs matched -- $motifs_matched")
		end
		cur_matching = copy(new_matching)
	end

	return cur_matching
end

function create_edge_idx_map(A_ten::Union{SymTensorUnweighted{Clique},ThirdOrderSymTensor})

	node_to_edge_indices = Vector{Vector{Int}}(undef,A_ten.n)
	for i in 1:A_ten.n
		node_to_edge_indices[i] = Vector{Int}(undef,0)
	end

	for edge_idx in 1:size(A_ten.indices,2)
		for i in 1:size(A_ten.indices,1)
			ip = A_ten.indices[i,edge_idx]
			push!(node_to_edge_indices[ip],edge_idx)
		end
	end
	return node_to_edge_indices
end

function local_search_profiled(A,B,A_ten,B_ten,U,V,matching,k)

	m = size(A,1)
	n = size(B,1)

	T_U = BallTree(U')
	T_V = BallTree(V')

	B_motifs = Set{Vector{Int}}()

	for idx = 1:size(B_ten.indices,2)
		push!(B_motifs,B_ten.indices[:,idx])
	end

	A_node_edge_indices = create_edge_idx_map(A_ten)

	seq_scoring_runtimes = 0.0
	top_scoring_runtimes = 0.0

	matched_matches = [m for m in matching if (m[1] != -1) && (m[2] != -1)]

	cur_seq_sim = seq_similarity(U,V,matched_matches)
	cur_motif_matched = TAME_score(B_motifs,A_ten,Set(size(A_ten.indices,2):-1:1), matching)
		#TODO: check if this can be done with a TAME_score function

	unprocessed_matches = sort(matched_matches,by=match->dot(U[match[1],:],V[match[2],:]),rev=true)

	prev_matching = copy(matching)
	updatable_matching = copy(matching)
	updatable_inverted_matching = Dict([(j,i) for (i,j) in updatable_matching])

	for (i,_) in unprocessed_matches

		ip = get(updatable_matching,i,-1)

		if ip != -1
			pref_B = Set(vcat(knn(T_V, V[ip,:], minimum((k,n)))[1],findnz(B[:,ip])[1]))
		else
			pref_B = Set()
		end

		if i != -1
			pref_A = Set(vcat(knn(T_U, U[i,:], minimum((k,m)))[1],findnz(A[:,i])[1]))
																		# assuming symmetric matrices
			start_A_edge_mask = Set(A_node_edge_indices[i])
		else
			pref_A = Set()
			start_A_edge_mask = Set()
		end

		#
		#  check the local neighborhood of ip in B
		#

		for jp in pref_B

			j = get(updatable_inverted_matching,jp,-1)

			# -- swap edges + compute change in  sequence scores -- #
			updatable_matching[i] = jp
			if j != -1 #if jp was matched

				updatable_matching[j] = ip
				A_edge_mask = union(start_A_edge_mask,Set(A_node_edge_indices[j]))

				# -- Compute sequence sim change -- #

				if ip != -1
					start_seq_sim,t = @timed dot(U[i,:],V[ip,:]) + dot(U[j,:],V[jp,:])
					seq_scoring_runtimes += t
					end_seq_sim,t =@timed dot(U[i,:],V[jp,:]) + dot(U[j,:],V[ip,:])
					seq_scoring_runtimes += t
				else
					start_seq_sim,t = @timed dot(U[j,:],V[jp,:])
					seq_scoring_runtimes += t
					end_seq_sim,t =@timed dot(U[i,:],V[jp,:])
					seq_scoring_runtimes += t
				end
				# doing it in loop means vectors
				# are more likely to be cached

			else

				if ip == -1
					continue # both are unmatched,
							 # no neighborhood to check in B
				end

				A_edge_mask = start_A_edge_mask

				start_seq_sim,t = @timed dot(U[i,:],V[ip,:])
				seq_scoring_runtimes += t
				end_seq_sim,t = @timed dot(U[i,:],V[jp,:])
				seq_scoring_runtimes += t

			end

			seq_sim_change = end_seq_sim - start_seq_sim

			#
			# Compute Change in topogical similarity
			#

			#(start_top_sim,start_motif_count),t = @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,prev_matching)
			(start_motif_count),t = @timed TAME_score(B_motifs,A_ten,A_edge_mask,prev_matching)

			top_scoring_runtimes += t

			#(end_top_sim,end_motif_count),t = @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,updatable_matching)
			(end_motif_count),t = @timed TAME_score(B_motifs,A_ten,A_edge_mask,updatable_matching)

			top_scoring_runtimes += t
			motif_match_change = (end_motif_count - start_motif_count)
			#top_sim_change = (end_top_sim - start_top_sim)

			#=
			if match_method == "top_sim"
				check_val = top_sim_change
			elseif match_method == "motif_count"
				check_val = motif_match_change
			end
			=#
			check_val = motif_match_change

			if check_val > 0 ||  (check_val == 0.0 && seq_sim_change > 0)

				cur_motif_matched += motif_match_change
				#cur_top_sim += top_sim_change
				cur_seq_sim += seq_sim_change

				# update the previous matching to be the same as cur
				prev_matching[i] = jp
				prev_matching[j] = ip

				# update invertible matchings
				updatable_inverted_matching[ip] = j
				updatable_inverted_matching[jp] = i
							#we can do this at the end bc
							#we only use them at the start

				ip = jp #update for future loops

			else  #change the matching back

				updatable_matching[i] = ip
				updatable_matching[j] = jp

			end

		end

		#
		#  check the local neighborhood of i in A
		#

		for j in pref_A

			jp = get(updatable_matching,j,-1)

			updatable_matching[j] = ip
			updatable_matching[i] = jp

			if jp != -1

				A_edge_mask = union(start_A_edge_mask,Set(A_node_edge_indices[j]))

				if ip != -1
					start_seq_sim,t = @timed dot(U[i,:],V[ip,:]) + dot(U[j,:],V[jp,:])
					seq_scoring_runtimes += t
					end_seq_sim,t =@timed dot(U[i,:],V[jp,:]) + dot(U[j,:],V[ip,:])
					seq_scoring_runtimes += t
				else
					start_seq_sim,t = @timed dot(U[j,:],V[jp,:])
					seq_scoring_runtimes += t
					end_seq_sim,t =@timed dot(U[i,:],V[jp,:])
					seq_scoring_runtimes += t
				end

			else
				if ip == -1
					continue # both are unmatched,
							 # no neighborhood to check in B
				end

				A_edge_mask = start_A_edge_mask

				start_seq_sim,t = @timed dot(U[i,:],V[ip,:])
				seq_scoring_runtimes += t
				end_seq_sim,t = @timed dot(U[j,:],V[ip,:])
				seq_scoring_runtimes += t
			end
			seq_sim_change = end_seq_sim - start_seq_sim

			#
			# Compute Change in topogical similarity
			#

			#(start_top_sim,start_motif_count),t = @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,prev_matching)
			(start_motif_count),t = @timed TAME_score(B_motifs,A_ten,A_edge_mask,prev_matching)
			top_scoring_runtimes += t

			#(end_top_sim, end_motif_count),t =   @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,updatable_matching)
			(end_motif_count),t =   @timed TAME_score(B_motifs,A_ten,A_edge_mask,updatable_matching)

			top_scoring_runtimes += t
			motif_match_change = (end_motif_count - start_motif_count)
			#top_sim_change = (end_top_sim - start_top_sim)

			#=
			if match_method == "top_sim"
				check_val = top_sim_change
			elseif match_method == "motif_count"
				check_val = motif_match_change
			end
			=#

			check_val = motif_match_change

			if check_val > 0 ||  (check_val == 0.0 && seq_sim_change > 0)

				cur_motif_matched += motif_match_change
				#cur_top_sim += top_sim_change
				cur_seq_sim += seq_sim_change

				# update the previous matching to be the same as cur
				prev_matching[i] = jp
				prev_matching[j] = ip

				updatable_inverted_matching[ip] = j
				updatable_inverted_matching[jp] = i

				ip = jp
				#matching = copy(updatable_matching)

			else #change back the matching
				updatable_matching[i] = ip
				updatable_matching[j] = jp
			end
		end

	end

	return updatable_matching, seq_scoring_runtimes, top_scoring_runtimes

end

function local_search(A,B,A_ten,B_ten,U,V,matching,k)#,match_method="top_sim")

	m = size(A,1)
	n = size(B,1)

	T_U = BallTree(U')
	T_V = BallTree(V')

	B_motifs = Set{Vector{Int}}()

	for idx = 1:size(B_ten.indices,2)
		push!(B_motifs,B_ten.indices[:,idx])
	end

	A_node_edge_indices = create_edge_idx_map(A_ten)

	matched_matches = [m for m in matching if (m[1] != -1) && (m[2] != -1)]

	cur_seq_sim = seq_similarity(U,V,matched_matches)
	cur_motif_matched = TAME_score(B_motifs, A_ten, Set(size(A_ten.indices,2):-1:1),matching)
													#matching doesn't reorient using this orientation

	#is it faster to precomputer X = UV'?
	unprocessed_matches = sort(matched_matches,by=match->dot(U[match[1],:],V[match[2],:]),rev=true)

	prev_matching = copy(matching)
	updatable_matching = copy(matching)
	updatable_inverted_matching = Dict([(j,i) for (i,j) in updatable_matching])

	for (i,_) in unprocessed_matches

		ip = get(updatable_matching,i,-1)

		if ip != -1
			pref_B = Set(vcat(knn(T_V, V[ip,:], minimum((k,n)))[1],findnz(B[ip,:])[1]))
			#append!(pref_B,sample(sample(1:B.n,10)))
		else
			pref_B = Set()
		end

		if i != -1
			pref_A = Set(vcat(knn(T_U, U[i,:], minimum((k,m)))[1],findnz(A[:,i])[1]))
																		# assuming symmetric matrices
			#append!(pref_A,sample(sample(1:A.n,10)))
			start_A_edge_mask = Set(A_node_edge_indices[i])
		else
			pref_A = Set()
			start_A_edge_mask = Set()
		end

		#
		#  check the local neighborhood of ip in B
		#

		for jp in pref_B

			j = get(updatable_inverted_matching,jp,-1)

			# -- swap edges + compute change in  sequence scores -- #
			updatable_matching[i] = jp
			if j != -1 #if jp was matched

				updatable_matching[j] = ip
				A_edge_mask = union(start_A_edge_mask,Set(A_node_edge_indices[j]))

				# -- Compute sequence sim change -- #
				if ip != -1
					start_seq_sim = dot(U[i,:],V[ip,:]) + dot(U[j,:],V[jp,:])
					end_seq_sim = dot(U[j,:],V[ip,:]) + dot(U[i,:],V[jp,:])
				else
					start_seq_sim = dot(U[j,:],V[jp,:])
					end_seq_sim = dot(U[i,:],V[jp,:])
				end

							# doing it in loop means vectors
							# are more likely to be cached

			else

				if ip == -1
					continue # both are unmatched,
							 # no neighborhood to check in B
				end
				A_edge_mask = start_A_edge_mask

				start_seq_sim = dot(U[i,:],V[ip,:])
				end_seq_sim = dot(U[i,:],V[jp,:])

			end

			seq_sim_change = end_seq_sim - start_seq_sim

			#
			# Compute Change in topogical similarity
			#

			start_motif_count = TAME_score(B_motifs,A_ten,A_edge_mask,prev_matching)
			end_motif_count = TAME_score(B_motifs,A_ten,A_edge_mask,updatable_matching)

			motif_match_change = (end_motif_count - start_motif_count)

			check_val = motif_match_change

			if check_val > 0 ||  (check_val == 0.0 && seq_sim_change > 0)

				cur_motif_matched += motif_match_change
				#cur_top_sim += top_sim_change
				cur_seq_sim += seq_sim_change

				# update the previous matching to be the same as cur
				prev_matching[i] = jp
				prev_matching[j] = ip

				# update invertible matchings
				updatable_inverted_matching[ip] = j
				updatable_inverted_matching[jp] = i
							#we can do this at the end bc
							#we only use them at the start

				ip = jp #update for future loops

			else  #change the matching back

				updatable_matching[i] = ip
				updatable_matching[j] = jp

			end

		end

		#
		#  check the local neighborhood of i in A
		#

		for j in pref_A

			jp = get(updatable_matching,j,-1)

			updatable_matching[j] = ip
			updatable_matching[i] = jp

			if jp != -1

				A_edge_mask = union(start_A_edge_mask,Set(A_node_edge_indices[j]))

				#start_seq_sim,t = @timed dot(U[i,:],V[ip,:]) + dot(U[j,:],V[jp,:])
				#seq_scoring_runtimes += t

				if ip != -1
					start_seq_sim = dot(U[i,:],V[ip,:]) + dot(U[j,:],V[jp,:])
					end_seq_sim = dot(U[j,:],V[ip,:]) + dot(U[i,:],V[jp,:])
				else
					start_seq_sim = dot(U[j,:],V[jp,:])
					end_seq_sim = dot(U[i,:],V[jp,:])
				end

				#seq_scoring_runtimes += t

			else

				if ip == -1
					continue # both are unmatched,
							 # no neighborhood to check in B
				end
				A_edge_mask = start_A_edge_mask

				start_seq_sim = dot(U[i,:],V[ip,:])
				end_seq_sim = dot(U[j,:],V[ip,:])

			end
			seq_sim_change = end_seq_sim - start_seq_sim

			#
			# Compute Change in topogical similarity
			#

			#=
			(start_top_sim,start_motif_count),t = @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,prev_matching)
			top_scoring_runtimes += t

			(end_top_sim, end_motif_count),t =   @timed top_similarity(B_motifs,A_ten,A_edge_mask,V,U,updatable_matching)
			top_scoring_runtimes += t
			=#

			start_motif_count = TAME_score(B_motifs,A_ten,A_edge_mask,prev_matching)
			end_motif_count = TAME_score(B_motifs,A_ten,A_edge_mask,updatable_matching)

			motif_match_change = (end_motif_count - start_motif_count)
			#top_sim_change = (end_top_sim - start_top_sim)

			#=
			if match_method == "top_sim"
				check_val = top_sim_change
			elseif match_method == "motif_count"
				check_val = motif_match_change
			end
			=#

			check_val = motif_match_change

			if check_val > 0 ||  (check_val == 0.0 && seq_sim_change > 0)

				cur_motif_matched += motif_match_change
				#cur_top_sim += top_sim_change
				cur_seq_sim += seq_sim_change

				# update the previous matching to be the same as cur
				prev_matching[i] = jp
				prev_matching[j] = ip

				updatable_inverted_matching[ip] = j
				updatable_inverted_matching[jp] = i

				ip = jp
				#matching = copy(updatable_matching)

			else #change back the matching
				updatable_matching[i] = ip
				updatable_matching[j] = jp
			end
		end

	end

	return updatable_matching

end

ext: jl

max_stars_repo_name: chuckcol/LambdaTAME

max_issues_repo_name: chuckcol/LambdaTAME

max_forks_repo_name: chuckcol/LambdaTAME


text:
@dict_readable struct TextDocumentIdentifier
    uri::DocumentUri
end

@dict_readable struct TextDocumentItem
    uri::DocumentUri
    languageId::String
    version::Int
    text::String
end

@dict_readable struct VersionedTextDocumentIdentifier
    uri::DocumentUri
    version::Union{Int,Nothing}
end

@dict_readable struct TextDocumentPositionParams
    textDocument::TextDocumentIdentifier
    position::Position
end

mutable struct DocumentFilter
    language::Union{String,Missing}
    scheme::Union{String,Missing}
    pattern::Union{String,Missing}
end

const DocumentSelector = Vector{DocumentFilter}

mutable struct TextDocumentEdit
    textDocument::VersionedTextDocumentIdentifier
    edits::Vector{TextEdit}
end

mutable struct WorkspaceEdit
    changes::Union{Any,Missing}
    documentChanges::Union{Vector{TextDocumentEdit},Missing}
end

@dict_readable struct DidOpenTextDocumentParams
    textDocument::TextDocumentItem
end

@dict_readable struct TextDocumentContentChangeEvent
    range::Union{Range,Missing}
    rangeLength::Union{Int,Missing}
    text::String
end

@dict_readable struct DidChangeTextDocumentParams
    textDocument::VersionedTextDocumentIdentifier
    contentChanges::Vector{TextDocumentContentChangeEvent}
end

@dict_readable struct DidSaveTextDocumentParams
    textDocument::TextDocumentIdentifier
    text::Union{String,Missing}
end

@dict_readable struct DidCloseTextDocumentParams
    textDocument::TextDocumentIdentifier
end

const TextDocumentSaveReason = Int
const TextDocumentSaveReasons = Dict(1 => "Manual", 2 => "AfterDelay", 3 => "FocusOut")

@dict_readable struct WillSaveTextDocumentParams
    textDocument::TextDocumentIdentifier
    reason::TextDocumentSaveReason
end


text:
module Domain

using Dates

export Person, Address, AddressType, EMAIL, WORK

# local function to generate an unique id
create_key(name::String) = string(hash(name * string(time())))

# enumerated type for an address.
@enum AddressType EMAIL WORK

struct Address
  id::String
  created::DateTime
  address_type::AddressType
  address::String
  #constructors
  Address(address_type, address) = new(create_key(address), now(), address_type, address)
end # Address

struct Person
  id::String
  created::DateTime
  name::String
  addresses::Array{Address, 1}
  #constructors
  Person(name) = new(create_key(name), now(), name, [])
  Person(name, addresses) = new(create_key(name), now(), name, addresses)
end

end


text:
type W2vData
  vocabulary::Array{UTF8String}
  projection::DenseMatrix{Float64}
end

nwords(wv::W2vData)  = size(wv.projection, 1)
projdim(wv::W2vData) = size(wv.projection, 2)

vocabulary(wv::W2vData) = wv.vocabulary
projection(wv::W2vData) = wv.projection

wordindex(wv::W2vData, word::UTF8String) = findfirst(vocabulary(wv), word)
wordindices(wv::W2vData, words::Array{UTF8String}) = [wordindex(wv, w) for w in words]

function projection(wv::W2vData, word::UTF8String)
  wi = wordindex(wv, word)
  projection(wv)[wi, :]
end

function projection(wv::W2vData, words::Array{UTF8String})
  wis = wordindices(wv, words)
  projection(wv)[wis, :]
end

function distance(wv::W2vData, word1::UTF8String, word2::UTF8String)
  vec1 = projection(wv, word1)
  vec2 = projection(wv, word2)
  sum(vec1 .* vec2)
end

function distance(wv::W2vData, word::UTF8String; n=15)
  wi = wordindex(wv, word)
  wi == 0 && return []

  vec = projection(wv, word)
  bestw = repeat([utf8("")]; inner=[n])
  bestd = -ones(Float64, n)

  for i = 1:nwords(wv)
    i == wi && continue

    w = vocabulary(wv)[i]
    d = sum(vec .* projection(wv)[i, :])

    for j = 1:n
      if d > bestd[j]
        if j < n
          bestd[(j+1):n] = bestd[j:(n-1)]
          bestw[(j+1):n] = bestw[j:(n-1)]
        end
        bestd[j] = d
        bestw[j] = w
        break
      end
    end
  end

  (bestw, bestd)
end

function nearest_words(wv::W2vData, vec::DenseMatrix{Float64}; n=15, reject_words::Array{UTF8String}=[utf8("")])
  vec /= norm(vec)

  bestw = repeat([utf8("")]; inner=[n])
  bestd = -ones(Float64, n)

  for i = 1:nwords(wv)
    w = vocabulary(wv)[i]
    in(w, reject_words) && continue

    d = sum(vec .* projection(wv)[i, :])

    for j = 1:n
      if d > bestd[j]
        if j < n
          bestd[(j+1):n] = bestd[j:(n-1)]
          bestw[(j+1):n] = bestw[j:(n-1)]
        end
        bestd[j] = d
        bestw[j] = w
        break
      end
    end
  end

  (bestw, bestd)
end

function analogy(wv::W2vData, words::Array{UTF8String}; n=15)
  length(words) < 3 && return ()

  v1 = projection(wv, words[1])
  v2 = projection(wv, words[2])
  v3 = projection(wv, words[3])
  vec = v2 - v1 + v3

  nearest_words(wv, vec; n=n, reject_words=words)
end

function load(filename::String, ::Type{W2vData})
  io = open(filename, "r")
  headline = readline(io)
  (n, dim) = map(integer, split(headline))

  vocabulary = Array(UTF8String, n)
  projection = Array(Float64, n, dim)

  for i = 1:n
    vocabulary[i] = read_word(io)
    projection[i, :] = read_vector(io, dim)
  end

  W2vData(vocabulary, projection)
end


text:
module PackageCompiler

using Base: active_project
using Libdl: Libdl
using Pkg: Pkg
using LazyArtifacts
using UUIDs: UUID, uuid1

export create_sysimage, create_app, create_library, audit_app, restore_default_sysimage

include("juliaconfig.jl")

const NATIVE_CPU_TARGET = "native"
const TLS_SYNTAX = VERSION >= v"1.7.0-DEV.1205" ? `-DNEW_DEFINE_FAST_TLS_SYNTAX` : ``

# See https://github.com/JuliaCI/julia-buildbot/blob/489ad6dee5f1e8f2ad341397dc15bb4fce436b26/master/inventory.py
function default_app_cpu_target()
    if Sys.ARCH === :i686
        return "pentium4;sandybridge,-xsaveopt,clone_all"
    elseif Sys.ARCH === :x86_64
        return "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)"
    elseif Sys.ARCH === :arm
        return "armv7-a;armv7-a,neon;armv7-a,neon,vfp4"
    elseif Sys.ARCH === :aarch64
        return "generic" # is this really the best here?
    elseif Sys.ARCH === :powerpc64le
        return "pwr8"
    else
        return "generic"
    end
end

current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file)

all_stdlibs() = readdir(Sys.STDLIB)
sysimage_modules() = map(x->x.name, Base._sysimage_modules)
stdlibs_in_sysimage() = intersect(all_stdlibs(), sysimage_modules())
stdlibs_not_in_sysimage() = setdiff(all_stdlibs(), sysimage_modules())

function load_all_deps(ctx)
    ctx_or_env = VERSION <= v"1.7.0-" ? ctx : ctx.env
    if isdefined(Pkg.Operations, :load_all_deps!)
        pkgs = Pkg.Types.PackageSpec[]
        Pkg.Operations.load_all_deps!(ctx_or_env, pkgs)
    else
        pkgs = Pkg.Operations.load_all_deps(ctx_or_env)
    end
    return pkgs
end
function source_path(ctx, pkg)
    if VERSION <= v"1.7.0-"
        Pkg.Operations.source_path(ctx, pkg)
    else
        Pkg.Operations.source_path(ctx.env.project_file, pkg)
    end
end

function bitflag()
    if Sys.ARCH == :i686
        return `-m32`
    elseif Sys.ARCH == :x86_64
        return `-m64`
    else
        return ``
    end
end

function march()
    if Sys.ARCH === :i686
        return "-march=pentium4"
    elseif Sys.ARCH === :x86_64
        return "-march=x86-64"
    elseif Sys.ARCH === :arm
        return "-march=armv7-a+simd"
    elseif Sys.ARCH === :aarch64
        return "-march=armv8-a+crypto+simd"
    elseif Sys.ARCH === :powerpc64le
        return nothing
    else
        return nothing
    end
end

# Overwriting an open file is problematic in Windows
# so move it out of the way first
function move_default_sysimage_if_windows()
    if Sys.iswindows() && isfile(default_sysimg_path())
        mv(default_sysimg_path(), tempname())
    end
end

function run_with_env(cmd, compiler)
    if Sys.iswindows()
        env = copy(ENV)
        env["PATH"] = string(env["PATH"], ";", dirname(compiler))
        run(Cmd(cmd; env=env))
    else
        run(cmd)
    end
end

function get_compiler()
    cc = get(ENV, "JULIA_CC", nothing)
    if cc !== nothing
        return cc
    end
    @static if Sys.iswindows() && Int==Int64
        return joinpath(LazyArtifacts.artifact"x86_64-w64-mingw32", "mingw64", "bin", "gcc.exe")
    elseif Sys.iswindows() && Int==Int32
        return joinpath(LazyArtifacts.artifact"i868-w64-mingw32", "mingw32", "bin", "gcc.exe")
    end
    if Sys.which("gcc") !== nothing
        return "gcc"
    elseif Sys.which("clang") !== nothing
        return "clang"
    end
    error("could not find a compiler, looked for `gcc` and `clang`")
end

function get_julia_cmd()
    julia_path = joinpath(Sys.BINDIR, Base.julia_exename())
    return `$julia_path --color=yes --startup-file=no`
end

function get_compat_version(version::VersionNumber, level::String)
    level == "full" ? version :
    level == "patch" ? VersionNumber(version.major, version.minor, version.patch) :
    level == "minor" ? VersionNumber(version.major, version.minor) :
    level == "major" ? VersionNumber(version.major) :
    error("Unknown level: $level")
end

function get_compat_version_str(version::VersionNumber, level::String)
    level == "full" ? "$(version)" :
    level == "patch" ? "$(version.major).$(version.minor).$(version.patch)" :
    level == "minor" ? "$(version.major).$(version.minor)" :
    level == "major" ? "$(version.major)" :
    error("Unknown level: $level")
end

function get_sysimg_file(name::String;
                     library_only::Bool=false,
                     version::Union{VersionNumber, Nothing}=nothing,
                     level::String="patch")

    dlext = Libdl.dlext
    (!library_only || Sys.iswindows()) && return "$name.$dlext"

    # For libraries on Unix/Apple, make sure the name starts with "lib"

    if !startswith(name, "lib")
        name = "lib" * name
    end

    if version === nothing
        return "$name.$dlext"
    end

    version = get_compat_version_str(version, level)
    sysimg_file = (
        Sys.isapple() ? "$name.$version.$dlext" :  # libname.1.2.3.dylib
        Sys.isunix() ? "$name.$dlext.$version" :   # libname.so.1.2.3
        error("Unable to determine sysimage_file; system is not Windows, Apple, or UNIX!")
    )

    return sysimg_file
end

function get_depot_path(root_dir::String, library_only::Bool)
    # Use <root>/share/julia as the depot path when creating libraries
    library_only && return joinpath(root_dir, "share", "julia")
    return root_dir
end

function get_soname(name::String;
                library_only::Bool=false,
                version::Union{VersionNumber, Nothing}=nothing,
                compat_level::String="major")

    (!Sys.isunix() || Sys.isapple()) && return nothing
    return get_sysimg_file(name, library_only=library_only, version=version, level=compat_level)
end

function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String})
    sysimg_source_path = Base.find_source_file("sysimg.jl")
    sysimg_content = read(sysimg_source_path, String)
    # replaces the hardcoded list of stdlibs in sysimg.jl with
    # the stdlibs that is given as argument
    content = replace(sysimg_content,
        r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]"))
    # Also replace a maximum call which fails for empty collections,
    # see https://github.com/JuliaLang/julia/pull/34727
    content = replace(content, "maximum(textwidth.(string.(stdlibs)))" =>
                               "reduce(max, textwidth.(string.(stdlibs)); init=0)")
    return content
end

function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String)
    tmp = mktempdir()
    sysimg_source_path = Base.find_source_file("sysimg.jl")
    base_dir = dirname(sysimg_source_path)
    tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji")
    tmp_sys_ji = joinpath(tmp, "sys.ji")
    compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl")

    @info "PackageCompiler: creating base system image (incremental=false)..."
    cd(base_dir) do
        # Create corecompiler.ji
        cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji
                                  -g0 -O0 $compiler_source_path`
        @debug "running $cmd"
        read(cmd)

        # Use that to create sys.ji
        new_sysimage_content = rewrite_sysimg_jl_only_needed_stdlibs(stdlibs)
        new_sysimage_content *= "\nempty!(Base.atexit_hooks)\n"
        new_sysimage_source_path = joinpath(tmp, "sysimage_packagecompiler_$(uuid1()).jl")
        write(new_sysimage_source_path, new_sysimage_content)
        try
            cmd = `$(get_julia_cmd()) --cpu-target $cpu_target
                                      --sysimage=$tmp_corecompiler_ji
                                      -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path`
            @debug "running $cmd"
            read(cmd)
        finally
            rm(new_sysimage_source_path; force=true)
        end
    end

    return tmp_sys_ji
end

function run_precompilation_script(project::String, sysimg::String, precompile_file::Union{String, Nothing})
    tracefile = tempname()
    if precompile_file === nothing
        arg = `-e ''`
    else
        arg = `$precompile_file`
    end
    touch(tracefile)
    cmd = `$(get_julia_cmd()) --sysimage=$(sysimg) --project=$project
            --compile=all --trace-compile=$tracefile $arg`
    @debug "run_precompilation_script: running $cmd"
    precompile_file === nothing || @info "===== Start precompile execution ====="
    run(cmd)  # `Run` this command so that we'll display stdout from the user's script.
    precompile_file === nothing || @info "===== End precompile execution ====="
    return tracefile
end

function create_sysimg_object_file(object_file::String, packages::Vector{String};
                            project::String,
                            base_sysimage::String,
                            precompile_execution_file::Vector{String},
                            precompile_statements_file::Vector{String},
                            cpu_target::String,
                            script::Union{Nothing, String},
                            isapp::Bool,
                            sysimage_build_args::Cmd)

    # Handle precompilation
    precompile_files = String[]
    @debug "running precompilation execution script..."
    for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file)
        tracefile = run_precompilation_script(project, base_sysimage, file)
        push!(precompile_files, tracefile)
    end
    append!(precompile_files, precompile_statements_file)

    precompile_code = """
        # This @eval prevents symbols from being put into Main
        @eval Module() begin
            using Base.Meta
            PrecompileStagingArea = Module()
            for (_pkgid, _mod) in Base.loaded_modules
                if !(_pkgid.name in ("Main", "Core", "Base"))
                    eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod))
                end
            end
            precompile_files = String[
                $(join(map(repr, precompile_files), "\n" * " " ^ 8))
            ]
            for file in precompile_files, statement in eachline(file)
                try
                    # println(statement)
                    # This is taken from https://github.com/JuliaLang/julia/blob/2c9e051c460dd9700e6814c8e49cc1f119ed8b41/contrib/generate_precompile.jl#L375-L393
                    ps = Meta.parse(statement)
                    isexpr(ps, :call) || continue
                    popfirst!(ps.args) # precompile(...)
                    ps.head = :tuple
                    l = ps.args[end]
                    if (isexpr(l, :tuple) || isexpr(l, :curly)) && length(l.args) > 0 # Tuple{...} or (...)
                        # XXX: precompile doesn't currently handle overloaded Vararg arguments very well.
                        # Replacing N with a large number works around it.
                        l = l.args[end]
                        if isexpr(l, :curly) && length(l.args) == 2 && l.args[1] === :Vararg # Vararg{T}
                            push!(l.args, 100) # form Vararg{T, 100} instead
                        end
                    end
                    # println(ps)
                    ps = Core.eval(PrecompileStagingArea, ps)
                    # XXX: precompile doesn't currently handle overloaded nospecialize arguments very well.
                    # Skipping them avoids the warning.
                    ms = length(ps) == 1 ? Base._methods_by_ftype(ps[1], 1, Base.get_world_counter()) : Base.methods(ps...)
                    ms isa Vector || continue
                    precompile(ps...)
                catch e
                    # See julia issue #28808
                    @debug "failed to execute \$statement"
                end
            end
        end # module
        """

    # include all packages into the sysimg
    julia_code = """
        Base.reinit_stdio()
        @eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
        @eval Sys STDLIB = $(repr(abspath(Sys.BINDIR, "../share/julia/stdlib", string('v', VERSION.major, '.', VERSION.minor))))
        Base.init_load_path()
        if isdefined(Base, :init_active_project)
            Base.init_active_project()
        end
        Base.init_depot_path()
        """

    for pkg in packages
        julia_code *= """
            import $pkg
            """
    end

    julia_code *= precompile_code

    if script !== nothing
        julia_code *= """
        include($(repr(abspath(script))))
        """
    end

    if isapp
        # If it is an app, there is only one packages
        @assert length(packages) == 1
        packages[1]
        app_start_code = """
        Base.@ccallable function julia_main()::Cint
            try
                $(packages[1]).julia_main()
            catch
                Core.print("julia_main() threw an unhandled exception")
                return 1
            end
        end
        """
        julia_code *= app_start_code
    end

    julia_code *= """
        empty!(LOAD_PATH)
        empty!(DEPOT_PATH)
        """

    # finally, make julia output the resulting object file
    @debug "creating object file at $object_file"
    @info "PackageCompiler: creating system image object file, this might take a while..."

    cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args
                              --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code`
    @debug "running $cmd"
    run(cmd)
end

default_sysimg_path() = abspath(Sys.BINDIR, "..", "lib", "julia", "sys." * Libdl.dlext)
default_sysimg_name() = basename(default_sysimg_path())
backup_default_sysimg_path() = default_sysimg_path() * ".backup"
backup_default_sysimg_name() = basename(backup_default_sysimg_path())

# TODO: Also check UUIDs for stdlibs, not only names
gather_stdlibs_project(project::String) = gather_stdlibs_project(create_pkg_context(project))
function gather_stdlibs_project(ctx)
    @assert ctx.env.manifest !== nothing
    sysimage_stdlibs = stdlibs_in_sysimage()
    non_sysimage_stdlibs = stdlibs_not_in_sysimage()
    sysimage_stdlibs_project = String[]
    non_sysimage_stdlibs_project = String[]
    for (uuid, pkg) in ctx.env.manifest
        if pkg.name in sysimage_stdlibs
            push!(sysimage_stdlibs_project, pkg.name)
        elseif pkg.name in non_sysimage_stdlibs
            push!(non_sysimage_stdlibs_project, pkg.name)
        end
    end
    return sysimage_stdlibs_project, non_sysimage_stdlibs_project
end

function check_packages_in_project(ctx, packages)
    packages_in_project = collect(keys(ctx.env.project.deps))
    if ctx.env.pkg !== nothing
        push!(packages_in_project, ctx.env.pkg.name)
    end
    packages_not_in_project = setdiff(string.(packages), packages_in_project)
    if !isempty(packages_not_in_project)
        error("package(s) $(join(packages_not_in_project, ", ")) not in project")
    end
end

"""
    create_sysimage(packages::Vector{String}; kwargs...)

Create a system image that includes the package(s) in `packages` given as a
string or vector). An attempt to automatically find a compiler will be done but
can also be given explicitly by setting the environment variable `JULIA_CC` to a
path to a compiler.

### Keyword arguments:

- `sysimage_path::Union{String,Nothing}`: The path to where
  the resulting sysimage should be saved. If set to `nothing` the keyword argument
  `replace_default` needs to be set to `true`.

- `project::String`: The project that should be active when the sysimage is created,
  defaults to the currently active project.

- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of
  files that contain code from which precompilation statements should be recorded.

- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of
  files that contain precompilation statements that should be included in the sysimage.

- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage
  of the current process otherwise build a new sysimage from scratch. Defaults to `true`.

- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file.
  Defaults to `false`, only set to `true` if you know the potential pitfalls.

- `replace_default::Bool`: If `true`, replace the default system image which is automatically
  used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref)

- `julia_init_c_file::String`: File to include in the system image with functions for
  initializing julia from external code. Used when creating a shared library.

- `version::VersionNumber`: Shared library version number (optional). Added to the sysimg
  `.so` name on Linux/UNIX, and the `.dylib` name on Apple platforms, and to set the internal
  `current_version` on Apple. Ignored on Windows.

- `compat_level::String`: compatibility level for library. One of "major", "minor".
  With `version`, used to determine the `compatibility_version` on Apple.

- `soname`: On linux, used to set the internal soname for the system image.

### Advanced keyword arguments

- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image.

- `script::String`: Path to a file that gets executed in the `--output-o` process.

- `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage,
  for example `-O1 --check-bounds=yes`.
"""
function create_sysimage(packages::Union{Symbol, Vector{String}, Vector{Symbol}}=String[];
                         sysimage_path::Union{String,Nothing}=nothing,
                         project::String=dirname(active_project()),
                         precompile_execution_file::Union{String, Vector{String}}=String[],
                         precompile_statements_file::Union{String, Vector{String}}=String[],
                         incremental::Bool=true,
                         filter_stdlibs=false,
                         replace_default::Bool=false,
                         base_sysimage::Union{Nothing, String}=nothing,
                         isapp::Bool=false,
                         julia_init_c_file=nothing,
                         version=nothing,
                         compat_level::String="major",
                         soname=nothing,
                         cpu_target::String=NATIVE_CPU_TARGET,
                         script::Union{Nothing, String}=nothing,
                         sysimage_build_args::Cmd=``)
    precompile_statements_file = abspath.(precompile_statements_file)
    precompile_execution_file = abspath.(precompile_execution_file)
    if replace_default==true
        if sysimage_path !== nothing
            error("cannot specify `sysimage_path` when `replace_default` is `true`")
        end
    end
    if sysimage_path === nothing
        if replace_default == false
            error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`")
        end
        # We will replace the default sysimage so just put it somewhere for now
        tmp = mktempdir()
        sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext))
    end
    if filter_stdlibs && incremental
        error("must use `incremental=false` to use `filter_stdlibs=true`")
    end

    # Functions lower down handles `packages` and precompilation file as arrays so convert here
    packages = string.(vcat(packages)) # Package names are often used as string inside Julia
    precompile_execution_file  = vcat(precompile_execution_file)
    precompile_statements_file = vcat(precompile_statements_file)

    # Instantiate the project
    ctx = create_pkg_context(project)

    @debug "instantiating and precompiling project at $(repr(project))"
    old_load_path = copy(LOAD_PATH)
    # Need to ensure that Pkg can find the source location of the packages:
    copy!(LOAD_PATH, [project])
    try
        Pkg.instantiate(ctx, verbose=true, allow_autoprecomp = true)
    finally
        copy!(LOAD_PATH, old_load_path)
    end

    check_packages_in_project(ctx, packages)

    if !incremental
        if base_sysimage !== nothing
            error("cannot specify `base_sysimage`  when `incremental=false`")
        end
        if filter_stdlibs
            sysimage_stdlibs, non_sysimage_stdlibs = gather_stdlibs_project(ctx)
        else
            sysimage_stdlibs = stdlibs_in_sysimage()
            non_sysimage_stdlibs = stdlibs_not_in_sysimage()
        end
        base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target)
    else
        if isnothing(base_sysimage)
            base_sysimage = current_process_sysimage_path()
        end
    end

    o_init_file = julia_init_c_file === nothing ? nothing : compile_c_init_julia(julia_init_c_file, sysimage_path)

    # Create the sysimage
    object_file = tempname() * ".o"
    create_sysimg_object_file(object_file, packages;
                              project,
                              base_sysimage,
                              precompile_execution_file,
                              precompile_statements_file,
                              cpu_target,
                              script,
                              isapp,
                              sysimage_build_args)
    create_sysimg_from_object_file(object_file,
                                   sysimage_path;
                                   o_init_file,
                                   compat_level,
                                   version,
                                   soname)

    # Maybe replace default sysimage
    if replace_default
        if !isfile(backup_default_sysimg_path())
            @debug "making a backup of default sysimg"
            cp(default_sysimg_path(), backup_default_sysimg_path())
        end
        move_default_sysimage_if_windows()
        mv(sysimage_path, default_sysimg_path(); force=true)
        @info "PackageCompiler: default sysimg replaced, restart Julia for the new sysimg to be in effect"
    end
    rm(object_file; force=true)
    return nothing
end

function compile_c_init_julia(julia_init_c_file::String, sysimage_path::String)
    @debug "Compiling $julia_init_c_file"
    compiler = get_compiler()
    m = something(march(), ``)
    flags = Base.shell_split(cflags())

    o_init_file = splitext(julia_init_c_file)[1] * ".o"
    cmd = `$compiler -c -O2 -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $TLS_SYNTAX $(bitflag()) $flags $m -o $o_init_file $julia_init_c_file`

    @debug "running $cmd"
    run_with_env(cmd, compiler)
    return o_init_file
end

function get_extra_linker_flags(version, compat_level, soname)
    current_ver_arg = ``
    compat_ver_arg = ``

    if version !== nothing
        compat_version = get_compat_version(version, compat_level)
        current_ver_arg = `-current_version $version`
        compat_ver_arg = `-compatibility_version $compat_version`
    end

    soname_arg = soname === nothing ? `` : `-Wl,-soname,$soname`
    rpath_args = rpath_sysimage()

    extra = (
        Sys.iswindows() ? `-Wl,--export-all-symbols` :
        Sys.isapple() ? `-fPIC $compat_ver_arg $current_ver_arg $rpath_args` :
        Sys.isunix() ? `-fPIC $soname_arg $rpath_args` :
        error("What kind of machine is this?")
    )

    return extra
end

function create_sysimg_from_object_file(input_object::String,
                                        sysimage_path::String;
                                        o_init_file=nothing,
                                        version=nothing,
                                        compat_level::String="major",
                                        soname=nothing)

    julia_libdir = dirname(Libdl.dlpath("libjulia"))

    o_files = [input_object]
    o_init_file !== nothing && push!(o_files, o_init_file)

    # Prevent compiler from stripping all symbols from the shared lib.
    # TODO: On clang on windows this is called something else
    if Sys.isapple()
        o_file_flags = `-Wl,-all_load $o_files`
    else
        o_file_flags = `-Wl,--whole-archive $o_files -Wl,--no-whole-archive`
    end

    extra = get_extra_linker_flags(version, compat_level, soname)

    compiler = get_compiler()
    m = something(march(), ``)
    private_libdir = if Base.DARWIN_FRAMEWORK # taken from Libdl tests
        if ccall(:jl_is_debugbuild, Cint, ()) != 0
            dirname(abspath(Libdl.dlpath(Base.DARWIN_FRAMEWORK_NAME * "_debug")))
        else
            joinpath(dirname(abspath(Libdl.dlpath(Base.DARWIN_FRAMEWORK_NAME))),"Frameworks")
        end
    elseif ccall(:jl_is_debugbuild, Cint, ()) != 0
        dirname(abspath(Libdl.dlpath("libjulia-internal-debug")))
    else
        dirname(abspath(Libdl.dlpath("libjulia-internal")))
    end
    cmd = `$compiler $(bitflag()) $m -shared -L$(julia_libdir) -L$(private_libdir) -o $sysimage_path $o_file_flags -ljulia-internal -ljulia $extra`
    @debug "running $cmd"
    run_with_env(cmd, compiler)
    return nothing
end

"""
    restore_default_sysimage()

Restores the default system image to the one that Julia shipped with.
Useful after running [`create_sysimage`](@ref) with `replace_default=true`.
"""
function restore_default_sysimage()
    if !isfile(backup_default_sysimg_path())
        error("did not find a backup sysimg")
    end
    move_default_sysimage_if_windows()
    mv(backup_default_sysimg_path(), default_sysimg_path(); force=true)
    @info "PackageCompiler: default sysimg restored, restart Julia for the new sysimg to be in effect"
    return nothing
end

function create_pkg_context(project)
    project_toml_path = Pkg.Types.projectfile_path(project; strict=true)
    if project_toml_path === nothing
        error("could not find project at $(repr(project))")
    end
    return Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path))
end

"""
    audit_app(app_dir::String)

Check for possible problems with regards to relocatability for
the project at `app_dir`.

!!! warning
    This cannot guarantee that the project is free of relocatability problems,
    it can only detect some known bad cases and warn about those.
"""
audit_app(app_dir::String) = audit_app(create_pkg_context(app_dir))
function audit_app(ctx::Pkg.Types.Context)
    # Check for build script usage
    if isfile(joinpath(dirname(ctx.env.project_file), "deps", "build.jl"))
        @warn "Project has a build script, this might indicate that it is not relocatable"
    end
    pkgs = load_all_deps(ctx)
    for pkg in pkgs
        pkg_source = source_path(ctx, pkg)
        pkg_source === nothing && continue
        if isfile(joinpath(pkg_source, "deps", "build.jl"))
            @warn "Package $(pkg.name) has a build script, this might indicate that it is not relocatable"
        end
    end
    return
end

"""
    create_app(package_dir::String, compiled_app::String; kwargs...)

Compile an app with the source in `package_dir` to the folder `compiled_app`.
The folder `package_dir` needs to contain a package where the package includes a
function with the signature

```julia
julia_main()::Cint
    # Perhaps do something based on ARGS
    ...
end
```

The executable will be placed in a folder called `bin` in `compiled_app` and
when the executable run the `julia_main` function is called.

Standard Julia arguments are set by passing them after a `--julia-args`
argument, for example:
```
\$ ./MyApp input.csv --julia-args -O3 -t8
```

An attempt to automatically find a compiler will be done but can also be given
explicitly by setting the environment variable `JULIA_CC` to a path to a
compiler.

### Keyword arguments:

- `app_name::String`: an alternative name for the compiled app. If not provided,
  the name of the package (as specified in `Project.toml`) is used.

- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of
  files that contain code from which precompilation statements should be recorded.

- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of
  files that contain precompilation statements that should be included in the sysimage
  for the app.

- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage
  of the current process otherwise build a new sysimage from scratch. Defaults to `false`.

- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file.
  Defaults to `false`, only set to `true` if you know the potential pitfalls.

- `audit::Bool`: Warn about eventual relocatability problems with the app, defaults
  to `true`.

- `force::Bool`: Remove the folder `compiled_app` if it exists before creating the app.

- `include_lazy_artifacts::Bool`: if lazy artifacts should be included in the bundled artifacts,
  defaults to `true`.

### Advanced keyword arguments

- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image.

- `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage,
  for example `-O1 --check-bounds=yes`.
"""
function create_app(package_dir::String,
                    app_dir::String;
                    app_name=nothing,
                    precompile_execution_file::Union{String, Vector{String}}=String[],
                    precompile_statements_file::Union{String, Vector{String}}=String[],
                    incremental=false,
                    filter_stdlibs=false,
                    audit=true,
                    force=false,
                    c_driver_program::String=joinpath(@__DIR__, "embedding_wrapper.c"),
                    cpu_target::String=default_app_cpu_target(),
                    include_lazy_artifacts::Bool=true,
                    sysimage_build_args::Cmd=``)

    _create_app(package_dir, app_dir, app_name, precompile_execution_file,
        precompile_statements_file, incremental, filter_stdlibs, audit, force, cpu_target;
        library_only=false, c_driver_program, julia_init_c_file=nothing,
        header_files=String[], version=nothing, compat_level="major",
        include_lazy_artifacts, sysimage_build_args)
end

"""
    create_library(package_dir::String, dest_dir::String; kwargs...)

Compile a library with the source in `package_dir` to the folder `dest_dir`.
The folder `package_dir` should to contain a package with C-callable functions,
e.g.

```
Base.@ccallable function julia_cg(fptr::Ptr{Cvoid}, cx::Ptr{Cdouble}, cb::Ptr{Cdouble}, len::Csize_t)::Cint
    try
        x = unsafe_wrap(Array, cx, (len,))
        b = unsafe_wrap(Array, cb, (len,))
        A = COp(fptr,len)
        cg!(x, A, b)
    catch
        Base.invokelatest(Base.display_error, Base.catch_stack())
        return 1
    end
    return 0
end
```

The library will be placed in the `lib` folder in `dest_dir` (or `bin` on Windows),
and can be linked to and called into from C/C++ or other languages that can use C libraries.

Note that any applications/programs linking to this library may need help finding
it at run time. Options include

* Installing all libraries somewhere in the library search path.
* Adding `/path/to/libname` to an appropriate library search path environment
  variable (`DYLD_LIBRARY_PATH` on OSX, `PATH` on Windows, or `LD_LIBRARY_PATH`
  on Linux/BSD/Unix).
* Running `install_name_tool -change libname /path/to/libname` (OSX)

To use any Julia exported functions, you *must* first call `init_julia(argc, argv)`,
where `argc` and `argv` are parameters that would normally be passed to `julia` on the
command line (e.g., to set up the number of threads or processes).

When your program is exiting, it is also suggested to call `shutdown_julia(retcode)`,
to allow Julia to cleanly clean up resources and call any finalizers. (This function
simply calls `jl_atexit_hook(retcode)`.)

An attempt to automatically find a compiler will be done but can also be given
explicitly by setting the environment variable `JULIA_CC` to a path to a
compiler.

### Keyword arguments:

- `lib_name::String`: an alternative name for the compiled library. If not provided,
  the name of the package (as specified in Project.toml) is used. `lib` will be
  prepended to the name if it is not already present.

- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of
  files that contain code from which precompilation statements should be recorded.

- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of
  files that contain precompilation statements that should be included in the sysimage
  for the library.

- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage
  of the current process otherwise build a new sysimage from scratch. Defaults to `false`.

- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file.
  Defaults to `false`, only set to `true` if you know the potential pitfalls.

- `audit::Bool`: Warn about eventual relocatability problems with the library, defaults
  to `true`.

- `force::Bool`: Remove the folder `compiled_lib` if it exists before creating the library.

- `header_files::Vector{String}`: A list of header files to include in the library bundle.

- `julia_init_c_file::String`: File to include in the system image with functions for
  initializing julia from external code.

- `version::VersionNumber`: Library version number. Added to the sysimg `.so` name
  on Linux, and the `.dylib` name on Apple platforms, and with `compat_level`, used to
  determine and set the `current_version`, `compatibility_version` (on Apple) and
  `soname` (on Linux/UNIX)

- `compat_level::String`: compatibility level for library. One of "major", "minor".
  Used to determine and set the `compatibility_version` (on Apple) and `soname` (on
  Linux/UNIX).

- `include_lazy_artifacts::Bool`: if lazy artifacts should be included in the bundled artifacts,
  defaults to `true`.

### Advanced keyword arguments

- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image.

- `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage,
  for example `-O1 --check-bounds=yes`.
"""
function create_library(package_dir::String,
                        dest_dir::String;
                        lib_name=nothing,
                        precompile_execution_file::Union{String, Vector{String}}=String[],
                        precompile_statements_file::Union{String, Vector{String}}=String[],
                        incremental=false,
                        filter_stdlibs=false,
                        audit=true,
                        force=false,
                        header_files::Vector{String} = String[],
                        julia_init_c_file::String=joinpath(@__DIR__, "julia_init.c"),
                        version=nothing,
                        compat_level="major",
                        cpu_target::String=default_app_cpu_target(),
                        include_lazy_artifacts::Bool=true,
                        sysimage_build_args::Cmd=``)

    julia_init_h_file::String=joinpath(@__DIR__, "julia_init.h")

    if !(julia_init_h_file in header_files)
        push!(header_files, julia_init_h_file)
    end

    if version isa String
        version = parse(VersionNumber, version)
    end

    _create_app(package_dir, dest_dir, lib_name, precompile_execution_file,
        precompile_statements_file, incremental, filter_stdlibs, audit, force, cpu_target;
        library_only=true, c_driver_program="", julia_init_c_file,
        header_files, version, compat_level, include_lazy_artifacts, sysimage_build_args)

end

function _create_app(package_dir::String,
                    dest_dir::String,
                    name,
                    precompile_execution_file,
                    precompile_statements_file,
                    incremental,
                    filter_stdlibs,
                    audit,
                    force,
                    cpu_target::String;
                    library_only::Bool,
                    c_driver_program::String,
                    julia_init_c_file,
                    header_files::Vector{String},
                    version,
                    compat_level::String,
                    include_lazy_artifacts::Bool,
                    sysimage_build_args::Cmd)
    isapp = !library_only

    precompile_statements_file = abspath.(precompile_statements_file)
    precompile_execution_file = abspath.(precompile_execution_file)
    package_dir = abspath(package_dir)
    ctx = create_pkg_context(package_dir)
    Pkg.instantiate(ctx, verbose=true, allow_autoprecomp = false)
    if isempty(ctx.env.manifest)
        @warn "it is not recommended to create an app without a preexisting manifest"
    end
    if ctx.env.pkg === nothing
        error("expected package to have a `name`-entry")
    end
    sysimg_name = ctx.env.pkg.name
    if name === nothing
        name = sysimg_name
    end

    sysimg_file = get_sysimg_file(name, library_only=library_only, version=version)
    soname = get_soname(name,
                        library_only=library_only,
                        version=version,
                        compat_level=compat_level)

    if isdir(dest_dir)
        if !force
            error("directory $(repr(dest_dir)) already exists, use `force=true` to overwrite (will completely",
                  " remove the directory)")
        end
        rm(dest_dir; force=true, recursive=true)
    end

    audit && audit_app(ctx)

    mkpath(dest_dir)

    bundle_julia_libraries(dest_dir, library_only)
    bundle_artifacts(ctx, dest_dir, library_only; include_lazy_artifacts=include_lazy_artifacts)

    library_only && bundle_headers(dest_dir, header_files)

    # TODO: Create in a temp dir and then move it into place?
    sysimage_path = Sys.isunix() ? joinpath(dest_dir, "lib") : joinpath(dest_dir, "bin")
    mkpath(sysimage_path)
    cd(sysimage_path) do
        if !incremental
            tmp = mktempdir()
            # Use workaround at https://github.com/JuliaLang/julia/issues/34064#issuecomment-563950633
            # by first creating a normal "empty" sysimage and then use that to finally create the one
            # with the @ccallable function
            tmp_base_sysimage = joinpath(tmp, "tmp_sys.so")
            create_sysimage(String[]; sysimage_path=tmp_base_sysimage, project=package_dir,
                            incremental=false, filter_stdlibs, cpu_target)

            create_sysimage([sysimg_name]; sysimage_path=sysimg_file, project=package_dir,
                            incremental=true,
                            precompile_execution_file,
                            precompile_statements_file,
                            cpu_target,
                            base_sysimage=tmp_base_sysimage,
                            isapp,
                            julia_init_c_file,
                            version,
                            soname,
                            sysimage_build_args)
        else
            create_sysimage([sysimg_name]; sysimage_path=sysimg_file, project=package_dir,
                                         incremental, filter_stdlibs,
                                         precompile_execution_file,
                                         precompile_statements_file,
                                         cpu_target,
                                         isapp,
                                         julia_init_c_file,
                                         version,
                                         soname,
                                         sysimage_build_args)
        end

        if Sys.isapple()
            cmd = `install_name_tool -id @rpath/$sysimg_file $sysimg_file`
            @debug "running $cmd"
            run(cmd)
        end

        if library_only && version !== nothing && Sys.isunix()
            compat_file = get_sysimg_file(name, library_only=library_only, version=version, level=compat_level)
            base_file = get_sysimg_file(name, library_only=library_only)
            @debug "creating symlinks for $compat_file and $base_file"
            symlink(sysimg_file, compat_file)
            symlink(sysimg_file, base_file)
        end
    end

    if !library_only
        executable_path = joinpath(dest_dir, "bin")
        mkpath(executable_path)
        cd(executable_path) do
            c_driver_program_path = abspath(c_driver_program)
            sysimage_path = Sys.iswindows() ? sysimg_file : joinpath("..", "lib", sysimg_file)
            create_executable_from_sysimg(; sysimage_path, executable_path=name,
                                         c_driver_program_path)
        end
    end

    return
end

function create_executable_from_sysimg(;sysimage_path::String,
                                        executable_path::String,
                                        c_driver_program_path::String,)
    flags = join((cflags(), ldflags(), ldlibs()), " ")
    flags = Base.shell_split(flags)
    wrapper = c_driver_program_path
    compiler = get_compiler()
    m = something(march(), ``)
    cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $TLS_SYNTAX $(bitflag()) $m -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $(rpath_executable()) $flags`
    @debug "running $cmd"
    run_with_env(cmd, compiler)
    return nothing
end

function bundle_julia_libraries(dest_dir, library_only)
    app_libdir = joinpath(dest_dir, Sys.isunix() ? "lib" : "bin")
    cp(julia_libdir(), app_libdir; force=true)
    # We do not want to bundle the sysimg (nor the backup sysimage):
    rm(joinpath(app_libdir, "julia", default_sysimg_name()); force=true)
    rm(joinpath(app_libdir, "julia", backup_default_sysimg_name()); force=true)
    # Remove debug symbol libraries
    if Sys.isapple()
        v = string(VERSION.major, ".", VERSION.minor)
        rm(joinpath(app_libdir, "libjulia.$v.dylib.dSYM"); force=true, recursive=true)
        rm(joinpath(app_libdir, "julia", "sys.dylib.dSYM"); force=true, recursive=true)
    end
    return
end

function bundle_artifacts(ctx, dest_dir, library_only; include_lazy_artifacts=true)
    @debug "bundling artifacts..."

    pkgs = load_all_deps(ctx)

    # Also want artifacts for the project itself
    @assert ctx.env.pkg !== nothing
    # This is kinda ugly...
    ctx.env.pkg.path = dirname(ctx.env.project_file)
    push!(pkgs, ctx.env.pkg)

    # Collect all artifacts needed for the project
    artifact_paths = Set{String}()
    for pkg in pkgs
        pkg_source_path = source_path(ctx, pkg)
        pkg_source_path === nothing && continue
        # Check to see if this package has an (Julia)Artifacts.toml
        for f in Pkg.Artifacts.artifact_names
            artifacts_toml_path = joinpath(pkg_source_path, f)
            if isfile(artifacts_toml_path)
                @debug "bundling artifacts for $(pkg.name)"
                artifact_dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml_path)
                for name in keys(artifact_dict)
                    if !include_lazy_artifacts &&
                            isa(artifact_dict[name], AbstractDict) &&
                            get(artifact_dict[name], "lazy", false)
                        @info "skipping lazy artifact \"$name\""
                        continue
                    end
                    meta = Pkg.Artifacts.artifact_meta(name, artifacts_toml_path)
                    meta === nothing && continue
                    @debug "  \"$name\""
                    push!(artifact_paths, Pkg.Artifacts.ensure_artifact_installed(name, artifacts_toml_path))
                end
                break
            end
        end
    end

    # Copy the artifacts needed to the app directory
    depot_path = get_depot_path(dest_dir, library_only)
    artifact_app_path = joinpath(depot_path, "artifacts")

    if !isempty(artifact_paths)
        mkpath(artifact_app_path)
    end
    for artifact_path in artifact_paths
        artifact_name = basename(artifact_path)
        cp(artifact_path, joinpath(artifact_app_path, artifact_name))
    end
    return
end

function bundle_headers(dest_dir, header_files)
    isempty(header_files) && return
    include_dir = joinpath(dest_dir, "include")
    mkpath(include_dir)

    for header_file in header_files
        new_file = joinpath(include_dir, basename(header_file))
        cp(header_file, new_file; force=true)
    end
    return
end

end # module


text:
using Memoize

const data = parse.(Int, split(readline(), ","))

@memoize function count_production(timer, remaining)
    n = 1
    while remaining >= 0
        remaining -= timer+1
        if remaining >= 0
            n += count_production(8, remaining)
            timer = 6
        end
    end
    n
end

# Part One and Two
count_production.(data, 256) |> sum |> println

max_stars_repo_name: pLOPeGG/JuliaOfCode

max_issues_repo_name: pLOPeGG/JuliaOfCode

max_forks_repo_name: pLOPeGG/JuliaOfCode


text:
### print integers
# Some credit to Formatting.jl (https://github.com/JuliaIO/Formatting.jl)

struct _Dec end
struct _Oct end
struct _Hex end
struct _HEX end
struct _Bin end

_mul(x::Integer, ::_Dec) = x * 10
_mul(x::Integer, ::_Bin) = x << 1
_mul(x::Integer, ::_Oct) = x << 3
_mul(x::Integer, ::Union{_Hex, _HEX}) = x << 4

_div(x::Integer, ::_Dec) = div(x, 10)
_div(x::Integer, ::_Bin) = x >> 1
_div(x::Integer, ::_Oct) = x >> 3
_div(x::Integer, ::Union{_Hex, _HEX}) = x >> 4

_ipre(op) = ""
_ipre(::_Hex) = "0x"
_ipre(::_HEX) = "0X"
_ipre(::_Oct) = "0o"
_ipre(::_Bin) = "0b"

_digitchar(x::Integer, ::_Bin) = Char(x == 0 ? '0' : '1')
_digitchar(x::Integer, ::_Dec) = Char('0' + x)
_digitchar(x::Integer, ::_Oct) = Char('0' + x)
_digitchar(x::Integer, ::_Hex) = Char(x < 10 ? '0' + x : 'a' + (x - 10))
_digitchar(x::Integer, ::_HEX) = Char(x < 10 ? '0' + x : 'A' + (x - 10))

function format_integer(val, op, hash::Bool)
    # prefix
    result = ""
    if hash
        result = _ipre(op)
    end

    # find largest power that fits in `val`, subtract it from `val`
    # and repeat the process until `val` is 0

    lowerbound = _div(val, op)
    b = one(val)
    while b <= lowerbound
        b = _mul(b, op)
    end
    remainder = val
    while b > 0
        (quotient, remainder) = divrem(remainder, b)
        result *= _digitchar(quotient, op)
        b = _div(b, op)
    end
    return result
end


text:
module OutlierDetectionTrees
    using OutlierDetectionInterface
    using OutlierDetectionInterface:SCORE_UNSUPERVISED
    const OD = OutlierDetectionInterface

    include("models/IForest.jl")

    const UUID = "6470b2ab-4fe8-498e-808d-6badd5c3da38"
    const MODELS = [:IForestDetector]

    for model in MODELS
        @eval begin
            OD.@default_frontend $model
            OD.@default_metadata $model $UUID
            export $model
        end
    end
end


text:
"""
Function to obtain the output current time series of a Dynamic Generator model out of the DAE Solution. It receives the simulation inputs,
the dynamic device and bus voltage. It is dispatched for device type to compute the specific current.

"""
function compute_output_current(
    sim::Simulation,
    dynamic_device::G,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
) where {G <: PSY.DynamicGenerator}

    #Obtain Data
    sys = get_system(sim)

    #Get machine
    machine = PSY.get_machine(dynamic_device)
    Sbase = PSY.get_base_power(sys)
    basepower = PSY.get_base_power(dynamic_device)
    base_power_ratio = basepower / Sbase
    return _machine_current(
        machine,
        PSY.get_name(dynamic_device),
        V_R,
        V_I,
        base_power_ratio,
        sim,
    )
end

"""
Function to obtain the output current time series of a Classic Machine model out of the DAE Solution. It is dispatched via the machine type.

"""
function _machine_current(
    machine::PSY.BaseMachine,
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))

    R = PSY.get_R(machine)
    Xd_p = PSY.get_Xd_p(machine)
    eq_p = PSY.get_eq_p(machine)

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ)
    I_I = similar(δ)

    for ix in 1:length(δ)
        v = δ[ix]
        V_d, V_q = ri_dq(v) * [V_R[ix]; V_I[ix]]
        #Obtain electric current
        i_dq[1] = (1.0 / (R^2 + Xd_p^2)) * (Xd_p * (eq_p .- V_q) - R * V_d)  #15.36
        i_dq[2] = (1.0 / (R^2 + Xd_p^2)) * (Xd_p * V_d + R * (eq_p .- V_q)) #15.36

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end

"""
Function to obtain the output current time series of a One-D-One-Q model out of the DAE Solution. It is dispatched via the machine type.
"""
function _machine_current(
    machine::PSY.OneDOneQMachine,
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))
    eq_p = post_proc_state_series(sim, (name, :eq_p))
    ed_p = post_proc_state_series(sim, (name, :ed_p))

    R = PSY.get_R(machine)
    Xd_p = PSY.get_Xd_p(machine)
    Xq_p = PSY.get_Xq_p(machine)

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ, Float64)
    I_I = similar(δ, Float64)

    for ix in 1:length(δ)
        v = δ[ix]
        V_d, V_q = ri_dq(v) * [V_R[ix]; V_I[ix]]
        #Obtain electric current
        i_dq[1] =
            (1.0 / (R^2 + Xd_p * Xq_p)) * (Xq_p * (eq_p[ix] - V_q) + R * (ed_p[ix] - V_d))  #15.32
        i_dq[2] =
            (1.0 / (R^2 + Xd_p * Xq_p)) * (-Xd_p * (ed_p[ix] - V_d) + R * (eq_p[ix] - V_q))  #15.32

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end

"""
Function to obtain the output current time series of a SimpleMarconato or SimpleAndersonFouad model out of the DAE Solution. It is dispatched via the machine type.
"""
function _machine_current(
    machine::PSY.Union{PSY.SimpleMarconatoMachine, PSY.SimpleAFMachine},
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))
    eq_pp = post_proc_state_series(sim, (name, :eq_pp))
    ed_pp = post_proc_state_series(sim, (name, :ed_pp))

    #Get parameters
    R = PSY.get_R(machine)
    Xd_pp = PSY.get_Xd_pp(machine)
    Xq_pp = PSY.get_Xq_pp(machine)

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ, Float64)
    I_I = similar(δ, Float64)

    for ix in 1:length(δ)
        v = δ[ix]
        V_d, V_q = ri_dq(v) * [V_R[ix]; V_I[ix]]
        #Obtain electric current
        i_dq[1] =
            (1.0 / (R^2 + Xd_pp * Xq_pp)) *
            (Xq_pp * (eq_pp[ix] - V_q) + R * (ed_pp[ix] - V_d))      #15.25
        i_dq[2] =
            (1.0 / (R^2 + Xd_pp * Xq_pp)) *
            (-Xd_pp * (ed_pp[ix] - V_d) + R * (eq_pp[ix] - V_q))      #15.25

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end

"""
Function to obtain the output current time series of a Marconato or AndersonFouad model out of the DAE Solution. It is dispatched via the machine type.
"""
function _machine_current(
    machine::Union{PSY.MarconatoMachine, PSY.AndersonFouadMachine},
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))
    eq_pp = post_proc_state_series(sim, (name, :eq_pp))
    ed_pp = post_proc_state_series(sim, (name, :ed_pp))
    ψd = post_proc_state_series(sim, (name, :ψd))
    ψq = post_proc_state_series(sim, (name, :ψq))

    #Get parameters
    Xd_pp = PSY.get_Xd_pp(machine)
    Xq_pp = PSY.get_Xq_pp(machine)

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ, Float64)
    I_I = similar(δ, Float64)

    for ix in 1:length(δ)
        v = δ[ix]

        #Obtain electric current
        i_dq[1] = (1.0 / Xd_pp) * (eq_pp[ix] - ψd[ix])      #15.18
        i_dq[2] = (1.0 / Xq_pp) * (-ed_pp[ix] - ψq[ix])     #15.18

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end

"""
Function to obtain the output current time series of a GENROU/GENROE model out of the DAE Solution. It is dispatched via the machine type.
"""
function _machine_current(
    machine::Union{PSY.RoundRotorQuadratic, PSY.RoundRotorExponential},
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))
    eq_p = post_proc_state_series(sim, (name, :eq_p))
    ed_p = post_proc_state_series(sim, (name, :ed_p))
    ψ_kd = post_proc_state_series(sim, (name, :ψ_kd))
    ψ_kq = post_proc_state_series(sim, (name, :ψ_kq))

    #Get parameters
    R = PSY.get_R(machine)
    Xd_p = PSY.get_Xd_p(machine)
    Xd_pp = PSY.get_Xd_pp(machine)
    Xq_pp = Xd_pp
    Xl = PSY.get_Xl(machine)
    γ_d1 = PSY.get_γ_d1(machine)
    γ_q1 = PSY.get_γ_q1(machine)
    γ_d2 = PSY.get_γ_d2(machine)

    ψq_pp = γ_q1 * ed_p + ψ_kq * (1 - γ_q1)
    ψd_pp = γ_d1 * eq_p + γ_d2 * (Xd_p - Xl) * ψ_kd

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ, Float64)
    I_I = similar(δ, Float64)

    for ix in 1:length(δ)
        v = δ[ix]
        V_d, V_q = ri_dq(v) * [V_R[ix]; V_I[ix]]

        #Obtain electric current
        i_dq[1] =
            (1.0 / (R^2 + Xq_pp * Xd_pp)) *
            (-R * (V_d - ψq_pp[ix]) + Xq_pp * (-V_q + ψd_pp[ix]))
        i_dq[2] =
            (1.0 / (R^2 + Xq_pp * Xd_pp)) *
            (Xd_pp * (V_d - ψq_pp[ix]) + R * (-V_q + ψd_pp[ix]))

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end

"""
Function to obtain the output current time series of a GENSAL/GENSAE model out of the DAE Solution. It is dispatched via the machine type.
"""
function _machine_current(
    machine::Union{PSY.SalientPoleQuadratic, PSY.SalientPoleExponential},
    name::String,
    V_R::Vector{Float64},
    V_I::Vector{Float64},
    base_power_ratio::Float64,
    sim::Simulation,
)
    δ = post_proc_state_series(sim, (name, :δ))
    eq_p = post_proc_state_series(sim, (name, :eq_p))
    ψ_kd = post_proc_state_series(sim, (name, :ψ_kd))
    ψq_pp = post_proc_state_series(sim, (name, :ψq_pp))

    #Get parameters
    R = PSY.get_R(machine)
    Xd_pp = PSY.get_Xd_pp(machine)
    γ_d1 = PSY.get_γ_d1(machine)
    γ_q1 = PSY.get_γ_q1(machine)

    ψd_pp = γ_d1 * eq_p + γ_q1 * ψ_kd

    i_dq = Vector{Float64}(undef, 2)
    I_R = similar(δ, Float64)
    I_I = similar(δ, Float64)

    for ix in 1:length(δ)
        v = δ[ix]
        V_d, V_q = ri_dq(v) * [V_R[ix]; V_I[ix]]

        #Obtain electric current
        i_dq[1] =
            (1.0 / (R^2 + Xd_pp^2)) * (-R * (V_d + ψq_pp[ix]) + Xd_pp * (ψd_pp[ix] - V_q))
        i_dq[2] =
            (1.0 / (R^2 + Xd_pp^2)) * (Xd_pp * (V_d + ψq_pp[ix]) + R * (ψd_pp[ix] - V_q))

        I_R[ix], I_I[ix] = base_power_ratio * dq_ri(v) * i_dq
    end
    return I_R, I_I
end


text:
module IJulia

# in the IPython front-end, enable verbose output via IJulia.set_verbose()
verbose = false
function set_verbose(v=true)
    global verbose::Bool = v
end

using ZMQ
using JSON
using REPLCompletions

uuid4() = repr(Base.Random.uuid4())

if length(ARGS) > 0
    global const profile = open(JSON.parse,ARGS[1])
else
    # generate profile and save
    let port0 = 5678
        global const profile = (String=>Any)[
            "ip" => "127.0.0.1",
            "transport" => "tcp",
            "stdin_port" => port0,
            "control_port" => port0+1,
            "hb_port" => port0+2,
            "shell_port" => port0+3,
            "iopub_port" => port0+4,
            "key" => uuid4()
        ]
        fname = "profile-$(getpid()).json"
        println("connect ipython with --existing $(pwd())/$fname")
        open(fname, "w") do f
            JSON.print(f, profile)
        end
    end
end

include("hmac.jl") # must go after profile is initialized
include("stdio.jl")
include("msg.jl")
include("history.jl")
include("handlers.jl")

const ctx = Context()
const publish = Socket(ctx, PUB)
const raw_input = Socket(ctx, ROUTER)
const requests = Socket(ctx, ROUTER)
const control = Socket(ctx, ROUTER)
const heartbeat = Socket(ctx, REP)

bind(publish, "$(profile["transport"])://$(profile["ip"]):$(profile["iopub_port"])")
bind(requests, "$(profile["transport"])://$(profile["ip"]):$(profile["shell_port"])")
bind(control, "$(profile["transport"])://$(profile["ip"]):$(profile["control_port"])")
bind(raw_input, "$(profile["transport"])://$(profile["ip"]):$(profile["stdin_port"])")
bind(heartbeat, "$(profile["transport"])://$(profile["ip"]):$(profile["hb_port"])")

include("heartbeat.jl")
start_heartbeat(heartbeat)

send_status("starting")

function eventloop(socket)
    try
        while true
            msg = recv_ipython(socket)
            try
                handlers[msg.header["msg_type"]](socket, msg)
            catch e
                # Try to keep going if we get an exception, but
                # send the exception traceback to the front-ends.
                # (Ignore SIGINT since this may just be a user-requested
                #  kernel interruption to interrupt long calculations.)
                if !isa(e, InterruptException)
                    content = pyerr_content(e, "KERNEL EXCEPTION")
                    map(s -> println(orig_STDERR, s), content["traceback"])
                    send_ipython(publish,
                                 execute_msg == nothing ?
                                 Msg([ "pyerr" ],
                                     [ "msg_id" => uuid4(),
                                     "username" => "jlkernel",
                                     "session" => uuid4(),
                                      "msg_type" => "pyerr" ], content) :
                                 msg_pub(execute_msg, "pyerr", content))
                end
            end
        end
    catch e
        # the IPython manager may send us a SIGINT if the user
        # chooses to interrupt the kernel; don't crash on this
        if isa(e, InterruptException)
            eventloop(socket)
        else
            rethrow()
        end
    end
end

function waitloop()
    @async eventloop(control)
    requests_task = @async eventloop(requests)
    while true
        try
            wait()
        catch e
            # send interrupts (user SIGINT) to the code-execution task
            if isa(e, InterruptException)
                @async Base.throwto(requests_task, e)
            else
                rethrow()
            end
        end
    end
end

end # IJulia