text:
using Oceananigans.Utils: instantiate
using Oceananigans.Grids: Face, Center

import Base: identity

@inline identity(i, j, k, grid, c) = @inbounds c[i, j, k]
@inline identity(i, j, k, grid, a::Number) = a

"""Evaluate the function `F` with signature `F(i, j, k, grid, args...)` at index `i, j, k` without
interpolation."""
@inline identity(i, j, k, grid, F::TF, args...) where TF<:Function = F(i, j, k, grid, args...)

# Utilities for inferring the interpolation function needed to
# interpolate a field from one location to the next.
interpolation_code(from, to) = interpolation_code(to)

interpolation_code(::Type{Face}) = :ᶠ
interpolation_code(::Type{Center}) = :ᶜ
interpolation_code(::Face) = :ᶠ
interpolation_code(::Center) = :ᶜ

# Intercept non-interpolations
interpolation_code(from::L, to::L) where L = :ᵃ
interpolation_code(::Nothing, to) = :ᵃ
interpolation_code(from, ::Nothing) = :ᵃ
interpolation_code(::Nothing, ::Nothing) = :ᵃ

for ξ in ("x", "y", "z")
    ▶sym = Symbol(:ℑ, ξ, :sym)
    @eval begin
        $▶sym(s::Symbol) = $▶sym(Val(s))
        $▶sym(::Union{Val{:ᶠ}, Val{:ᶜ}}) = $ξ
        $▶sym(::Val{:ᵃ}) = ""
    end
end

"""
    interpolation_operator(from, to)

Returns the function to interpolate a field `from = (XA, YZ, ZA)`, `to = (XB, YB, ZB)`,
where the `XA`s and `XB`s are `Face()` or `Center()` instances.
"""
function interpolation_operator(from, to)
    from, to = instantiate.(from), instantiate.(to)
    x, y, z = (interpolation_code(X, Y) for (X, Y) in zip(from, to))

    if all(ξ === :ᵃ for ξ in (x, y, z))
        return identity
    else
        return eval(Symbol(:ℑ, ℑxsym(x), ℑysym(y), ℑzsym(z), x, y, z))
    end
end

"""
    interpolation_operator(::Nothing, to)

Return the `identity` interpolator function. This is needed to obtain the interpolation
operator for fields that have no intrinsic location, like numbers or functions.
"""
interpolation_operator(::Nothing, to) = identity

assumed_field_location(name) = name === :u  ? (Face, Center, Center) :
                               name === :v  ? (Center, Face, Center) :
                               name === :w  ? (Center, Center, Face) :
                               name === :uh ? (Face, Center, Center) :
                               name === :vh ? (Center, Face, Center) :
                                              (Center, Center, Center)

"""
    index_and_interp_dependencies(X, Y, Z, dependencies, model_field_names)

Returns a tuple of indices and interpolation functions to the location `X, Y, Z`
for each name in `dependencies`.

The indices correspond to the position of each dependency within `model_field_names`.

The interpolation functions interpolate the dependent field to `X, Y, Z`.
"""
function index_and_interp_dependencies(X, Y, Z, dependencies, model_field_names)
    interps = Tuple(interpolation_operator(assumed_field_location(name), (X, Y, Z))
                    for name in dependencies)

    indices = ntuple(length(dependencies)) do i
        name = dependencies[i]
        findfirst(isequal(name), model_field_names)
    end

    return indices, interps
end


text:
using DynamicalSystems, BenchmarkTools, StaticArrays

ds = Systems.towel()

function lyapunovspectrum_normal(ds::DiscreteDS, N::Real; Ttr::Real = 100)

    u = evolve(ds, Ttr)
    D = length(u)
    eom = ds.eom
    jac = ds.jacob

    # Initialization
    λ = zeros(eltype(u), D)
    Q = @SMatrix eye(eltype(u), D)
    K = copy(Q)
    # Main algorithm
    for i in 1:N
        u = eom(u)
        K = jac(u)*Q

        Q, R = DynamicalSystems.qr_sq(K)
        for i in 1:D
            λ[i] += log(abs(R[i, i]))
        end
    end
    λ./N
end

function lyapunovspectrum_static(ds::DiscreteDS, N::Real; Ttr::Real = 100)

    u = evolve(ds, Ttr)
    D = length(u)
    eom = ds.eom
    jac = ds.jacob

    # Initialization
    λ = zeros(eltype(u), D)
    Q = @SMatrix eye(eltype(u), D)
    K = copy(Q)
    # Main algorithm
    for i in 1:N
        u = eom(u)
        K = jac(u)*Q

        Q, R = qr(K)
        for i in 1:D
            λ[i] += log(abs(R[i, i]))
        end
    end
    λ./N
end

N = 100000
lyapunovspectrum_normal(ds, 100000)
lyapunovspectrum_static(ds, 100000)

a = @btime lyapunovspectrum_normal(ds, 100000);
b = @btime lyapunovspectrum_static(ds, 100000);

# qr static is 3 times faster and does 0 allocations.

ext: jl


text:
function helper_fn(z, attempt)
	single_data_name, test_fold = z
	proc_id = myid();

	helper_start_time = time()

	transaction_counts = convert(Array{Int64}, readcsv( string("../tcm_data/IRI_data_assortments/", single_data_name, "_transaction_counts.csv") ) )
	assortments = convert(Array{Int64}, readcsv( string("../tcm_data/IRI_data_assortments/", single_data_name, "_assortments.csv") ) )
	folds  = convert(Array{Int64}, readcsv( string("../tcm_data/IRI_data_assortments/", single_data_name, "_folds.csv") ) )

	train_assortments = assortments[ find(folds .!= test_fold), :]
	test_assortments = assortments[ find(folds .== test_fold), :]

	train_transaction_counts = transaction_counts[ find(folds .!= test_fold), :]
	test_transaction_counts = transaction_counts[ find(folds .== test_fold), :]

	num_train_transactions = sum(train_transaction_counts)
	num_test_transactions = sum(test_transaction_counts)

	num_train_assortments = size(train_assortments, 1)
	num_test_assortments = size(test_assortments, 1)

	N = 10

	time_limit = 1200

	perTol = 1e-6

	## RANKING-BASED MODEL ##

	ranking_lambda, orderings, ranking_loglik, ranking_elapsed_time = tcm_estimateRanking(N, train_assortments, train_transaction_counts, time_limit, perTol)
	ranking_K = length(ranking_lambda)

	test_ranking_predict, test_ranking_predict_by_class = tcm_predictRanking(N, ranking_K, ranking_lambda, orderings, test_assortments)
	ranking_KL, ranking_noinf_count, ranking_KL_avg = tcm_evaluateKL(N, test_ranking_predict, test_transaction_counts)

	avg_CS_size, weighted_avg_CS_size, max_CS_size = tcm_rankingStats(N, ranking_lambda, orderings)

	row_string = string(single_data_name,  ",",
							test_fold, ",",
							num_train_assortments, ",",
							num_test_assortments, ",",
							num_train_transactions, ",",
							num_test_transactions, ",",
							proc_id, ",",
							attempt, ",",
							"ranking", ",",
							ranking_loglik, ",",
							ranking_K, ",",
							perTol, ",",
							ranking_KL, ",",
							ranking_noinf_count, ",",
							ranking_KL_avg, ",",
							ranking_elapsed_time, ",",
							avg_CS_size, ",",
							weighted_avg_CS_size, ",",
							max_CS_size)
	row_string = string(row_string, "\n");

	@spawnat 1 begin
		print(ranking_outcsvhandle, row_string);
		flush(ranking_outcsvhandle);
	end

	## FOREST-LL, COLD-START ##

	LL_list = [4,8,16,32,64]

	initial_left, initial_right, initial_product, initial_isLeaf = tcm_createBasicForest(N)

	for leaf_limit in LL_list

		lambda, forest_left, forest_right, forest_product, forest_isLeaf, forest_loglik, forest_elapsed_time = tcm_estimateForest_leafLimit(N, train_assortments, train_transaction_counts, time_limit, perTol, leaf_limit,
																												initial_left, initial_right, initial_product, initial_isLeaf)
		K = length(lambda)

		test_forest_predict, test_forest_predict_by_class = tcm_predictForest(N, K, lambda, forest_left, forest_right, forest_product, forest_isLeaf, test_assortments)
		forest_KL, forest_noinf_count, forest_KL_avg = tcm_evaluateKL(N, test_forest_predict, test_transaction_counts)
		#forest_KL_avg = forest_KL / sum(test_transaction_counts) * 100

		simple_avg_leaves, weighted_avg_leaves, max_leaves = tcm_leafStats(lambda, forest_isLeaf)
		simple_avg_depth, weighted_avg_depth, max_depth = tcm_depthStats(lambda, forest_left, forest_right, forest_isLeaf)

		row_string = string(single_data_name,  ",",
							test_fold, ",",
							num_train_assortments, ",",
							num_test_assortments, ",",
							num_train_transactions, ",",
							num_test_transactions, ",",
							proc_id, ",",
							attempt, ",",
							"forest-LL-cold", ",",
							leaf_limit, ",",
							forest_loglik, ",",
							K, ",",
							perTol, ",",
							forest_KL, ",",
							forest_noinf_count, ",",
							forest_KL_avg, ",",
							forest_elapsed_time, ",",
							simple_avg_leaves, ",",
							weighted_avg_leaves, ",",
							max_leaves, ",",
							simple_avg_depth, ",",
							weighted_avg_depth, ",",
							max_depth)
		row_string = string(row_string, "\n");

		@spawnat 1 begin
			print(forest_outcsvhandle, row_string);
			flush(forest_outcsvhandle);

			# for k in 1:K
			# 	temp_tree = [forest_left[k] forest_right[k] forest_product[k] forest_isLeaf[k]]
			# 	forest_tree_path = string(expdirpath, "forest_cold_", single_data_name, "_", test_fold, "_depth", depth_limit, "_tree", k, ".csv")
			# 	writecsv(forest_tree_path, temp_tree)
			# end

			# forest_lambda_path = string(expdirpath, "forest_cold_", single_data_name, "_", test_fold, "_depth", depth_limit, "_lambda.csv")
			# writecsv(forest_lambda_path, lambda)
		end
	end

	## FOREST-LL, WARM-START ##
	LL_list = [4,8,16,32,64]

	initial_left, initial_right, initial_product, initial_isLeaf = tcm_convertRankingToForest(N, orderings)

	for leaf_limit in LL_list

		lambda, forest_left, forest_right, forest_product, forest_isLeaf, forest_loglik, forest_elapsed_time = tcm_estimateForest_leafLimit(N, train_assortments, train_transaction_counts, time_limit, perTol, leaf_limit,
																												initial_left, initial_right, initial_product, initial_isLeaf)
		K = length(lambda)

		test_forest_predict, test_forest_predict_by_class = tcm_predictForest(N, K, lambda, forest_left, forest_right, forest_product, forest_isLeaf, test_assortments)
		forest_KL, forest_noinf_count, forest_KL_avg = tcm_evaluateKL(N, test_forest_predict, test_transaction_counts)
		#forest_KL_avg = forest_KL / sum(test_transaction_counts) * 100

		simple_avg_leaves, weighted_avg_leaves, max_leaves = tcm_leafStats(lambda, forest_isLeaf)
		simple_avg_depth, weighted_avg_depth, max_depth = tcm_depthStats(lambda, forest_left, forest_right, forest_isLeaf)

		row_string = string(single_data_name,  ",",
							test_fold, ",",
							num_train_assortments, ",",
							num_test_assortments, ",",
							num_train_transactions, ",",
							num_test_transactions, ",",
							proc_id, ",",
							attempt, ",",
							"forest-LL-ranking-warm", ",",
							leaf_limit, ",",
							forest_loglik, ",",
							K, ",",
							perTol, ",",
							forest_KL, ",",
							forest_noinf_count, ",",
							forest_KL_avg, ",",
							forest_elapsed_time, ",",
							simple_avg_leaves, ",",
							weighted_avg_leaves, ",",
							max_leaves, ",",
							simple_avg_depth, ",",
							weighted_avg_depth, ",",
							max_depth)
		row_string = string(row_string, "\n");

		@spawnat 1 begin
			print(forest_outcsvhandle, row_string);
			flush(forest_outcsvhandle);

			# for k in 1:K
			# 	temp_tree = [forest_left[k] forest_right[k] forest_product[k] forest_isLeaf[k]]
			# 	forest_tree_path = string(expdirpath, "forest_ranking_warm_", single_data_name, "_", test_fold, "_depth", depth_limit, "_tree", k, ".csv")
			# 	writecsv(forest_tree_path, temp_tree)
			# end

			# forest_lambda_path = string(expdirpath, "forest_ranking_warm_", single_data_name, "_", test_fold, "_depth", depth_limit, "_lambda.csv")
			# writecsv(forest_lambda_path, lambda)
		end
	end

	## FOREST-CV, WARM-START ##
	#depth_list = [3,4,5,6,7]
	LL_list = [4,8,16,32,64]

	initial_left, initial_right, initial_product, initial_isLeaf = tcm_convertRankingToForest(N, orderings)

	nfolds = 4
	train_folds = folds[ find(folds .!= test_fold)]
	best_LL, best_KL_avg_cv, elapsed_time_cv = tcm_estimateForest_leafLimit_kcv_weighted_ass(N, LL_list, nfolds, train_folds, train_assortments, train_transaction_counts, time_limit, perTol, initial_left, initial_right, initial_product, initial_isLeaf)

	row_string = string(single_data_name,  ",",
							test_fold, ",",
							num_train_assortments, ",",
							num_test_assortments, ",",
							num_train_transactions, ",",
							num_test_transactions, ",",
							proc_id, ",",
							attempt, ",",
							"forest-LL-cv-ranking-warm", ",",
							best_LL, ",",
							perTol, ",",
							best_KL_avg_cv, ",",
							elapsed_time_cv)
	row_string = string(row_string, "\n");

	@spawnat 1 begin
		print(forest_cv_outcsvhandle, row_string);
		flush(forest_cv_outcsvhandle);
	end

	## FOREST-CV, COLD-START ##
	#depth_list = [3,4,5,6,7]
	LL_list = [4,8,16,32,64]

	initial_left, initial_right, initial_product, initial_isLeaf = tcm_createBasicForest(N)

	nfolds = 4
	train_folds = folds[ find(folds .!= test_fold)]
	best_LL, best_KL_avg_cv, elapsed_time_cv = tcm_estimateForest_leafLimit_kcv_weighted_ass(N, LL_list, nfolds, train_folds, train_assortments, train_transaction_counts, time_limit, perTol, initial_left, initial_right, initial_product, initial_isLeaf)

	row_string = string(single_data_name,  ",",
							test_fold, ",",
							num_train_assortments, ",",
							num_test_assortments, ",",
							num_train_transactions, ",",
							num_test_transactions, ",",
							proc_id, ",",
							attempt, ",",
							"forest-LL-cv-cold", ",",
							best_LL, ",",
							perTol, ",",
							best_KL_avg_cv, ",",
							elapsed_time_cv)
	row_string = string(row_string, "\n");

	@spawnat 1 begin
		print(forest_cv_outcsvhandle, row_string);
		flush(forest_cv_outcsvhandle);
	end

	helper_end_time = time()

	return helper_end_time - helper_start_time;

end

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_licenses: MIT Unlicense


text:
using JSON, Query, NamedTuples, DataFrames

s = """
{
    "Students": [
        {
            "Name": "John",
            "Parents": [ {"name": "Paul"}, {"name": "Mary"}]
        },
        {
            "Name": "Steward",
            "Parents": [ {"name": "George"} ]
        },
        {
            "Name": "Felix",
            "Parents": [ {"name": "Greg"}, {"name": "Susan"}]
        },
        {
            "Name": "Sara",
            "Parents": [ {"name": "Susan"}]
        }
    ]
}
"""

q = @from student in JSON.parse(s)["Students"] begin
    @from parent in student["Parents"]
    @select {Student=student["Name"], Parent=parent["name"]}
    @collect DataFrame
end

println(q)


text:
# Copyright (c) 2021 Idiap Research Institute, http://www.idiap.ch/
#  Niccolò Antonello <nantonel@idiap.ch>

export topsortperm
"""
  `topsortperm(fst::WFST, i = get_initial(fst); filter=arc->true)`

Requires `fst` to be acyclic.

Returns the topological permutation of states of `fst` starting from the `i`-th state.

Modify the `filter` function to perform the operation considering only specific arcs.

"""
function topsortperm(fst::WFST, i = get_initial(fst;single=true); kwargs...)
  if isempty(fst)
    return Bool[]
  else
    perm=zeros(Int,size(fst,1))
    acyclic=true
    perm[1] = i
    cnt = 2

    dfs = DFS(fst,i; kwargs...)

    for (p,s,n,d,e,a) in dfs
      if d # discovering new nodes
        perm[cnt] = n
        cnt += 1
      else
        if !e && !dfs.is_completed[n]
          throw(ErrorException("Topological sort failed, fst is not acyclic"))
        end
      end
    end
    return perm
  end
end

export topsort
"""
  `topsort(fst::WFST, i = get_initial(fst); filter=arc->true)`

Requires `fst` to be acyclic.

Returns an equivalent WFST to `fst` which is topologically sorted starting from the `i`-th state.

Modify the `filter` function to perform the operation considering only specific arcs.

"""
function topsort(fst::WFST, i = get_initial(fst;single=true); kwargs...)
  perm = topsortperm(fst,i;kwargs...)
  return permute_states(fst,perm)
end


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

@doc raw"""CustomResourceSubresources defines the status and scale subresources for CustomResources.

    IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources(;
        scale=nothing,
        status=nothing,
    )

    - scale::IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceScale : scale indicates the custom resource should serve a `/scale` subresource that returns an `autoscaling/v1` Scale object.
    - status::IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceStatus : status indicates the custom resource should serve a `/status` subresource. When enabled: 1. requests to the custom resource primary endpoint ignore changes to the `status` stanza of the object. 2. requests to the custom resource `/status` subresource ignore changes to anything other than the `status` stanza of the object.
"""
mutable struct IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources <: SwaggerModel
    scale::Any # spec type: Union{ Nothing, IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceScale } # spec name: scale
    status::Any # spec type: Union{ Nothing, IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceStatus } # spec name: status

    function IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources(;scale=nothing, status=nothing)
        o = new()
        validate_property(IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources, Symbol("scale"), scale)
        setfield!(o, Symbol("scale"), scale)
        validate_property(IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources, Symbol("status"), status)
        setfield!(o, Symbol("status"), status)
        o
    end
end # type IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources

const _property_map_IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources = Dict{Symbol,Symbol}(Symbol("scale")=>Symbol("scale"), Symbol("status")=>Symbol("status"))
const _property_types_IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources = Dict{Symbol,String}(Symbol("scale")=>"IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceScale", Symbol("status")=>"IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresourceStatus")
Base.propertynames(::Type{ IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources }) = collect(keys(_property_map_IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources))
Swagger.property_type(::Type{ IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources[name]))}
Swagger.field_name(::Type{ IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources }, property_name::Symbol) =  _property_map_IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources[property_name]

function check_required(o::IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources)
    true
end

function validate_property(::Type{ IoK8sApiextensionsApiserverPkgApisApiextensionsV1CustomResourceSubresources }, name::Symbol, val)
end


text:
module ReferenceSequences

export ReferenceSequence

using Bio.Seq
using IndexableBitVectors

include("nmask.jl")
include("refseq.jl")

end # module


text:
using BenchmarkTools
using QContinuousMeasurement
using Logging

# To remove output
disable_logging(Logging.Warn)

# Define a parent BenchmarkGroup to contain our suite
const SUITE = BenchmarkGroup()

# Precompile
Eff_QFI_HD_Dicke(2, 1, .01, 0.001);

# Add some benchmarks to the "trig" group
for n in (20, 40, 60)
    SUITE[n] = @benchmarkable Eff_QFI_HD_Dicke($n, 1, .5, 0.001, outsteps=10)
end


text:
module TestBags

using Test
using AsyncFinalizers.Internal: @record, SingleReaderDualBag, takemany!, takemanyto!

function test_serial()
    bag = SingleReaderDualBag{Any}()
    put!(bag, 1)
    put!(bag, 2)
    put!(bag, 3)
    items = takemany!(bag)
    @test sort!(items) == 1:3
end

function check_concurrent(ntasks; nitems = 10000)
    bag = SingleReaderDualBag{Int}()
    items = eltype(bag)[]
    haserror = Threads.Atomic{Bool}(false)
    @record(:check_concurrent_sync_begin)
    @sync begin
        for tid in 1:ntasks
            Threads.@spawn try
                @record(:check_concurrent_writer_begin, tid)
                for i in tid:ntasks:nitems
                    put!(bag, i)
                    haserror[] && break
                end
                @record(:check_concurrent_writer_end, tid)
            catch err
                haserror[] = true
                @error(
                    "check_concurrent: writer $tid failed",
                    exception = (err, catch_backtrace())
                )
            end
        end
        takemanyto!(items, bag)
        while length(items) < nitems
            @record(:check_concurrent_polling, ntaken = length(items))
            takemanyto!(items, bag)
            haserror[] && break
        end
    end
    @record(:check_concurrent_sync_end)
    @test sort!(items) == 1:nitems
end

function test_concurrent()
    @testset for ntasks in unique!([
        1,
        Threads.nthreads(),
        cld(Threads.nthreads(), 2),
        2 * Threads.nthreads(),
    ])
        check_concurrent(ntasks)
    end
end

end  # module


text:
using HDF5, JLD, Base.Test, Compat, LegacyStrings
import Compat.String

# Define variables of different types
x = 3.7
A = reshape(collect(1:15), 3, 5)
Aarray = Vector{Float64}[[1.2,1.3],[2.2,2.3,2.4]]
str = "Hello"
stringsA = Compat.ASCIIString["It", "was", "a", "dark", "and", "stormy", "night"]
stringsU = Compat.UTF8String["It", "was", "a", "dark", "and", "stormy", "night"]
strings16 = convert(Array{UTF16String}, stringsA)
strings16_2d = reshape(strings16[1:6], (2,3))
empty_string = ""
empty_string_array = Compat.ASCIIString[]
empty_array_of_strings = Compat.ASCIIString[""]
tf = true
TF = A .> 10
B = [-1.5 sqrt(2) NaN 6;
     0.0  Inf eps() -Inf]
AB = Any[A, B]
t = (3, "cat")
c = Complex64(3,7)
cint = 1+im  # issue 108
C = reinterpret(Complex128, B, (4,))
emptyA = zeros(0,2)
emptyB = zeros(2,0)
try
    global MyStruct
    type MyStruct
        len::Int
        data::Array{Float64}
        MyStruct(len::Int) = new(len)
        MyStruct(len::Int, data::Array{Float64}) = new(len, data)
    end
catch
end
ms = MyStruct(2, [3.2, -1.7])
msempty = MyStruct(5, Float64[])
sym = :TestSymbol
syms = [:a, :b]
d = Dict([(syms[1],"aardvark"), (syms[2], "banana")])
ex = quote
    function incrementby1(x::Int)
        x+1
    end
end
T = UInt8
char = 'x'
unicode_char = '\U10ffff'
α = 22
β = Any[[1, 2], [3, 4]]  # issue #93
vv = Vector{Int}[[1,2,3]]  # issue #123
typevar = Array{Int}[[1]]
typevar_lb = Vector{TypeVar(:U, Integer)}[[1]]
typevar_ub = Vector{TypeVar(:U, Int, Any)}[[1]]
typevar_lb_ub = Vector{TypeVar(:U, Int, Real)}[[1]]
undef = cell(1)
undefs = cell(2, 2)
ms_undef = MyStruct(0)
# Immutable type:
rng = 1:5
# Type with a pointer field (#84)
immutable ObjWithPointer
    a::Ptr{Void}
end
objwithpointer = ObjWithPointer(0)
# Custom BitsType (#99)
bitstype 64 MyBT
bt = reinterpret(MyBT, @compat Int64(55))
# Symbol arrays (#100)
sa_asc = [:a, :b]
sa_utf8 = [:α, :β]
# SubArray (to test tuple type params)
subarray = sub([1:5;], 1:5)
# Array of empty tuples (to test tuple type params)
arr_empty_tuple = (@compat Tuple{})[]
immutable EmptyImmutable end
emptyimmutable = EmptyImmutable()
arr_emptyimmutable = [emptyimmutable]
type EmptyType end
emptytype = EmptyType()
arr_emptytype = [emptytype]
immutable EmptyII
    x::EmptyImmutable
end
emptyii = EmptyII(EmptyImmutable())
immutable EmptyIT
    x::EmptyType
end
emptyit = EmptyIT(EmptyType())
type EmptyTI
    x::EmptyImmutable
end
emptyti = EmptyTI(EmptyImmutable())
type EmptyTT
    x::EmptyType
end
emptytt = EmptyTT(EmptyType())
immutable EmptyIIOtherField
    x::EmptyImmutable
    y::Float64
end
emptyiiotherfield = EmptyIIOtherField(EmptyImmutable(), 5.0)

# Unicode type field names (#118)
type MyUnicodeStruct☺{τ}
    α::τ
    ∂ₓα::τ
    MyUnicodeStruct☺(α::τ, ∂ₓα::τ) = new(α, ∂ₓα)
end
unicodestruct☺ = MyUnicodeStruct☺{Float64}(1.0, -1.0)
# Arrays of matrices (#131)
array_of_matrices = Matrix{Int}[[1 2; 3 4], [5 6; 7 8]]
# Tuple of arrays and bitstype
tup = (1, 2, [1, 2], [1 2; 3 4], bt)
# Empty tuple
empty_tup = ()
# Non-pointer-free immutable
immutable MyImmutable{T}
    x::Int
    y::Vector{T}
    z::Bool
end
nonpointerfree_immutable_1 = MyImmutable(1, [1., 2., 3.], false)
nonpointerfree_immutable_2 = MyImmutable(2, Any[3., 4., 5.], true)
immutable MyImmutable2
    x::Vector{Int}
    MyImmutable2() = new()
end
nonpointerfree_immutable_3 = MyImmutable2()
# Immutable with a non-concrete datatype (issue #143)
immutable Vague
    name::Compat.String
end
vague = Vague("foo")
# Immutable with a union of BitsTypes
@compat immutable BitsUnion
    x::Union{Int64, Float64}
end
bitsunion = BitsUnion(5.0)
# Immutable with a union of Types
@compat immutable TypeUnionField
    x::Union{Type{Int64}, Type{Float64}}
end
typeunionfield = TypeUnionField(Int64)
# Generic union type field
@compat immutable GenericUnionField
    x::Union{Vector{Int},Int}
end
genericunionfield = GenericUnionField(1)
# Array references
arr_contained = [1, 2, 3]
arr_ref = typeof(arr_contained)[]
push!(arr_ref, arr_contained, arr_contained)
# Object references
type ObjRefType
    x::ObjRefType
    y::ObjRefType
    ObjRefType() = new()
    ObjRefType(x, y) = new(x, y)
end
ref1 = ObjRefType()
obj_ref = ObjRefType(ObjRefType(ref1, ref1), ObjRefType(ref1, ref1))
# Immutable that requires padding between elements in array
immutable PaddingTest
    x::Int64
    y::Int8
end
padding_test = PaddingTest[PaddingTest(i, i) for i = 1:8]
# Empty arrays of various types and sizes
empty_arr_1 = Int[]
empty_arr_2 = Array(Int, 56, 0)
empty_arr_3 = Any[]
empty_arr_4 = cell(0, 97)
# Moderately big dataset (which will be mmapped)
bigdata = [1:10000;]
# BigFloats and BigInts
bigints = big(3).^(1:100)
bigfloats = big(3.2).^(1:100)
# None
@compat none = Union{}
@compat nonearr = Array(Union{}, 5)
# nothing/Void
scalar_nothing = nothing
@compat vector_nothing = Union{Int,Void}[1,nothing]

# some data big enough to ensure that compression is used:
Abig = kron(eye(10), reshape(1:400, 20, 20))
Bbig = Any[i for i=1:3000]
Sbig = "A test string "^1000

# Bitstype type parameters
type BitsParams{x}; end
bitsparamfloat  = BitsParams{1.0}()
bitsparambool   = BitsParams{true}()
bitsparamsymbol = BitsParams{:x}()
bitsparamint    = BitsParams{1}()
bitsparamuint   = BitsParams{0x01}()
bitsparamint16  = BitsParams{@compat Int16(1)}()

# Tuple of tuples
tuple_of_tuples = (1, 2, (3, 4, [5, 6]), [7, 8])

iseq(x,y) = isequal(x,y)
iseq(x::MyStruct, y::MyStruct) = (x.len == y.len && x.data == y.data)
iseq(x::MyImmutable, y::MyImmutable) = (isequal(x.x, y.x) && isequal(x.y, y.y) && isequal(x.z, y.z))
@compat iseq(x::Union{EmptyTI, EmptyTT}, y::Union{EmptyTI, EmptyTT}) = isequal(x.x, y.x)
iseq(c1::Array{Base.Sys.CPUinfo}, c2::Array{Base.Sys.CPUinfo}) = length(c1) == length(c2) && all([iseq(c1[i], c2[i]) for i = 1:length(c1)])
function iseq(c1::Base.Sys.CPUinfo, c2::Base.Sys.CPUinfo)
    for n in fieldnames(Base.Sys.CPUinfo)
        if getfield(c1, n) != getfield(c2, n)
            return false
        end
    end
    true
end
iseq(x::MyUnicodeStruct☺, y::MyUnicodeStruct☺) = (x.α == y.α && x.∂ₓα == y.∂ₓα)
@compat iseq(x::Array{Union{}}, y::Array{Union{}}) = size(x) == size(y)
macro check(fid, sym)
    ex = quote
        let tmp
            try
                tmp = read($fid, $(string(sym)))
            catch e
                warn("Error reading ", $(string(sym)))
                rethrow(e)
            end
            if !iseq(tmp, $sym)
                written = $sym
                error("For ", $(string(sym)), ", read value $tmp does not agree with written value $written")
            end
            written_type = typeof($sym)
            if typeof(tmp) != written_type
                error("For ", $(string(sym)), ", read type $(typeof(tmp)) does not agree with written type $(written_type)")
            end
        end
    end
    esc(ex)
end

# Test for equality of expressions, skipping line numbers
checkexpr(a, b) = @assert a == b
function checkexpr(a::Expr, b::Expr)
    @assert a.head == b.head
    i = 1
    j = 1
    while i <= length(a.args) && j <= length(b.args)
        if (isa(a.args[i], Expr) && a.args[i].head == :line) || isa(a.args[i], LineNumberNode)
            i += 1
            continue
        end
        if (isa(b.args[j], Expr) && b.args[j].head == :line) || isa(b.args[j], LineNumberNode)
            j += 1
            continue
        end
        checkexpr(a.args[i], b.args[j])
        i += 1
        j += 1
    end
    @assert i >= length(a.args) && j >= length(b.args)
end

for fname in ("v0.4.13-julia-0.3.jld", "v0.4.14-julia-0.4.0-dev+4483.jld")
    fidr = jldopen(joinpath(dirname(@__FILE__), fname), "r")
    @check fidr x
    @check fidr A
    dsetA = fidr["A"]
    @test ndims(dsetA) == ndims(A)
    @test size(dsetA) == size(A)
    @test size(dsetA, 1) == size(A, 1)
    @test size(dsetA, 2) == size(A, 2)
    @test size(dsetA, 3) == size(A, 3)
    @check fidr Aarray
    @check fidr str
    @check fidr stringsA
    @check fidr stringsU
    @check fidr strings16
    @check fidr strings16_2d
    @check fidr empty_string
    @check fidr empty_string_array
    @check fidr empty_array_of_strings
    @check fidr tf
    @check fidr TF
    @check fidr AB
    @check fidr t
    @check fidr c
    @check fidr cint
    @check fidr C
    @check fidr emptyA
    @check fidr emptyB
    @check fidr ms
    @check fidr msempty
    @check fidr sym
    @check fidr syms
    @check fidr d
    exr = read(fidr, "ex")   # line numbers are stripped, don't expect equality
    checkexpr(ex, exr)
    @check fidr T
    @check fidr char
    @check fidr unicode_char
    @check fidr α
    @check fidr β
    @check fidr vv
    @check fidr rng
    @check fidr typevar
    @check fidr typevar_lb
    @check fidr typevar_ub
    @check fidr typevar_lb_ub

    # Special cases for reading undefs
    undef = read(fidr, "undef")
    if !isa(undef, Array{Any, 1}) || length(undef) != 1 || isdefined(undef, 1)
        error("For undef, read value does not agree with written value")
    end
    undefs = read(fidr, "undefs")
    if !isa(undefs, Array{Any, 2}) || length(undefs) != 4 || any(map(i->isdefined(undefs, i), 1:4))
        error("For undefs, read value does not agree with written value")
    end
    ms_undef = read(fidr, "ms_undef")
    if !isa(ms_undef, MyStruct) || ms_undef.len != 0 || isdefined(ms_undef, :data)
        error("For ms_undef, read value does not agree with written value")
    end

    @check fidr bt
    @check fidr sa_asc
    @check fidr sa_utf8
    # @check fidr subarray
    @check fidr arr_empty_tuple
    @check fidr emptyimmutable
    @check fidr emptytype
    @check fidr arr_emptyimmutable
    @check fidr arr_emptytype
    @check fidr emptyii
    @check fidr emptyit
    @check fidr emptyti
    @check fidr emptytt
    @check fidr emptyiiotherfield
    @check fidr unicodestruct☺
    @check fidr array_of_matrices
    @check fidr tup
    @check fidr empty_tup
    @check fidr nonpointerfree_immutable_1
    @check fidr nonpointerfree_immutable_2
    @check fidr nonpointerfree_immutable_3
    vaguer = read(fidr, "vague")
    @test typeof(vaguer) == typeof(vague) && vaguer.name == vague.name
    @check fidr bitsunion
    @check fidr typeunionfield
    @check fidr genericunionfield

    arr = read(fidr, "arr_ref")
    @test arr == arr_ref
    @test arr[1] === arr[2]

    obj = read(fidr, "obj_ref")
    @test obj.x.x === obj.x.y == obj.y.x === obj.y.y
    @test obj.x !== obj.y

    @check fidr padding_test
    @check fidr empty_arr_1
    @check fidr empty_arr_2
    @check fidr empty_arr_3
    @check fidr empty_arr_4
    @check fidr bigdata
    @check fidr bigfloats
    @check fidr bigints
    @check fidr none
    @check fidr nonearr
    @check fidr scalar_nothing
    @check fidr vector_nothing
    @check fidr Abig
    @check fidr Bbig
    @check fidr Sbig
    @check fidr bitsparamfloat
    @check fidr bitsparambool
    @check fidr bitsparamsymbol
    @check fidr bitsparamint
    @check fidr bitsparamuint
    @check fidr tuple_of_tuples

    x1 = read(fidr, "group1/x")
    @assert x1 == Any[1]
    x2 = read(fidr, "group2/x")
    @assert x2 == Any[2]

    close(fidr)
end


text:
#==============================================================================================#
#                              runtests.jl -- TypeTree test file                               #
#==============================================================================================#

using TypeTree
using Test

#==============================================================================================#
#                                          Test Data                                           #
#==============================================================================================#

#----------------------------------------------------------------------------------------------#
#                                        Abstract types                                        #
#----------------------------------------------------------------------------------------------#

abstract type tyTree <: Any end
abstract type A <: tyTree end
abstract type Aa <: A end
abstract type Ab <: A end
abstract type B <: tyTree end
abstract type Ba <: B end

#----------------------------------------------------------------------------------------------#
#                                        Concrete types                                        #
#----------------------------------------------------------------------------------------------#

struct Aα <: A end
struct Aaα <: Aa end
struct Aaβ <: Aa end
struct Abα <: Ab end

#==============================================================================================#
#                                            Tests                                             #
#==============================================================================================#

@testset "quiz.type.abstract:                                                     " begin
    # All types
    @test length(tt(A)) == 7
    @test length(tt(Aa)) == 3
    @test length(tt(Ab)) == 2
    @test length(tt(B)) == 2
    @test length(tt(Ba)) == 1
    @test length(tt(tyTree)) == 10
    # Abstract types only
    @test length(tt(A, concrete=false)) == 3
    @test length(tt(Aa, concrete=false)) == 1
    @test length(tt(Ab, concrete=false)) == 1
    @test length(tt(B, concrete=false)) == 2
    @test length(tt(Ba, concrete=false)) == 1
    @test length(tt(tyTree, concrete=false)) == 6
end


text:
using Fronts
using Fronts.PorousModels
using Test
using Fronts._Diff: derivative
using OrdinaryDiffEq: ODEFunction, ODEProblem
using OrdinaryDiffEq.DiffEqBase: NullParameters
using StaticArrays: @SVector, SVector

@testset "Fronts.jl" begin
    include("test_dirichlet.jl")
    include("test_flowrate.jl")
    include("test_cauchy.jl")
    include("test_richards.jl")
    include("test_transform.jl")
    include("test_inverse.jl")
    include("test_PorousModels.jl")
end


text:
function get_dstdtc(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getDSTDTC", jdouble, (AbsoluteDate,), arg0)
end

function get_f10(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getF10", jdouble, (AbsoluteDate,), arg0)
end

function get_f10b(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getF10B", jdouble, (AbsoluteDate,), arg0)
end

function get_max_date(obj::JB2008InputParameters)
    return jcall(obj, "getMaxDate", AbsoluteDate, ())
end

function get_min_date(obj::JB2008InputParameters)
    return jcall(obj, "getMinDate", AbsoluteDate, ())
end

function get_s10(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getS10", jdouble, (AbsoluteDate,), arg0)
end

function get_s10b(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getS10B", jdouble, (AbsoluteDate,), arg0)
end

function get_xm10(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getXM10", jdouble, (AbsoluteDate,), arg0)
end

function get_xm10b(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getXM10B", jdouble, (AbsoluteDate,), arg0)
end

function get_y10(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getY10", jdouble, (AbsoluteDate,), arg0)
end

function get_y10b(obj::JB2008InputParameters, arg0::AbsoluteDate)
    return jcall(obj, "getY10B", jdouble, (AbsoluteDate,), arg0)
end


text:
using CUDA
using CUDAKernels
using KernelAbstractions: @kernel, @index
using Adapt: adapt_structure

using Oceananigans.Grids: on_architecture
using Oceananigans.Architectures: device, GPU, CPU, AbstractMultiArchitecture
using Oceananigans.Utils: work_layout

function set!(Φ::NamedTuple; kwargs...)
    for (fldname, value) in kwargs
        ϕ = getproperty(Φ, fldname)
        set!(ϕ, value)
    end
    return nothing
end

set!(u::Field, v) = u .= v # fallback

function set!(u::Field, f::Function)
    if architecture(u) isa GPU
        cpu_grid = on_architecture(CPU(), u.grid)
        u_cpu = Field(location(u), cpu_grid)
        f_field = field(location(u), f, cpu_grid)
        set!(u_cpu, f_field)
        set!(u, u_cpu)
    elseif architecture(u) isa CPU
        f_field = field(location(u), f, u.grid)
        set!(u, f_field)
    end

    return nothing
end

function set!(u::Field, f::Union{Array, CuArray, OffsetArray})
    f = arch_array(architecture(u), f)
    u .= f
    return nothing
end

function set!(u::Field, v::Field)
    if architecture(u) === architecture(v)
        parent(u) .= parent(v)
    else
        u_parent = parent(u)
        v_parent = parent(v)
        # If u_parent is a view, we have to convert to an Array.
        # If u_parent or v_parent is on the GPU, we don't expect
        # SubArray.
        u_parent isa SubArray && (u_parent = Array(u_parent))
        v_parent isa SubArray && (v_parent = Array(v_parent))
        copyto!(u_parent, v_parent)
    end

    return nothing
end


text:
module Types
struct Bucket
end

struct SamplingLimit
	min::UInt16
	center::UInt16
	max::UInt16
end

struct SamplingDescription
	encoded_FOV::NTuple{Float32,3}
	recon_FOV::NTuple{Float32,3}
	encoded_matrix::NTuple{UInt16,3}
	recon_matrix::NTuple{UInt16,3}
	sampling_limits::NTuple{SamplingLimit,3}
end

struct ReconBuffer
	data::Array{ComplexF32}
	trajectory::Optional{Array{Float32}}
	density::Optiona{Array{Float32}}
	headers::Vector{MRD.AcquisitionHeader}
	sampling::SamplingDescription
end

struct ReconBit
	data::ReconBuffer
	reference::Optional{ReconBuffer}
end
end


text:
const bin_dir = joinpath(@__DIR__, "bin/")
const so_dir = joinpath(@__DIR__, "so/")
const wrapperjlfile = "_wrapper.jl"
# export bin_dir

isapple  = Sys.isapple()
islinux = Sys.islinux()

function download_or_cp_file(rf::RemoteFile, pof::String="",
    rb::RemoteFiles.AbstractBackend=RemoteFiles.CURL())
    try
        download(rb, rf; quiet=true)
    catch
        fof = string(pof, "/", rf.file)
        if isfile(fof)
            cp(fof, path(rf), force=true)
        else
            throw(error("can't get file:\n      $(path(rf)) \n from $(rf.uri)"))
        end
    end
    return path(rf)
end

function _check_so(so::String)
    if isfile(so) return end
    @show so
    run(`make -C $(dirname(so)) all`)
end

# sys_suffix = isapple ? "_macos" : "_linux";


text:
"""File for processing the data as CSV output from executable .jl files"""

"""DO NOT run this file as a whole"""
"""This file is meant for block-by-block execution in REPL"""
"""Source files may need to be manually transferred to folders or source paths changed"""

using Pkg
Pkg.activate("Process")

using CSV
using DataFrames
using ProgressMeter

EDdatapath = "EDdata_2030" # CAN BE CHANGED TO BAUPATH
processpath = "Processed_output_2030"

BAUpath = "output_2015/Output_BAU/BAU_1-8760/"
DSOMpath = "output_2015/Output_DSOM/DSOM_1-8760/"
TSOMpath = "output_2015/Output_TSOM/TSOM_1-8760/"
LoadShiftpath = "output_2015/Output_LoadShift/"
ED = "ED"
BAU = "BAU"
TSOM = "TSOM"
DSOM = "DSOM"
Global = "Global"

# dailyfunctions - calculate corresponding hours for each day
    function dayhours(day::Int64)
        starthour = (day-1)*24+1
        endhour = (day)*24
        return (starthour:endhour)
    end
    function starthour(day::Int64)
        starthour = (day-1)*24+1
        return starthour
    end
    function endhour(day::Int64)
        endhour = (day)*24
        return endhour
    end
# dailyfunctions END
# calculates day by day the cost over next 14 days and saves the highest one. returns cost and startday
    function highestcost2weeks(daily_df::DataFrame) #
        highcost = 0
        index = Int64
        for i in 1:(nrow(daily_df)-14)
            currentcost = sum(daily_df[(i:i+13), :Global_Cost])
            if currentcost > highcost
                highcost = currentcost
                index = i
            end
        end
        return [highcost,index]
    end

### Read Nodes and info files
        numberofnodes = 485

### Read ED generations, calculate generation by tech after ED over the whole year and save

        ED_P_df = CSV.read(joinpath(EDdatapath, "ED_P.csv"), DataFrame)
        ED_P_R_df = CSV.read(joinpath(EDdatapath, "ED_P_R.csv"), DataFrame)
        ED_P_S_df = CSV.read(joinpath(EDdatapath, "ED_P_S.csv"), DataFrame)
        ED_all_df = vcat(ED_P_df, ED_P_R_df, ED_P_S_df)
        pp = names(unstack(ED_P_df, :fuel, :output))[4:end]
        res = names(unstack(ED_P_R_df, :fuel, :output))[4:end]
        stor = names(unstack(ED_P_S_df, :fuel, :output))[4:end]
        g = vcat(pp,res,stor)
        ED_yearlong_tech_dict = Dict()
        for p in pp
            ED_yearlong_tech_dict[p] = sum(subset(ED_P_df, :fuel => x -> x .== p).output)/(1*10^6)
        end
        for r in res
            ED_yearlong_tech_dict[r] = sum(subset(ED_P_R_df, :fuel => x -> x .== r).output)/(1*10^6)
        end
        for s in stor
            ED_yearlong_tech_dict[s] = sum(subset(ED_P_S_df, :fuel => x -> x .== s).output)/(1*10^6)
        end
        ED_yearlong_tech_df = DataFrame(tech = g, TotalGen = (ED_yearlong_tech_dict[p] for p in g) |> collect)

    CSV.write(joinpath(processpath, ED, "ED_yearlong_Generation_by_Tech_in_TWh.csv"), ED_yearlong_tech_df)     # Save

### Read CM changes per Case, calculate generation by tech after CM over the whole year
        ED_P_df = CSV.read(joinpath(EDdatapath, "ED_P.csv"), DataFrame)
        ED_P_R_df = CSV.read(joinpath(EDdatapath, "ED_P_R.csv"), DataFrame)
        ED_P_S_df = CSV.read(joinpath(EDdatapath, "ED_P_S.csv"), DataFrame)
        ED_all_df = vcat(ED_P_df, ED_P_R_df, ED_P_S_df)

        CM_P_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_up.csv"), DataFrame)
        CM_P_dn_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_dn.csv"), DataFrame)
        CM_P_R_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_R_up.csv"), DataFrame)
        CM_P_R_dn_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_R_dn.csv"), DataFrame)
        CM_P_S_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_S_up.csv"), DataFrame)

        pp = names(unstack(CM_P_up_BAU_df, :fuel, :output))[4:end]
        res = names(unstack(CM_P_R_up_BAU_df, :fuel, :output))[4:end]
        stor = names(unstack(CM_P_S_up_BAU_df, :fuel, :output))[4:end]
        g = vcat(pp,res,stor)

        Gen_BAU_P_df = copy(ED_P_df)
        Gen_BAU_P_df.output = ED_P_df.output + CM_P_up_BAU_df.output + CM_P_dn_BAU_df.output
        Gen_BAU_P_R_df = copy(ED_P_R_df)
        Gen_BAU_P_R_df.output = ED_P_R_df.output + CM_P_R_up_BAU_df.output + CM_P_R_dn_BAU_df.output
        Gen_BAU_P_S_df = copy(ED_P_S_df)
        Gen_BAU_P_S_df.output = ED_P_S_df.output + CM_P_S_up_BAU_df.output

        sum(Gen_BAU_P_R_df.output)
        sum(ED_P_R_df.output)
        sum(Gen_BAU_P_df.output)
        sum(ED_P_df.output)

        Gen_BAU_Total_Dict = Dict()
        for p in pp
            Gen_BAU_Total_Dict[p] = sum(subset(Gen_BAU_P_df, :fuel => x -> x .== p).output)/(1*10^6)
        end
        for r in res
            Gen_BAU_Total_Dict[r] = sum(subset(Gen_BAU_P_R_df, :fuel => x -> x .== r).output)/(1*10^6)
        end
        for s in stor
            Gen_BAU_Total_Dict[s] = sum(subset(Gen_BAU_P_S_df, :fuel => x -> x .== s).output)/(1*10^6)
        end
        BAU_Gen_yearlong_tech_df = DataFrame(tech = g, TotalGen = (Gen_BAU_Total_Dict[p] for p in g) |> collect)

        CM_P_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_up.csv"), DataFrame)
        CM_P_dn_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_dn.csv"), DataFrame)
        CM_P_R_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_R_up.csv"), DataFrame)
        CM_P_R_dn_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_R_dn.csv"), DataFrame)
        CM_P_S_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_S_up.csv"), DataFrame)

        Gen_TSOM_P_df = copy(ED_P_df)
        Gen_TSOM_P_df.output = ED_P_df.output + CM_P_up_TSOM_df.output + CM_P_dn_TSOM_df.output
        Gen_TSOM_P_R_df = copy(ED_P_R_df)
        Gen_TSOM_P_R_df.output = ED_P_R_df.output + CM_P_R_up_TSOM_df.output + CM_P_R_dn_TSOM_df.output
        Gen_TSOM_P_S_df = copy(ED_P_S_df)
        Gen_TSOM_P_S_df.output = ED_P_S_df.output + CM_P_S_up_TSOM_df.output

        Gen_TSOM_Total_Dict = Dict()
        for p in pp
            Gen_TSOM_Total_Dict[p] = sum(subset(Gen_TSOM_P_df, :fuel => x -> x .== p).output)/(1*10^6)
        end
        for r in res
            Gen_TSOM_Total_Dict[r] = sum(subset(Gen_TSOM_P_R_df, :fuel => x -> x .== r).output)/(1*10^6)
        end
        for s in stor
            Gen_TSOM_Total_Dict[s] = sum(subset(Gen_TSOM_P_S_df, :fuel => x -> x .== s).output)/(1*10^6)
        end
        TSOM_Gen_yearlong_tech_df = DataFrame(tech = g, TotalGen = (Gen_TSOM_Total_Dict[p] for p in g) |> collect)

        CM_P_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_up.csv"), DataFrame)
        CM_P_dn_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_dn.csv"), DataFrame)
        CM_P_R_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_R_up.csv"), DataFrame)
        CM_P_R_dn_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_R_dn.csv"), DataFrame)
        CM_P_S_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_S_up.csv"), DataFrame)

        Gen_DSOM_P_df = copy(ED_P_df)
        Gen_DSOM_P_df.output = ED_P_df.output + CM_P_up_DSOM_df.output + CM_P_dn_DSOM_df.output
        Gen_DSOM_P_R_df = copy(ED_P_R_df)
        Gen_DSOM_P_R_df.output = ED_P_R_df.output + CM_P_R_up_DSOM_df.output + CM_P_R_dn_DSOM_df.output
        Gen_DSOM_P_S_df = copy(ED_P_S_df)
        Gen_DSOM_P_S_df.output = ED_P_S_df.output + CM_P_S_up_DSOM_df.output

        Gen_DSOM_Total_Dict = Dict()
        for p in pp
            Gen_DSOM_Total_Dict[p] = sum(subset(Gen_DSOM_P_df, :fuel => x -> x .== p).output)/(1*10^6)
        end
        for r in res
            Gen_DSOM_Total_Dict[r] = sum(subset(Gen_DSOM_P_R_df, :fuel => x -> x .== r).output)/(1*10^6)
        end
        for s in stor
            Gen_DSOM_Total_Dict[s] = sum(subset(Gen_DSOM_P_S_df, :fuel => x -> x .== s).output)/(1*10^6)
        end

        DSOM_Gen_yearlong_tech_df = DataFrame(tech = g, TotalGen = (Gen_DSOM_Total_Dict[p] for p in g) |> collect)
    CSV.write(joinpath(processpath, "BAU_Gen_yearlong_tech.csv"), BAU_Gen_yearlong_tech_df)
    CSV.write(joinpath(processpath, "TSOM_Gen_yearlong_tech.csv"), TSOM_Gen_yearlong_tech_df)
    CSV.write(joinpath(processpath, "DSOM_Gen_yearlong_tech.csv"), DSOM_Gen_yearlong_tech_df)

    sum(ED_yearlong_tech_df.TotalGen)
    sum(BAU_Gen_yearlong_tech_df.TotalGen)
    sum(TSOM_Gen_yearlong_tech_df.TotalGen)
    sum(DSOM_Gen_yearlong_tech_df.TotalGen)

### Calculate Daily and hourly nodal Cost and Volume of RD (BAU)

            working_df = CSV.read(joinpath(BAUpath, "C_V_RD_N__1-8760.csv"), DataFrame)
            nodal_hourlycost_df = unstack(working_df[!,[:node, :time, :CostTotal]], :time, :CostTotal)
            nodal_hourlyvolume_df = unstack(working_df[!,[:node, :time, :VolumeTotal]], :time, :VolumeTotal)
            nodal_dailycost_df = DataFrame(node=nodal_hourlycost_df.node)
            for i in 1:365
                hours=(i-1)*24+2:i*24+1
                nodal_dailycost_df[!, string(i)] = sum(eachcol(nodal_hourlycost_df[!,hours]))
            end

            nodal_dailyvolume_df = DataFrame(node=nodal_hourlyvolume_df.node)
            for i in 1:365
                hours=(i-1)*24+2:i*24+1
                nodal_dailyvolume_df[!, string(i)] = sum(eachcol(nodal_hourlyvolume_df[!,hours]))
            end

            nodal_twoweek_cost = DataFrame(node=Int64[], RDcost = Float64[])
            for i in 1:nrow(nodal_dailycost_df)
                n = nodal_dailycost_df[i,1]
                rdcost = sum(nodal_dailycost_df[i,322:335])
                push!(nodal_twoweek_cost, [n rdcost])
            end
            nodal_twoweek_RDcost = CSV.read("NewData/nodeCoordinates.csv", DataFrame)
            nodal_twoweek_RDcost.RDcost = nodal_twoweek_cost.RDcost
            CSV.write(joinpath(processpath, BAU, "nodal_twoweek_RDcost.csv"), nodal_twoweek_RDcost)

            nodal_annual_cost = DataFrame(node=Int64[], RDcost = Float64[])
            for i in 1:nrow(nodal_dailycost_df)
                n = nodal_dailycost_df[i,1]
                rdcost = sum(nodal_dailycost_df[i,2:end])
                push!(nodal_annual_cost, [n rdcost])
            end
            nodal_annual_RDcost = CSV.read("NewData/nodeCoordinates.csv", DataFrame)
            nodal_annual_RDcost.RDcost = nodal_annual_cost.RDcost
            CSV.write(joinpath(processpath, BAU, "nodal_annual_RDcost.csv"), nodal_annual_RDcost)

    nodal_dailyvolume_df
    CSV.write(joinpath(processpath, BAU, "hourly_nodal_RD_volume.csv"), nodal_hourlyvolume_df)
    CSV.write(joinpath(processpath, BAU, "hourly_nodal_RD_cost.csv"), nodal_hourlycost_df)
    CSV.write(joinpath(processpath, BAU, "daily_nodal_RD_volume.csv"), nodal_dailyvolume_df)
    CSV.write(joinpath(processpath, BAU, "daily_nodal_RD_cost.csv"), nodal_dailycost_df)

### Load total cost of RD for scenarios TOTAL COST OVER YEAR

    BAU_info_df = CSV.read(joinpath(BAUpath, "CM_info.csv"), DataFrame)
    TSOM_info_df = CSV.read(joinpath(TSOMpath, "CM_info.csv"), DataFrame)
    DSOM_info_df = CSV.read(joinpath(DSOMpath, "CM_info.csv"), DataFrame)

    Total_daily_RD_cost_bycase_df = DataFrame(day=1:365, BAU =BAU_info_df.objective_value, TSOM = TSOM_info_df.objective_value, DSOM = DSOM_info_df.objective_value)
    Total_daily_RD_cost_bycase_df = DataFrame(day=1:365, BAU =BAU_info_df.objective_value, DSOM = DSOM_info_df.objective_value)

    Total_annual_RD_cost_bycase_df = DataFrame(case=["BAU", "DSOM", "TSOM"], cost = [sum(Total_daily_RD_cost_bycase_df.BAU), sum(Total_daily_RD_cost_bycase_df.DSOM), sum(Total_daily_RD_cost_bycase_df.TSOM),])
    CSV.write(joinpath(processpath, Global, "Total_annual_RD_cost_bycase.csv"), Total_annual_RD_cost_bycase_df)

    (sum(Total_daily_RD_cost_bycase_df.BAU)-sum(Total_daily_RD_cost_bycase_df.TSOM))/sum(Total_daily_RD_cost_bycase_df.BAU)*100
    (sum(Total_daily_RD_cost_bycase_df.BAU)-sum(Total_daily_RD_cost_bycase_df.DSOM))/sum(Total_daily_RD_cost_bycase_df.BAU)*100
    sum(Total_daily_RD_cost_bycase_df.BAU)/(10^6)
    sum(Total_daily_RD_cost_bycase_df.DSOM)/(10^6)
    sum(Total_daily_RD_cost_bycase_df.TSOM)/(10^6)
    sum(Total_daily_RD_cost_bycase_df[321:334, :BAU])
    (sum(Total_daily_RD_cost_bycase_df[321:334, :BAU])-sum(Total_daily_RD_cost_bycase_df[321:334, :TSOM]))/sum(Total_daily_RD_cost_bycase_df[321:334, :BAU])
    (sum(Total_daily_RD_cost_bycase_df[321:334, :BAU])-sum(Total_daily_RD_cost_bycase_df[321:334, :DSOM]))/sum(Total_daily_RD_cost_bycase_df[321:334, :BAU])

### Output new Load profiles, Load Duration Curves

    LoadShift_TSOM_twoweeks_df = subset(CSV.read(joinpath(TSOMpath, "CM_LoadShift.csv"), DataFrame), :time => x -> 7681 .<= x .<= 8016)
    LoadShift_TSOM_twoweeks_df.ShiftUp = zeros(nrow(LoadShift_TSOM_twoweeks_df))
    LoadShift_TSOM_twoweeks_df.ShiftDn = zeros(nrow(LoadShift_TSOM_twoweeks_df))
    for i in 1:nrow(LoadShift_TSOM_twoweeks_df)
        if LoadShift_TSOM_twoweeks_df[i, 5] > 0
                LoadShift_TSOM_twoweeks_df[i, 6] = LoadShift_TSOM_twoweeks_df[i, 5]
        elseif LoadShift_TSOM_twoweeks_df[i, 5] < 0
                LoadShift_TSOM_twoweeks_df[i, 7] = -LoadShift_TSOM_twoweeks_df[i, 5]
        end
    end
    sum(LoadShift_TSOM_twoweeks_df.ShiftUp)
    sum(LoadShift_TSOM_twoweeks_df.ShiftDn)
    LoadShift_DSOM_twoweeks_df = subset(CSV.read(joinpath(LoadShiftpath, "Loadshift.csv"), DataFrame), :hour => x -> 7681 .<= x .<= 8016)
    sum(LoadShift_DSOM_twoweeks_df.ShiftUp)
    sum(LoadShift_DSOM_twoweeks_df.ShiftDn)
    LoadShift_DSOM_df = CSV.read(joinpath(LoadShiftpath, "LoadShift.csv"), DataFrame)
    LoadShift_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_Loadshift.csv"), DataFrame)

    Loads_yearlong_BYCASE_df = DataFrame(time = 1:8760,
        OriginalLoad = (sum(LoadShift_DSOM_df[(h-1)*numberofnodes+1:h*numberofnodes, :OriginalLoad]) for h in 1:8760) |> collect,
        LoadDSOM = (sum(LoadShift_DSOM_df[(h-1)*numberofnodes+1:h*numberofnodes, :NewLoad]) for h in 1:8760) |> collect
        )

    LoadDuration_yearlong_BYCASE_df = DataFrame(id=1:8760,
        BAU = sort((sum(LoadShift_DSOM_df[(h-1)*numberofnodes+1:h*numberofnodes, :OriginalLoad]) for h in 1:8760) |> collect, rev = true),
        DSOM = sort((sum(LoadShift_DSOM_df[(h-1)*numberofnodes+1:h*numberofnodes, :NewLoad]) for h in 1:8760) |> collect, rev = true),
        TSOM = sort((sum(LoadShift_TSOM_df[(h-1)*numberofnodes+1:h*numberofnodes, :ShiftedLoad]) for h in 1:8760) |> collect, rev = true)
        )

    CSV.write(joinpath(processpath, Global, "LoadDurationCurve_yearlong_BYCASE.csv"), LoadDuration_yearlong_BYCASE_df)

    Loads_twoweeks_BYCASE_df = DataFrame(time = 7681:8016,
        OriginalLoad = (sum(LoadShift_DSOM_twoweeks_df[(h-1)*numberofnodes+1:h*numberofnodes, :OriginalLoad]) for h in 1:336) |> collect,
        LoadTSOM = (sum(LoadShift_TSOM_twoweeks_df[(h-1)*numberofnodes+1:h*numberofnodes, :ShiftedLoad]) for h in 1:336) |> collect,
        LoadDSOM = (sum(LoadShift_DSOM_twoweeks_df[(h-1)*numberofnodes+1:h*numberofnodes, :NewLoad]) for h in 1:336) |> collect
        )

    CSV.write(joinpath(processpath, Global, "Loads_twoweeks_BYCASE.csv"), Loads_twoweeks_BYCASE_df)

### Calculate daily RD for days 321-334 by fuel, GESAMTVOLUMEN RD, RD Profiles

    CM_P_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_up.csv"), DataFrame)
    CM_P_dn_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_dn.csv"), DataFrame)
    CM_P_R_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_R_up.csv"), DataFrame)
    CM_P_R_dn_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_R_dn.csv"), DataFrame)
    # CM_P_S_up_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_S_up.csv"), DataFrame)
    CM_LL_BAU_df = CSV.read(joinpath(BAUpath, "CM_P_load_lost.csv"), DataFrame)
    CM_all_up_BAU_df = vcat(CM_P_up_BAU_df, CM_P_R_up_BAU_df
    #, CM_P_S_up_BAU_df
    )
    CM_all_dn_BAU_df = vcat(CM_P_dn_BAU_df, CM_P_R_dn_BAU_df)

    CM_twoweek_RDup_BAU_df = subset(CM_all_up_BAU_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_RDdn_BAU_df = subset(CM_all_dn_BAU_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_LL_BAU_df = subset(CM_LL_BAU_df, :time => x -> 7681 .<= x .<= 8016)

    sum(subset(CM_P_R_dn_BAU_df, :time => x -> 7681 .<= x .<= 8016).output)

    TwoWeekVol_RD_BAU = DataFrame(SummeUp = sum(CM_twoweek_RDup_BAU_df.output), SummeLL = sum(CM_twoweek_LL_BAU_df.output), SummeDn = sum(CM_twoweek_RDdn_BAU_df.output))
    TotalVol_RD_BAU = DataFrame(SummeUp = sum(CM_all_up_BAU_df.output), SummeLL = sum(CM_LL_BAU_df.output), SummeDn = sum(CM_all_dn_BAU_df.output))

    CSV.write(joinpath(processpath, BAU, "BAU_RD_Gesamtvolumen.csv"), TotalVol_RD_BAU)
    sum(CM_all_up_BAU_df.output)+sum(CM_LL_BAU_df.output)
    sum(CM_all_dn_BAU_df.output)

    g = unique(CM_twoweek_RDup_BAU_df.fuel)

    RDprofile_BAU_twoweeks_df = DataFrame(hour=Int64[], fuel=String[], RD_up=Float64[], RD_down=Float64[], RD_net=Float64[])
    begin
            prog = Progress(14*24)
        for h in 7681:8016
            for z in g
                up = sum(subset(CM_twoweek_RDup_BAU_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
                dn = sum(subset(CM_twoweek_RDdn_BAU_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
                net = up + dn
            push!(RDprofile_BAU_twoweeks_df, [h, string(z), up, dn, net])
            end
            ll = sum(subset(CM_twoweek_LL_BAU_df, :time => y -> y .== h).output)
            push!(RDprofile_BAU_twoweeks_df, [h "LostLoad" ll 0 ll])
            next!(prog)
        end
    end

    RDprofile_BAU_twoweeks_df = vcat(rename!(RDprofile_BAU_twoweeks_df[!, [:hour, :fuel, :RD_up]], :RD_up => :RD), rename!(RDprofile_BAU_twoweeks_df[!, [:hour, :fuel, :RD_down]], :RD_down => :RD))

    CM_P_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_up.csv"), DataFrame)
    CM_P_dn_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_dn.csv"), DataFrame)
    CM_P_R_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_R_up.csv"), DataFrame)
    CM_P_R_dn_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_R_dn.csv"), DataFrame)
    # CM_P_S_up_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_S_up.csv"), DataFrame)
    CM_LL_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_P_load_lost.csv"), DataFrame)
    CM_LS_TSOM_df = CSV.read(joinpath(TSOMpath, "CM_Loadshift.csv"), DataFrame)
    CM_all_up_TSOM_df = vcat(CM_P_up_TSOM_df, CM_P_R_up_TSOM_df
    #, CM_P_S_up_TSOM_df
    )
    CM_all_dn_TSOM_df = vcat(CM_P_dn_TSOM_df, CM_P_R_dn_TSOM_df)

    CM_LS_TSOM_df.ShiftUp = zeros(nrow(CM_LS_TSOM_df))
    CM_LS_TSOM_df.ShiftDn = zeros(nrow(CM_LS_TSOM_df))
    for i in 1:nrow(CM_LS_TSOM_df)
        if CM_LS_TSOM_df[i, 5] > 0
                CM_LS_TSOM_df[i, 6] = CM_LS_TSOM_df[i, 5]
        elseif CM_LS_TSOM_df[i, 5] < 0
                CM_LS_TSOM_df[i, 7] = -CM_LS_TSOM_df[i, 5]
        end
    end

    sum(CM_LS_TSOM_df.ShiftDn)

    # TWO WEEK SUM OF CURTAILMENT
    sum(subset(CM_P_R_dn_TSOM_df, :time => x -> 7681 .<= x .<= 8016).output)

    CM_twoweek_RDup_TSOM_df = subset(CM_all_up_TSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_RDdn_TSOM_df = subset(CM_all_dn_TSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_LL_TSOM_df = subset(CM_LL_TSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_LS_TSOM_df = subset(CM_LS_TSOM_df, :time => x -> 7681 .<= x .<= 8016)

    TwoWeekVol_RD_TSOM = DataFrame(SummeUp = sum(CM_twoweek_RDup_TSOM_df.output), SummeLL = sum(CM_twoweek_LL_TSOM_df.output), SummeDn = sum(CM_twoweek_RDdn_TSOM_df.output))

    TotalVol_RD_TSOM = DataFrame(SummeUp = sum(CM_all_up_TSOM_df.output), SummeLL = sum(CM_LL_TSOM_df.output), SummeDn = sum(CM_all_dn_TSOM_df.output))

    CSV.write(joinpath(processpath, TSOM, "TSOM_RD_Gesamtvolumen.csv"), TotalVol_RD_TSOM)
    sum(CM_all_up_TSOM_df.output)+sum(CM_LL_TSOM_df.output)
    sum(CM_all_dn_TSOM_df.output)

    RDprofile_TSOM_twoweeks_df = DataFrame(hour=Int64[], fuel=String[], RD_up=Float64[], RD_down=Float64[], RD_net=Float64[])

    g = unique(CM_twoweek_RDup_TSOM_df.fuel)

    begin
        prog = Progress(14*24)
        for h in 7681:8016
            for z in g
                up = sum(subset(CM_twoweek_RDup_TSOM_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
                dn = sum(subset(CM_twoweek_RDdn_TSOM_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
                net = up + dn
                push!(RDprofile_TSOM_twoweeks_df, [h, string(z), up, dn, net])
            end
            ll = sum(subset(CM_twoweek_LL_TSOM_df, :time => y -> y .== h).output)
            lsup = -sum(subset(CM_twoweek_LS_TSOM_df, :time => y -> y .== h).ShiftUp)
            lsdn = sum(subset(CM_twoweek_LS_TSOM_df, :time => y -> y .== h).ShiftDn)
            push!(RDprofile_TSOM_twoweeks_df, [h "LostLoad" ll 0 ll])
            push!(RDprofile_TSOM_twoweeks_df, [h "LoadShift" lsup lsdn lsup+lsdn])
            next!(prog)
        end
    end

    RDprofile_TSOM_twoweeks_df = vcat(rename!(RDprofile_TSOM_twoweeks_df[!, [:hour, :fuel, :RD_up]], :RD_up => :RD_tot), rename!(RDprofile_TSOM_twoweeks_df[!, [:hour, :fuel, :RD_down]], :RD_down => :RD_tot))

    CM_P_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_up.csv"), DataFrame)
    CM_P_dn_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_dn.csv"), DataFrame)
    CM_P_R_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_R_up.csv"), DataFrame)
    CM_P_R_dn_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_R_dn.csv"), DataFrame)
    # CM_P_S_up_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_S_up.csv"), DataFrame)
    CM_LL_DSOM_df = CSV.read(joinpath(DSOMpath, "CM_P_load_lost.csv"), DataFrame)
    CM_LS_DSOM_df = CSV.read(joinpath(LoadShiftpath, "Loadshift.csv"), DataFrame)
    CM_all_up_DSOM_df = vcat(CM_P_up_DSOM_df, CM_P_R_up_DSOM_df
    #, CM_P_S_up_DSOM_df
    )
    CM_all_dn_DSOM_df = vcat(CM_P_dn_DSOM_df, CM_P_R_dn_DSOM_df)

    # TWO WEEK SUM OF CURTAILMENT
    sum(subset(CM_P_R_dn_DSOM_df, :time => x -> 7681 .<= x .<= 8016).output)

    CM_twoweek_RDup_DSOM_df = subset(CM_all_up_DSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_RDdn_DSOM_df = subset(CM_all_dn_DSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_LL_DSOM_df = subset(CM_LL_DSOM_df, :time => x -> 7681 .<= x .<= 8016)
    CM_twoweek_LS_DSOM_df = subset(CM_LS_DSOM_df, :hour => x -> 7681 .<= x .<= 8016)

    TwoWeekVol_RD_DSOM = DataFrame(SummeUp = sum(CM_twoweek_RDup_DSOM_df.output), SummeLL = sum(CM_twoweek_LL_DSOM_df.output), SummeDn = sum(CM_twoweek_RDdn_DSOM_df.output))
    TotalVol_RD_DSOM = DataFrame(SummeUp = sum(CM_all_up_DSOM_df.output), SummeLL = sum(CM_LL_DSOM_df.output), SummeDn = sum(CM_all_dn_DSOM_df.output))

    CSV.write(joinpath(processpath, DSOM, "DSOM_RD_Gesamtvolumen.csv"), TotalVol_RD_DSOM)

    g = unique(CM_twoweek_RDup_DSOM_df.fuel)

    RDprofile_DSOM_twoweeks_df = DataFrame(hour=Int64[], fuel=String[], RD_up=Float64[], RD_down=Float64[], RD_net=Float64[])

    for h in 7681:8016
        for z in g
            up = sum(subset(CM_twoweek_RDup_DSOM_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
            dn = sum(subset(CM_twoweek_RDdn_DSOM_df, :fuel => x -> x .== z, :time => y -> y .== h).output)
            net = up + dn
        push!(RDprofile_DSOM_twoweeks_df, [h, string(z), up, dn, net])
        end
        ll = sum(subset(CM_twoweek_LL_DSOM_df, :time => y -> y .== h).output)
        lsup = -sum(subset(CM_twoweek_LS_DSOM_df, :hour => y -> y .== h).ShiftUp)
        lsdn = sum(subset(CM_twoweek_LS_DSOM_df, :hour => y -> y .== h).ShiftDn)
        push!(RDprofile_DSOM_twoweeks_df, [h "LostLoad" ll 0 ll])
        push!(RDprofile_DSOM_twoweeks_df, [h "LoadShift" lsup lsdn lsup+lsdn])
    end

    RDprofile_DSOM_twoweeks_df = vcat(rename!(RDprofile_DSOM_twoweeks_df[!, [:hour, :fuel, :RD_up]], :RD_up => :RD_tot), rename!(RDprofile_DSOM_twoweeks_df[!, [:hour, :fuel, :RD_down]], :RD_down => :RD_tot))

    TotalVol_RD_Global_yearlong_BYCASE_df = insertcols!(vcat(TotalVol_RD_BAU, TotalVol_RD_DSOM, TotalVol_RD_TSOM), 1, :Case => ["BAU","DSOM","TSOM"])

    CSV.write(joinpath(processpath, Global, "Total_RD_Vol_yearlong_BYCASE.csv"), TotalVol_RD_Global_yearlong_BYCASE_df)
    CSV.write(joinpath(processpath, BAU, "RDprofile_BAU_twoweeks.csv"), RDprofile_BAU_twoweeks_df)
    CSV.write(joinpath(processpath, "Sens_quart", "RDprofile_DSOM_twoweeks.csv"), RDprofile_DSOM_twoweeks_df)
    CSV.write(joinpath(processpath, "Sens_quart", "RDprofile_TSOM_twoweeks.csv"), RDprofile_TSOM_twoweeks_df)

    (sum(CM_all_dn_BAU_df.output) - sum(CM_all_dn_DSOM_df.output))/sum(CM_all_dn_BAU_df.output)
    (sum(CM_all_dn_BAU_df.output) - sum(CM_all_dn_TSOM_df.output))

    BAU_info_df = CSV.read(joinpath(BAUpath, "CM_info.csv"), DataFrame)
    TSOM_info_df = CSV.read(joinpath(TSOMpath, "CM_info.csv"), DataFrame)
    DSOM_info_df = CSV.read(joinpath(DSOMpath, "CM_info.csv"), DataFrame)

    """CALCULATE SAVE PER MWh OF LOAD SHIFTING"""

    Save_per_MWh_BYCASE_df = DataFrame(Case=["TSOM", "DSOM"], Save=
        [(sum(BAU_info_df.objective_value)-sum(TSOM_info_df.objective_value))/sum(CM_LS_TSOM_df.ShiftUp), (sum(BAU_info_df.objective_value)-sum(DSOM_info_df.objective_value))/sum(CM_LS_DSOM_df.ShiftUp)])
    Save_per_MWh_BYCASE_twoweeks_df = DataFrame(Case=["TSOM", "DSOM"], Save=
        [(sum(BAU_info_df.objective_value[321:334])-sum(TSOM_info_df.objective_value[321:334]))/sum(CM_LS_TSOM_df.ShiftUp[320*numberofnodes*24+1:334*numberofnodes*24]),
        (sum(BAU_info_df.objective_value[321:334])-sum(DSOM_info_df.objective_value[321:334]))/sum(CM_LS_DSOM_df.ShiftUp[320*numberofnodes*24+1:334*numberofnodes*24])])
    CSV.write(joinpath(processpath, "Sens_quart", "Save_per_MWh_twoweeks_BYCASE.csv"), Save_per_MWh_BYCASE_twoweeks_df)

nod_vol_dict = Dict((nodal_dailyvolume_df[i, 1] => Dict((j => nodal_dailyvolume_df[i, j+1]) for j in 1:365)) for i in 1:nrow(nodal_dailyvolume_df))
nod_cost_dict = Dict((nodal_dailycost_df[i, 1] => Dict((j => nodal_dailycost_df[i, j+1]) for j in 1:365)) for i in 1:nrow(nodal_dailycost_df))

### Calculate Daily and hourly global Cost and Volume of RD, sum for year and twoweeks

        working_df = CSV.read(joinpath(BAUpath, "C_V_RD_N__1-8760.csv"), DataFrame)
        daily_global_BAU_df = DataFrame(day=1:365,
            Global_Cost=((sum((working_df[s, :CostTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            )
        hourly_global_BAU_df = DataFrame(hour=1:8760,
            Total_RD_Cost=((sum((working_df[s, :CostTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_PP=((sum((working_df[s, :VolumePP]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            )
            highcostweeks=highestcost2weeks(daily_global_BAU_df)
            highcoststartday = Int64(highcostweeks[2])
            period_string = string(highcoststartday) * "_to_" * string(highcoststartday+13)
            print("the highest cost over two weeks is " * string(highcostweeks[1]) * " in the two weeks starting with day " * string(highcoststartday))
        working_df = CSV.read(joinpath(TSOMpath, "C_V_RD_N.csv"), DataFrame)
        daily_global_TSOM_df = DataFrame(day=1:365,
            Global_Cost=((sum((working_df[s, :CostTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            )
        hourly_global_TSOM_df = DataFrame(hour=1:8760,
            Total_RD_Cost=((sum((working_df[s, :CostTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_PP=((sum((working_df[s, :VolumePP]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            )
        working_df = CSV.read(joinpath(DSOMpath, "C_V_RD_N__1-8760.csv"), DataFrame)
        daily_global_DSOM_df = DataFrame(day=1:365,
            Global_Cost=((sum((working_df[s, :CostTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
            )
        hourly_global_DSOM_df = DataFrame(hour=1:8760,
            Total_RD_Cost=((sum((working_df[s, :CostTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Vol_PP=((sum((working_df[s, :VolumePP]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in ((i-1)*numberofnodes+1:i*numberofnodes)) for i in 1:8760) |> collect),
            )

        GlobalVol_Curtailment_BAU_hourly_twoweeks_df = subset(hourly_global_BAU_df[!, [:hour, :Global_Vol_Res]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)
        GlobalVol_Curtailment_BAU_hourly_twoweeks_df[!, 2] = GlobalVol_Curtailment_BAU_hourly_twoweeks_df[!, 2]*-1
        GlobalVol_Curtailment_DSOM_hourly_twoweeks_df = subset(hourly_global_DSOM_df[!, [:hour, :Global_Vol_Res]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)
        GlobalVol_Curtailment_DSOM_hourly_twoweeks_df[!, 2] = GlobalVol_Curtailment_DSOM_hourly_twoweeks_df[!, 2]*-1
        GlobalVol_Curtailment_TSOM_hourly_twoweeks_df = subset(hourly_global_TSOM_df[!, [:hour, :Global_Vol_Res]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)
        GlobalVol_Curtailment_TSOM_hourly_twoweeks_df[!, 2] = GlobalVol_Curtailment_TSOM_hourly_twoweeks_df[!, 2]*-1

        sum(GlobalVol_Curtailment_BAU_hourly_twoweeks_df.Global_Vol_Res)
        sum(GlobalVol_Curtailment_DSOM_hourly_twoweeks_df.Global_Vol_Res)
        sum(GlobalVol_Curtailment_TSOM_hourly_twoweeks_df.Global_Vol_Res)

        GlobalCost_BAU_hourly_twoweeks = subset(hourly_global_BAU_df[!, [:hour, :Total_RD_Cost]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)
        GlobalCost_DSOM_hourly_twoweeks = subset(hourly_global_DSOM_df[!, [:hour, :Total_RD_Cost]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)
        GlobalCost_TSOM_hourly_twoweeks = subset(hourly_global_TSOM_df[!, [:hour, :Total_RD_Cost]], :hour => x -> (321-1)*24+1 .<= x .<= 334*24)

        GlobalVol_BAU_Curtailment_yearlong = sum(daily_global_BAU_df.Global_Vol_Res)
        GlobalVol_DSOM_Curtailment_yearlong = sum(daily_global_DSOM_df.Global_Vol_Res)
        GlobalVol_TSOM_Curtailment_yearlong = sum(daily_global_TSOM_df.Global_Vol_Res)
        Annual_Curtailment_BYCASE_df = DataFrame(BAU=[GlobalVol_BAU_Curtailment_yearlong], DSOM=[GlobalVol_DSOM_Curtailment_yearlong], TSOM=[GlobalVol_TSOM_Curtailment_yearlong],)

        CSV.write(joinpath(processpath, Global, "Annual_Curtailment_BYCASE.csv"), Annual_Curtailment_BYCASE_df)

        CSV.write(joinpath(processpath, BAU, "GlobalVol_Curtailment_BAU_hourly_twoweeks.csv"), GlobalVol_Curtailment_BAU_hourly_twoweeks_df)
        CSV.write(joinpath(processpath, DSOM, "GlobalVol_Curtailment_DSOM_hourly_twoweeks.csv"), GlobalVol_Curtailment_DSOM_hourly_twoweeks_df)
        CSV.write(joinpath(processpath, TSOM, "GlobalVol_Curtailment_TSOM_hourly_twoweeks.csv"), GlobalVol_Curtailment_TSOM_hourly_twoweeks_df)

        CSV.write(joinpath(processpath, BAU, "GlobalCost_BAU_hourly_twoweeks.csv"), GlobalCost_BAU_hourly_twoweeks)
        CSV.write(joinpath(processpath, DSOM, "GlobalCost_DSOM_hourly_twoweeks.csv"), GlobalCost_DSOM_hourly_twoweeks)
        CSV.write(joinpath(processpath, TSOM, "GlobalCost_TSOM_hourly_twoweeks.csv"), GlobalCost_TSOM_hourly_twoweeks)

### Calculate total RD cost over two weeks

    Global_RD_Cost_twoweeks_df = DataFrame()
    working_df = CSV.read(joinpath(BAUpath, "C_V_RD_N__1-8760.csv"), DataFrame)
    Global_RD_Cost_twoweeks_df[!, :BAU] = [sum(subset(working_df, :time => x -> (321-1)*24+1 .<= x .<= 334*24).CostTotal)]
    working_df = CSV.read(joinpath(DSOMpath, "C_V_RD_N__1-8760.csv"), DataFrame)
    Global_RD_Cost_twoweeks_df[!, :DSOM] = [sum(subset(working_df, :time => x -> (321-1)*24+1 .<= x .<= 334*24).CostTotal)]
    working_df = CSV.read(joinpath(TSOMpath, "C_V_RD_N.csv"), DataFrame)
    Global_RD_Cost_twoweeks_df[!, :TSOM] = [sum(subset(working_df, :time => x -> (321-1)*24+1 .<= x .<= 334*24).CostTotal)]
    working_df=nothing
    (Global_RD_Cost_twoweeks_df.BAU[1]-Global_RD_Cost_twoweeks_df.DSOM[1])/Global_RD_Cost_twoweeks_df.BAU[1]
    (Global_RD_Cost_twoweeks_df.BAU[1]-Global_RD_Cost_twoweeks_df.TSOM[1])/Global_RD_Cost_twoweeks_df.BAU[1]

    CSV.write(joinpath(processpath, Global, "GlobalCost_compare_twoweeks.csv"), Global_RD_Cost_twoweeks_df)

    (Global_RD_Cost_twoweeks_df.BAU[1]-Global_RD_Cost_twoweeks_df.TSOM[1])/Global_RD_Cost_twoweeks_df.BAU[1]
    (Global_RD_Cost_twoweeks_df.BAU[1]-Global_RD_Cost_twoweeks_df.DSOM[1])/Global_RD_Cost_twoweeks_df.BAU[1]

daily_global_BAU_df = DataFrame(day=1:365,
    Global_Cost=((sum((working_df[s, :CostTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
    Global_Vol_Res=((sum((working_df[s, :VolumeRes]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
    Global_Volume=((sum((working_df[s, :VolumeTotal]) for s in (endhour(i-1)*numberofnodes+1):endhour(i)*numberofnodes) for i in 1:365) |> collect),
    )

sum(subset(working_df, :time => x -> (321-1)*24+1 .<= x .<= 334*24).CostTotal)

highcost_hourly_df = copy(working_df[numberofnodes*24*(highcoststartday-1)+1:(numberofnodes*24*(highcoststartday+13)), [:node, :time, :CostTotal]])
highcost_hourly_df = unstack(highcost_hourly_df, :time, :CostTotal)
highcost_hourly_df[!, :total] = ((sum(highcost_hourly_df[i, 2:end]) for i in 1:numberofnodes) |> collect)
sum(highcost_hourly_df[!, :total])

nodes_df[!, :totalcost] = highcost_hourly_df[!, :total]
nodes_df
nodes_sorted_df = sort(nodes_df, [:totalcost], rev=true)

sum(nodes_df[!, :totalcost])
sum(nodes_sorted_df[!, :totalcost])

daily_global_df

GlobalVol_RD_twoweeks_BYCASE_df = vcat(TwoWeekVol_RD_BAU, TwoWeekVol_RD_DSOM, TwoWeekVol_RD_TSOM)
GlobalVol_RD_twoweeks_BYCASE_df.Case = ["BAU","DSOM","TSOM"]
GlobalVol_RD_twoweeks_BYCASE_df
CSV.write(joinpath(processpath, Global, "GlobalVol_RD_twoweeks_BYCASE.csv"), GlobalVol_RD_twoweeks_BYCASE_df)

sum(subset(CM_LL_BAU_df, :output => x -> x .> 0).output)

))


text:
### A Pluto.jl notebook ###
# v0.17.2

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el)
        el
    end
end

# ╔═╡ 6e33e012-4b3d-11eb-30ea-2b5f82c16b8e
begin
	psg = "C:/Users/dusty/Documents/PlaymatProjects/PlaymatSimulator"
	gd = "C:/Users/dusty/Documents/PlaymatProjects/PlaymatGames/Mage"

	import Pkg
	Pkg.activate(psg)

	using PlaymatSimulator
	using Serialization
	using PlutoUI
	using Colors
end

# ╔═╡ fcaed323-b589-4b6c-a052-e0eedb4ec552
md"""
# 🧙 Welcome to Mage.jl
Choose a game variant below to get started ☮
"""

# ╔═╡ 65e7c338-4b42-11eb-014a-017404d2040a
begin
	GAMES = ["EDH"=>"EDH"]
	#[ gn=>gn for gn in readdir(GAME_DIR) if !(occursin(".", gn)) ]

	pushfirst!(GAMES,""=>"")

	md"""
	Select a game to play: $( @bind GAME_NAME Select(GAMES) )
	"""
end

# ╔═╡ e86a6936-4b43-11eb-33b9-3ddefed762c7
if GAME_NAME != ""
	DECKS = [ gn=>gn for gn in readdir("$GAME_NAME/decks") if !(occursin(".", gn)) ]
	pushfirst!(DECKS,""=>"")

	md"""
	Select a deck to play: $( @bind DECK_NAME Select(DECKS) )
	"""
end

# ╔═╡ e11fc3c8-4b44-11eb-0c25-3d787ef9da0a
if GAME_NAME != "" && DECK_NAME != ""
	loadGame = true

	md"""
	##### Simulating an $GAME_NAME game with a $DECK_NAME deck, please wait...

	*press alt+F4 to quit*
	"""
else
	loadGame = false
	nothing
end

# ╔═╡ eb1a3e94-4cca-11eb-2b39-a5794006c375
if @isdefined(loadGame) && loadGame

	UserSettings = Dict(
		:GAME_NAME => GAME_NAME,
		:DECK_NAME => DECK_NAME,
	)

	serialize("../../PlaymatSimulator/tmp/user_selection.jls", UserSettings)
end;

# ╔═╡ 2f0b4ab3-3039-4d97-b369-b71c9fda33a6
if @isdefined(loadGame) && loadGame && @isdefined(UserSettings) && "$gd/$GAME_NAME/$GAME_NAME.jl" |> isfile
	rungame("$gd/$GAME_NAME/$GAME_NAME.jl")
end

# ╔═╡ 1816db80-4b62-11eb-0a61-91f0662ae287
html"""

"""

# ╔═╡ 386f2d76-4ef1-11eb-032b-1715050b4952
md"""
*Want to develop your own PlaymatSimulator games? Check out the docs!
"""

# ╔═╡ ff137f4c-cb47-4b30-b261-0d54ba88bac7
# page/cell width control
"""


text:
# code for Video 5.3:
julia> using RDatasets, Distributions, DecisionTree, MLBase
julia> iris = dataset("datasets", "iris")
julia> features = convert(Array, iris[:, 1:4]) # measurements of length
# 1) sepal_length, 2) sepal_width, 3) petal_length, 4) petal_width
julia> labels = convert(Array, iris[:, 5])     # species

### Train model, make predictions on test records
julia> train = rand(Bernoulli(0.75), nrow(iris)) .== 1
julia> test = !train

julia> model = build_tree(labels[train], features[train, :])
Decision Tree
Leaves: 9
Depth:  5

julia> model = prune_tree(model, 0.7)
Decision Tree
Leaves: 8
Depth:  5

julia> print_tree(model)
Feature 3, Threshold 3.0
L-> setosa : 35/35
R-> Feature 4, Threshold 1.8
    L-> Feature 3, Threshold 5.1
        L-> Feature 1, Threshold 5.0
            L-> Feature 2, Threshold 2.5
                L-> versicolor : 1/1
                R-> virginica : 1/1
            R-> versicolor : 36/36
        R-> virginica : 3/4
    R-> Feature 3, Threshold 4.9
        L-> Feature 1, Threshold 6.0
            L-> versicolor : 1/1
            R-> virginica : 2/2
        R-> virginica : 39/39

julia> predictions = apply_tree(model, features[test,:])
37-element Array{Any,1}:
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "setosa"
 "versicolor"
 "versicolor"
 "versicolor"
 ⋮
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"
 "virginica"

julia> species = ["setosa", "virginica", "versicolor"]

 # let's calculate the ROC of these predictions:
julia> lm = labelmap(species)
LabelMap (with 3 labels):
[1] setosa
[2] virginica
[3] versicolor

julia> labels_test = labelencode(lm, labels[test])
37-element Array{Int64,1}:
 1
 1
 1
 1
 1
 1
 1
 1
 1
 3
 3
 3
 ⋮
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2

julia> predictions_test = labelencode(lm, convert(Array{ASCIIString}, predictions))
37-element Array{Int64,1}:
 1
 1
 1
 1
 1
 1
 1
 1
 1
 3
 3
 3
 ⋮
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2

julia> cor(labels_test, predictions_test) # 0.910473114440612
julia> R2(labels_test, predictions_test) # 0.8145363408521302

julia> for i in 1:length(labels_test)
         println(labels_test[i], " - ", predictions_test[i])
       end
1 - 1
1 - 1
1 - 1
1 - 1
1 - 1
1 - 1
1 - 1
1 - 1
1 - 1
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 3
3 - 1       <-- wrong prediction!
3 - 3
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2
2 - 2

julia> using Winston
julia> scatter(labels_test, predictions_test, "^")

julia> r = roc(labels_test, predictions_test)
MLBase.ROCNums{Int64}
  p = 37
  n = 0
  tp = 36
  tn = 0
  fp = 0
  fn = 0

true_positive_rate(r) # 0.972972972972973
true_negative_rate(r) # NaN
precision(r)          # 1.0
recall(r)             # 0.972972972972973

julia> nfoldCV_tree(labels[train], features[train,:], 0.7, 10)

Fold 1
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 4  0  0
 0  6  0
 0  0  2

Accuracy: 1.0
Kappa:    1.0

Fold 2
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 3  0  0
 0  5  1
 0  0  3

Accuracy: 0.9166666666666666
Kappa:    0.870967741935484

Fold 3
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 8  0  0
 0  1  0
 0  0  3

 Accuracy: 1.0
Kappa:    1.0

Fold 4
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 1  0  0
 0  5  0
 0  1  5

Accuracy: 0.9166666666666666
Kappa:    0.8554216867469879

Fold 5
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 3  0  0
 0  2  0
 0  2  5

Accuracy: 0.8333333333333334
Kappa:    0.7391304347826089

Fold 6
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 3  0  0
 0  3  1
 0  0  5

Accuracy: 0.9166666666666666
Kappa:    0.870967741935484

Fold 7
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 3  0  0
 1  2  0
 0  1  5

Accuracy: 0.8333333333333334
Kappa:    0.7419354838709679

Fold 8
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 6  0  0
 0  1  2
 0  2  1

 Accuracy: 0.6666666666666666
Kappa:    0.4666666666666666

Fold 9
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 4  0  0
 0  5  0
 0  1  2

Accuracy: 0.9166666666666666
Kappa:    0.8695652173913044

Fold 10
Classes:  Any["setosa","versicolor","virginica"]
Matrix:   3x3 Array{Int64,2}:
 6  0  0
 0  4  0
 0  0  2

Accuracy: 1.0
Kappa:    1.0

Mean Accuracy: 0.9
10-element Array{Float64,1}:
 1.0
 0.916667
 1.0
 0.916667
 0.833333
 0.916667
 0.833333
 0.666667
 0.916667
 1.0

# If we do this for a range of purities:
accuracy_cv = DataFrame()
accuracy_cv[:purity]   = collect(0.1:0.025:1.0)
accuracy_cv[:accuracy] = map(x -> mean(nfoldCV_tree(labels[train], features[train,:], x, 10)), accuracy_cv[:purity])

## Folding results ommited ##

Mean Accuracy: 0.9166666666666666
37-element DataArrays.DataArray{Any,1}:
 0.183333
 0.258333
 0.233333
 0.25
 0.191667
 0.216667
 0.216667
 0.2
 0.225
 0.233333
 0.483333
 0.583333
 ⋮
 0.916667
 0.908333
 0.908333
 0.908333
 0.908333
 0.925
 0.916667
 0.908333
 0.925
 0.916667
 0.916667
 0.916667

julia> using Gadfly
julia> plot(accuracy_cv, x = :purity, y = :accuracy, Geom.point) # see image accuracy_cv.png

max_stars_repo_path: Section 5/5_3.jl

max_stars_repo_name: lytemar/Julia-for-Data-Science-Video

max_issues_repo_path: Section 5/5_3.jl

max_issues_repo_name: lytemar/Julia-for-Data-Science-Video

max_forks_repo_path: Section 5/5_3.jl

max_forks_repo_name: lytemar/Julia-for-Data-Science-Video


text:
using OffsetArrays

@testset "Iteration Bound Tests" begin
  function masktest_incr1_none1start!(y,x)
    @turbo for i ∈ 0:20
      y[i] = x[i] + 2
    end
  end
  function masktest_incr2_none1start!(y,x)
    @turbo for i ∈ 0:2:20
      y[i] = x[i] + 2
    end
  end

  x = OffsetVector(rand(24),0:23);
  y = copy(x);
  masktest_incr1_none1start!(y,x)
  @test y == x .+ ifelse.(axes(x,1) .≤ 20, 2, 0)
  @turbo y .= x;
  @test y == x
  masktest_incr2_none1start!(y,x)
  @test y == x .+ ifelse.((axes(x,1) .≤ 20) .& iseven.(axes(x,1)), 2, 0)

  # issue #290
  function my_gemm_noturbo!(out, s::Matrix{UInt8}, V, srows, scols, Vcols, μ)
    k = srows >> 2
    rem = srows & 3
    fill!(out, 0)

    for c in 1:Vcols
      for i in 1:scols
        for l in 1:k
          block = s[l, i]
          for p in 1:4
            Aij = (block >> (2 * (p - 1))) & 3
            out[i, c] += (((Aij >= 2) + (Aij == 3) + (Aij == 1) * μ[i]) *
                          V[4 * (l - 1) + p, c])
          end
        end
      end
    end
    nothing
  end
  function my_gemm!(out, s::Matrix{UInt8}, V, srows, scols, Vcols, μ)
    k = srows >> 2
    rem = srows & 3
    fill!(out, 0)

    @avx for c in 1:Vcols
      for i in 1:scols
        for l in 1:k
          block = s[l, i]
          for p in 1:4
            Aij = (block >> (2 * (p - 1))) & 3
            out[i, c] += (((Aij >= 2) + (Aij == 3) + (Aij == 1) * μ[i]) *
                          V[4 * (l - 1) + p, c])
          end
        end
      end
    end
    nothing
  end

  out_true = Matrix{Float64}(undef, 100, 100);
  out_test1 = similar(out_true);
  # out_test2 = zeros(100, 100)
  μ = rand(100);
  s = rand(UInt8, 100, 100);
  V = rand(400, 100);

  my_gemm_noturbo!(out_true, s, V, 400, 100, 100, μ)
  my_gemm!(out_test1, s, V, 400, 100, 100, μ);
  @test out_true ≈ out_test1
end


text:
using JSON
using Reactive
using Compat
import Compat.String

import Base: writemime
import Interact: update_view, Slider, Widget, InputWidget, Latex, HTML, recv_msg,
                 statedict, viewdict, Layout, Box,
                 Progress, Checkbox, Button, ToggleButton, Textarea, Textbox, Options

export mimewritable, writemime

const ijulia_js = readstring(joinpath(dirname(@__FILE__), "ijulia.js"))

if displayable("text/html")
    display("text/html", """
     <div id="interact-js-shim">
         <script charset="utf-8">$(ijulia_js)</script>
         <script>
             window.interactLoadedFlag = true
            \$("#interact-js-shim").bind("destroyed", function () {
                if (window.interactLoadedFlag) {
                    console.warn("JavaScript required by Interact will be removed if you remove this cell or run using Interact more than once.")
                }
            })
            \$([IPython.events]).on("kernel_starting.Kernel kernel_restarting.Kernel", function () { window.interactLoadedFlag = false })
        </script>
     </div>""")
end

import IJulia
import IJulia: metadata, display_dict
using  IJulia.CommManager
import IJulia.CommManager: register_comm
import Base: show, mimewritable

const comms = Dict{Signal, Comm}()

function get_data_dict(value, mimetypes)
    dict = Dict{Compat.ASCIIString, Compat.String}()
    for m in mimetypes
        if mimewritable(m, value)
            dict[m] = stringmime(m, value)
        elseif m == "text/latex" && mimewritable("application/x-latex", value)
            dict[string("text/latex")] =
                stringmime("application/x-latex", value)
        else
            warn("IPython seems to be requesting an unavailable mime type")
        end
    end
    return dict
end

function init_comm(x::Signal)
    if !haskey(comms, x)
        subscriptions = Dict{Compat.ASCIIString, Int}()
        function handle_subscriptions(msg)
            if haskey(msg.content, "data")
                action = get(msg.content["data"], "action", "")
                if action == "subscribe_mime"
                    mime = msg.content["data"]["mime"]
                    subscriptions[mime] = get(subscriptions, mime, 0) + 1
                elseif action == "unsubscribe_mime"
                    mime = msg.content["data"]["mime"]
                    subscriptions[mime] = get(subscriptions, mime, 1) - 1
                end
            end
        end
        # One Comm channel per signal object
        comm = Comm(:Signal)
        comms[x] = comm   # Backend -> Comm
        # Listen for mime type registrations
        comm.on_msg = handle_subscriptions
        # prevent resending the first time?
        function notify(value)
            mimes = keys(filter((k,v) -> v > 0, subscriptions))
            if length(mimes) > 0
                send_comm(comm, @compat Dict(:value =>
                                 get_data_dict(value, mimes)))
            end
        end
        preserve(map(notify, x))
    else
        comm = comms[x]
    end

    return comm
end

function metadata(x::Signal)
    comm = init_comm(x)
    return @compat Dict("reactive"=>true,
                        "comm_id"=>comm.id)
end

function IJulia.display_dict(x::Signal)
    IJulia.display_dict(value(x))
end

# Render the value of a signal.
mimewritable(m::MIME, s::Signal) =
    mimewritable(m, s.value)

# fixes ambiguity warnings
@compat function Base.show(io::IO, m::MIME"text/plain", s::Signal)
    Base.show(io, m, s.value)
end

@compat function Base.show(io::IO, m::MIME"text/csv", s::Signal)
    Base.show(io, m, s.value)
end

@compat function Base.show(io::IO, m::MIME"text/tab-separated-values", s::Signal)
    Base.show(io, m, s.value)
end

@compat function Base.show(io::IO, m::MIME, s::Signal)
    Base.show(io, m, s.value)
end

@compat function Base.show(io::IO, ::MIME"text/html", w::Widget)
    #widget display is handled in metadata
    nothing
end

## This is for our own widgets.
function register_comm(comm::Comm{:InputWidget}, msg)
    w_id = msg.content["data"]["widget_id"]
    comm.on_msg = (msg) -> recv_msg(w, msg.content["data"]["value"])
end

JSON.lower(s::Signal) = s.value

##################### IPython IPEP 23: Backbone.js Widgets #################

## ButtonView ✓
## CheckboxView ✓
## DropdownView ✓
## FloatSliderView ✓
## FloatTextView ✓
## IntSliderView ✓
## IntTextView ✓
## ProgressView
## RadioButtonsView ✓
## SelectView ✓
## TextareaView ✓
## TextView ✓
## ToggleButtonsView ✓
## ToggleButtonView ✓
## AccordionView W
## ContainerView W
## PopupView W
## TabView W

# Interact -> IJulia view names
widget_class(::HTML) = "HTML"
widget_class(::Layout) = "Layout"
widget_class(::Box) = "Box"
widget_class(::Latex) = "Label"
widget_class(::Progress) = "Progress"
widget_class{T<:Integer}(::Slider{T}) = "IntSlider"
widget_class(::Button) = "Button"
widget_class(::Textarea) = "Textarea"
widget_class{T<:AbstractFloat}(::Slider{T}) = "FloatSlider"
widget_class{T<:Integer}(::Textbox{T}) = "IntText"
widget_class(::Checkbox) = "Checkbox"
widget_class(::ToggleButton) = "ToggleButton"
widget_class{T<:AbstractFloat}(::Textbox{T}) = "FloatText"
widget_class(::Textbox) = "Text"
widget_class{view}(::Options{view}) = string(view)
widget_class(w, suffix) = widget_class(w) * suffix
view_name(w) = widget_class(w, "View")
model_name(w) = widget_class(w, "Model")

"""
Update output widgets
"""
update!(p::Progress, val) = begin
    p.value = val;
    update_view(p)
end

function metadata(x::Widget)
    display_widget(x)
    Dict()
end

function metadata{T <: Widget}(x::Signal{T})
    create_widget_signal(x)
    Dict()
end

function add_ipy4_state!(state)
    state[:_view_module] = "jupyter-js-widgets"
    state[:_model_module] = "jupyter-js-widgets"
end

const widget_comms = Dict{Widget, Comm}()
function update_view(w::Widget; prevw=w)
    if typeof(w) != typeof(prevw)
        #If the widget type has changed, a new widget must be set up and the old
        #one removed.
        remove_view(prevw)
        display_widget(w)
    else
        #w is same type as prevw
        if w !== prevw
            #new widget instance takes over the comm of the old instnace
            wire_comms(w, widget_comms[prevw])
            delete!(widget_comms, prevw)
        end
        #update all existing views of the widget
        haskey(widget_comms, w) && send_comm(widget_comms[w], view_state(w))
    end
end

function remove_view(prevw::Widget)
    #closing the comm removes ALL widget(s) associated with that comm
    close_comm(widget_comms[prevw])
    delete!(widget_comms, prevw)
    nothing
end

function view_state(w::Widget; src::Widget=w)
    msg = viewdict(src)
    msg[:method] = "update"
    state = Dict()
    state[:msg_throttle] = 3
    state[:_view_name] = view_name(src)
    state[:_model_name] = model_name(src)
    state[:model_name] =  model_name(src)
    :label in fieldnames(w) && (state[:description] = w.label)
    state[:visible] = true
    state[:disabled] = false
    state[:readout] = true
    add_ipy4_state!(state)
    msg[:state] = merge!(state, statedict(src))
    msg
end

function init_widget_dict(w::Widget)
    merge!(viewdict(w),
        Dict{Symbol, Any}(
            :model_name => model_name(w),
            :_model_module => "jupyter-js-widgets",
            :_model_name => model_name(w), # Jupyter 4.0 missing (https://github.com/ipython/ipywidgets/pull/84)
            :_view_module => "jupyter-js-widgets",
            :_view_name => view_name(w),
        ))
end

"""
`display_widget(w::Widget)`
Creates the widget on the front-end and displays it
Sets `widget_comms[w]` to the widget's comm
returns the widget's Comm object
"""
function display_widget(w::Widget)
    comm = create_view(w::Widget)
    send_comm(comm, Dict(:method=>"display"))
end

function create_view(w::Widget)
    if haskey(widget_comms, w)
        comm = widget_comms[w]
    else
        #create the widget on the front-end by opening its comm
        comm = Comm("jupyter.widget", data=init_widget_dict(w))
        wire_comms(w, comm)
    end
    send_comm(comm, view_state(w)) #set/update the widget's state
    comm
end

function create_view(b::Box)
    layout_comm = create_view(b.layout)
    foreach(b.children) do childw
        create_view(childw)
    end
    invoke(create_view, (Widget,), b)
end

function wire_comms(w, comm)
    # dispatch messages to widget's handler
    widget_comms[w] = comm
    comm.on_msg = msg -> handle_msg(w, msg)
end

#used to avoid double/triple creation of updaters, without this multiple widgets
#can appear on update if a Signal{Widget} is `display`ed more than once
const sigwidg_has_updater = WeakKeyDict{Signal, Bool}()
"""
Display the current value of a Signal{Widget} and ensure it stays up-to-date
"""
function create_widget_signal{T<:Widget}(s::Signal{T})
    local prev_widg = value(s)
    display_widget(value(s))
    if !haskey(sigwidg_has_updater, s)
        map(s, init=nothing) do x
            update_view(x; prevw=prev_widg)
            prev_widg = x
            nothing
        end |> preserve
        sigwidg_has_updater[s] = true
    end
end

include("statedict.jl")
include("handle_msg.jl")