text:
macro test_ecode(error_codes, expr)
    quote
        try
            $expr
            @test false
        catch e
            if e isa AWSException
                @test e.code in [$error_codes;]
            else
                rethrow(e)
            end
        end
    end
end

@testset "Load Credentials" begin
    user = aws_user_arn(aws)
    @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user)
    aws.region = "us-east-1"

    @test_ecode("InvalidAction", AWSServices.iam("GetFoo"))

    @test_ecode(
        ["AccessDenied", "NoSuchEntity"],
        AWSServices.iam("GetUser", Dict("UserName" => "notauser"))
    )

    @test_ecode("ValidationError", AWSServices.iam("GetUser", Dict("UserName" => "@#!%%!")))

    @test_ecode(
        ["AccessDenied", "EntityAlreadyExists"],
        AWSServices.iam("CreateUser", Dict("UserName" => "root"))
    )
end

@testset "_role_session_name" begin
    @test AWS._role_session_name("prefix-", "name", "-suffix") == "prefix-name-suffix"
    @test AWS._role_session_name("a"^22, "b"^22, "c"^22) == "a"^22 * "b"^20 * "c"^22
end

@testset "ec2_instance_metadata" begin
    @testset "connect timeout" begin
        apply(Patches._instance_metadata_timeout_patch) do
            @test AWS.ec2_instance_metadata("/latest/meta-data") === nothing
        end
    end
end

@testset "ec2_instance_region" begin
    @testset "available" begin
        patch = @patch function HTTP.request(method::String, url; kwargs...)
            return HTTP.Response("ap-atlantis-1")  # Made up region
        end

        apply(patch) do
            @test AWS.ec2_instance_region() == "ap-atlantis-1"
        end
    end

    @testset "unavailable" begin
        apply(Patches._instance_metadata_timeout_patch) do
            @test AWS.ec2_instance_region() === nothing
        end
    end
end

@testset "aws_get_profile_settings" begin
    @testset "no profile" begin
        @test aws_get_profile_settings("foo", Inifile()) === nothing
    end
end

@testset "_aws_get_role" begin
    profile = "foobar"
    ini = Inifile()

    @testset "settings early exit" begin
        apply(Patches.get_profile_settings_empty_patch) do
            @test AWS._aws_get_role(profile, ini) === nothing
        end
    end

    @testset "source_profile early exit" begin
        apply(Patches.get_profile_settings_empty_patch) do
            @test AWS._aws_get_role(profile, ini) === nothing
        end
    end

    @testset "default profile" begin
        access_key_id = "assumed_access_key_id"
        config_dir = joinpath(@__DIR__, "configs", "default-role")

        patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id)

        cred = withenv(
            "AWS_CONFIG_FILE" => joinpath(config_dir, "config"),
            "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"),
            "AWS_ACCESS_KEY_ID" => nothing,
            "AWS_SECRET_ACCESS_KEY" => nothing,
        ) do
            ini = read(Inifile(), ENV["AWS_CONFIG_FILE"])
            apply(patch) do
                AWS._aws_get_role("default", ini)
            end
        end

        @test cred.access_key_id == access_key_id
    end
end

@testset "AWSCredentials" begin
    @testset "Defaults" begin
        creds = AWSCredentials("access_key_id", "secret_key")
        @test creds.token == ""
        @test creds.user_arn == ""
        @test creds.account_number == ""
        @test creds.expiry == typemax(DateTime)
        @test creds.renew === nothing
    end

    @testset "Renewal" begin
        # Credentials shouldn't throw an error if no renew function is supplied
        creds = AWSCredentials("access_key_id", "secret_key"; renew=nothing)
        newcreds = check_credentials(creds; force_refresh=true)

        # Creds should remain unchanged if no renew function exists
        @test creds === newcreds
        @test creds.access_key_id == "access_key_id"
        @test creds.secret_key == "secret_key"
        @test creds.renew === nothing

        # Creds should error if the renew function returns nothing
        creds = AWSCredentials("access_key_id", "secret_key"; renew=() -> nothing)
        @test_throws NoCredentials check_credentials(creds; force_refresh=true)

        # Creds should remain unchanged
        @test creds.access_key_id == "access_key_id"
        @test creds.secret_key == "secret_key"

        # Creds should take on value of a returned AWSCredentials except renew function
        function gen_credentials()
            i = 0
            return () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i"))
        end

        creds = AWSCredentials(
            "access_key_id", "secret_key"; renew=gen_credentials(), expiry=now(UTC)
        )

        @test creds.renew !== nothing
        renewed = creds.renew()

        @test creds.access_key_id == "access_key_id"
        @test creds.secret_key == "secret_key"
        @test creds.expiry <= now(UTC)
        @test AWS._will_expire(creds)

        @test renewed.access_key_id === "NEW_ID_1"
        @test renewed.secret_key == "NEW_KEY_1"
        @test renewed.renew === nothing
        @test renewed.expiry == typemax(DateTime)
        @test !AWS._will_expire(renewed)
        renew = creds.renew

        # Check renewal on time out
        newcreds = check_credentials(creds; force_refresh=false)
        @test creds === newcreds
        @test creds.access_key_id == "NEW_ID_2"
        @test creds.secret_key == "NEW_KEY_2"
        @test creds.renew !== nothing
        @test creds.renew === renew
        @test creds.expiry == typemax(DateTime)
        @test !AWS._will_expire(creds)

        # Check renewal doesn't happen if not forced or timed out
        newcreds = check_credentials(creds; force_refresh=false)
        @test creds === newcreds
        @test creds.access_key_id == "NEW_ID_2"
        @test creds.secret_key == "NEW_KEY_2"
        @test creds.renew !== nothing
        @test creds.renew === renew
        @test creds.expiry == typemax(DateTime)

        # Check forced renewal works
        newcreds = check_credentials(creds; force_refresh=true)
        @test creds === newcreds
        @test creds.access_key_id == "NEW_ID_3"
        @test creds.secret_key == "NEW_KEY_3"
        @test creds.renew !== nothing
        @test creds.renew === renew
        @test creds.expiry == typemax(DateTime)
    end

    mktempdir() do dir
        config_file = joinpath(dir, "config")
        creds_file = joinpath(dir, "creds")
        write(
            config_file,
            """
            [profile test]
            output = json
            region = us-east-1

            [profile test:dev]
            source_profile = test
            role_arn = arn:aws:iam::123456789000:role/Dev

            [profile test:sub-dev]
            source_profile = test:dev
            role_arn = arn:aws:iam::123456789000:role/SubDev

            [profile test2]
            aws_access_key_id = WRONG_ACCESS_ID
            aws_secret_access_key = WRONG_ACCESS_KEY
            output = json
            region = us-east-1

            [profile test3]
            source_profile = test:dev
            role_arn = arn:aws:iam::123456789000:role/test3

            [profile test4]
            aws_access_key_id = RIGHT_ACCESS_ID4
            aws_secret_access_key = RIGHT_ACCESS_KEY4
            source_profile = test:dev
            role_arn = arn:aws:iam::123456789000:role/test3
            """,
        )

        write(
            creds_file,
            """
            [test]
            aws_access_key_id = TEST_ACCESS_ID
            aws_secret_access_key = TEST_ACCESS_KEY

            [test2]
            aws_access_key_id = RIGHT_ACCESS_ID2
            aws_secret_access_key = RIGHT_ACCESS_KEY2

            [test3]
            aws_access_key_id = RIGHT_ACCESS_ID3
            aws_secret_access_key = RIGHT_ACCESS_KEY3
            """,
        )

        withenv(
            "AWS_SHARED_CREDENTIALS_FILE" => creds_file,
            "AWS_CONFIG_FILE" => config_file,
            "AWS_DEFAULT_PROFILE" => "test",
            "AWS_ACCESS_KEY_ID" => nothing,
        ) do
            @testset "Loading" begin
                # Check credentials load
                config = AWSConfig()
                creds = config.credentials

                @test creds isa AWSCredentials

                @test creds.access_key_id == "TEST_ACCESS_ID"
                @test creds.secret_key == "TEST_ACCESS_KEY"
                @test creds.renew !== nothing

                # Check credential file takes precedence over config
                withenv("AWS_DEFAULT_PROFILE" => "test2") do
                    config = AWSConfig()
                    creds = config.credentials

                    @test creds.access_key_id == "RIGHT_ACCESS_ID2"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY2"
                end

                # Check credentials take precedence over role
                withenv("AWS_DEFAULT_PROFILE" => "test3") do
                    config = AWSConfig()
                    creds = config.credentials

                    @test creds.access_key_id == "RIGHT_ACCESS_ID3"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY3"
                end

                withenv("AWS_DEFAULT_PROFILE" => "test4") do
                    config = AWSConfig()
                    creds = config.credentials

                    @test creds.access_key_id == "RIGHT_ACCESS_ID4"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY4"
                end
            end

            @testset "Refresh" begin
                withenv("AWS_DEFAULT_PROFILE" => "test") do
                    # Check credentials refresh on timeout
                    config = AWSConfig()
                    creds = config.credentials
                    creds.access_key_id = "EXPIRED_ACCESS_ID"
                    creds.secret_key = "EXPIRED_ACCESS_KEY"
                    creds.expiry = now(UTC)

                    @test creds.renew !== nothing
                    renew = creds.renew

                    @test renew() isa AWSCredentials

                    creds = check_credentials(config.credentials)

                    @test creds.access_key_id == "TEST_ACCESS_ID"
                    @test creds.secret_key == "TEST_ACCESS_KEY"
                    @test creds.expiry > now(UTC)

                    # Check renew function remains unchanged
                    @test creds.renew !== nothing
                    @test creds.renew === renew

                    # Check force_refresh
                    creds.access_key_id = "WRONG_ACCESS_KEY"
                    creds = check_credentials(creds; force_refresh=true)
                    @test creds.access_key_id == "TEST_ACCESS_ID"
                end
            end

            @testset "Profile" begin
                # Check profile kwarg
                withenv("AWS_DEFAULT_PROFILE" => "test") do
                    creds = AWSCredentials(; profile="test2")
                    @test creds.access_key_id == "RIGHT_ACCESS_ID2"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY2"

                    config = AWSConfig(; profile="test2")
                    creds = config.credentials
                    @test creds.access_key_id == "RIGHT_ACCESS_ID2"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY2"

                    # Check profile persists on renewal
                    creds.access_key_id = "WRONG_ACCESS_ID2"
                    creds.secret_key = "WRONG_ACCESS_KEY2"
                    creds = check_credentials(creds; force_refresh=true)

                    @test creds.access_key_id == "RIGHT_ACCESS_ID2"
                    @test creds.secret_key == "RIGHT_ACCESS_KEY2"
                end
            end

            @testset "Assume Role" begin
                # Check we try to assume a role
                withenv("AWS_DEFAULT_PROFILE" => "test:dev") do
                    @test_ecode("InvalidClientTokenId", AWSConfig())
                end

                # Check we try to assume a role
                withenv("AWS_DEFAULT_PROFILE" => "test:sub-dev") do
                    oldout = stdout
                    r, w = redirect_stdout()

                    @test_ecode("InvalidClientTokenId", AWSConfig())
                    redirect_stdout(oldout)
                    close(w)
                    output = String(read(r))
                    occursin("Assuming \"test:dev\"", output)
                    occursin("Assuming \"test\"", output)
                    close(r)
                end
            end
        end
    end
end

@testset "Retrieving AWS Credentials" begin
    test_values = Dict{String,Any}(
        "Default-Profile" => "default",
        "Test-Profile" => "test",
        "Test-Config-Profile" => "profile test",

        # Default profile values, needs to match due to AWSCredentials.jl check_credentials()
        "AccessKeyId" => "Default-Key",
        "SecretAccessKey" => "Default-Secret",
        "Test-AccessKeyId" => "Test-Key",
        "Test-SecretAccessKey" => "Test-Secret",
        "Token" => "Test-Token",
        "InstanceProfileArn" => "Test-Arn",
        "RoleArn" => "Test-Arn",
        "Expiration" => now(UTC),
        "URI" => "/Test-URI/",
        "Security-Credentials" => "Test-Security-Credentials",
    )

    _http_request_patch = @patch function HTTP.request(method::String, url; kwargs...)
        security_credentials = test_values["Security-Credentials"]
        uri = test_values["URI"]
        url = string(url)

        if url == "http://169.254.169.254/latest/meta-data/iam/info"
            instance_profile_arn = test_values["InstanceProfileArn"]
            return HTTP.Response("{\"InstanceProfileArn\": \"$instance_profile_arn\"}")
        elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
            return HTTP.Response(test_values["Security-Credentials"])
        elseif url ==
               "http://169.254.169.254/latest/meta-data/iam/security-credentials/$security_credentials" ||
            url == "http://169.254.170.2$uri"
            my_dict = JSON.json(test_values)
            response = HTTP.Response(my_dict)
            return response
        else
            return nothing
        end
    end

    @testset "~/.aws/config - Default Profile" begin
        mktemp() do config_file, config_io
            write(
                config_io,
                """
                [$(test_values["Default-Profile"])]
                aws_access_key_id=$(test_values["AccessKeyId"])
                aws_secret_access_key=$(test_values["SecretAccessKey"])
                """,
            )
            close(config_io)

            withenv("AWS_CONFIG_FILE" => config_file) do
                default_profile = dot_aws_config()

                @test default_profile.access_key_id == test_values["AccessKeyId"]
                @test default_profile.secret_key == test_values["SecretAccessKey"]
            end
        end
    end

    @testset "~/.aws/config - Specified Profile" begin
        mktemp() do config_file, config_io
            write(
                config_io,
                """
                [$(test_values["Test-Config-Profile"])]
                aws_access_key_id=$(test_values["Test-AccessKeyId"])
                aws_secret_access_key=$(test_values["Test-SecretAccessKey"])
                """,
            )
            close(config_io)

            withenv("AWS_CONFIG_FILE" => config_file) do
                specified_result = dot_aws_config(test_values["Test-Profile"])

                @test specified_result.access_key_id == test_values["Test-AccessKeyId"]
                @test specified_result.secret_key == test_values["Test-SecretAccessKey"]
            end
        end
    end

    @testset "~/.aws/creds - Default Profile" begin
        mktemp() do creds_file, creds_io
            write(
                creds_io,
                """
                [$(test_values["Default-Profile"])]
                aws_access_key_id=$(test_values["AccessKeyId"])
                aws_secret_access_key=$(test_values["SecretAccessKey"])
                """,
            )
            close(creds_io)

            withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do
                specified_result = dot_aws_credentials()

                @test specified_result.access_key_id == test_values["AccessKeyId"]
                @test specified_result.secret_key == test_values["SecretAccessKey"]
            end
        end
    end

    @testset "~/.aws/creds - Specified Profile" begin
        mktemp() do creds_file, creds_io
            write(
                creds_io,
                """
                [$(test_values["Test-Profile"])]
                aws_access_key_id=$(test_values["Test-AccessKeyId"])
                aws_secret_access_key=$(test_values["Test-SecretAccessKey"])
                """,
            )
            close(creds_io)

            withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do
                specified_result = dot_aws_credentials(test_values["Test-Profile"])

                @test specified_result.access_key_id == test_values["Test-AccessKeyId"]
                @test specified_result.secret_key == test_values["Test-SecretAccessKey"]
            end
        end
    end

    @testset "Environment Variables" begin
        withenv(
            "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"],
            "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"],
        ) do
            aws_creds = env_var_credentials()
            @test aws_creds.access_key_id == test_values["AccessKeyId"]
            @test aws_creds.secret_key == test_values["SecretAccessKey"]
        end
    end

    @testset "Instance - EC2" begin
        role_name = "foobar"
        role_arn = "arn:aws:sts::1234:assumed-role/$role_name"
        access_key = "access-key-$(randstring(6))"
        secret_key = "secret-key-$(randstring(6))"
        session_token = "session-token-$(randstring(6))"
        session_name = "$role_name-session"
        patch = Patches._assume_role_patch(
            "AssumeRole";
            access_key=access_key,
            secret_key=secret_key,
            session_token=session_token,
            role_arn=role_arn,
        )

        apply([patch, _http_request_patch]) do
            result = ec2_instance_credentials("default")
            @test result.access_key_id == test_values["AccessKeyId"]
            @test result.secret_key == test_values["SecretAccessKey"]
            @test result.token == test_values["Token"]
            @test result.user_arn == test_values["InstanceProfileArn"]
            @test result.expiry == test_values["Expiration"]
            @test result.renew !== nothing

            result = mktemp() do config_file, config_io
                write(
                    config_io,
                    """
                    [profile $role_name]
                    credential_source = Ec2InstanceMetadata
                    role_arn = $role_arn
                    """,
                )
                close(config_io)

                withenv(
                    "AWS_CONFIG_FILE" => config_file,
                    "AWS_ROLE_SESSION_NAME" => session_name,
                ) do
                    ec2_instance_credentials(role_name)
                end
            end

            @test result.access_key_id == access_key
            @test result.secret_key == secret_key
            @test result.token == session_token
            @test result.user_arn == "$(role_arn)/$(session_name)"
            @test result.renew !== nothing
        end
    end

    @testset "Instance - ECS" begin
        withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => test_values["URI"]) do
            apply(_http_request_patch) do
                result = ecs_instance_credentials()
                @test result.access_key_id == test_values["AccessKeyId"]
                @test result.secret_key == test_values["SecretAccessKey"]
                @test result.token == test_values["Token"]
                @test result.user_arn == test_values["RoleArn"]
                @test result.expiry == test_values["Expiration"]
                @test result.renew == ecs_instance_credentials
            end
        end
    end

    @testset "Web Identity File" begin
        @test credentials_from_webtoken() == nothing

        mktempdir() do dir
            web_identity_file = joinpath(dir, "web_identity")
            write(web_identity_file, "foobar")
            session_name = "foobar-session"

            access_key = "access-key-$(randstring(6))"
            secret_key = "secret-key-$(randstring(6))"
            session_token = "session-token-$(randstring(6))"
            role_arn = "arn:aws:sts::1234:assumed-role/foobar"

            patch = Patches._assume_role_patch(
                "AssumeRoleWithWebIdentity";
                access_key=access_key,
                secret_key=secret_key,
                session_token=session_token,
                role_arn=role_arn,
            )

            withenv(
                "AWS_ROLE_ARN" => "foobar",
                "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file,
                "AWS_ROLE_SESSION_NAME" => session_name,
            ) do
                apply(patch) do
                    result = credentials_from_webtoken()

                    @test result.access_key_id == access_key
                    @test result.secret_key == secret_key
                    @test result.token == session_token
                    @test result.user_arn == "$(role_arn)/$(session_name)"
                    @test result.renew == credentials_from_webtoken
                    expiry = result.expiry
                    sleep(0.1)
                    result = check_credentials(result)

                    @test result.access_key_id == access_key
                    @test result.secret_key == secret_key
                    @test result.token == session_token
                    @test result.user_arn == "$(role_arn)/$(session_name)"
                    @test result.renew == credentials_from_webtoken
                    @test expiry != result.expiry
                end
            end

            session_name = "AWS.jl-role-foobar-20210101T000000Z"
            patches = [
                patch
                @patch Dates.now(::Type{UTC}) = DateTime(2021)
            ]

            withenv(
                "AWS_ROLE_ARN" => "foobar",
                "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file,
                "AWS_ROLE_SESSION_NAME" => nothing,
            ) do
                apply(patches) do
                    result = credentials_from_webtoken()
                    @test result.user_arn == "$(role_arn)/$(session_name)"
                end
            end
        end
    end

    @testset "Credentials Not Found" begin
        patches = [
            @patch HTTP.request(method::String, url; kwargs...) = nothing
            Patches._cred_file_patch
            Patches._config_file_patch
        ]

        withenv(
            "AWS_ACCESS_KEY_ID" => nothing,
            "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing,
        ) do
            apply(patches) do
                @test_throws NoCredentials AWSConfig()
            end
        end
    end

    @testset "Helper functions" begin
        @testset "Check Credentials - EnvVars" begin
            withenv(
                "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"],
                "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"],
            ) do
                testAWSCredentials = AWSCredentials(
                    test_values["AccessKeyId"],
                    test_values["SecretAccessKey"];
                    expiry=Dates.now(UTC) - Minute(10),
                    renew=env_var_credentials,
                )

                result = check_credentials(testAWSCredentials; force_refresh=true)
                @test result.access_key_id == testAWSCredentials.access_key_id
                @test result.secret_key == testAWSCredentials.secret_key
                @test result.expiry == typemax(DateTime)
                @test result.renew == testAWSCredentials.renew
            end
        end
    end
end

@testset "aws_get_region" begin
    mktempdir() do dir
        config_str = """
            [default]
            region = us-west-2

            [profile test]
            region = ap-northeast-1
            """
        config_file = joinpath(dir, "config")
        write(config_file, config_str)
        ini = read(Inifile(), IOBuffer(config_str))

        @testset "environmental variable" begin
            withenv("AWS_DEFAULT_REGION" => "us-gov-east-1") do
                @test aws_get_region(; config=ini, profile="default") == "us-gov-east-1"
                @test aws_get_region() == "us-gov-east-1"
            end
        end

        @testset "default profile" begin
            withenv("AWS_DEFAULT_REGION" => nothing) do
                @test aws_get_region(; config=ini, profile="default") == "us-west-2"
                @test aws_get_region(; config=config_file, profile="default") == "us-west-2"
            end

            withenv(
                "AWS_DEFAULT_REGION" => nothing,
                "AWS_CONFIG_FILE" => config_file,
                "AWS_PROFILE" => nothing,
                "AWS_DEFAULT_PROFILE" => nothing,
            ) do
                @test aws_get_region() == "us-west-2"
            end
        end

        @testset "specified profile" begin
            withenv("AWS_DEFAULT_REGION" => nothing) do
                @test aws_get_region(; config=ini, profile="test") == "ap-northeast-1"
                @test aws_get_region(; config=config_file, profile="test") ==
                    "ap-northeast-1"
            end

            withenv(
                "AWS_DEFAULT_REGION" => nothing,
                "AWS_CONFIG_FILE" => config_file,
                "AWS_PROFILE" => "test",
            ) do
                @test aws_get_region() == "ap-northeast-1"
            end
        end

        @testset "unknown profile" begin
            withenv("AWS_DEFAULT_REGION" => nothing) do
                @test aws_get_region(; config=ini, profile="unknown") == AWS.DEFAULT_REGION
                @test aws_get_region(; config=config_file, profile="unknown") ==
                    AWS.DEFAULT_REGION
            end

            withenv(
                "AWS_DEFAULT_REGION" => nothing,
                "AWS_CONFIG_FILE" => config_file,
                "AWS_PROFILE" => "unknown",
            ) do
                @test aws_get_region() == AWS.DEFAULT_REGION
            end
        end

        @testset "default keyword" begin
            default = nothing
            withenv("AWS_DEFAULT_REGION" => nothing) do
                @test aws_get_region(; config=ini, profile="unknown", default=default) ===
                    default
                @test aws_get_region(;
                    config=config_file, profile="unknown", default=default
                ) === default
            end

            withenv(
                "AWS_DEFAULT_REGION" => nothing,
                "AWS_CONFIG_FILE" => config_file,
                "AWS_PROFILE" => "unknown",
            ) do
                @test aws_get_region(; default=default) === default
            end
        end

        @testset "no such config file" begin
            withenv("AWS_DEFAULT_REGION" => nothing, "AWS_CONFIG_FILE" => tempname()) do
                @test aws_get_region() == AWS.DEFAULT_REGION
            end
        end
    end
end


text:
import GLAbstraction: free!
########### Initialization
const currentdio = Base.RefValue{Diorama}()

getcurrentdio() = currentdio[]
iscurrentdio(x) = x == currentdio[]
function makecurrentdio(x)
    return currentdio[] = x
end

Overseer.ledger(dio::Diorama) = dio.ledger

function Diorama(extra_stages::Stage...; name = :Glimpse, kwargs...) #Defaults
    m = Ledger(Stage(:start, [Timer(), EventPoller()]),
               Stage(:setup,
                     [MousePicker(), PolygonMesher(), DensityMesher(), VectorMesher(),
                      FunctionMesher(), FunctionColorizer(), IDColorGenerator(), Uploader(),
                      InstancedUploader(), TextUploader()]), extra_stages...,
               Stage(:simulation, [Oscillator(), Mover(), Editor(), UniformCalculator()]),
               Stage(:rendering,
                     [PreRenderer(), Resizer(), LineRenderer(), UniformUploader(),
                      DefaultRenderer(), DepthPeelingRenderer(), FinalRenderer(),
                      TextRenderer(), GuiRenderer()]),
               Stage(:stop, [CameraOperator(), Sleeper()]))

    #assemble all rendering, canvas and camera components
    t = Diorama(name, m, stages(m))
    e = Entity(t)
    t[UpdatedComponents][e] = UpdatedComponents()
    t[e] = DioEntity()
    t[e] = TimingData()
    fetch(glimpse_call() do
              c = Canvas(name; kwargs...)
              make_current(c)
              m[e] = c
              wh = size(c)
              m[e] = IOTarget(GLA.FrameBuffer(wh,
                                              GLA.Texture(RGBAf0, wh;
                                                          internalformat = GL_RGBA),
                                              GLA.Texture(RGBAf0, wh;
                                                          internalformat = GL_RGBA),
                                              GLA.Texture(GLA.Depth{Float32}, wh)),
                              c.background)
              m[e] = FullscreenVao()
              m[e] = UpdatedComponents(DataType[])
              m[e] = Mouse(div.(wh, 2)..., 0, 0, (0, 0), (0, 0), GLFW.MOUSE_BUTTON_1,
                           GLFW.RELEASE)
              m[e] = Keyboard(GLFW.KEY_UNKNOWN, 0, GLFW.RELEASE)
              for v in assemble_camera3d(Int32.(size(c))...)
                  m[e] = v
              end
              return Entity(t, DioEntity(), Spatial(; position = Point3f0(200.0f0)),
                            PointLight(), UniformColor(RGBf0(1.0, 1.0, 1.0)))
          end)

    fetch(glimpse_call(() -> Overseer.prepare(t)))
    return t
end

function Base.show(io::IO, d::Diorama)
    return println(io, "Diorama with $(length(entities(d))) entities.")
end
# "Darken all the lights in the dio by a certain amount"
# darken!(dio::Diorama, percentage)  = darken!.(dio.lights, percentage)
# lighten!(dio::Diorama, percentage) = lighten!.(dio.lights, percentage)

#This is kind of like a try catch command to execute only when a valid canvas is attached to the diorama
#i.e All GL calls should be inside one of these otherwise it might be bad.
function canvas_command(command::Function, dio::Diorama, catchcommand = x -> nothing)
    canvas = dio.ledger[Canvas][1]
    if canvas !== nothing
        glimpse_call(() -> command(canvas))
    else
        glimpse_call(() -> catchcommand(canvas))
    end
end

function expose(dio::Diorama; kwargs...)
    if dio.loop === nothing
        canvas_command(dio,
                       x -> m[Entity(m[DioEntity], 1)] = Overseer.Entity(dio,
                                                                         Canvas(dio.name;
                                                                                kwargs...))) do x
            make_current(x)
            expose(x)
            return renderloop(dio)
        end
    end
    return dio
end

function Overseer.update(dio::Diorama, init = false)
    timer = singleton(dio, TimingData).timer
    mesg = init ? "Init" : "Running"
    @timeit timer mesg for stage in dio.renderloop_stages
        for sys in last(stage)
            timeit(() -> update(sys, dio), timer, string(typeof(sys)))
        end
    end
end

#TODO move control over this to diorama itself
function renderloop(dio)
    return dio.loop = canvas_command(dio) do canvas
        Overseer.prepare(dio)
        try
            @info "Initializing render..."
            update(dio, true)
            @info "Rendering..."
            while !should_close(canvas)
                pollevents(canvas)
                singleton(dio, Camera3D).locked = false
                update(dio)
                empty!(singleton(dio, UpdatedComponents))
            end
            close(canvas)
            dio.loop = nothing
        catch e
            close(canvas)
            for stage in stages(dio)
                Overseer.update(stage, dio)
            end
            throw(e)
        end
    end
end

function reload(dio::Diorama)
    close(dio)
    canvas_command(dio) do canvas
        while isopen(canvas) && dio.loop !== nothing
            sleep(0.01)
        end
        dio.reupload = true
        return expose(dio)
    end
end

function close(dio::Diorama)
    canvas_command(canvas -> should_close!(canvas, true), dio)
    if dio.loop === nothing
        canvas_command(canvas -> close(canvas), dio)
    end
end

function free!(dio::Diorama)
    close(dio)
    canvas_command(dio) do c
        for c in components(dio)
            if eltype(c) <: Vao
                for v in c
                    finalize(v)
                end
            end
        end
        finalize(c)
    end
end

isrendering(dio::Diorama) = dio.loop !== nothing

Base.size(dio::Diorama) = canvas_command(c -> windowsize(c), dio, x -> (0, 0))
function set_background_color!(dio::Diorama, color)
    return canvas_command(c -> set_background_color!(c, color), dio)
end
background_color(dio::Diorama) = canvas_command(c -> c.background, dio)

print_debug_timers(dio::Diorama) = print_timer(singleton(dio, TimingData).timer)
reset_debug_timers!(dio::Diorama) = reset_timer!(singleton(dio, TimingData).timer)

###########
# manipulations
# set_rotation_speed!(dio::Diorama, rotation_speed::Number) = dio.camera.rotation_speed = Float32(rotation_speed)

function center_cameras(dio::Diorama)
    spat = dio[Spatial]
    cam = dio[Camera3D]
    lights = dio[PointLight]
    center = zero(Point3f0)
    e_counter = 0
    for e in @entities_in(spat && !cam && !lights)
        center += e.position
        e_counter += 1
    end
    center /= e_counter

    for e in @entities_in(spat && cam)
        current_pos  = e.position
        current_dist = norm(e.lookat - current_pos)
        # we want to keep the distance to the lookat constant
        u_forward = unitforward(current_pos, center)
        new_pos = center - current_dist * u_forward
        e[Spatial] = Spatial(new_pos, e.velocity)
        e.lookat = Vec3f0(center)
    end
end

function push_system(dio::Diorama, s::System)
    return (dio.ledger = push_system(dio.ledger, s); glimpse_call(() -> Overseer.prepare(dio)))
end

function insert_system(dio::Diorama, id::Integer, s::System)
    return (dio.ledger = insert_system(dio.ledger, id, s); glimpse_call(() -> Overseer.prepare(dio)))
end

debug_timer(dio::Diorama) = singleton(dio, TimingData).timer

function Base.empty!(dio::Diorama)
    dio_entities = dio[DioEntity]
    for e in valid_entities(dio)
        if !(e in dio_entities)
            schedule_delete!(dio.ledger, e)
        end
    end
    return delete_scheduled!(dio.ledger)
end

#TODO make a sanitize camera stuff
function center_camera!(dio::Diorama, p::Point3f0)
    camera          = singleton(dio, Camera3D)
    camera_entity   = Entity(first(dio[Camera3D].indices))
    eye             = dio[Spatial][camera_entity].position
    lookat          = camera.lookat
    shift           = p - lookat
    camera.lookat   = p
    new_eye         = eye + shift
    camera.view     = lookatmat(new_eye, p, camera.up)
    camera.projview = camera.proj * camera.view
    return dio[Spatial][camera_entity] = Spatial(dio[Spatial][camera_entity];
                                                 position = new_eye)
end

function Base.setindex!(dio::Diorama, v::T, e::Entity) where {T<:ComponentData}
    setindex!(dio.ledger, v, e)
    return update_component!(dio, T)
end

function update_component!(dio::Diorama, ::Type{T}) where {T<:ComponentData}
    uc = singleton(dio, UpdatedComponents)
    if !in(T, uc.components)
        push!(uc, T)
    end
end

function reload_shaders(dio::Diorama)
    return glimpse_call(() -> for prog in components(dio, RenderProgram)
                            ProgType = eltype(prog)
                            dio[Entity(1)] = ProgType(Program(shaders(ProgType)))
                        end)
end

function register_update(dio::Diorama, ::Type{T}) where {T<:ComponentData}
    return T ∈ singleton(dio, UpdatedComponents) ? nothing :
           push!(singleton(dio, UpdatedComponents), T)
end

ext: jl


text:
# LSTM

@polly function lstm_forward(nt,np,ns,nq,s_F,inp_F,c_F,U,W,IFOG)
	for t=1:nt
		fill!(IFOG,0.0)
		for r=1:ns
			for p=1:np,k=1:4
				IFOG[r,k] +=U[r,p,k] * inp_F[t,p]
			end
			if t>1
				for s=1:ns,k=1:4
					IFOG[r,k] +=W[r,s] * s_F[t-1,s]
				end
			end
		end
		if t>1
			for s=1:ns
				c_F[t,s] = c_F[t-1,s] * IFOG[s,2] + IFOG[s,1]*IFOG[s,4]
			end
		end
		for s=1:ns
			s_F[t,s] = c_F[t,s] * IFOG[s,3]
		end
	end
end

@polly function lstm_backward(nt,np,ns,nq,s_F,inp_F,c_F,U,W,IFOG,del_S,del_C,del_U,del_W,del_IFOG)

	for t=nt:-1:1
		for s=1:ns
			del_IFOG[s,3] = c_F[t,s] * del_S[t,s]
		end
		for s=1:ns
			del_C[t,s] += IFOG[s,3] * del_S[t,s]
		end
		if t>1
			for s=1:ns
				del_IFOG[s,2] = c_F[t-1,s] * del_C[t,s]
				del_C[t-1,s] = IFOG[s,2] * del_C[t,s]
			end
		end
		for s=1:ns
			del_IFOG[s,1] = IFOG[s,4] * del_C[t,s]
			del_IFOG[s,4] = IFOG[s,1] * del_C[t,s]
		end

		for s=1:ns
			for p=1:np
				for k=1:4
					del_U[s,p,k] = inp_F[t,p] * IFOG[s,k]
				end
			end
		end

		if t>1
			for s=1:ns
				for r=1:ns
					for k=1:4
						del_W[s,r,k] = s_F[t-1,r] * IFOG[s,k]
					end
				end
			end
		end

		if t>1
			for s=1:ns
				for r=1:ns
					for k=1:4
						del_S[t-1,r] += W[r,s,k] * del_IFOG[s,k]
					end
				end
			end
		end
	end
end #function end

let
	nt = 5
	np = 700
	nq = 600
	ns = 650

	s_F = zeros(Float32,nt,ns)
	inp_F = zeros(Float32,nt,np)
	c_F = zeros(Float32,nt,ns)
	U = zeros(Float32,ns,np,4)
	W = zeros(Float32,ns,ns,4)
	IFOG = zeros(Float32,ns,4)
	del_S = zeros(Float32,nt,ns)
	del_C = zeros(Float32,nt,ns)
	del_U = zeros(Float32,ns,np,4)
	del_W = zeros(Float32,ns,ns,4)
	del_IFOG = zeros(Float32,ns,4)

	#init array

	for a=1:nt, b=1:ns
		s_F[a,b] = Float32((a*b) % nt)
		del_S[a,b] = Float32((a*(b+2)) % ns)
		del_C[a,b] = Float32(((a+2)*(b+5)) % nt)
		c_F[a,b] = Float32((a*b) % ns)
	end

	for a=1:ns, b=1:np, c=1:4
		U[a,b,c] = Float32((a*(b+c)) % np)
		del_U[a,b,c] = Float32(((a+c)*b) % ns)
	end

	for a=1:ns,b=1:ns,c=1:4
		W[a,b,c] = Float32(((a+1)*(b+c)) % ns)
		del_W[a,b,c] = Float32(((a+c)*(b+1)) % nt)
	end

	for b=1:ns,c=1:4
		IFOG[b,c] = (b+2*c+1) / nt
	end

	for a=1:nt,b=1:np
		inp_F[a,b] = ((a*b) % np)
	end

	if (length(ARGS)==0) || ARGS[1]=="forward"
		SUITE["lstm"]=@benchmarkable lstm_forward($nt,$np,$ns,$nq,s_F,inp_F,c_F,U,W,IFOG) setup = (s_F=copy($s_F); inp_F=copy($inp_F); c_F=copy($c_F); U=copy($U); W=copy($W); IFOG=copy($IFOG))
	else
		SUITE["lstm"]=@benchmarkable lstm_backward($nt,$np,$ns,$nq,s_F,inp_F,c_F,U,W,IFOG,del_S,del_C,del_U,del_W,del_IFOG) setup = (s_F=copy($s_F); inp_F=copy($inp_F); c_F=copy($c_F); U=copy($U); W=copy($W); IFOG=copy($IFOG))
	end
end

max_stars_repo_name: akileshbadrinaaraayanan/IITH

max_issues_repo_name: akileshbadrinaaraayanan/IITH

max_forks_repo_name: akileshbadrinaaraayanan/IITH


text:
module GroupLens
using DataDeps

export @datadep_str

function __init__()
    register(DataDep("MovieLens-latest",
        """
        DataSet: Latest extract of data from MovieLens
        Source: GroupLens project at U. of Minnesota
        Website: https://grouplens.org/datasets/movielens/latest/

        Movie ratings from the MovieLens web site.  This data set contains about 27,000,000 ratings
        and 1,100,000 tag applications applied to 58,000 movies by 280,000 users.
        """,
        "http://files.grouplens.org/datasets/movielens/ml-latest.zip",
        "b9c23b665ee348bd1fadfadca688b8750c575f3af76d3441cd50cba87ad2c4df",
        post_fetch_method=unpack
    ))
    register(DataDep("MovieLens-latest-small",
        """
        DataSet: Latest small extract of data from MovieLens
        Source: GroupLens project at U. of Minnesota
        Website: https://grouplens.org/datasets/movielens/latest/

        100,000 ratings and 3,600 tag applications applied to 9,000 movies by 600 users.
        Last updated 9/2018.
        """,
        "http://files.grouplens.org/datasets/movielens/ml-latest-small.zip",
        "696d65a3dfceac7c45750ad32df2c259311949efec81f0f144fdfb91ebc9e436",
        post_fetch_method=unpack
    ))
    register(DataDep("MovieLens-20m",
        """
        DataSet: Stable benchmark dataset from MovieLens
        Source: GroupLens project at U. of Minnesota
        Website: https://grouplens.org/datasets/movielens/20m/

        Stable benchmark dataset. 20 million ratings and 465,000 tag applications applied
        to 27,000 movies by 138,000 users. Includes tag genome data with 12 million
        relevance scores across 1,100 tags.
        Released 4/2015;
        updated 10/2016 to update links.csv and add tag genome data.
        """,
        "http://files.grouplens.org/datasets/movielens/ml-20m.zip",
        "96f243c338a8665f6bcc89c53edf6ee39162a846940de6b7c8c48aeada765ff3",
        post_fetch_method=unpack
    ))
    register(DataDep("Book-Crossing",
        """
        DataSet: Book ratings from the Book-Crossing community
        Source: Cai-Nicolas Ziegler at DBIS Freiberg
        Website: http://www2.informatik.uni-freiburg.de/~cziegler/BX/

        Collected by Cai-Nicolas Ziegler in a 4-week crawl (August / September 2004)
        from the Book-Crossing community with kind permission from Ron Hornbaker,
        CTO of Humankind Systems. Contains 278,858 users (anonymized but with demographic
        information) providing 1,149,780 ratings (explicit / implicit) about 271,379 books.
        """,
        "http://www2.informatik.uni-freiburg.de/~cziegler/BX/BX-CSV-Dump.zip",
        "adc7b47afb47c8e29e5f0b70a9816569b3367c5588064ba7a916f16e3ef5d060",
        post_fetch_method=unpack
    ))
    register(DataDep("MovieLens-20m-trailers",
        """
        DataSet: Links to YouTube IDs for trailers for movies in MovieLens-20m
        Source: GroupLens project at U. of Minnesota
        Website: https://grouplens.org/datasets/movielens/20m-youtube/

        This amendment to the MovieLens 20M Dataset is a CSV file that maps
        MovieLens Movie IDs to YouTube IDs representing movie trailers.
        It contains 25,623 YouTube IDs.
        Released 5/2018.
        """,
        "http://files.grouplens.org/datasets/movielens/ml-20m-youtube.zip",
        "7f2fbf681afeec7b50c91b739885fcee4ed8c9c7ba43358a38f094e095eccb5d",
        post_fetch_method=unpack
    ))

end

end # module


text:
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Usage:
julia --project=[...] lasso_example.jl data_source lambda
"""

include("src/AGD_distance_restart.jl")

gr()
fntsm = Plots.font("serif", pointsize=12)
fntlg = Plots.font("serif", pointsize=18)
default(titlefont=fntlg, guidefont=fntlg, tickfont=fntsm, legendfont=fntsm)

@assert length(ARGS) == 2
data_source = joinpath(@__DIR__, "data", ARGS[1])
println("Data source: ", data_source)
lambda = parse(Float64, ARGS[2])
println("Lambda: ", lambda)
original_data = load_libsvm_file(data_source)
data = preprocess_regression_problem(original_data)

# Solve a problem of the form
# 0.5 * \| A x - b \|_2^2 + lambda * | x |_1
lasso_function = create_general_regression_function(
    data=data,
    loss=create_squared_function(1.0),
    regularizer=create_L1_proximal_operator(lambda),
  )
v_0 = zeros(lasso_function.dim);

parameters = default_basic_agd_parameters();
parameters.max_iter=5000;

basic_history, fval_restart_history, our_adaptive_history = run_algs(
  lasso_function, v_0, parameters
);

optimality_threshold = 1e-8
approximate_minimum = min(
  minimum(basic_history.fval_history),
  minimum(fval_restart_history.fval_history),
  minimum(our_adaptive_history.fval_history),
) - optimality_threshold / 10.0

BEST_FIXED_FREQUENCY = true
if BEST_FIXED_FREQUENCY
  best_fixed_frequency_result =
    pick_best_fixed_restart_frequency(lasso_function, v_0, parameters;
      set_of_frequencies = [128, 256, 512, 1024, 2048],
      approximate_optimality_threshold = approximate_minimum + optimality_threshold,
  )
  best_fixed_frequency_history = best_fixed_frequency_result.history
  print("best_fixed_frequency = ", best_fixed_frequency_result.frequency)
end

basic_history.fval_history .-= approximate_minimum;
fval_restart_history.fval_history .-= approximate_minimum;
our_adaptive_history.fval_history .-= approximate_minimum;
if BEST_FIXED_FREQUENCY
  best_fixed_frequency_history.fval_history .-= approximate_minimum
end

plot_algs(
  basic_history,
  fval_restart_history,
  our_adaptive_history,
  y_min=optimality_threshold,
)

if BEST_FIXED_FREQUENCY
  plot_history!(best_fixed_frequency_history;
      label="Best fixed ($(best_fixed_frequency_result.frequency))",
      color=3,
    )
end

println("")
println("Plot name: ", ARGS[1])
savefig(joinpath(@__DIR__, "plots", ARGS[1] * ".pdf"))


text:
using VariationalInequalitySolver
using ADNLPModels, NLPModels
using Random, Test

Random.seed!(1234)

@testset "VariationalInequalitySolver.jl" begin
  nls = ADNLSModel(x -> [x[1] - 1; 10 * (x[2] - x[1]^2)], [-1.2; 1.0], 2)
  vi = NLSVIModel(nls)
  xr = rand(2)
  @test project(vi, xr) == xr
  sol1 = ProjectionVI(vi, xr)
  sol2 = ProjectedReflectedGradientVI(vi, xr)
  @show sol1, sol2
end

@testset "Test API AbstractVIModel" begin
  for T in [Float16, Float32, Float64]
    nls = ADNLSModel(x -> [x[1] - 1; 10 * (x[2] - x[1]^2)], T[-1.2; 1.0], 2)
    vi = NLSVIModel(nls)

    @test get_meta(vi) == vi.meta

    xr = ones(T, 2)
    @test residual(vi, xr) == T[0; 0]
    J = T[1 0; -20 10]
    @test jac_residual(vi, xr) == J
    v = ones(T, 2)
    @test jprod_residual(vi, xr, v) == T[1; -10]
    @test jtprod_residual(vi, xr, v) == T[-19; 10]
    J = jac_op_residual(vi, xr)
    @test J * v == T[1; -10]
    @test J' * v == T[-19; 10]
    @test hess_residual(vi, xr, xr) == T[-20 0; 0 0]
    @test hprod_residual(vi, xr, 2, v) == T[-20; 0]
    H = hess_op_residual(vi, xr, 2)
    @test H * v == T[-20; 0]
  end
end


text:
using Documenter, ClimateTasks

makedocs(;
    modules=[ClimateTasks],
    format=Documenter.HTML(),
    pages=[
        "Home" => "index.md",
    ],
    repo="https://github.com/gaelforget/ClimateTasks.jl/blob/{commit}{path}#L{line}",
    sitename="ClimateTasks.jl",
    authors="gaelforget <gforget@mit.edu>",
)

deploydocs(;
    repo="github.com/gaelforget/ClimateTasks.jl",
)