{"action":"created","comment":{"url":"https://api.github.com/repos/mlpack/mlpack/pulls/comments/114827028","pull_request_review_id":36319991,"id":114827028,"diff_hunk":"@@ -0,0 +1,172 @@\n+/**\n+ * @file mountain_car.hpp\n+ * @author Shangtong Zhang\n+ *\n+ * This file is an implementation of Mountain Car task:\n+ * https://gym.openai.com/envs/MountainCar-v0\n+ *\n+ * TODO: provide an option to use dynamics directly from OpenAI gym.\n+ *\n+ * mlpack is free software; you may redistribute it and/or modify it under the\n+ * terms of the 3-clause BSD license.  You should have received a copy of the\n+ * 3-clause BSD license along with mlpack.  If not, see\n+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.\n+ */\n+\n+#ifndef MLPACK_METHODS_RL_ENVIRONMENT_MOUNTAIN_CAR_HPP\n+#define MLPACK_METHODS_RL_ENVIRONMENT_MOUNTAIN_CAR_HPP\n+\n+#include <mlpack/prereqs.hpp>\n+\n+namespace mlpack {\n+namespace rl {\n+\n+/**\n+ * Implementation of Mountain Car task.\n+ */\n+class MountainCar\n+{\n+ public:\n+\n+  /**\n+   * Implementation of state of Mountain Car.\n+   * Each state is a (velocity, position) pair.\n+   */\n+  class State\n+  {\n+   public:\n+    //! Construct a state instance.\n+    State(): data(2, arma::fill::zeros) { }\n+\n+    /**\n+     * Construct a state based on given data.\n+     * @param data desired internal data.\n+     */\n+    State(const arma::colvec& data): data(data) { }\n+\n+    //! Encode the state to a column vector.\n+    const arma::colvec& Encode() const { return data; }\n+\n+    //! Modify the internal representation of the state.\n+    arma::colvec& Data() { return data; }\n+\n+    //! Get velocity.\n+    double Velocity() const { return data[0]; }\n+\n+    //! Modify velocity.\n+    double& Velocity() { return data[0]; }\n+\n+    //! Get position.\n+    double Position() const { return data[1]; }\n+\n+    //! Modify position.\n+    double& Position() { return data[1]; }\n+\n+   private:\n+    //! Locally-stored velocity and position.\n+    arma::colvec data;\n+  };\n+\n+  /**\n+   * Implementation of action of Mountain Car.\n+   */\n+  enum Action\n+  {\n+    backward,\n+    stop,\n+    forward,\n+\n+    //! Track the size of the action space.\n+    size\n+  };\n+\n+  /**\n+   * Construct a Mountain Car instance.\n+   * @param positionMin minimum legal position.\n+   * @param positionMax maximum legal position.\n+   * @param velocityMin minimum legal velocity.\n+   * @param velocityMax maximum legal velocity.\n+   */\n+  MountainCar(double positionMin = -1.2, double positionMax = 0.5, double velocityMin = -0.07, double velocityMax = 0.07):\n+      positionMin(positionMin), positionMax(positionMax), velocityMin(velocityMin), velocityMax(velocityMax) { }\n+\n+  /**\n+   * Dynamics of Mountain Car.\n+   * Get reward and next state based on current state and current action.\n+   * @param state Current state.\n+   * @param action Current action.\n+   * @param nextState Next state.\n+   * @return reward, it's always -1.0.\n+   */\n+  double Sample(const State& state, const Action& action, State& nextState) const\n+  {\n+    int direction = action - 1;\n+    nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * std::cos(3 * state.Position());\n+    nextState.Velocity() = std::min(std::max(nextState.Velocity(), velocityMin), velocityMax);\n+\n+    nextState.Position() = state.Position() + nextState.Velocity();\n+    nextState.Position() = std::min(std::max(nextState.Position(), positionMin), positionMax);\n+\n+    if (std::abs(nextState.Position() - positionMin) <= 1e-5)\n+    {\n+      nextState.Velocity() = 0.0;\n+    }\n+\n+    return -1.0;\n+  }\n+\n+  /**\n+   * Dynamics of Mountain Car.\n+   * Get reward based on current state and current action.\n+   * @param state Current state.\n+   * @param action Current action.\n+   * @return reward, it's always -1.0.\n+   */\n+  double Sample(const State& state, const Action& action) const\n+  {\n+    State nextState;\n+    return Sample(state, action, nextState);\n+  }\n+\n+  /**\n+   * Initial position is randomly generated within [-0.6, -0.4].\n+   * Initial velocity is 0.\n+   * @return Initial state for each episode.\n+   */\n+  State InitialSample() const","path":"src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp","position":136,"original_position":136,"commit_id":"1acd5d181510f10c63b9e4ba04067466213835d8","original_commit_id":"1acd5d181510f10c63b9e4ba04067466213835d8","user":{"login":"zoq","id":4209744,"avatar_url":"https://avatars0.githubusercontent.com/u/4209744?v=3","gravatar_id":"","url":"https://api.github.com/users/zoq","html_url":"https://github.com/zoq","followers_url":"https://api.github.com/users/zoq/followers","following_url":"https://api.github.com/users/zoq/following{/other_user}","gists_url":"https://api.github.com/users/zoq/gists{/gist_id}","starred_url":"https://api.github.com/users/zoq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zoq/subscriptions","organizations_url":"https://api.github.com/users/zoq/orgs","repos_url":"https://api.github.com/users/zoq/repos","events_url":"https://api.github.com/users/zoq/events{/privacy}","received_events_url":"https://api.github.com/users/zoq/received_events","type":"User","site_admin":false},"body":"Thanks for your thoughts and clarification; besides some minor style issues, I think this is ready to be merged. I'll go and fix the issues once I merge this in.\r\n","created_at":"2017-05-04T16:29:22Z","updated_at":"2017-05-04T16:29:22Z","html_url":"https://github.com/mlpack/mlpack/pull/989#discussion_r114827028","pull_request_url":"https://api.github.com/repos/mlpack/mlpack/pulls/989","_links":{"self":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/comments/114827028"},"html":{"href":"https://github.com/mlpack/mlpack/pull/989#discussion_r114827028"},"pull_request":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/989"}}},"pull_request":{"url":"https://api.github.com/repos/mlpack/mlpack/pulls/989","id":117391592,"html_url":"https://github.com/mlpack/mlpack/pull/989","diff_url":"https://github.com/mlpack/mlpack/pull/989.diff","patch_url":"https://github.com/mlpack/mlpack/pull/989.patch","issue_url":"https://api.github.com/repos/mlpack/mlpack/issues/989","number":989,"state":"open","locked":false,"title":"Implement two classical control problems for testing reinforcement learning method","user":{"login":"ShangtongZhang","id":6715118,"avatar_url":"https://avatars1.githubusercontent.com/u/6715118?v=3","gravatar_id":"","url":"https://api.github.com/users/ShangtongZhang","html_url":"https://github.com/ShangtongZhang","followers_url":"https://api.github.com/users/ShangtongZhang/followers","following_url":"https://api.github.com/users/ShangtongZhang/following{/other_user}","gists_url":"https://api.github.com/users/ShangtongZhang/gists{/gist_id}","starred_url":"https://api.github.com/users/ShangtongZhang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ShangtongZhang/subscriptions","organizations_url":"https://api.github.com/users/ShangtongZhang/orgs","repos_url":"https://api.github.com/users/ShangtongZhang/repos","events_url":"https://api.github.com/users/ShangtongZhang/events{/privacy}","received_events_url":"https://api.github.com/users/ShangtongZhang/received_events","type":"User","site_admin":false},"body":"","created_at":"2017-04-25T02:06:27Z","updated_at":"2017-05-04T16:29:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"489d844dbe750c742c822ff4eef2340dc270c9bd","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/mlpack/mlpack/pulls/989/commits","review_comments_url":"https://api.github.com/repos/mlpack/mlpack/pulls/989/comments","review_comment_url":"https://api.github.com/repos/mlpack/mlpack/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mlpack/mlpack/issues/989/comments","statuses_url":"https://api.github.com/repos/mlpack/mlpack/statuses/1acd5d181510f10c63b9e4ba04067466213835d8","head":{"label":"ShangtongZhang:rl-env","ref":"rl-env","sha":"1acd5d181510f10c63b9e4ba04067466213835d8","user":{"login":"ShangtongZhang","id":6715118,"avatar_url":"https://avatars1.githubusercontent.com/u/6715118?v=3","gravatar_id":"","url":"https://api.github.com/users/ShangtongZhang","html_url":"https://github.com/ShangtongZhang","followers_url":"https://api.github.com/users/ShangtongZhang/followers","following_url":"https://api.github.com/users/ShangtongZhang/following{/other_user}","gists_url":"https://api.github.com/users/ShangtongZhang/gists{/gist_id}","starred_url":"https://api.github.com/users/ShangtongZhang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ShangtongZhang/subscriptions","organizations_url":"https://api.github.com/users/ShangtongZhang/orgs","repos_url":"https://api.github.com/users/ShangtongZhang/repos","events_url":"https://api.github.com/users/ShangtongZhang/events{/privacy}","received_events_url":"https://api.github.com/users/ShangtongZhang/received_events","type":"User","site_admin":false},"repo":{"id":28815204,"name":"mlpack","full_name":"ShangtongZhang/mlpack","owner":{"login":"ShangtongZhang","id":6715118,"avatar_url":"https://avatars1.githubusercontent.com/u/6715118?v=3","gravatar_id":"","url":"https://api.github.com/users/ShangtongZhang","html_url":"https://github.com/ShangtongZhang","followers_url":"https://api.github.com/users/ShangtongZhang/followers","following_url":"https://api.github.com/users/ShangtongZhang/following{/other_user}","gists_url":"https://api.github.com/users/ShangtongZhang/gists{/gist_id}","starred_url":"https://api.github.com/users/ShangtongZhang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ShangtongZhang/subscriptions","organizations_url":"https://api.github.com/users/ShangtongZhang/orgs","repos_url":"https://api.github.com/users/ShangtongZhang/repos","events_url":"https://api.github.com/users/ShangtongZhang/events{/privacy}","received_events_url":"https://api.github.com/users/ShangtongZhang/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ShangtongZhang/mlpack","description":"mlpack: a scalable C++ machine learning library","fork":true,"url":"https://api.github.com/repos/ShangtongZhang/mlpack","forks_url":"https://api.github.com/repos/ShangtongZhang/mlpack/forks","keys_url":"https://api.github.com/repos/ShangtongZhang/mlpack/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ShangtongZhang/mlpack/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ShangtongZhang/mlpack/teams","hooks_url":"https://api.github.com/repos/ShangtongZhang/mlpack/hooks","issue_events_url":"https://api.github.com/repos/ShangtongZhang/mlpack/issues/events{/number}","events_url":"https://api.github.com/repos/ShangtongZhang/mlpack/events","assignees_url":"https://api.github.com/repos/ShangtongZhang/mlpack/assignees{/user}","branches_url":"https://api.github.com/repos/ShangtongZhang/mlpack/branches{/branch}","tags_url":"https://api.github.com/repos/ShangtongZhang/mlpack/tags","blobs_url":"https://api.github.com/repos/ShangtongZhang/mlpack/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ShangtongZhang/mlpack/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ShangtongZhang/mlpack/git/refs{/sha}","trees_url":"https://api.github.com/repos/ShangtongZhang/mlpack/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ShangtongZhang/mlpack/statuses/{sha}","languages_url":"https://api.github.com/repos/ShangtongZhang/mlpack/languages","stargazers_url":"https://api.github.com/repos/ShangtongZhang/mlpack/stargazers","contributors_url":"https://api.github.com/repos/ShangtongZhang/mlpack/contributors","subscribers_url":"https://api.github.com/repos/ShangtongZhang/mlpack/subscribers","subscription_url":"https://api.github.com/repos/ShangtongZhang/mlpack/subscription","commits_url":"https://api.github.com/repos/ShangtongZhang/mlpack/commits{/sha}","git_commits_url":"https://api.github.com/repos/ShangtongZhang/mlpack/git/commits{/sha}","comments_url":"https://api.github.com/repos/ShangtongZhang/mlpack/comments{/number}","issue_comment_url":"https://api.github.com/repos/ShangtongZhang/mlpack/issues/comments{/number}","contents_url":"https://api.github.com/repos/ShangtongZhang/mlpack/contents/{+path}","compare_url":"https://api.github.com/repos/ShangtongZhang/mlpack/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ShangtongZhang/mlpack/merges","archive_url":"https://api.github.com/repos/ShangtongZhang/mlpack/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ShangtongZhang/mlpack/downloads","issues_url":"https://api.github.com/repos/ShangtongZhang/mlpack/issues{/number}","pulls_url":"https://api.github.com/repos/ShangtongZhang/mlpack/pulls{/number}","milestones_url":"https://api.github.com/repos/ShangtongZhang/mlpack/milestones{/number}","notifications_url":"https://api.github.com/repos/ShangtongZhang/mlpack/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ShangtongZhang/mlpack/labels{/name}","releases_url":"https://api.github.com/repos/ShangtongZhang/mlpack/releases{/id}","deployments_url":"https://api.github.com/repos/ShangtongZhang/mlpack/deployments","created_at":"2015-01-05T14:11:32Z","updated_at":"2017-03-01T18:16:08Z","pushed_at":"2017-05-01T21:32:35Z","git_url":"git://github.com/ShangtongZhang/mlpack.git","ssh_url":"git@github.com:ShangtongZhang/mlpack.git","clone_url":"https://github.com/ShangtongZhang/mlpack.git","svn_url":"https://github.com/ShangtongZhang/mlpack","homepage":null,"size":149280,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mlpack:master","ref":"master","sha":"7f7561d9cbda272055868c9d3c3bba08d9f0f2af","user":{"login":"mlpack","id":10216045,"avatar_url":"https://avatars3.githubusercontent.com/u/10216045?v=3","gravatar_id":"","url":"https://api.github.com/users/mlpack","html_url":"https://github.com/mlpack","followers_url":"https://api.github.com/users/mlpack/followers","following_url":"https://api.github.com/users/mlpack/following{/other_user}","gists_url":"https://api.github.com/users/mlpack/gists{/gist_id}","starred_url":"https://api.github.com/users/mlpack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mlpack/subscriptions","organizations_url":"https://api.github.com/users/mlpack/orgs","repos_url":"https://api.github.com/users/mlpack/repos","events_url":"https://api.github.com/users/mlpack/events{/privacy}","received_events_url":"https://api.github.com/users/mlpack/received_events","type":"Organization","site_admin":false},"repo":{"id":28149409,"name":"mlpack","full_name":"mlpack/mlpack","owner":{"login":"mlpack","id":10216045,"avatar_url":"https://avatars3.githubusercontent.com/u/10216045?v=3","gravatar_id":"","url":"https://api.github.com/users/mlpack","html_url":"https://github.com/mlpack","followers_url":"https://api.github.com/users/mlpack/followers","following_url":"https://api.github.com/users/mlpack/following{/other_user}","gists_url":"https://api.github.com/users/mlpack/gists{/gist_id}","starred_url":"https://api.github.com/users/mlpack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mlpack/subscriptions","organizations_url":"https://api.github.com/users/mlpack/orgs","repos_url":"https://api.github.com/users/mlpack/repos","events_url":"https://api.github.com/users/mlpack/events{/privacy}","received_events_url":"https://api.github.com/users/mlpack/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mlpack/mlpack","description":"mlpack: a scalable C++ machine learning library -- ","fork":false,"url":"https://api.github.com/repos/mlpack/mlpack","forks_url":"https://api.github.com/repos/mlpack/mlpack/forks","keys_url":"https://api.github.com/repos/mlpack/mlpack/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mlpack/mlpack/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mlpack/mlpack/teams","hooks_url":"https://api.github.com/repos/mlpack/mlpack/hooks","issue_events_url":"https://api.github.com/repos/mlpack/mlpack/issues/events{/number}","events_url":"https://api.github.com/repos/mlpack/mlpack/events","assignees_url":"https://api.github.com/repos/mlpack/mlpack/assignees{/user}","branches_url":"https://api.github.com/repos/mlpack/mlpack/branches{/branch}","tags_url":"https://api.github.com/repos/mlpack/mlpack/tags","blobs_url":"https://api.github.com/repos/mlpack/mlpack/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mlpack/mlpack/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mlpack/mlpack/git/refs{/sha}","trees_url":"https://api.github.com/repos/mlpack/mlpack/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mlpack/mlpack/statuses/{sha}","languages_url":"https://api.github.com/repos/mlpack/mlpack/languages","stargazers_url":"https://api.github.com/repos/mlpack/mlpack/stargazers","contributors_url":"https://api.github.com/repos/mlpack/mlpack/contributors","subscribers_url":"https://api.github.com/repos/mlpack/mlpack/subscribers","subscription_url":"https://api.github.com/repos/mlpack/mlpack/subscription","commits_url":"https://api.github.com/repos/mlpack/mlpack/commits{/sha}","git_commits_url":"https://api.github.com/repos/mlpack/mlpack/git/commits{/sha}","comments_url":"https://api.github.com/repos/mlpack/mlpack/comments{/number}","issue_comment_url":"https://api.github.com/repos/mlpack/mlpack/issues/comments{/number}","contents_url":"https://api.github.com/repos/mlpack/mlpack/contents/{+path}","compare_url":"https://api.github.com/repos/mlpack/mlpack/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mlpack/mlpack/merges","archive_url":"https://api.github.com/repos/mlpack/mlpack/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mlpack/mlpack/downloads","issues_url":"https://api.github.com/repos/mlpack/mlpack/issues{/number}","pulls_url":"https://api.github.com/repos/mlpack/mlpack/pulls{/number}","milestones_url":"https://api.github.com/repos/mlpack/mlpack/milestones{/number}","notifications_url":"https://api.github.com/repos/mlpack/mlpack/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mlpack/mlpack/labels{/name}","releases_url":"https://api.github.com/repos/mlpack/mlpack/releases{/id}","deployments_url":"https://api.github.com/repos/mlpack/mlpack/deployments","created_at":"2014-12-17T18:16:59Z","updated_at":"2017-05-03T07:33:05Z","pushed_at":"2017-05-03T17:20:18Z","git_url":"git://github.com/mlpack/mlpack.git","ssh_url":"git@github.com:mlpack/mlpack.git","clone_url":"https://github.com/mlpack/mlpack.git","svn_url":"https://github.com/mlpack/mlpack","homepage":"http://www.mlpack.org/","size":239256,"stargazers_count":1483,"watchers_count":1483,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":561,"mirror_url":null,"open_issues_count":89,"forks":561,"open_issues":89,"watchers":1483,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/989"},"html":{"href":"https://github.com/mlpack/mlpack/pull/989"},"issue":{"href":"https://api.github.com/repos/mlpack/mlpack/issues/989"},"comments":{"href":"https://api.github.com/repos/mlpack/mlpack/issues/989/comments"},"review_comments":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/989/comments"},"review_comment":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mlpack/mlpack/pulls/989/commits"},"statuses":{"href":"https://api.github.com/repos/mlpack/mlpack/statuses/1acd5d181510f10c63b9e4ba04067466213835d8"}}}}
{"actor":{"display_login":"zoq"}}