{"action":"created","comment":{"url":"https://api.github.com/repos/acdha/mountstatus/pulls/comments/136105731","pull_request_review_id":59598401,"id":136105731,"diff_hunk":"@@ -0,0 +1,272 @@\n+/*\n+    Paranoid mount monitor for POSIX operating systems\n+\n+    The general idea is that some classes of storage failure require care to\n+    detect because any access to the mountpoint, including your monitoring code,\n+    will block and in the case of certain kernel bugs, that may either\n+    irrecoverable or until repeated TCP + NFS timeouts expire after multiple\n+    days. Asynchronous APIs could help except that e.g. the Linux async APIs\n+    don't include calls like stat(2).\n+\n+    We try to avoid this situation by using an external child process with a\n+    timeout. If it fails to respond by the deadline, we'll send it a SIGKILL\n+    and avoid further checks until the process disappears.\n+\n+    The major improvements of the Rust version compared to the older C version\n+    are the use of persistent state to avoid having more than one check pending\n+    for any given mountpoint and the ability to send metrics to a Prometheus\n+    push-gateway so they will be alertable even if the local system is severely\n+    degraded.\n+ */\n+\n+extern crate argparse;\n+extern crate hostname;\n+extern crate libc;\n+extern crate syslog;\n+extern crate wait_timeout;\n+\n+#[macro_use]\n+extern crate cfg_if;\n+\n+#[macro_use]\n+extern crate prometheus;\n+#[macro_use]\n+extern crate lazy_static;\n+\n+use std::collections::HashMap;\n+use std::process;\n+use std::str;\n+use std::thread;\n+use std::time::{Duration, Instant};\n+\n+use argparse::{ArgumentParser, Store, StoreOption};\n+use syslog::Facility;\n+use wait_timeout::ChildExt;\n+\n+lazy_static! {\n+    static ref TOTAL_MOUNTS: prometheus::Gauge = register_gauge!(\n+        \"total_mountpoints\",\n+        \"Total number of mountpoints\"\n+    ).unwrap();\n+\n+    static ref DEAD_MOUNTS: prometheus::Gauge = register_gauge!(\n+        \"dead_mountpoints\",\n+        \"Number of unresponsive mountpoints\"\n+    ).unwrap();\n+}\n+\n+mod get_mounts;\n+\n+#[derive(Debug)]\n+struct MountStatus {\n+    last_checked: Instant,\n+    alive: bool,\n+    check_process: Option<process::Child>,\n+}\n+\n+fn handle_syslog_error(err: std::io::Error) -> usize {\n+    // Convenience function allowing all of our syslog calls to use .unwrap_or_else\n+    eprintln!(\"Syslog failed: {}\", err);\n+    0\n+}\n+\n+fn main() {\n+    let mut poll_interval = 60;\n+    let mut prometheus_push_gateway: Option<String> = None;\n+\n+    {\n+        // this block limits scope of borrows by ap.refer() method\n+        let mut ap = ArgumentParser::new();\n+        ap.set_description(concat!(\n+            \"Monitor the status of mounted filesystems and report inaccessible mounts.\",\n+            \" Dead mounts will be reported to the local syslog server and optionally\",\n+            \" a Prometheus push-gateway service.\"\n+        ));\n+\n+        ap.refer(&mut prometheus_push_gateway).add_option(\n+            &[\"--prometheus-push-gateway\"],\n+            StoreOption,\n+            \"Location of the Prometheus push-gateway server to send metrics to\",\n+        );\n+        ap.refer(&mut poll_interval).add_option(\n+            &[\"--poll-interval\"],\n+            Store,\n+            \"Number of seconds to wait before checking mounts\",\n+        );\n+        ap.parse_args_or_exit();\n+    }\n+\n+    let poll_interval_duration = Duration::from_secs(poll_interval);\n+\n+    println!(\n+        \"mount_status_monitor checking mounts every {} seconds\",\n+        poll_interval_duration.as_secs()\n+    );\n+\n+    let syslog = syslog::unix(Facility::LOG_DAEMON).unwrap_or_else(|err| {\n+        eprintln!(\"Unable to connect to syslog: {}\", err);\n+        std::process::exit(1);\n+    });\n+\n+    let mut mount_statuses = HashMap::<String, MountStatus>::new();\n+\n+    loop {\n+        check_mounts(&mut mount_statuses, &syslog);\n+\n+        // We calculate these values each time because a filesystem may have been\n+        // mounted or unmounted since the last check:\n+        let total_mounts = mount_statuses.len();\n+        let dead_mounts = mount_statuses\n+            .iter()\n+            .map(|(_, status)| !status.alive)\n+            .filter(|&i| i)\n+            .count();\n+\n+        // TODO: consider making this debug or sending it to stdout?\n+        syslog\n+            .info(format!(\n+                \"Checked {} mounts; {} are dead\",\n+                total_mounts,\n+                dead_mounts\n+            ))\n+            .unwrap_or_else(handle_syslog_error);\n+\n+        if prometheus_push_gateway.is_some() {\n+            let prometheus_instance = hostname::get_hostname().unwrap();\n+\n+            // The Prometheus metrics are defined as floats so we need to convert;\n+            // for monitoring the precision loss in general is fine and it's\n+            // exceedingly unlikely to be relevant when counting the number of\n+            // mountpoints:\n+            TOTAL_MOUNTS.set(total_mounts as f64);\n+            DEAD_MOUNTS.set(dead_mounts as f64);\n+\n+            if let Err(e) = prometheus::push_metrics(\n+                \"mount_status_monitor\",\n+                labels!{\"instance\".to_owned() => prometheus_instance.to_owned(), },\n+                prometheus_push_gateway.as_ref().unwrap(),\n+                prometheus::gather(),\n+            )\n+            {\n+                eprintln!(\"Unable to send pushgateway metrics: {}\", e);\n+            }\n+        }\n+\n+        // Wait before checking again:\n+        thread::sleep(poll_interval_duration);\n+    }\n+}\n+\n+fn check_mounts(mount_statuses: &mut HashMap<String, MountStatus>, logger: &syslog::Logger) {\n+    let mount_points = get_mounts::get_mount_points();\n+\n+    // Remove any mount status entries which are no longer in the current list of mountpoints:\n+    mount_statuses.retain(|ref k, _| {\n+        mount_points.iter().position(|i| *i == **k).is_some()","path":"src/main.rs","position":165,"original_position":165,"commit_id":"4686ebb8cc563278d80c0905153680521fa60220","original_commit_id":"4686ebb8cc563278d80c0905153680521fa60220","user":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"body":"@Dr-Emann I wasn't sure how much I should care about that given that the list of mounts is usually well under a hundred items but it might warrant further thought.","created_at":"2017-08-30T15:34:46Z","updated_at":"2017-08-30T15:34:46Z","html_url":"https://github.com/acdha/mountstatus/pull/2#discussion_r136105731","pull_request_url":"https://api.github.com/repos/acdha/mountstatus/pulls/2","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/comments/136105731"},"html":{"href":"https://github.com/acdha/mountstatus/pull/2#discussion_r136105731"},"pull_request":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/2"}},"in_reply_to_id":136096535},"pull_request":{"url":"https://api.github.com/repos/acdha/mountstatus/pulls/2","id":138286346,"html_url":"https://github.com/acdha/mountstatus/pull/2","diff_url":"https://github.com/acdha/mountstatus/pull/2.diff","patch_url":"https://github.com/acdha/mountstatus/pull/2.patch","issue_url":"https://api.github.com/repos/acdha/mountstatus/issues/2","number":2,"state":"open","locked":false,"title":"Port to Rust","user":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"body":"This branch converts the existing C codebase to Rust with some simplifications. Using a modern language made it easier to add some advanced features:\r\n\r\n* We keep track of past checks and won't repeat a check for a dead mountpoint until the child process has exited\r\n* Poll interval is now configurable\r\n* Total and dead mount counts can optionally be [pushed to Prometheus](https://prometheus.io/docs/instrumenting/pushing/)\r\n\r\nAs part of this work, the model has been simplified to remove the concept of daemonization and only support running under supervisors such as launchd, upstart, or systemd. This also allows running as an unprivileged user as long as you're comfortable with the possibility of errors due to a mountpoint not being accessible by the running user (perhaps this should tested at launch for a one-time info report?).\r\n\r\nStill todo:\r\n* [ ] Documentation overhaul\r\n* [ ] Use Docker to build RPM/Deb packages\r\n* [ ] Update provided upstart and systemd files\r\n* [ ] Add launchd\r\n* [ ] Decide on an automated testing strategy — perhaps using Docker + a FUSE filesystem which can be sent `kill -STOP` rather than setting up a fake NFS server?","created_at":"2017-08-29T21:17:58Z","updated_at":"2017-08-30T15:34:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c7452331cc04a6876fa6472e2f7c3a8e0dfc7caf","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/acdha/mountstatus/pulls/2/commits","review_comments_url":"https://api.github.com/repos/acdha/mountstatus/pulls/2/comments","review_comment_url":"https://api.github.com/repos/acdha/mountstatus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/acdha/mountstatus/issues/2/comments","statuses_url":"https://api.github.com/repos/acdha/mountstatus/statuses/4686ebb8cc563278d80c0905153680521fa60220","head":{"label":"acdha:corrosion","ref":"corrosion","sha":"4686ebb8cc563278d80c0905153680521fa60220","user":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"repo":{"id":128567,"name":"mountstatus","full_name":"acdha/mountstatus","owner":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/acdha/mountstatus","description":"MountStatusMonitor: paranoid monitor for POSIX filesystem mounts (Linux, OS X, FreeBSD)","fork":false,"url":"https://api.github.com/repos/acdha/mountstatus","forks_url":"https://api.github.com/repos/acdha/mountstatus/forks","keys_url":"https://api.github.com/repos/acdha/mountstatus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/acdha/mountstatus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/acdha/mountstatus/teams","hooks_url":"https://api.github.com/repos/acdha/mountstatus/hooks","issue_events_url":"https://api.github.com/repos/acdha/mountstatus/issues/events{/number}","events_url":"https://api.github.com/repos/acdha/mountstatus/events","assignees_url":"https://api.github.com/repos/acdha/mountstatus/assignees{/user}","branches_url":"https://api.github.com/repos/acdha/mountstatus/branches{/branch}","tags_url":"https://api.github.com/repos/acdha/mountstatus/tags","blobs_url":"https://api.github.com/repos/acdha/mountstatus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/acdha/mountstatus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/acdha/mountstatus/git/refs{/sha}","trees_url":"https://api.github.com/repos/acdha/mountstatus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/acdha/mountstatus/statuses/{sha}","languages_url":"https://api.github.com/repos/acdha/mountstatus/languages","stargazers_url":"https://api.github.com/repos/acdha/mountstatus/stargazers","contributors_url":"https://api.github.com/repos/acdha/mountstatus/contributors","subscribers_url":"https://api.github.com/repos/acdha/mountstatus/subscribers","subscription_url":"https://api.github.com/repos/acdha/mountstatus/subscription","commits_url":"https://api.github.com/repos/acdha/mountstatus/commits{/sha}","git_commits_url":"https://api.github.com/repos/acdha/mountstatus/git/commits{/sha}","comments_url":"https://api.github.com/repos/acdha/mountstatus/comments{/number}","issue_comment_url":"https://api.github.com/repos/acdha/mountstatus/issues/comments{/number}","contents_url":"https://api.github.com/repos/acdha/mountstatus/contents/{+path}","compare_url":"https://api.github.com/repos/acdha/mountstatus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/acdha/mountstatus/merges","archive_url":"https://api.github.com/repos/acdha/mountstatus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/acdha/mountstatus/downloads","issues_url":"https://api.github.com/repos/acdha/mountstatus/issues{/number}","pulls_url":"https://api.github.com/repos/acdha/mountstatus/pulls{/number}","milestones_url":"https://api.github.com/repos/acdha/mountstatus/milestones{/number}","notifications_url":"https://api.github.com/repos/acdha/mountstatus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/acdha/mountstatus/labels{/name}","releases_url":"https://api.github.com/repos/acdha/mountstatus/releases{/id}","deployments_url":"https://api.github.com/repos/acdha/mountstatus/deployments","created_at":"2009-02-14T00:11:17Z","updated_at":"2017-06-08T17:41:12Z","pushed_at":"2017-08-30T05:24:23Z","git_url":"git://github.com/acdha/mountstatus.git","ssh_url":"git@github.com:acdha/mountstatus.git","clone_url":"https://github.com/acdha/mountstatus.git","svn_url":"https://github.com/acdha/mountstatus","homepage":"","size":49,"stargazers_count":9,"watchers_count":9,"language":"C","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":3,"forks":3,"open_issues":3,"watchers":9,"default_branch":"master"}},"base":{"label":"acdha:master","ref":"master","sha":"69a73a446ff57a66e0e1749c0acf12cd314123f7","user":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"repo":{"id":128567,"name":"mountstatus","full_name":"acdha/mountstatus","owner":{"login":"acdha","id":46565,"avatar_url":"https://avatars3.githubusercontent.com/u/46565?v=4","gravatar_id":"","url":"https://api.github.com/users/acdha","html_url":"https://github.com/acdha","followers_url":"https://api.github.com/users/acdha/followers","following_url":"https://api.github.com/users/acdha/following{/other_user}","gists_url":"https://api.github.com/users/acdha/gists{/gist_id}","starred_url":"https://api.github.com/users/acdha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acdha/subscriptions","organizations_url":"https://api.github.com/users/acdha/orgs","repos_url":"https://api.github.com/users/acdha/repos","events_url":"https://api.github.com/users/acdha/events{/privacy}","received_events_url":"https://api.github.com/users/acdha/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/acdha/mountstatus","description":"MountStatusMonitor: paranoid monitor for POSIX filesystem mounts (Linux, OS X, FreeBSD)","fork":false,"url":"https://api.github.com/repos/acdha/mountstatus","forks_url":"https://api.github.com/repos/acdha/mountstatus/forks","keys_url":"https://api.github.com/repos/acdha/mountstatus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/acdha/mountstatus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/acdha/mountstatus/teams","hooks_url":"https://api.github.com/repos/acdha/mountstatus/hooks","issue_events_url":"https://api.github.com/repos/acdha/mountstatus/issues/events{/number}","events_url":"https://api.github.com/repos/acdha/mountstatus/events","assignees_url":"https://api.github.com/repos/acdha/mountstatus/assignees{/user}","branches_url":"https://api.github.com/repos/acdha/mountstatus/branches{/branch}","tags_url":"https://api.github.com/repos/acdha/mountstatus/tags","blobs_url":"https://api.github.com/repos/acdha/mountstatus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/acdha/mountstatus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/acdha/mountstatus/git/refs{/sha}","trees_url":"https://api.github.com/repos/acdha/mountstatus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/acdha/mountstatus/statuses/{sha}","languages_url":"https://api.github.com/repos/acdha/mountstatus/languages","stargazers_url":"https://api.github.com/repos/acdha/mountstatus/stargazers","contributors_url":"https://api.github.com/repos/acdha/mountstatus/contributors","subscribers_url":"https://api.github.com/repos/acdha/mountstatus/subscribers","subscription_url":"https://api.github.com/repos/acdha/mountstatus/subscription","commits_url":"https://api.github.com/repos/acdha/mountstatus/commits{/sha}","git_commits_url":"https://api.github.com/repos/acdha/mountstatus/git/commits{/sha}","comments_url":"https://api.github.com/repos/acdha/mountstatus/comments{/number}","issue_comment_url":"https://api.github.com/repos/acdha/mountstatus/issues/comments{/number}","contents_url":"https://api.github.com/repos/acdha/mountstatus/contents/{+path}","compare_url":"https://api.github.com/repos/acdha/mountstatus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/acdha/mountstatus/merges","archive_url":"https://api.github.com/repos/acdha/mountstatus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/acdha/mountstatus/downloads","issues_url":"https://api.github.com/repos/acdha/mountstatus/issues{/number}","pulls_url":"https://api.github.com/repos/acdha/mountstatus/pulls{/number}","milestones_url":"https://api.github.com/repos/acdha/mountstatus/milestones{/number}","notifications_url":"https://api.github.com/repos/acdha/mountstatus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/acdha/mountstatus/labels{/name}","releases_url":"https://api.github.com/repos/acdha/mountstatus/releases{/id}","deployments_url":"https://api.github.com/repos/acdha/mountstatus/deployments","created_at":"2009-02-14T00:11:17Z","updated_at":"2017-06-08T17:41:12Z","pushed_at":"2017-08-30T05:24:23Z","git_url":"git://github.com/acdha/mountstatus.git","ssh_url":"git@github.com:acdha/mountstatus.git","clone_url":"https://github.com/acdha/mountstatus.git","svn_url":"https://github.com/acdha/mountstatus","homepage":"","size":49,"stargazers_count":9,"watchers_count":9,"language":"C","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":3,"forks":3,"open_issues":3,"watchers":9,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/2"},"html":{"href":"https://github.com/acdha/mountstatus/pull/2"},"issue":{"href":"https://api.github.com/repos/acdha/mountstatus/issues/2"},"comments":{"href":"https://api.github.com/repos/acdha/mountstatus/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/acdha/mountstatus/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/acdha/mountstatus/statuses/4686ebb8cc563278d80c0905153680521fa60220"}},"author_association":"OWNER"}}
{"actor":{"display_login":"acdha"}}