{"action":"created","comment":{"url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/comments/131507912","pull_request_review_id":54501279,"id":131507912,"diff_hunk":"@@ -123,13 +124,53 @@ def get_file_list_curated\n     end\n     file_list_curated\n   end\n+  def get_file_list_ALL\n+    file_list_curated = Hash.new\n+    puts \"snapshots_to_consider: \" + get_all_snapshots_to_consider.lines.count.to_s\n+    get_all_snapshots_to_consider.each_line do |line|\n+      unless line.include?('/')\n+        print \"**\" + line\n+\tnext\n+      end\n+      file_timestamp = line[0..13].to_i\n+      file_url = line[15..-2]\n+      file_id = file_url.split('/')[3..-1].join('/')\n+      file_id = [file_timestamp, file_id].join('/')\n+      file_id = CGI::unescape file_id \n+      file_id = file_id.tidy_bytes unless file_id == \"\"\n+      if file_id.nil?\n+        puts \"Malformed file url, ignoring: #{file_url}\"\n+      else\n+        if match_exclude_filter(file_url)\n+          puts \"File url matches exclude filter, ignoring: #{file_url}\"\n+        elsif not match_only_filter(file_url)\n+          puts \"File url doesn't match only filter, ignoring: #{file_url}\"\n+        elsif file_list_curated[file_id]\n+         #puts \"__\" + file_id","path":"lib/wayback_machine_downloader.rb","position":34,"original_position":34,"commit_id":"5bd9fbffdd4d6f37e9263008e04bb88b72346b62","original_commit_id":"5bd9fbffdd4d6f37e9263008e04bb88b72346b62","user":{"login":"hartator","id":307597,"avatar_url":"https://avatars2.githubusercontent.com/u/307597?v=4","gravatar_id":"","url":"https://api.github.com/users/hartator","html_url":"https://github.com/hartator","followers_url":"https://api.github.com/users/hartator/followers","following_url":"https://api.github.com/users/hartator/following{/other_user}","gists_url":"https://api.github.com/users/hartator/gists{/gist_id}","starred_url":"https://api.github.com/users/hartator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hartator/subscriptions","organizations_url":"https://api.github.com/users/hartator/orgs","repos_url":"https://api.github.com/users/hartator/repos","events_url":"https://api.github.com/users/hartator/events{/privacy}","received_events_url":"https://api.github.com/users/hartator/received_events","type":"User","site_admin":false},"body":"I am working on this right now. It's actually working.\r\n\r\nIn this code `file_id` is not just the file id, but` file_id = [file_timestamp, file_id]`. I'll edit this part to make it clearer for futur readers. I was confused a bit too. ","created_at":"2017-08-05T00:09:50Z","updated_at":"2017-08-05T00:09:51Z","html_url":"https://github.com/hartator/wayback-machine-downloader/pull/89#discussion_r131507912","pull_request_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89","_links":{"self":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/comments/131507912"},"html":{"href":"https://github.com/hartator/wayback-machine-downloader/pull/89#discussion_r131507912"},"pull_request":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89"}}},"pull_request":{"url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89","id":112685857,"html_url":"https://github.com/hartator/wayback-machine-downloader/pull/89","diff_url":"https://github.com/hartator/wayback-machine-downloader/pull/89.diff","patch_url":"https://github.com/hartator/wayback-machine-downloader/pull/89.patch","issue_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/89","number":89,"state":"open","locked":false,"title":"insaner - download all","user":{"login":"insaner","id":6283329,"avatar_url":"https://avatars1.githubusercontent.com/u/6283329?v=4","gravatar_id":"","url":"https://api.github.com/users/insaner","html_url":"https://github.com/insaner","followers_url":"https://api.github.com/users/insaner/followers","following_url":"https://api.github.com/users/insaner/following{/other_user}","gists_url":"https://api.github.com/users/insaner/gists{/gist_id}","starred_url":"https://api.github.com/users/insaner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/insaner/subscriptions","organizations_url":"https://api.github.com/users/insaner/orgs","repos_url":"https://api.github.com/users/insaner/repos","events_url":"https://api.github.com/users/insaner/events{/privacy}","received_events_url":"https://api.github.com/users/insaner/received_events","type":"User","site_admin":false},"body":"Add option to download all archived snapshots (not just the latest and not just a single timestamp) ie, issue #28 ","created_at":"2017-03-27T07:47:11Z","updated_at":"2017-08-05T00:09:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ac165c259fe1268b71bff269741cfe98b9c65171","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89/commits","review_comments_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89/comments","review_comment_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/89/comments","statuses_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/statuses/5bd9fbffdd4d6f37e9263008e04bb88b72346b62","head":{"label":"insaner:insaner-download-all","ref":"insaner-download-all","sha":"5bd9fbffdd4d6f37e9263008e04bb88b72346b62","user":{"login":"insaner","id":6283329,"avatar_url":"https://avatars1.githubusercontent.com/u/6283329?v=4","gravatar_id":"","url":"https://api.github.com/users/insaner","html_url":"https://github.com/insaner","followers_url":"https://api.github.com/users/insaner/followers","following_url":"https://api.github.com/users/insaner/following{/other_user}","gists_url":"https://api.github.com/users/insaner/gists{/gist_id}","starred_url":"https://api.github.com/users/insaner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/insaner/subscriptions","organizations_url":"https://api.github.com/users/insaner/orgs","repos_url":"https://api.github.com/users/insaner/repos","events_url":"https://api.github.com/users/insaner/events{/privacy}","received_events_url":"https://api.github.com/users/insaner/received_events","type":"User","site_admin":false},"repo":{"id":79898629,"name":"wayback-machine-downloader","full_name":"insaner/wayback-machine-downloader","owner":{"login":"insaner","id":6283329,"avatar_url":"https://avatars1.githubusercontent.com/u/6283329?v=4","gravatar_id":"","url":"https://api.github.com/users/insaner","html_url":"https://github.com/insaner","followers_url":"https://api.github.com/users/insaner/followers","following_url":"https://api.github.com/users/insaner/following{/other_user}","gists_url":"https://api.github.com/users/insaner/gists{/gist_id}","starred_url":"https://api.github.com/users/insaner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/insaner/subscriptions","organizations_url":"https://api.github.com/users/insaner/orgs","repos_url":"https://api.github.com/users/insaner/repos","events_url":"https://api.github.com/users/insaner/events{/privacy}","received_events_url":"https://api.github.com/users/insaner/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/insaner/wayback-machine-downloader","description":"Download an entire website from the Wayback Machine.","fork":true,"url":"https://api.github.com/repos/insaner/wayback-machine-downloader","forks_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/forks","keys_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/keys{/key_id}","collaborators_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/teams","hooks_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/hooks","issue_events_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/issues/events{/number}","events_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/events","assignees_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/assignees{/user}","branches_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/branches{/branch}","tags_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/tags","blobs_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/git/refs{/sha}","trees_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/git/trees{/sha}","statuses_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/statuses/{sha}","languages_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/languages","stargazers_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/stargazers","contributors_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/contributors","subscribers_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/subscribers","subscription_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/subscription","commits_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/commits{/sha}","git_commits_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/git/commits{/sha}","comments_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/comments{/number}","issue_comment_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/issues/comments{/number}","contents_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/contents/{+path}","compare_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/compare/{base}...{head}","merges_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/merges","archive_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/downloads","issues_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/issues{/number}","pulls_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/pulls{/number}","milestones_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/milestones{/number}","notifications_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/labels{/name}","releases_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/releases{/id}","deployments_url":"https://api.github.com/repos/insaner/wayback-machine-downloader/deployments","created_at":"2017-01-24T09:49:59Z","updated_at":"2017-01-24T09:50:01Z","pushed_at":"2017-01-24T10:03:08Z","git_url":"git://github.com/insaner/wayback-machine-downloader.git","ssh_url":"git@github.com:insaner/wayback-machine-downloader.git","clone_url":"https://github.com/insaner/wayback-machine-downloader.git","svn_url":"https://github.com/insaner/wayback-machine-downloader","homepage":"","size":115,"stargazers_count":0,"watchers_count":0,"language":"Ruby","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hartator:master","ref":"master","sha":"e95ade807994052e98568f23a4a2c6e1a9aacbec","user":{"login":"hartator","id":307597,"avatar_url":"https://avatars2.githubusercontent.com/u/307597?v=4","gravatar_id":"","url":"https://api.github.com/users/hartator","html_url":"https://github.com/hartator","followers_url":"https://api.github.com/users/hartator/followers","following_url":"https://api.github.com/users/hartator/following{/other_user}","gists_url":"https://api.github.com/users/hartator/gists{/gist_id}","starred_url":"https://api.github.com/users/hartator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hartator/subscriptions","organizations_url":"https://api.github.com/users/hartator/orgs","repos_url":"https://api.github.com/users/hartator/repos","events_url":"https://api.github.com/users/hartator/events{/privacy}","received_events_url":"https://api.github.com/users/hartator/received_events","type":"User","site_admin":false},"repo":{"id":39706711,"name":"wayback-machine-downloader","full_name":"hartator/wayback-machine-downloader","owner":{"login":"hartator","id":307597,"avatar_url":"https://avatars2.githubusercontent.com/u/307597?v=4","gravatar_id":"","url":"https://api.github.com/users/hartator","html_url":"https://github.com/hartator","followers_url":"https://api.github.com/users/hartator/followers","following_url":"https://api.github.com/users/hartator/following{/other_user}","gists_url":"https://api.github.com/users/hartator/gists{/gist_id}","starred_url":"https://api.github.com/users/hartator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hartator/subscriptions","organizations_url":"https://api.github.com/users/hartator/orgs","repos_url":"https://api.github.com/users/hartator/repos","events_url":"https://api.github.com/users/hartator/events{/privacy}","received_events_url":"https://api.github.com/users/hartator/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/hartator/wayback-machine-downloader","description":"Download an entire website from the Wayback Machine.","fork":false,"url":"https://api.github.com/repos/hartator/wayback-machine-downloader","forks_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/forks","keys_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/teams","hooks_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/hooks","issue_events_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/events{/number}","events_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/events","assignees_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/assignees{/user}","branches_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/branches{/branch}","tags_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/tags","blobs_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/git/refs{/sha}","trees_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/statuses/{sha}","languages_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/languages","stargazers_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/stargazers","contributors_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/contributors","subscribers_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/subscribers","subscription_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/subscription","commits_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/commits{/sha}","git_commits_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/git/commits{/sha}","comments_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/comments{/number}","issue_comment_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/comments{/number}","contents_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/contents/{+path}","compare_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/merges","archive_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/downloads","issues_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues{/number}","pulls_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls{/number}","milestones_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/milestones{/number}","notifications_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/labels{/name}","releases_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/releases{/id}","deployments_url":"https://api.github.com/repos/hartator/wayback-machine-downloader/deployments","created_at":"2015-07-25T23:45:34Z","updated_at":"2017-08-04T15:28:12Z","pushed_at":"2017-07-11T19:58:21Z","git_url":"git://github.com/hartator/wayback-machine-downloader.git","ssh_url":"git@github.com:hartator/wayback-machine-downloader.git","clone_url":"https://github.com/hartator/wayback-machine-downloader.git","svn_url":"https://github.com/hartator/wayback-machine-downloader","homepage":"","size":132,"stargazers_count":845,"watchers_count":845,"language":"Ruby","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":118,"mirror_url":null,"open_issues_count":22,"forks":118,"open_issues":22,"watchers":845,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89"},"html":{"href":"https://github.com/hartator/wayback-machine-downloader/pull/89"},"issue":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/89"},"comments":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/issues/89/comments"},"review_comments":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89/comments"},"review_comment":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/pulls/89/commits"},"statuses":{"href":"https://api.github.com/repos/hartator/wayback-machine-downloader/statuses/5bd9fbffdd4d6f37e9263008e04bb88b72346b62"}}}}
{"actor":{"display_login":"hartator"}}