{"action":"created","comment":{"url":"https://api.github.com/repos/pulibrary/orangelight/pulls/comments/157576035","pull_request_review_id":84240923,"id":157576035,"diff_hunk":"@@ -31,6 +32,17 @@ namespace :pulsearch do\n     end\n   end\n \n+  desc 'Generate a robots.txt file'\n+  task :robots_txt do |_t, args|\n+    file_path = args[:file_path] || Rails.root.join('public', 'robots.txt')\n+    robots = RobotsGeneratorService.new(path: file_path, disallowed_paths: Rails.configuration.robots.disallowed_paths)\n+    robots.insert_group(user_agent: '*')\n+    robots.insert_crawl_delay(10)\n+    robots.insert_sitemap(Rails.configuration.robots.sitemap_url)\n+    robots.generate\n+    robots.write\n+  end","path":"lib/tasks/pulsearch.rake","position":20,"original_position":20,"commit_id":"9402999ad592ef41f6c7a5196d790209f9f64dda","original_commit_id":"9402999ad592ef41f6c7a5196d790209f9f64dda","user":{"login":"jrgriffiniii","id":1443986,"avatar_url":"https://avatars0.githubusercontent.com/u/1443986?v=4","gravatar_id":"","url":"https://api.github.com/users/jrgriffiniii","html_url":"https://github.com/jrgriffiniii","followers_url":"https://api.github.com/users/jrgriffiniii/followers","following_url":"https://api.github.com/users/jrgriffiniii/following{/other_user}","gists_url":"https://api.github.com/users/jrgriffiniii/gists{/gist_id}","starred_url":"https://api.github.com/users/jrgriffiniii/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jrgriffiniii/subscriptions","organizations_url":"https://api.github.com/users/jrgriffiniii/orgs","repos_url":"https://api.github.com/users/jrgriffiniii/repos","events_url":"https://api.github.com/users/jrgriffiniii/events{/privacy}","received_events_url":"https://api.github.com/users/jrgriffiniii/received_events","type":"User","site_admin":false},"body":"> is this code that was useful to generate the original static version, that we don't need to maintain?\r\nI felt that this was the preferred approach, with the hope that any adjustment to the Routes for the App would require the config. being adjusted (please see https://github.com/pulibrary/orangelight/pull/1050/files#diff-b29c2e1a89929db60557b9de48f7b633).  At the time that this PR was issued, I could not determine a straightforward approach to accessing all Rails routes and finding some more automated way to blacklist certain patterns, so unfortunately, the robots.txt rules will require some curation.","created_at":"2017-12-18T19:21:41Z","updated_at":"2017-12-18T19:21:41Z","html_url":"https://github.com/pulibrary/orangelight/pull/1050#discussion_r157576035","pull_request_url":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/comments/157576035"},"html":{"href":"https://github.com/pulibrary/orangelight/pull/1050#discussion_r157576035"},"pull_request":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050"}},"in_reply_to_id":157573428},"pull_request":{"url":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050","id":158704098,"html_url":"https://github.com/pulibrary/orangelight/pull/1050","diff_url":"https://github.com/pulibrary/orangelight/pull/1050.diff","patch_url":"https://github.com/pulibrary/orangelight/pull/1050.patch","issue_url":"https://api.github.com/repos/pulibrary/orangelight/issues/1050","number":1050,"state":"open","locked":false,"title":"[WIP] Integrates the `sitemap_generator` Gem and implements the `RobotsGeneratorService` to generate a robots.txt","user":{"login":"jrgriffiniii","id":1443986,"avatar_url":"https://avatars0.githubusercontent.com/u/1443986?v=4","gravatar_id":"","url":"https://api.github.com/users/jrgriffiniii","html_url":"https://github.com/jrgriffiniii","followers_url":"https://api.github.com/users/jrgriffiniii/followers","following_url":"https://api.github.com/users/jrgriffiniii/following{/other_user}","gists_url":"https://api.github.com/users/jrgriffiniii/gists{/gist_id}","starred_url":"https://api.github.com/users/jrgriffiniii/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jrgriffiniii/subscriptions","organizations_url":"https://api.github.com/users/jrgriffiniii/orgs","repos_url":"https://api.github.com/users/jrgriffiniii/repos","events_url":"https://api.github.com/users/jrgriffiniii/events{/privacy}","received_events_url":"https://api.github.com/users/jrgriffiniii/received_events","type":"User","site_admin":false},"body":"Resolves #1000","created_at":"2017-12-15T22:11:32Z","updated_at":"2017-12-18T19:21:41Z","closed_at":null,"merged_at":null,"merge_commit_sha":"299af81772869626272b868a9785f68041d7c3cf","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050/commits","review_comments_url":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050/comments","review_comment_url":"https://api.github.com/repos/pulibrary/orangelight/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pulibrary/orangelight/issues/1050/comments","statuses_url":"https://api.github.com/repos/pulibrary/orangelight/statuses/9402999ad592ef41f6c7a5196d790209f9f64dda","head":{"label":"pulibrary:issues-1000-jrgriffiniii-sitemap","ref":"issues-1000-jrgriffiniii-sitemap","sha":"9402999ad592ef41f6c7a5196d790209f9f64dda","user":{"login":"pulibrary","id":1827800,"avatar_url":"https://avatars3.githubusercontent.com/u/1827800?v=4","gravatar_id":"","url":"https://api.github.com/users/pulibrary","html_url":"https://github.com/pulibrary","followers_url":"https://api.github.com/users/pulibrary/followers","following_url":"https://api.github.com/users/pulibrary/following{/other_user}","gists_url":"https://api.github.com/users/pulibrary/gists{/gist_id}","starred_url":"https://api.github.com/users/pulibrary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pulibrary/subscriptions","organizations_url":"https://api.github.com/users/pulibrary/orgs","repos_url":"https://api.github.com/users/pulibrary/repos","events_url":"https://api.github.com/users/pulibrary/events{/privacy}","received_events_url":"https://api.github.com/users/pulibrary/received_events","type":"Organization","site_admin":false},"repo":{"id":21954918,"name":"orangelight","full_name":"pulibrary/orangelight","owner":{"login":"pulibrary","id":1827800,"avatar_url":"https://avatars3.githubusercontent.com/u/1827800?v=4","gravatar_id":"","url":"https://api.github.com/users/pulibrary","html_url":"https://github.com/pulibrary","followers_url":"https://api.github.com/users/pulibrary/followers","following_url":"https://api.github.com/users/pulibrary/following{/other_user}","gists_url":"https://api.github.com/users/pulibrary/gists{/gist_id}","starred_url":"https://api.github.com/users/pulibrary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pulibrary/subscriptions","organizations_url":"https://api.github.com/users/pulibrary/orgs","repos_url":"https://api.github.com/users/pulibrary/repos","events_url":"https://api.github.com/users/pulibrary/events{/privacy}","received_events_url":"https://api.github.com/users/pulibrary/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pulibrary/orangelight","description":"PUL Blacklight Project","fork":false,"url":"https://api.github.com/repos/pulibrary/orangelight","forks_url":"https://api.github.com/repos/pulibrary/orangelight/forks","keys_url":"https://api.github.com/repos/pulibrary/orangelight/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pulibrary/orangelight/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pulibrary/orangelight/teams","hooks_url":"https://api.github.com/repos/pulibrary/orangelight/hooks","issue_events_url":"https://api.github.com/repos/pulibrary/orangelight/issues/events{/number}","events_url":"https://api.github.com/repos/pulibrary/orangelight/events","assignees_url":"https://api.github.com/repos/pulibrary/orangelight/assignees{/user}","branches_url":"https://api.github.com/repos/pulibrary/orangelight/branches{/branch}","tags_url":"https://api.github.com/repos/pulibrary/orangelight/tags","blobs_url":"https://api.github.com/repos/pulibrary/orangelight/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pulibrary/orangelight/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pulibrary/orangelight/git/refs{/sha}","trees_url":"https://api.github.com/repos/pulibrary/orangelight/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pulibrary/orangelight/statuses/{sha}","languages_url":"https://api.github.com/repos/pulibrary/orangelight/languages","stargazers_url":"https://api.github.com/repos/pulibrary/orangelight/stargazers","contributors_url":"https://api.github.com/repos/pulibrary/orangelight/contributors","subscribers_url":"https://api.github.com/repos/pulibrary/orangelight/subscribers","subscription_url":"https://api.github.com/repos/pulibrary/orangelight/subscription","commits_url":"https://api.github.com/repos/pulibrary/orangelight/commits{/sha}","git_commits_url":"https://api.github.com/repos/pulibrary/orangelight/git/commits{/sha}","comments_url":"https://api.github.com/repos/pulibrary/orangelight/comments{/number}","issue_comment_url":"https://api.github.com/repos/pulibrary/orangelight/issues/comments{/number}","contents_url":"https://api.github.com/repos/pulibrary/orangelight/contents/{+path}","compare_url":"https://api.github.com/repos/pulibrary/orangelight/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pulibrary/orangelight/merges","archive_url":"https://api.github.com/repos/pulibrary/orangelight/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pulibrary/orangelight/downloads","issues_url":"https://api.github.com/repos/pulibrary/orangelight/issues{/number}","pulls_url":"https://api.github.com/repos/pulibrary/orangelight/pulls{/number}","milestones_url":"https://api.github.com/repos/pulibrary/orangelight/milestones{/number}","notifications_url":"https://api.github.com/repos/pulibrary/orangelight/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pulibrary/orangelight/labels{/name}","releases_url":"https://api.github.com/repos/pulibrary/orangelight/releases{/id}","deployments_url":"https://api.github.com/repos/pulibrary/orangelight/deployments","created_at":"2014-07-17T19:35:14Z","updated_at":"2017-12-13T17:30:38Z","pushed_at":"2017-12-18T18:57:31Z","git_url":"git://github.com/pulibrary/orangelight.git","ssh_url":"git@github.com:pulibrary/orangelight.git","clone_url":"https://github.com/pulibrary/orangelight.git","svn_url":"https://github.com/pulibrary/orangelight","homepage":"","size":7561,"stargazers_count":10,"watchers_count":10,"language":"Ruby","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"archived":false,"open_issues_count":142,"license":null,"forks":2,"open_issues":142,"watchers":10,"default_branch":"master"}},"base":{"label":"pulibrary:master","ref":"master","sha":"a0c52d35af5d0f2014787202eb5e0102efc9d753","user":{"login":"pulibrary","id":1827800,"avatar_url":"https://avatars3.githubusercontent.com/u/1827800?v=4","gravatar_id":"","url":"https://api.github.com/users/pulibrary","html_url":"https://github.com/pulibrary","followers_url":"https://api.github.com/users/pulibrary/followers","following_url":"https://api.github.com/users/pulibrary/following{/other_user}","gists_url":"https://api.github.com/users/pulibrary/gists{/gist_id}","starred_url":"https://api.github.com/users/pulibrary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pulibrary/subscriptions","organizations_url":"https://api.github.com/users/pulibrary/orgs","repos_url":"https://api.github.com/users/pulibrary/repos","events_url":"https://api.github.com/users/pulibrary/events{/privacy}","received_events_url":"https://api.github.com/users/pulibrary/received_events","type":"Organization","site_admin":false},"repo":{"id":21954918,"name":"orangelight","full_name":"pulibrary/orangelight","owner":{"login":"pulibrary","id":1827800,"avatar_url":"https://avatars3.githubusercontent.com/u/1827800?v=4","gravatar_id":"","url":"https://api.github.com/users/pulibrary","html_url":"https://github.com/pulibrary","followers_url":"https://api.github.com/users/pulibrary/followers","following_url":"https://api.github.com/users/pulibrary/following{/other_user}","gists_url":"https://api.github.com/users/pulibrary/gists{/gist_id}","starred_url":"https://api.github.com/users/pulibrary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pulibrary/subscriptions","organizations_url":"https://api.github.com/users/pulibrary/orgs","repos_url":"https://api.github.com/users/pulibrary/repos","events_url":"https://api.github.com/users/pulibrary/events{/privacy}","received_events_url":"https://api.github.com/users/pulibrary/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pulibrary/orangelight","description":"PUL Blacklight Project","fork":false,"url":"https://api.github.com/repos/pulibrary/orangelight","forks_url":"https://api.github.com/repos/pulibrary/orangelight/forks","keys_url":"https://api.github.com/repos/pulibrary/orangelight/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pulibrary/orangelight/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pulibrary/orangelight/teams","hooks_url":"https://api.github.com/repos/pulibrary/orangelight/hooks","issue_events_url":"https://api.github.com/repos/pulibrary/orangelight/issues/events{/number}","events_url":"https://api.github.com/repos/pulibrary/orangelight/events","assignees_url":"https://api.github.com/repos/pulibrary/orangelight/assignees{/user}","branches_url":"https://api.github.com/repos/pulibrary/orangelight/branches{/branch}","tags_url":"https://api.github.com/repos/pulibrary/orangelight/tags","blobs_url":"https://api.github.com/repos/pulibrary/orangelight/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pulibrary/orangelight/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pulibrary/orangelight/git/refs{/sha}","trees_url":"https://api.github.com/repos/pulibrary/orangelight/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pulibrary/orangelight/statuses/{sha}","languages_url":"https://api.github.com/repos/pulibrary/orangelight/languages","stargazers_url":"https://api.github.com/repos/pulibrary/orangelight/stargazers","contributors_url":"https://api.github.com/repos/pulibrary/orangelight/contributors","subscribers_url":"https://api.github.com/repos/pulibrary/orangelight/subscribers","subscription_url":"https://api.github.com/repos/pulibrary/orangelight/subscription","commits_url":"https://api.github.com/repos/pulibrary/orangelight/commits{/sha}","git_commits_url":"https://api.github.com/repos/pulibrary/orangelight/git/commits{/sha}","comments_url":"https://api.github.com/repos/pulibrary/orangelight/comments{/number}","issue_comment_url":"https://api.github.com/repos/pulibrary/orangelight/issues/comments{/number}","contents_url":"https://api.github.com/repos/pulibrary/orangelight/contents/{+path}","compare_url":"https://api.github.com/repos/pulibrary/orangelight/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pulibrary/orangelight/merges","archive_url":"https://api.github.com/repos/pulibrary/orangelight/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pulibrary/orangelight/downloads","issues_url":"https://api.github.com/repos/pulibrary/orangelight/issues{/number}","pulls_url":"https://api.github.com/repos/pulibrary/orangelight/pulls{/number}","milestones_url":"https://api.github.com/repos/pulibrary/orangelight/milestones{/number}","notifications_url":"https://api.github.com/repos/pulibrary/orangelight/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pulibrary/orangelight/labels{/name}","releases_url":"https://api.github.com/repos/pulibrary/orangelight/releases{/id}","deployments_url":"https://api.github.com/repos/pulibrary/orangelight/deployments","created_at":"2014-07-17T19:35:14Z","updated_at":"2017-12-13T17:30:38Z","pushed_at":"2017-12-18T18:57:31Z","git_url":"git://github.com/pulibrary/orangelight.git","ssh_url":"git@github.com:pulibrary/orangelight.git","clone_url":"https://github.com/pulibrary/orangelight.git","svn_url":"https://github.com/pulibrary/orangelight","homepage":"","size":7561,"stargazers_count":10,"watchers_count":10,"language":"Ruby","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"archived":false,"open_issues_count":142,"license":null,"forks":2,"open_issues":142,"watchers":10,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050"},"html":{"href":"https://github.com/pulibrary/orangelight/pull/1050"},"issue":{"href":"https://api.github.com/repos/pulibrary/orangelight/issues/1050"},"comments":{"href":"https://api.github.com/repos/pulibrary/orangelight/issues/1050/comments"},"review_comments":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050/comments"},"review_comment":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pulibrary/orangelight/pulls/1050/commits"},"statuses":{"href":"https://api.github.com/repos/pulibrary/orangelight/statuses/9402999ad592ef41f6c7a5196d790209f9f64dda"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jrgriffiniii"}}