{"action":"created","comment":{"url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/comments/94619684","pull_request_review_id":15149880,"id":94619684,"diff_hunk":"@@ -147,66 +144,89 @@ def render_POST(self, request, **kwargs):\n         \"\"\"\n         request_body = request.content.getvalue()\n         try:\n-            request_data = demjson.decode(request_body)\n+            api_params = demjson.decode(request_body)\n         except ValueError as e:\n             message = \"Invalid JSON in POST body. {}\"\n             message.format(e.pretty_description())\n             raise Error('400', message=message)\n \n-        log.msg(\"{}\".format(request_data))\n-        spider_data = self.get_required_argument(request_data, \"request\")\n-        error_msg = \"Missing required key 'url' in 'request' object\"\n-        self.get_required_argument(spider_data, \"url\", error_msg=error_msg)\n-\n-        return self.prepare_crawl(request_data, spider_data, **kwargs)\n-\n-    def get_required_argument(self, request_data, name, error_msg=None):\n+        log.msg(\"{}\".format(api_params))\n+        try:\n+            scrapy_request_args = extract_scrapy_request_args(\n+                api_params.get(\"request\", {}), raise_error=True\n+            )\n+        except ValueError as e:\n+            raise Error(400, e.message)\n+\n+        if not api_params.get(\"start_requests\"):\n+            self.get_required_argument(api_params, \"request\")","path":"scrapyrt/resources.py","position":null,"original_position":90,"commit_id":"a6ad46ac5c84bad10f7de9ca1c61f663751c15e2","original_commit_id":"9c2f456b8e2c3d8ae56a98318a736078122bad6c","user":{"login":"chekunkov","id":744331,"avatar_url":"https://avatars.githubusercontent.com/u/744331?v=3","gravatar_id":"","url":"https://api.github.com/users/chekunkov","html_url":"https://github.com/chekunkov","followers_url":"https://api.github.com/users/chekunkov/followers","following_url":"https://api.github.com/users/chekunkov/following{/other_user}","gists_url":"https://api.github.com/users/chekunkov/gists{/gist_id}","starred_url":"https://api.github.com/users/chekunkov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chekunkov/subscriptions","organizations_url":"https://api.github.com/users/chekunkov/orgs","repos_url":"https://api.github.com/users/chekunkov/repos","events_url":"https://api.github.com/users/chekunkov/events{/privacy}","received_events_url":"https://api.github.com/users/chekunkov/received_events","type":"User","site_admin":false},"body":"@pawelmhm what do you think about this one?","created_at":"2017-01-04T16:50:51Z","updated_at":"2017-01-04T16:50:51Z","html_url":"https://github.com/scrapinghub/scrapyrt/pull/38#discussion_r94619684","pull_request_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38","_links":{"self":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/comments/94619684"},"html":{"href":"https://github.com/scrapinghub/scrapyrt/pull/38#discussion_r94619684"},"pull_request":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38"}}},"pull_request":{"url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38","id":71909241,"html_url":"https://github.com/scrapinghub/scrapyrt/pull/38","diff_url":"https://github.com/scrapinghub/scrapyrt/pull/38.diff","patch_url":"https://github.com/scrapinghub/scrapyrt/pull/38.patch","issue_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/38","number":38,"state":"open","locked":false,"title":"Start requests","user":{"login":"pawelmhm","id":2700942,"avatar_url":"https://avatars.githubusercontent.com/u/2700942?v=3","gravatar_id":"","url":"https://api.github.com/users/pawelmhm","html_url":"https://github.com/pawelmhm","followers_url":"https://api.github.com/users/pawelmhm/followers","following_url":"https://api.github.com/users/pawelmhm/following{/other_user}","gists_url":"https://api.github.com/users/pawelmhm/gists{/gist_id}","starred_url":"https://api.github.com/users/pawelmhm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pawelmhm/subscriptions","organizations_url":"https://api.github.com/users/pawelmhm/orgs","repos_url":"https://api.github.com/users/pawelmhm/repos","events_url":"https://api.github.com/users/pawelmhm/events{/privacy}","received_events_url":"https://api.github.com/users/pawelmhm/received_events","type":"User","site_admin":false},"body":"add support for start_requests. This is still somewhat early beta, but I'm opening PR to start design discussion and get early feedback. \n### Core of changes\n\nIf user provides argument: start_requests=True it will enable start_requests in spider and url argument won't be required. If there is no start_requests url argument is required and API will return 400 Bad Requests (as it does currently). This behavior should be common acrosss GET and POST endpoints.\n### Other changes\n- Improve tests in test_resource_crawl so that they test both GET and POST for similar thing. This should improve test coverage.\n- Add start_requests_spider that will be initialized from string template, formatted with start_url taken from mockserver. This required changing order of things in setUp method in tests/servers.py and required to add site argument(so that we can easily obtain site.url())\n- separate Scrapy.Request arguments from API arguments. This is needed because it makes validation easier, and also allows us to remove API arguments from arguments passed to scrapy.Request. It is important in GET handler. \n- slighly improved validation of invalid Request arguments - 400 errors can be raised in either crawler manager (if there is incorrect value of arguments) or in API if there is incorrect name of argument. \n","created_at":"2016-05-30T14:32:52Z","updated_at":"2017-01-04T16:50:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"21f7dcb41fa2989a6cd86f2c148cd8a4e6dc1ce9","assignee":{"login":"chekunkov","id":744331,"avatar_url":"https://avatars.githubusercontent.com/u/744331?v=3","gravatar_id":"","url":"https://api.github.com/users/chekunkov","html_url":"https://github.com/chekunkov","followers_url":"https://api.github.com/users/chekunkov/followers","following_url":"https://api.github.com/users/chekunkov/following{/other_user}","gists_url":"https://api.github.com/users/chekunkov/gists{/gist_id}","starred_url":"https://api.github.com/users/chekunkov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chekunkov/subscriptions","organizations_url":"https://api.github.com/users/chekunkov/orgs","repos_url":"https://api.github.com/users/chekunkov/repos","events_url":"https://api.github.com/users/chekunkov/events{/privacy}","received_events_url":"https://api.github.com/users/chekunkov/received_events","type":"User","site_admin":false},"assignees":[{"login":"chekunkov","id":744331,"avatar_url":"https://avatars.githubusercontent.com/u/744331?v=3","gravatar_id":"","url":"https://api.github.com/users/chekunkov","html_url":"https://github.com/chekunkov","followers_url":"https://api.github.com/users/chekunkov/followers","following_url":"https://api.github.com/users/chekunkov/following{/other_user}","gists_url":"https://api.github.com/users/chekunkov/gists{/gist_id}","starred_url":"https://api.github.com/users/chekunkov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chekunkov/subscriptions","organizations_url":"https://api.github.com/users/chekunkov/orgs","repos_url":"https://api.github.com/users/chekunkov/repos","events_url":"https://api.github.com/users/chekunkov/events{/privacy}","received_events_url":"https://api.github.com/users/chekunkov/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38/commits","review_comments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38/comments","review_comment_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/38/comments","statuses_url":"https://api.github.com/repos/scrapinghub/scrapyrt/statuses/a6ad46ac5c84bad10f7de9ca1c61f663751c15e2","head":{"label":"scrapinghub:start_requests","ref":"start_requests","sha":"a6ad46ac5c84bad10f7de9ca1c61f663751c15e2","user":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"repo":{"id":28868495,"name":"scrapyrt","full_name":"scrapinghub/scrapyrt","owner":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scrapinghub/scrapyrt","description":"Scrapy realtime","fork":false,"url":"https://api.github.com/repos/scrapinghub/scrapyrt","forks_url":"https://api.github.com/repos/scrapinghub/scrapyrt/forks","keys_url":"https://api.github.com/repos/scrapinghub/scrapyrt/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scrapinghub/scrapyrt/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scrapinghub/scrapyrt/teams","hooks_url":"https://api.github.com/repos/scrapinghub/scrapyrt/hooks","issue_events_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/events{/number}","events_url":"https://api.github.com/repos/scrapinghub/scrapyrt/events","assignees_url":"https://api.github.com/repos/scrapinghub/scrapyrt/assignees{/user}","branches_url":"https://api.github.com/repos/scrapinghub/scrapyrt/branches{/branch}","tags_url":"https://api.github.com/repos/scrapinghub/scrapyrt/tags","blobs_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/refs{/sha}","trees_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scrapinghub/scrapyrt/statuses/{sha}","languages_url":"https://api.github.com/repos/scrapinghub/scrapyrt/languages","stargazers_url":"https://api.github.com/repos/scrapinghub/scrapyrt/stargazers","contributors_url":"https://api.github.com/repos/scrapinghub/scrapyrt/contributors","subscribers_url":"https://api.github.com/repos/scrapinghub/scrapyrt/subscribers","subscription_url":"https://api.github.com/repos/scrapinghub/scrapyrt/subscription","commits_url":"https://api.github.com/repos/scrapinghub/scrapyrt/commits{/sha}","git_commits_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/commits{/sha}","comments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/comments{/number}","issue_comment_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/comments{/number}","contents_url":"https://api.github.com/repos/scrapinghub/scrapyrt/contents/{+path}","compare_url":"https://api.github.com/repos/scrapinghub/scrapyrt/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scrapinghub/scrapyrt/merges","archive_url":"https://api.github.com/repos/scrapinghub/scrapyrt/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scrapinghub/scrapyrt/downloads","issues_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues{/number}","pulls_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls{/number}","milestones_url":"https://api.github.com/repos/scrapinghub/scrapyrt/milestones{/number}","notifications_url":"https://api.github.com/repos/scrapinghub/scrapyrt/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scrapinghub/scrapyrt/labels{/name}","releases_url":"https://api.github.com/repos/scrapinghub/scrapyrt/releases{/id}","deployments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/deployments","created_at":"2015-01-06T15:07:16Z","updated_at":"2017-01-03T13:36:58Z","pushed_at":"2017-01-04T16:35:33Z","git_url":"git://github.com/scrapinghub/scrapyrt.git","ssh_url":"git@github.com:scrapinghub/scrapyrt.git","clone_url":"https://github.com/scrapinghub/scrapyrt.git","svn_url":"https://github.com/scrapinghub/scrapyrt","homepage":"","size":119,"stargazers_count":226,"watchers_count":226,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":47,"mirror_url":null,"open_issues_count":21,"forks":47,"open_issues":21,"watchers":226,"default_branch":"master"}},"base":{"label":"scrapinghub:master","ref":"master","sha":"019e6ac9d4a3d28c8e66988d01be23177a7fd6b3","user":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"repo":{"id":28868495,"name":"scrapyrt","full_name":"scrapinghub/scrapyrt","owner":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scrapinghub/scrapyrt","description":"Scrapy realtime","fork":false,"url":"https://api.github.com/repos/scrapinghub/scrapyrt","forks_url":"https://api.github.com/repos/scrapinghub/scrapyrt/forks","keys_url":"https://api.github.com/repos/scrapinghub/scrapyrt/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scrapinghub/scrapyrt/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scrapinghub/scrapyrt/teams","hooks_url":"https://api.github.com/repos/scrapinghub/scrapyrt/hooks","issue_events_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/events{/number}","events_url":"https://api.github.com/repos/scrapinghub/scrapyrt/events","assignees_url":"https://api.github.com/repos/scrapinghub/scrapyrt/assignees{/user}","branches_url":"https://api.github.com/repos/scrapinghub/scrapyrt/branches{/branch}","tags_url":"https://api.github.com/repos/scrapinghub/scrapyrt/tags","blobs_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/refs{/sha}","trees_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scrapinghub/scrapyrt/statuses/{sha}","languages_url":"https://api.github.com/repos/scrapinghub/scrapyrt/languages","stargazers_url":"https://api.github.com/repos/scrapinghub/scrapyrt/stargazers","contributors_url":"https://api.github.com/repos/scrapinghub/scrapyrt/contributors","subscribers_url":"https://api.github.com/repos/scrapinghub/scrapyrt/subscribers","subscription_url":"https://api.github.com/repos/scrapinghub/scrapyrt/subscription","commits_url":"https://api.github.com/repos/scrapinghub/scrapyrt/commits{/sha}","git_commits_url":"https://api.github.com/repos/scrapinghub/scrapyrt/git/commits{/sha}","comments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/comments{/number}","issue_comment_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/comments{/number}","contents_url":"https://api.github.com/repos/scrapinghub/scrapyrt/contents/{+path}","compare_url":"https://api.github.com/repos/scrapinghub/scrapyrt/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scrapinghub/scrapyrt/merges","archive_url":"https://api.github.com/repos/scrapinghub/scrapyrt/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scrapinghub/scrapyrt/downloads","issues_url":"https://api.github.com/repos/scrapinghub/scrapyrt/issues{/number}","pulls_url":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls{/number}","milestones_url":"https://api.github.com/repos/scrapinghub/scrapyrt/milestones{/number}","notifications_url":"https://api.github.com/repos/scrapinghub/scrapyrt/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scrapinghub/scrapyrt/labels{/name}","releases_url":"https://api.github.com/repos/scrapinghub/scrapyrt/releases{/id}","deployments_url":"https://api.github.com/repos/scrapinghub/scrapyrt/deployments","created_at":"2015-01-06T15:07:16Z","updated_at":"2017-01-03T13:36:58Z","pushed_at":"2017-01-04T16:35:33Z","git_url":"git://github.com/scrapinghub/scrapyrt.git","ssh_url":"git@github.com:scrapinghub/scrapyrt.git","clone_url":"https://github.com/scrapinghub/scrapyrt.git","svn_url":"https://github.com/scrapinghub/scrapyrt","homepage":"","size":119,"stargazers_count":226,"watchers_count":226,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":47,"mirror_url":null,"open_issues_count":21,"forks":47,"open_issues":21,"watchers":226,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38"},"html":{"href":"https://github.com/scrapinghub/scrapyrt/pull/38"},"issue":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/38"},"comments":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/issues/38/comments"},"review_comments":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38/comments"},"review_comment":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/pulls/38/commits"},"statuses":{"href":"https://api.github.com/repos/scrapinghub/scrapyrt/statuses/a6ad46ac5c84bad10f7de9ca1c61f663751c15e2"}}}}
{"actor":{"display_login":"chekunkov"}}