{"action":"created","comment":{"url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/comments/136864102","pull_request_review_id":60461745,"id":136864102,"diff_hunk":"@@ -204,34 +208,69 @@ def files_in_country(country_code):\n                            order_by=order_by,\n                            current_country=country_code)\n \n-\n-@pages_blueprint.route('/files/download/<filename>')\n-def files_download(filename):\n-    try:\n-        report = current_app.db_session.query(Report) \\\n-                        .filter(Report.textname.like(\"%\" + filename)).first()\n-        # XXX\n-        # We have duplicate measurements :(\n-        # So the below exception actually happens. This should be resolved\n-        # in the data processing pipeline.\n-        #   Report.filename == filename).one()\n-    except NoResultFound:\n-        raise NotFound(\"No file with that filename found\")\n-    except MultipleResultsFound:\n-        # This should never happen.\n-        raise HTTPException(\"Duplicate measurement detected\")\n-\n-    if report is None:\n+def decompress_autoclaved(autoclaved_filename, frames_off, frames_size, intra_off, intra_size):\n+    def generator():\n+        url = urljoin(current_app.config['AUTOCLAVED_BASE_URL'], autoclaved_filename)\n+\n+        current_app.logger.debug(\"Fetching autoclaved from: %s\" % url)\n+        current_app.logger.debug(\"  frames_off: %d\" % frames_off)\n+        current_app.logger.debug(\"  frames_size: %d\" % frames_size)\n+        current_app.logger.debug(\"  intra_off: %d\" % intra_off)\n+        current_app.logger.debug(\"  intra_size: %d\" % intra_size)\n+        try:\n+            r = requests.get(url, headers={\"Range\": \"bytes={}-{}\".format(frames_off, frames_off + frames_size)},\n+                                stream=True)\n+            streamed_data = 0\n+            for chunk in lz4framed.Decompressor(r.raw):\n+                current_app.logger.debug(\"chunking away 1\")\n+                d = chunk\n+                current_app.logger.debug(\"chunking away 2\")\n+                if streamed_data == 0 and intra_off > 0:\n+                    d = chunk[intra_off:]\n+                if streamed_data > (intra_size + len(d)):\n+                    d = chunk[:(streamed_data - intra_size)]\n+                yield d\n+                streamed_data += len(d)\n+            current_app.logger.debug(\"sent: %d\" % streamed_data)\n+        except Exception as exc:\n+            current_app.logger.error(\"failed to fetch streamed data: %s\" % exc)\n+            raise HTTPException(\"Failed to fetch data\")\n+    return generator\n+\n+@pages_blueprint.route('/files/download/<path:textname>')\n+def files_download(textname):\n+    q = current_app.db_session.query(\n+            Measurement.id.label('m_id'),\n+            Measurement.report_no.label('report_no'),\n+            Measurement.frame_off.label('frame_off'),\n+            Measurement.frame_size.label('frame_size'),\n+            Measurement.intra_off.label('intra_off'),\n+            Measurement.intra_size.label('intra_size'),\n+            Report.report_no.label('r_report_no'),\n+            Report.autoclaved_no.label('r_autoclaved_no'),\n+            Autoclaved.filename.label('a_filename'),\n+            Autoclaved.autoclaved_no.label('a_autoclaved_no'),\n+    ).filter(Report.textname == textname) \\\n+        .join(Report, Report.report_no == Measurement.report_no) \\\n+        .join(Autoclaved, Autoclaved.autoclaved_no == Report.autoclaved_no) \\\n+        .order_by(Measurement.frame_off.asc(), Measurement.intra_off.asc())\n+\n+    msmts = q.all()\n+    if len(msmts) == 0:\n+        current_app.logger.debug(\"Could not find %s\" % textname)\n         raise NotFound(\"No file with that filename found\")\n \n-    bucket_date, filename = report.textname.split(\"/\")\n-    return redirect(\n-        get_download_url(\n-            current_app,\n-            bucket_date,\n-            filename\n-        )\n-    )\n+    autoclaved_filename = msmts[0].a_filename\n+    intra_off = msmts[0].intra_off\n+    intra_size = 0\n+    for msmt in msmts:\n+        if autoclaved_filename != msmt.a_filename:\n+            raise HTTPException(\"Autoclaved file is spanned across multiple files\")\n+        intra_size += msmt.intra_size\n+    frames_off = msmts[0].frame_off\n+    frames_size = msmts[-1].frame_off - frames_off + msmts[-1].frame_size","path":"measurements/pages/__init__.py","position":112,"original_position":112,"commit_id":"571818490684ba3bdd05d91c29224fd71902b36d","original_commit_id":"571818490684ba3bdd05d91c29224fd71902b36d","user":{"login":"hellais","id":424620,"avatar_url":"https://avatars0.githubusercontent.com/u/424620?v=4","gravatar_id":"","url":"https://api.github.com/users/hellais","html_url":"https://github.com/hellais","followers_url":"https://api.github.com/users/hellais/followers","following_url":"https://api.github.com/users/hellais/following{/other_user}","gists_url":"https://api.github.com/users/hellais/gists{/gist_id}","starred_url":"https://api.github.com/users/hellais/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellais/subscriptions","organizations_url":"https://api.github.com/users/hellais/orgs","repos_url":"https://api.github.com/users/hellais/repos","events_url":"https://api.github.com/users/hellais/events{/privacy}","received_events_url":"https://api.github.com/users/hellais/received_events","type":"User","site_admin":false},"body":"> Mumble: so, it's correct to say that frames variables are compressed and intra variables are uncompressed? If so, would you mind renaming them? I am a bit lost with the current naming.\r\n\r\nI used this naming as it's consistent with the naming inside of the DB schema. I think naming them differently would not lead to further clarity as it's something fairly specific (i.e. you need to know the schema of the DB to know what this is doing).\r\n\r\nI can renames `frames_size` to `all_frames_size` (and `intras_size` to `all_intras_size`) if that makes it clearer though.","created_at":"2017-09-04T19:25:23Z","updated_at":"2017-09-04T19:25:23Z","html_url":"https://github.com/TheTorProject/ooni-measurements/pull/28#discussion_r136864102","pull_request_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/comments/136864102"},"html":{"href":"https://github.com/TheTorProject/ooni-measurements/pull/28#discussion_r136864102"},"pull_request":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28"}},"in_reply_to_id":136846024},"pull_request":{"url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28","id":137004360,"html_url":"https://github.com/TheTorProject/ooni-measurements/pull/28","diff_url":"https://github.com/TheTorProject/ooni-measurements/pull/28.diff","patch_url":"https://github.com/TheTorProject/ooni-measurements/pull/28.patch","issue_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/28","number":28,"state":"open","locked":false,"title":"Feature/download url","user":{"login":"hellais","id":424620,"avatar_url":"https://avatars0.githubusercontent.com/u/424620?v=4","gravatar_id":"","url":"https://api.github.com/users/hellais","html_url":"https://github.com/hellais","followers_url":"https://api.github.com/users/hellais/followers","following_url":"https://api.github.com/users/hellais/following{/other_user}","gists_url":"https://api.github.com/users/hellais/gists{/gist_id}","starred_url":"https://api.github.com/users/hellais/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellais/subscriptions","organizations_url":"https://api.github.com/users/hellais/orgs","repos_url":"https://api.github.com/users/hellais/repos","events_url":"https://api.github.com/users/hellais/events{/privacy}","received_events_url":"https://api.github.com/users/hellais/received_events","type":"User","site_admin":false},"body":"This depends on https://github.com/TheTorProject/ooni-measurements/pull/27\r\n\r\nIt adds support for using the new pipeline for serving download URLs","created_at":"2017-08-22T15:04:08Z","updated_at":"2017-09-04T19:25:23Z","closed_at":null,"merged_at":null,"merge_commit_sha":"49ecf5dc181d849f9d0bcc2824b69a7eae00113d","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28/commits","review_comments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28/comments","review_comment_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/comments{/number}","comments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/28/comments","statuses_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/statuses/571818490684ba3bdd05d91c29224fd71902b36d","head":{"label":"TheTorProject:feature/download_url","ref":"feature/download_url","sha":"571818490684ba3bdd05d91c29224fd71902b36d","user":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"repo":{"id":71924459,"name":"ooni-measurements","full_name":"TheTorProject/ooni-measurements","owner":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/TheTorProject/ooni-measurements","description":null,"fork":false,"url":"https://api.github.com/repos/TheTorProject/ooni-measurements","forks_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/forks","keys_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/teams","hooks_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/hooks","issue_events_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/events{/number}","events_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/events","assignees_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/assignees{/user}","branches_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/branches{/branch}","tags_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/tags","blobs_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/refs{/sha}","trees_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/statuses/{sha}","languages_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/languages","stargazers_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/stargazers","contributors_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/contributors","subscribers_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/subscribers","subscription_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/subscription","commits_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/commits{/sha}","git_commits_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/commits{/sha}","comments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/comments{/number}","issue_comment_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/comments{/number}","contents_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/contents/{+path}","compare_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/merges","archive_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/downloads","issues_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues{/number}","pulls_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls{/number}","milestones_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/milestones{/number}","notifications_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/labels{/name}","releases_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/releases{/id}","deployments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/deployments","created_at":"2016-10-25T17:59:28Z","updated_at":"2017-08-02T20:56:49Z","pushed_at":"2017-09-04T19:09:26Z","git_url":"git://github.com/TheTorProject/ooni-measurements.git","ssh_url":"git@github.com:TheTorProject/ooni-measurements.git","clone_url":"https://github.com/TheTorProject/ooni-measurements.git","svn_url":"https://github.com/TheTorProject/ooni-measurements","homepage":null,"size":2649,"stargazers_count":5,"watchers_count":5,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":9,"forks":3,"open_issues":9,"watchers":5,"default_branch":"master"}},"base":{"label":"TheTorProject:master","ref":"master","sha":"4a294fe6a6e6829fa52200f101f8ba900e369724","user":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"repo":{"id":71924459,"name":"ooni-measurements","full_name":"TheTorProject/ooni-measurements","owner":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/TheTorProject/ooni-measurements","description":null,"fork":false,"url":"https://api.github.com/repos/TheTorProject/ooni-measurements","forks_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/forks","keys_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/teams","hooks_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/hooks","issue_events_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/events{/number}","events_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/events","assignees_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/assignees{/user}","branches_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/branches{/branch}","tags_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/tags","blobs_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/refs{/sha}","trees_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/statuses/{sha}","languages_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/languages","stargazers_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/stargazers","contributors_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/contributors","subscribers_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/subscribers","subscription_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/subscription","commits_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/commits{/sha}","git_commits_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/git/commits{/sha}","comments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/comments{/number}","issue_comment_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/comments{/number}","contents_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/contents/{+path}","compare_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/merges","archive_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/downloads","issues_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues{/number}","pulls_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls{/number}","milestones_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/milestones{/number}","notifications_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/labels{/name}","releases_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/releases{/id}","deployments_url":"https://api.github.com/repos/TheTorProject/ooni-measurements/deployments","created_at":"2016-10-25T17:59:28Z","updated_at":"2017-08-02T20:56:49Z","pushed_at":"2017-09-04T19:09:26Z","git_url":"git://github.com/TheTorProject/ooni-measurements.git","ssh_url":"git@github.com:TheTorProject/ooni-measurements.git","clone_url":"https://github.com/TheTorProject/ooni-measurements.git","svn_url":"https://github.com/TheTorProject/ooni-measurements","homepage":null,"size":2649,"stargazers_count":5,"watchers_count":5,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":9,"forks":3,"open_issues":9,"watchers":5,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28"},"html":{"href":"https://github.com/TheTorProject/ooni-measurements/pull/28"},"issue":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/28"},"comments":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/issues/28/comments"},"review_comments":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28/comments"},"review_comment":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/pulls/28/commits"},"statuses":{"href":"https://api.github.com/repos/TheTorProject/ooni-measurements/statuses/571818490684ba3bdd05d91c29224fd71902b36d"}},"author_association":"OWNER"}}
{"actor":{"display_login":"hellais"}}