{"action":"created","comment":{"url":"https://api.github.com/repos/etalab/geozones/pulls/comments/95768180","pull_request_review_id":16136477,"id":95768180,"diff_hunk":"@@ -307,121 +353,72 @@ def attach_epci(db, filename):\n                     {'level': town.id, 'code': insee},\n                     {'$addToSet': {'parents': epci_id}}):\n                 processed += 1\n-    success('Attached {0} french town to their EPCI'.format(processed))\n-\n-    # info('Attach EPCI to counties and regions')\n-    # processed = 0\n-    # pipeline = [\n-    #     {'$match': {'level': town.id: 'parents': {'$regex': epci.id}}},\n-    #     {'$unwind': '$parents'},\n-    #     {'$match': {'parents': {'$regex': '^(' + county.id + '|' + region.id + ')'}}},\n-    #     {'$group': {'_id': '$parents', 'population': {'$sum': '$population'}}}\n-    # ]\n-    # for result in db.aggregate(pipeline):\n-    #     if db.find_one_and_update({'_id': result['_id']},\n-    #         {'$set': {'population': result['population']}}):\n-    #         processed += 1\n-    # success('Computed population for {0} french districts'.format(processed))\n-\n-    # processed = 0\n-    # for siren, region in epci_region.items():\n-    #     insee_region = region.replace('r', '')\n-    #     region_id = 'fr/region/{0}'.format(insee_region)\n-    #     if db.find_one_and_update({'level': epci.id, 'code': siren},\n-    #         {'$addToSet': {'parents': region_id, 'parents': region_id}}):\n-    #         processed += 1\n-    # success('Attached {0} french EPCI to their region'.format(processed))\n-\n-\n-@town.postprocessor('http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2016/txt/comsimp2016.zip')\n+    success('Attached {0} french town to their EPCI', processed)\n+\n+\n+@town.postprocessor('https://www.insee.fr/fr/statistiques/fichier/2114819/france2016-txt.zip')  # NOQA\n def process_insee_cog(db, filename):\n     '''Use informations from INSEE COG to attach parents.\n     http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement.asp\n     '''\n     info('Processing INSEE COG')\n     processed = 0\n-    counties = {}\n     districts = {}\n     with ZipFile(filename) as cogzip:\n-        with cogzip.open('comsimp2016.txt') as tsvfile:\n+        with cogzip.open('france2016.txt') as tsvfile:\n             tsvio = io.TextIOWrapper(tsvfile, encoding='cp1252')\n             reader = csv.DictReader(tsvio, delimiter='\\t')\n             for row in reader:\n-                # Lower everything, from 2B to 2b for instance.\n-                region_code = row['REG'].lower()\n-                county_code = row['DEP'].lower()\n-                district_code = row['AR'].lower()\n-                town_code = row['COM'].lower()\n-                insee_code = ''.join((county_code, town_code))\n-\n-                region_id = 'fr/region/{0}'.format(region_code)\n-                county_id = 'fr/county/{0}'.format(county_code)\n-\n-                parents = [region_id, county_id]\n-\n-                if county_id not in counties:\n-                    counties[county_id] = region_id\n+                region_code = row['REG']\n+                county_code = row['DEP']\n+                district_code = row['AR']\n+                region = retrieve_current_region(db, region_code)\n+                county = retrieve_current_county(db, county_code)\n \n                 if district_code:\n                     district_code = ''.join((county_code, district_code))\n                     district_id = 'fr/district/{0}'.format(district_code)\n-                    parents.append(district_id)\n-                    if district_id not in districts:\n-                        districts[district_id] = [region_id, county_id]\n-\n-                if db.find_one_and_update(\n-                        {'level': town.id, 'code': insee_code},\n-                        {'$addToSet': {'parents': {'$each': parents}}}):\n-                    processed += 1\n-    success('Attached {0} french towns to their parents'.format(processed))\n+                    if district_id not in districts and region and county:\n+                        districts[district_id] = [region['_id'], county['_id']]\n \n-    processed = 0\n     for district_id, parents in districts.items():\n         if db.find_one_and_update(\n                 {'_id': district_id},\n                 {'$addToSet': {\n                     'parents': {'$each': parents},\n                 }}):\n             processed += 1\n-    success('Attached {0} french districts to their parents'.format(processed))\n-\n-    processed = 0\n-    for county_id, parent in counties.items():\n-        if db.find_one_and_update(\n-                {'_id': county_id},\n-                {'$addToSet': {'parents': parent}}):\n-            processed += 1\n-    success('Attached {0} french counties to their parents'.format(processed))\n+    success('Attached {0} french districts to their parents', processed)\n \n \n @town.postprocessor()\n def town_with_districts(db, filename):\n     info('Attaching Paris town districts')\n-    paris = db.find_one({'_id': 'fr/town/75056'})\n+    paris = db.find_one({'_id': 'COM75056@1942-01-01'})","path":"france.py","position":500,"original_position":500,"commit_id":"2e223b84c4607ab0834e082f044b055171e57218","original_commit_id":"2e223b84c4607ab0834e082f044b055171e57218","user":{"login":"noirbizarre","id":15725,"avatar_url":"https://avatars.githubusercontent.com/u/15725?v=3","gravatar_id":"","url":"https://api.github.com/users/noirbizarre","html_url":"https://github.com/noirbizarre","followers_url":"https://api.github.com/users/noirbizarre/followers","following_url":"https://api.github.com/users/noirbizarre/following{/other_user}","gists_url":"https://api.github.com/users/noirbizarre/gists{/gist_id}","starred_url":"https://api.github.com/users/noirbizarre/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/noirbizarre/subscriptions","organizations_url":"https://api.github.com/users/noirbizarre/orgs","repos_url":"https://api.github.com/users/noirbizarre/repos","events_url":"https://api.github.com/users/noirbizarre/events{/privacy}","received_events_url":"https://api.github.com/users/noirbizarre/received_events","type":"User","site_admin":false},"body":"We are loosing the prefix in identifiers (`fr/town` or `fr/commune`) ?","created_at":"2017-01-12T11:01:45Z","updated_at":"2017-01-12T11:03:46Z","html_url":"https://github.com/etalab/geozones/pull/22#discussion_r95768180","pull_request_url":"https://api.github.com/repos/etalab/geozones/pulls/22","_links":{"self":{"href":"https://api.github.com/repos/etalab/geozones/pulls/comments/95768180"},"html":{"href":"https://github.com/etalab/geozones/pull/22#discussion_r95768180"},"pull_request":{"href":"https://api.github.com/repos/etalab/geozones/pulls/22"}}},"pull_request":{"url":"https://api.github.com/repos/etalab/geozones/pulls/22","id":75280144,"html_url":"https://github.com/etalab/geozones/pull/22","diff_url":"https://github.com/etalab/geozones/pull/22.diff","patch_url":"https://github.com/etalab/geozones/pull/22.patch","issue_url":"https://api.github.com/repos/etalab/geozones/issues/22","number":22,"state":"open","locked":false,"title":"Integrate historical data for regions, departements and communes (in French)","user":{"login":"davidbgk","id":3556,"avatar_url":"https://avatars.githubusercontent.com/u/3556?v=3","gravatar_id":"","url":"https://api.github.com/users/davidbgk","html_url":"https://github.com/davidbgk","followers_url":"https://api.github.com/users/davidbgk/followers","following_url":"https://api.github.com/users/davidbgk/following{/other_user}","gists_url":"https://api.github.com/users/davidbgk/gists{/gist_id}","starred_url":"https://api.github.com/users/davidbgk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davidbgk/subscriptions","organizations_url":"https://api.github.com/users/davidbgk/orgs","repos_url":"https://api.github.com/users/davidbgk/repos","events_url":"https://api.github.com/users/davidbgk/events{/privacy}","received_events_url":"https://api.github.com/users/davidbgk/received_events","type":"User","site_admin":false},"body":"In order to generate user friendly IDs/URLs, we are moving from `county` to `departement` and from `town` to `commune` for French related zones.\n\nThe aggregate bug has been resolved in the meantime with manual accumulation of geometries.\n\nRefs opendatateam/udata#499\n\nWiP because there is still an issue with D(R)OM-(T|C)OM aggregation when loaded within udata.\n","created_at":"2016-06-27T09:25:42Z","updated_at":"2017-01-12T11:03:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"668b2a290950739c6606d4676595d0a6eb64730b","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/etalab/geozones/pulls/22/commits","review_comments_url":"https://api.github.com/repos/etalab/geozones/pulls/22/comments","review_comment_url":"https://api.github.com/repos/etalab/geozones/pulls/comments{/number}","comments_url":"https://api.github.com/repos/etalab/geozones/issues/22/comments","statuses_url":"https://api.github.com/repos/etalab/geozones/statuses/2e223b84c4607ab0834e082f044b055171e57218","head":{"label":"davidbgk:french-names","ref":"french-names","sha":"2e223b84c4607ab0834e082f044b055171e57218","user":{"login":"davidbgk","id":3556,"avatar_url":"https://avatars.githubusercontent.com/u/3556?v=3","gravatar_id":"","url":"https://api.github.com/users/davidbgk","html_url":"https://github.com/davidbgk","followers_url":"https://api.github.com/users/davidbgk/followers","following_url":"https://api.github.com/users/davidbgk/following{/other_user}","gists_url":"https://api.github.com/users/davidbgk/gists{/gist_id}","starred_url":"https://api.github.com/users/davidbgk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davidbgk/subscriptions","organizations_url":"https://api.github.com/users/davidbgk/orgs","repos_url":"https://api.github.com/users/davidbgk/repos","events_url":"https://api.github.com/users/davidbgk/events{/privacy}","received_events_url":"https://api.github.com/users/davidbgk/received_events","type":"User","site_admin":false},"repo":{"id":45543608,"name":"geozones","full_name":"davidbgk/geozones","owner":{"login":"davidbgk","id":3556,"avatar_url":"https://avatars.githubusercontent.com/u/3556?v=3","gravatar_id":"","url":"https://api.github.com/users/davidbgk","html_url":"https://github.com/davidbgk","followers_url":"https://api.github.com/users/davidbgk/followers","following_url":"https://api.github.com/users/davidbgk/following{/other_user}","gists_url":"https://api.github.com/users/davidbgk/gists{/gist_id}","starred_url":"https://api.github.com/users/davidbgk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davidbgk/subscriptions","organizations_url":"https://api.github.com/users/davidbgk/orgs","repos_url":"https://api.github.com/users/davidbgk/repos","events_url":"https://api.github.com/users/davidbgk/events{/privacy}","received_events_url":"https://api.github.com/users/davidbgk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/davidbgk/geozones","description":"Simple spatial/administrative referential","fork":true,"url":"https://api.github.com/repos/davidbgk/geozones","forks_url":"https://api.github.com/repos/davidbgk/geozones/forks","keys_url":"https://api.github.com/repos/davidbgk/geozones/keys{/key_id}","collaborators_url":"https://api.github.com/repos/davidbgk/geozones/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/davidbgk/geozones/teams","hooks_url":"https://api.github.com/repos/davidbgk/geozones/hooks","issue_events_url":"https://api.github.com/repos/davidbgk/geozones/issues/events{/number}","events_url":"https://api.github.com/repos/davidbgk/geozones/events","assignees_url":"https://api.github.com/repos/davidbgk/geozones/assignees{/user}","branches_url":"https://api.github.com/repos/davidbgk/geozones/branches{/branch}","tags_url":"https://api.github.com/repos/davidbgk/geozones/tags","blobs_url":"https://api.github.com/repos/davidbgk/geozones/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/davidbgk/geozones/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/davidbgk/geozones/git/refs{/sha}","trees_url":"https://api.github.com/repos/davidbgk/geozones/git/trees{/sha}","statuses_url":"https://api.github.com/repos/davidbgk/geozones/statuses/{sha}","languages_url":"https://api.github.com/repos/davidbgk/geozones/languages","stargazers_url":"https://api.github.com/repos/davidbgk/geozones/stargazers","contributors_url":"https://api.github.com/repos/davidbgk/geozones/contributors","subscribers_url":"https://api.github.com/repos/davidbgk/geozones/subscribers","subscription_url":"https://api.github.com/repos/davidbgk/geozones/subscription","commits_url":"https://api.github.com/repos/davidbgk/geozones/commits{/sha}","git_commits_url":"https://api.github.com/repos/davidbgk/geozones/git/commits{/sha}","comments_url":"https://api.github.com/repos/davidbgk/geozones/comments{/number}","issue_comment_url":"https://api.github.com/repos/davidbgk/geozones/issues/comments{/number}","contents_url":"https://api.github.com/repos/davidbgk/geozones/contents/{+path}","compare_url":"https://api.github.com/repos/davidbgk/geozones/compare/{base}...{head}","merges_url":"https://api.github.com/repos/davidbgk/geozones/merges","archive_url":"https://api.github.com/repos/davidbgk/geozones/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/davidbgk/geozones/downloads","issues_url":"https://api.github.com/repos/davidbgk/geozones/issues{/number}","pulls_url":"https://api.github.com/repos/davidbgk/geozones/pulls{/number}","milestones_url":"https://api.github.com/repos/davidbgk/geozones/milestones{/number}","notifications_url":"https://api.github.com/repos/davidbgk/geozones/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/davidbgk/geozones/labels{/name}","releases_url":"https://api.github.com/repos/davidbgk/geozones/releases{/id}","deployments_url":"https://api.github.com/repos/davidbgk/geozones/deployments","created_at":"2015-11-04T14:20:34Z","updated_at":"2016-02-04T09:04:39Z","pushed_at":"2016-12-13T04:57:06Z","git_url":"git://github.com/davidbgk/geozones.git","ssh_url":"git@github.com:davidbgk/geozones.git","clone_url":"https://github.com/davidbgk/geozones.git","svn_url":"https://github.com/davidbgk/geozones","homepage":null,"size":286,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"etalab:master","ref":"master","sha":"64fb425bda7da86dec297cf32478a5dda0fbfe3c","user":{"login":"etalab","id":5313610,"avatar_url":"https://avatars.githubusercontent.com/u/5313610?v=3","gravatar_id":"","url":"https://api.github.com/users/etalab","html_url":"https://github.com/etalab","followers_url":"https://api.github.com/users/etalab/followers","following_url":"https://api.github.com/users/etalab/following{/other_user}","gists_url":"https://api.github.com/users/etalab/gists{/gist_id}","starred_url":"https://api.github.com/users/etalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/etalab/subscriptions","organizations_url":"https://api.github.com/users/etalab/orgs","repos_url":"https://api.github.com/users/etalab/repos","events_url":"https://api.github.com/users/etalab/events{/privacy}","received_events_url":"https://api.github.com/users/etalab/received_events","type":"Organization","site_admin":false},"repo":{"id":33991469,"name":"geozones","full_name":"etalab/geozones","owner":{"login":"etalab","id":5313610,"avatar_url":"https://avatars.githubusercontent.com/u/5313610?v=3","gravatar_id":"","url":"https://api.github.com/users/etalab","html_url":"https://github.com/etalab","followers_url":"https://api.github.com/users/etalab/followers","following_url":"https://api.github.com/users/etalab/following{/other_user}","gists_url":"https://api.github.com/users/etalab/gists{/gist_id}","starred_url":"https://api.github.com/users/etalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/etalab/subscriptions","organizations_url":"https://api.github.com/users/etalab/orgs","repos_url":"https://api.github.com/users/etalab/repos","events_url":"https://api.github.com/users/etalab/events{/privacy}","received_events_url":"https://api.github.com/users/etalab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/etalab/geozones","description":"Simple spatial/administrative referential","fork":false,"url":"https://api.github.com/repos/etalab/geozones","forks_url":"https://api.github.com/repos/etalab/geozones/forks","keys_url":"https://api.github.com/repos/etalab/geozones/keys{/key_id}","collaborators_url":"https://api.github.com/repos/etalab/geozones/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/etalab/geozones/teams","hooks_url":"https://api.github.com/repos/etalab/geozones/hooks","issue_events_url":"https://api.github.com/repos/etalab/geozones/issues/events{/number}","events_url":"https://api.github.com/repos/etalab/geozones/events","assignees_url":"https://api.github.com/repos/etalab/geozones/assignees{/user}","branches_url":"https://api.github.com/repos/etalab/geozones/branches{/branch}","tags_url":"https://api.github.com/repos/etalab/geozones/tags","blobs_url":"https://api.github.com/repos/etalab/geozones/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/etalab/geozones/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/etalab/geozones/git/refs{/sha}","trees_url":"https://api.github.com/repos/etalab/geozones/git/trees{/sha}","statuses_url":"https://api.github.com/repos/etalab/geozones/statuses/{sha}","languages_url":"https://api.github.com/repos/etalab/geozones/languages","stargazers_url":"https://api.github.com/repos/etalab/geozones/stargazers","contributors_url":"https://api.github.com/repos/etalab/geozones/contributors","subscribers_url":"https://api.github.com/repos/etalab/geozones/subscribers","subscription_url":"https://api.github.com/repos/etalab/geozones/subscription","commits_url":"https://api.github.com/repos/etalab/geozones/commits{/sha}","git_commits_url":"https://api.github.com/repos/etalab/geozones/git/commits{/sha}","comments_url":"https://api.github.com/repos/etalab/geozones/comments{/number}","issue_comment_url":"https://api.github.com/repos/etalab/geozones/issues/comments{/number}","contents_url":"https://api.github.com/repos/etalab/geozones/contents/{+path}","compare_url":"https://api.github.com/repos/etalab/geozones/compare/{base}...{head}","merges_url":"https://api.github.com/repos/etalab/geozones/merges","archive_url":"https://api.github.com/repos/etalab/geozones/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/etalab/geozones/downloads","issues_url":"https://api.github.com/repos/etalab/geozones/issues{/number}","pulls_url":"https://api.github.com/repos/etalab/geozones/pulls{/number}","milestones_url":"https://api.github.com/repos/etalab/geozones/milestones{/number}","notifications_url":"https://api.github.com/repos/etalab/geozones/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/etalab/geozones/labels{/name}","releases_url":"https://api.github.com/repos/etalab/geozones/releases{/id}","deployments_url":"https://api.github.com/repos/etalab/geozones/deployments","created_at":"2015-04-15T12:06:35Z","updated_at":"2016-12-09T18:13:37Z","pushed_at":"2016-12-13T04:57:06Z","git_url":"git://github.com/etalab/geozones.git","ssh_url":"git@github.com:etalab/geozones.git","clone_url":"https://github.com/etalab/geozones.git","svn_url":"https://github.com/etalab/geozones","homepage":null,"size":273,"stargazers_count":21,"watchers_count":21,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":5,"forks":5,"open_issues":5,"watchers":21,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/etalab/geozones/pulls/22"},"html":{"href":"https://github.com/etalab/geozones/pull/22"},"issue":{"href":"https://api.github.com/repos/etalab/geozones/issues/22"},"comments":{"href":"https://api.github.com/repos/etalab/geozones/issues/22/comments"},"review_comments":{"href":"https://api.github.com/repos/etalab/geozones/pulls/22/comments"},"review_comment":{"href":"https://api.github.com/repos/etalab/geozones/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/etalab/geozones/pulls/22/commits"},"statuses":{"href":"https://api.github.com/repos/etalab/geozones/statuses/2e223b84c4607ab0834e082f044b055171e57218"}}}}
{"actor":{"display_login":"noirbizarre"}}