{"action":"created","comment":{"url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/comments/118703612","pull_request_review_id":40522904,"id":118703612,"diff_hunk":"@@ -1,196 +1,250 @@\n import abc\n-import json\n-import time\n-import types\n-import logging\n+import collections\n import datetime\n-from typing import Tuple\n-from typing import Union\n-from typing import Iterator\n+import logging\n+import types\n+import warnings\n \n import pendulum\n import requests\n \n from django.conf import settings\n+from django.utils import timezone\n \n+from share.harvest.exceptions import HarvesterDisabledError\n+from share.harvest.ratelimit import RateLimittedProxy\n+from share.harvest.serialization import DeprecatedDefaultSerializer\n+from share.models import RawDatum\n \n-logger = logging.getLogger(__name__)\n \n+logger = logging.getLogger(__name__)\n+FetchResult = collections.namedtuple('FetchResult', ('identifier', 'datum'))\n \n-class RateLimittedProxy:\n \n-    def __init__(self, proxy_to, calls, per_second):\n-        self._proxy_to = proxy_to\n-        self._allowance = calls\n-        self._calls = calls\n-        self._last_call = 0\n-        self._per_second = per_second\n-        self._cache = {}\n+class BaseHarvester(metaclass=abc.ABCMeta):\n+    \"\"\"\n \n-    def _check_limit(self):\n-        if self._allowance > 1:\n-            return\n-        wait = self._per_second - (time.time() - self._last_call)\n-        if wait > 0:\n-            logger.debug('Rate limitting %s. Sleeping for %s', self._proxy_to, wait)\n-            time.sleep(wait)\n-        self._allowance = self._calls\n-        logger.debug('Access granted for %s', self._proxy_to)\n+    Fetch:\n+        Aquire and serialize data from a remote source, respecting rate limits.\n+        fetch* methods return a generator that yield FetchResult objects\n \n-    def _called(self):\n-        self._allowance -= 1\n-        self._last_call = time.time()\n+    Harvest:\n+        Fetch and store data, respecting global rate limits.\n+        harvest* methods return a generator that yield RawDatum objects\n \n-    def __call__(self, *args, **kwargs):\n-        self._check_limit()\n-        ret = self._proxy_to(*args, **kwargs)\n-        self._called()\n-        return ret\n+    \"\"\"\n \n-    def __getattr__(self, name):\n-        return self._cache.setdefault(name, self.__class__(getattr(self._proxy_to, name), self._calls, self._per_second))\n+    SERIALIZER_CLASS = DeprecatedDefaultSerializer\n \n+    def __init__(self, source_config, pretty=False, **kwargs):\n+        \"\"\"\n \n-class BaseHarvester(metaclass=abc.ABCMeta):\n+        Args:\n+            source_config (SourceConfig):\n+            pretty (bool, optional): Defaults to False.\n+            **kwargs: Custom kwargs, generally from the source_config. Stored in self.kwargs\n \n-    def __init__(self, source_config, **kwargs):\n-        self.last_call = 0\n-        self.new_raw_ids = []\n-        self.old_raw_ids = []\n-        self.config = source_config\n+        \"\"\"\n         self.kwargs = kwargs\n+        self.config = source_config\n+        self.serializer = self.SERIALIZER_CLASS(pretty)\n \n-        session = requests.Session()\n-        session.headers.update({'User-Agent': settings.SHARE_USER_AGENT})\n+        self.session = requests.Session()\n+        self.session.headers.update({'User-Agent': settings.SHARE_USER_AGENT})\n         # TODO Make rate limit apply across threads\n-        self.requests = RateLimittedProxy(session, self.config.rate_limit_allowance, self.config.rate_limit_period)\n+        self.requests = RateLimittedProxy(self.session, self.config.rate_limit_allowance, self.config.rate_limit_period)\n \n-    @abc.abstractmethod\n-    def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum, **kwargs) -> Iterator[Tuple[str, Union[str, dict, bytes]]]:\n-        \"\"\"Fetch date from this provider inside of the given date range.\n+    def fetch_id(self, identifier):\n+        \"\"\"Fetch a document by provider ID.\n \n-        Any HTTP[S] requests MUST be sent using the self.requests client.\n-        It will automatically in force rate limits\n+        Optional to implement, intended for dev and manual testing.\n \n         Args:\n-            start_date (datetime):\n-            end_date (datetime):\n+            identifier (str): Unique ID the provider uses to identify works.\n \n         Returns:\n-            Iterator<Tuple<str, str|dict|bytes>>: The fetched data paired with\n-            the unique ID that this provider uses.\n+            FetchResult\n \n-            [\n-                ('1', {'my': 'doc'}),\n-                ('2', {'your': 'doc'}),\n-            ]\n         \"\"\"\n-        raise NotImplementedError()\n+        raise NotImplementedError('{!r} does not support fetching by ID'.format(self))\n+\n+    def fetch(self, today=False, **kwargs):\n+        \"\"\"Fetch data from today.\n+\n+        Yields:\n+            FetchResult\n \n-    def fetch_by_id(self, provider_id):\n         \"\"\"\n-        Fetch a document by provider ID.\n+        return self.fetch_date_range(datetime.date.today() - datetime.timedelta(days=1), datetime.date.today(), **kwargs)\n \n-        Optional to implement, intended for dev and manual testing.\n+    def fetch_date(self, date: datetime.date, **kwargs):\n+        \"\"\"Fetch data from the specified date.\n \n-        Args:\n-            provider_id (str): Unique ID the provider uses to identify works.\n+        Yields:\n+            FetchResult\n+        \"\"\"\n+        return self.fetch_date_range(date - datetime.timedelta(days=1), date, **kwargs)\n \n-        Returns:\n-            str|dict|bytes: Fetched data.\n+    def fetch_date_range(self, start, end, limit=None, **kwargs):\n+        \"\"\"Fetch data from the specified date range.\n+\n+        Yields:\n+            FetchResult\n \n         \"\"\"\n-        raise NotImplementedError()\n+        if not isinstance(start, datetime.date):\n+            raise TypeError('start must be a datetime.date. Got {!r}'.format(start))\n+\n+        if not isinstance(end, datetime.date):\n+            raise TypeError('end must be a datetime.date. Got {!r}'.format(end))\n+\n+        if start >= end:\n+            raise ValueError('start must be before end. {!r} > {!r}'.format(start, end))\n+\n+        if limit == 0:\n+            return  # No need to do anything\n \n-    def shift_range(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> pendulum.Pendulum:\n-        \"\"\"Most providers will not need this method.\n+        # Cast to datetimes for compat reasons\n+        start = pendulum.Pendulum.instance(datetime.datetime.combine(start, datetime.time(0, 0, 0, 0, timezone.utc)))\n+        end = pendulum.Pendulum.instance(datetime.datetime.combine(end, datetime.time(0, 0, 0, 0, timezone.utc)))\n \n-        For providers that should be collecting data at an offset, see figshare.\n+        if hasattr(self, 'shift_range'):\n+            warnings.warn(\n+                '{!r} implements a deprecated interface. '\n+                'Handle date transforms in _do_fetch. '\n+                'shift_range will no longer be called in SHARE 2.9.0'.format(self),\n+                DeprecationWarning\n+            )\n+            start, end = self.shift_range(start, end)\n+\n+        data_gen = self._do_fetch(start, end, **kwargs)\n+\n+        if not isinstance(data_gen, types.GeneratorType) and len(data_gen) != 0:\n+            raise TypeError('{!r}._do_fetch must return a GeneratorType for optimal performance and memory usage'.format(self))\n+\n+        for i, (identifier, datum) in enumerate(data_gen):\n+            yield FetchResult(identifier, self.serializer.serialize(datum))\n+\n+            if limit is not None and i >= limit:\n+                break\n+\n+    def harvest_id(self, identifier):\n+        \"\"\"Harvest a document by ID.\n+\n+        Note:\n+            Dependant on whether or not fetch_id is implemented.\n \n         Args:\n-            start_date (datetime):\n-            end_date (datetime):\n+            identifier (str): Unique ID the provider uses to identify works.\n \n         Returns:\n-            (datetime, datetime): The shifted date range\n+            RawDatum\n+\n         \"\"\"\n-        return start_date, end_date\n+        res = self.fetch_by_id(identifier)\n+        return RawDatum.objects.store_data(res.identifier, res.datum, self.config)\n \n-    def harvest(self, start_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], end_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], shift_range: bool=True, limit: int=None, **kwargs) -> list:\n-        from share.models import RawDatum\n-        start_date, end_date = self._validate_dates(start_date, end_date)\n+    def harvest(self, **kwargs):\n+        \"\"\"Fetch data from yesterday.\n \n-        rawdata = ((identifier, self.encode_data(datum)) for identifier, datum in self.do_harvest(start_date, end_date, **kwargs))\n-        assert isinstance(rawdata, types.GeneratorType), 'do_harvest did not return a generator type, found {!r}. Make sure to use the yield keyword'.format(type(rawdata))\n+        Yields:\n+            RawDatum\n \n-        yield from RawDatum.objects.store_chunk(self.config, rawdata, limit=limit)\n+        \"\"\"\n+        return self.harvest_date(datetime.date.today(), **kwargs)\n \n-    def raw(self, start_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], end_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], shift_range: bool=True, limit: int=None, **kwargs) -> list:\n-        start_date, end_date = self._validate_dates(start_date, end_date)\n-        count, harvest = 0, self.do_harvest(start_date, end_date, **kwargs)\n-        assert isinstance(harvest, types.GeneratorType), 'do_harvest did not return a generator type, found {!r}. Make sure to use the yield keyword'.format(type(harvest))\n+    def harvest_date(self, date, **kwargs):\n+        \"\"\"Harvest data from the specified date.\n \n-        for doc_id, datum in harvest:\n-            yield doc_id, self.encode_data(datum, pretty=True)\n-            count += 1\n-            if limit and count >= limit:\n-                break\n+        Yields:\n+            RawDatum\n+\n+        \"\"\"\n+        return self.harvest_date_range(date - datetime.timedelta(days=1), date, **kwargs)\n+\n+    def harvest_date_range(self, start, end, limit=None, force=False, ignore_disabled=False, **kwargs):\n+        \"\"\"Fetch data from the specified date range.\n \n-    def harvest_by_id(self, doc_id):\n-        from share.models import RawDatum\n-        datum = self.fetch_by_id(doc_id)\n-        return RawDatum.objects.store_data(doc_id, self.encode_data(datum), self.config)\n-\n-    def encode_data(self, data, pretty=False) -> str:\n-        if isinstance(data, str):\n-            return data\n-        if isinstance(data, bytes):\n-            logger.warning(\n-                '%r.encode_data got a bytes instance. '\n-                'do_harvest should be returning str types as only the harvester will know how to properly encode the bytes'\n-                'defaulting to decoding as utf-8',\n-                self,\n-            )\n-            return data.decode('utf-8')\n-        if isinstance(data, dict):\n-            return self.encode_json(data, pretty=pretty)\n-        raise Exception('Unable to properly encode data blob {!r}. Data should be a dict, bytes, or str objects.'.format(data))\n-\n-    def encode_json(self, data: dict, pretty: bool=False) -> str:\n-        \"\"\"Orders a python dict recursively so it will always hash to the\n-        same value. Used for Dedupping harvest results\n         Args:\n-            data (dict):\n+            start (date):\n+            end (date):\n+            limit (int, optional): The maximum number of unique data to harvest. Defaults to None.\n+            force (bool, optional): Disable all safety checks, unexpected exceptions will still be raised. Defaults to False.\n+            ignore_disabled (bool, optional): Don't check if this Harvester or Source is disabled or deleted. Defaults to False.\n+            **kwargs: Forwared to _do_fetch.\n+\n+        Yields:\n+            RawDatum\n \n-        Returns:\n-            str: json.dumpsed ordered dictionary\n         \"\"\"\n-        return json.dumps(data, sort_keys=True, indent=4 if pretty else None)\n+        if self.serializer.pretty:\n+            raise ValueError('To ensure that data is optimally deduplicated, harvests may not occur while using a pretty serializer.')\n \n-    def _validate_dates(self, start_date, end_date):\n-        assert not (bool(start_date) ^ bool(end_date)), 'Must specify both a start and end date or neither'\n-        assert isinstance(start_date, (datetime.timedelta, datetime.datetime, pendulum.Pendulum)) and isinstance(end_date, (datetime.timedelta, datetime.datetime, pendulum.Pendulum)), 'start_date and end_date must be either datetimes or timedeltas'\n-        assert not (isinstance(start_date, datetime.timedelta) and isinstance(end_date, datetime.timedelta)), 'Only one of start_date and end_date may be a timedelta'\n+        if (self.config.disabled or self.config.source.is_deleted) and not (force or ignore_disabled):\n+            raise HarvesterDisabledError('Harvester {!r} is disabled. Either enable it, run with force=True, or ignore_disabled=True.'.format(self.config))\n \n-        if isinstance(start_date, datetime.datetime):\n-            start_date = pendulum.instance(start_date)\n+        with self.config.acquire_lock(required=not force):\n+            logger.info('Harvesting %s - %s from %r', start, end, self.config)\n+            yield from RawDatum.objects.store_chunk(self.config, self.fetch_date_range(start, end, **kwargs), limit=limit)\n \n-        if isinstance(end_date, datetime.datetime):\n-            end_date = pendulum.instance(end_date)\n+    def harvest_from_log(self, harvest_log, **kwargs):\n+        \"\"\"Harvest data as specified by the given harvest_log.\n \n-        if isinstance(start_date, datetime.timedelta):\n-            start_date = pendulum.instance(end_date + start_date)\n+        Args:\n+            harvest_log (HarvestLog): The HarvestLog that describes the parameters of this harvest\n+            limit (int, optional): The maximum number of unique data to harvest. Defaults to None.\n+            **kwargs: Forwared to harvest_date_range.\n \n-        if isinstance(end_date, datetime.timedelta):\n-            end_date = pendulum.instance(start_date + end_date)\n+        Yields:\n+            RawDatum\n \n-        og_start, og_end = start_date, end_date\n-        start_date, end_date = self.shift_range(start_date, end_date)\n-        assert isinstance(start_date, pendulum.Pendulum) and isinstance(end_date, pendulum.Pendulum), 'transpose_time_window must return a tuple of 2 datetimes'\n+        \"\"\"\n+        error = None\n+        datum_ids = []\n+        logger.info('Harvesting %r', harvest_log)\n+\n+        with harvest_log.handle(self.VERSION):\n+            try:\n+                for datum in self.harvest_date_range(harvest_log.start_date, harvest_log.end_date, **kwargs):\n+                    datum_ids.append(datum.id)\n+                    yield datum\n+            except Exception as e:\n+                error = e\n+                raise error\n+            finally:\n+                try:\n+                    harvest_log.raw_data.add(*datum_ids)","path":"share/harvest/base.py","position":346,"original_position":346,"commit_id":"cbdb9c428daba7e68ba7275f57bb5c1d11e41505","original_commit_id":"cbdb9c428daba7e68ba7275f57bb5c1d11e41505","user":{"login":"aaxelb","id":6776190,"avatar_url":"https://avatars0.githubusercontent.com/u/6776190?v=3","gravatar_id":"","url":"https://api.github.com/users/aaxelb","html_url":"https://github.com/aaxelb","followers_url":"https://api.github.com/users/aaxelb/followers","following_url":"https://api.github.com/users/aaxelb/following{/other_user}","gists_url":"https://api.github.com/users/aaxelb/gists{/gist_id}","starred_url":"https://api.github.com/users/aaxelb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aaxelb/subscriptions","organizations_url":"https://api.github.com/users/aaxelb/orgs","repos_url":"https://api.github.com/users/aaxelb/repos","events_url":"https://api.github.com/users/aaxelb/events{/privacy}","received_events_url":"https://api.github.com/users/aaxelb/received_events","type":"User","site_admin":false},"body":"It seems like we should associate raws with the log right when they're created, in `store_chunk`, rather than waiting until the entire harvest is done, transform tasks are already created, and everything.","created_at":"2017-05-26T13:38:14Z","updated_at":"2017-05-26T13:38:14Z","html_url":"https://github.com/CenterForOpenScience/SHARE/pull/647#discussion_r118703612","pull_request_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647","_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/comments/118703612"},"html":{"href":"https://github.com/CenterForOpenScience/SHARE/pull/647#discussion_r118703612"},"pull_request":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647"}}},"pull_request":{"url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647","id":119381834,"html_url":"https://github.com/CenterForOpenScience/SHARE/pull/647","diff_url":"https://github.com/CenterForOpenScience/SHARE/pull/647.diff","patch_url":"https://github.com/CenterForOpenScience/SHARE/pull/647.patch","issue_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/647","number":647,"state":"open","locked":false,"title":"[Feature] Celery 4","user":{"login":"chrisseto","id":5532905,"avatar_url":"https://avatars1.githubusercontent.com/u/5532905?v=3","gravatar_id":"","url":"https://api.github.com/users/chrisseto","html_url":"https://github.com/chrisseto","followers_url":"https://api.github.com/users/chrisseto/followers","following_url":"https://api.github.com/users/chrisseto/following{/other_user}","gists_url":"https://api.github.com/users/chrisseto/gists{/gist_id}","starred_url":"https://api.github.com/users/chrisseto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chrisseto/subscriptions","organizations_url":"https://api.github.com/users/chrisseto/orgs","repos_url":"https://api.github.com/users/chrisseto/repos","events_url":"https://api.github.com/users/chrisseto/events{/privacy}","received_events_url":"https://api.github.com/users/chrisseto/received_events","type":"User","site_admin":false},"body":"Keeping a load of messages in RabbitMQ has gotten a bit messy. This change introduces a task that will check the database for harvests that need to be run. It will be added to the beat schedule to run every 5 minutes or so.\r\n\r\n~~A lot of the celery bits are a bit messy right now. They'll be fixed in the upgrade to celery 4.~~\r\nThis PR also contains the upgrade to celery 4\r\n\r\n```\r\n---------- coverage: platform darwin, python 3.5.2-final-0 -----------\r\nName                             Stmts   Miss  Cover   Missing\r\n--------------------------------------------------------------\r\nshare/harvest/__init__.py            1      0   100%\r\nshare/harvest/base.py               96      9    91%   104, 124, 139-140, 158, 219-223\r\nshare/harvest/exceptions.py          6      0   100%\r\nshare/harvest/ratelimit.py          30     15    50%   19-26, 29-30, 33-36, 39\r\nshare/harvest/serialization.py      28      2    93%   15, 46\r\n--------------------------------------------------------------\r\nTOTAL                              161     26    84%\r\n```","created_at":"2017-05-08T05:59:39Z","updated_at":"2017-05-26T13:38:14Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6bc38d6f45a7f5065510a0c6cc90c9fe9f57f863","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647/commits","review_comments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647/comments","review_comment_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/647/comments","statuses_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/statuses/cbdb9c428daba7e68ba7275f57bb5c1d11e41505","head":{"label":"CenterForOpenScience:feature/pull-harvesting","ref":"feature/pull-harvesting","sha":"cbdb9c428daba7e68ba7275f57bb5c1d11e41505","user":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars2.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"repo":{"id":21280081,"name":"SHARE","full_name":"CenterForOpenScience/SHARE","owner":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars2.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CenterForOpenScience/SHARE","description":"SHARE is building a free, open, data set about research and scholarly activities across their life cycle.","fork":false,"url":"https://api.github.com/repos/CenterForOpenScience/SHARE","forks_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/forks","keys_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/teams","hooks_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/hooks","issue_events_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/events{/number}","events_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/events","assignees_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/assignees{/user}","branches_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/branches{/branch}","tags_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/tags","blobs_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/refs{/sha}","trees_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/statuses/{sha}","languages_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/languages","stargazers_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/stargazers","contributors_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/contributors","subscribers_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/subscribers","subscription_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/subscription","commits_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/commits{/sha}","git_commits_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/commits{/sha}","comments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/comments{/number}","issue_comment_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/comments{/number}","contents_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/contents/{+path}","compare_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/merges","archive_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/downloads","issues_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues{/number}","pulls_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls{/number}","milestones_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/milestones{/number}","notifications_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/labels{/name}","releases_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/releases{/id}","deployments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/deployments","created_at":"2014-06-27T15:19:21Z","updated_at":"2017-05-04T19:08:47Z","pushed_at":"2017-05-25T20:13:24Z","git_url":"git://github.com/CenterForOpenScience/SHARE.git","ssh_url":"git@github.com:CenterForOpenScience/SHARE.git","clone_url":"https://github.com/CenterForOpenScience/SHARE.git","svn_url":"https://github.com/CenterForOpenScience/SHARE","homepage":"http://share-research.readthedocs.io/en/latest/index.html","size":40328,"stargazers_count":53,"watchers_count":53,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":41,"mirror_url":null,"open_issues_count":26,"forks":41,"open_issues":26,"watchers":53,"default_branch":"develop"}},"base":{"label":"CenterForOpenScience:develop","ref":"develop","sha":"f1e7c07bb28ca6bcda3fa2fcaae44bd7f4199532","user":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars2.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"repo":{"id":21280081,"name":"SHARE","full_name":"CenterForOpenScience/SHARE","owner":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars2.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CenterForOpenScience/SHARE","description":"SHARE is building a free, open, data set about research and scholarly activities across their life cycle.","fork":false,"url":"https://api.github.com/repos/CenterForOpenScience/SHARE","forks_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/forks","keys_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/teams","hooks_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/hooks","issue_events_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/events{/number}","events_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/events","assignees_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/assignees{/user}","branches_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/branches{/branch}","tags_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/tags","blobs_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/refs{/sha}","trees_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/statuses/{sha}","languages_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/languages","stargazers_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/stargazers","contributors_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/contributors","subscribers_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/subscribers","subscription_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/subscription","commits_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/commits{/sha}","git_commits_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/git/commits{/sha}","comments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/comments{/number}","issue_comment_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/comments{/number}","contents_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/contents/{+path}","compare_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/merges","archive_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/downloads","issues_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues{/number}","pulls_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls{/number}","milestones_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/milestones{/number}","notifications_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/labels{/name}","releases_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/releases{/id}","deployments_url":"https://api.github.com/repos/CenterForOpenScience/SHARE/deployments","created_at":"2014-06-27T15:19:21Z","updated_at":"2017-05-04T19:08:47Z","pushed_at":"2017-05-25T20:13:24Z","git_url":"git://github.com/CenterForOpenScience/SHARE.git","ssh_url":"git@github.com:CenterForOpenScience/SHARE.git","clone_url":"https://github.com/CenterForOpenScience/SHARE.git","svn_url":"https://github.com/CenterForOpenScience/SHARE","homepage":"http://share-research.readthedocs.io/en/latest/index.html","size":40328,"stargazers_count":53,"watchers_count":53,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":41,"mirror_url":null,"open_issues_count":26,"forks":41,"open_issues":26,"watchers":53,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647"},"html":{"href":"https://github.com/CenterForOpenScience/SHARE/pull/647"},"issue":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/647"},"comments":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/issues/647/comments"},"review_comments":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647/comments"},"review_comment":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/pulls/647/commits"},"statuses":{"href":"https://api.github.com/repos/CenterForOpenScience/SHARE/statuses/cbdb9c428daba7e68ba7275f57bb5c1d11e41505"}}}}
{"actor":{"display_login":"aaxelb"}}