@staticmethod
    def disease_stats_update_aggregates(records=None, all=False):
        """
            This will calculate the disease_stats_aggregates for the specified
            records. Either all (when rebuild_all is invoked) or for the
            individual parameter(s) at the specified location(s) when run
            onaccept/onapprove.
            @ToDo: onapprove/onaccept wrapper function.

            This will get the raw data from disease_stats_data and generate
            a disease_stats_aggregate record for the given time period.

            The reason for doing this is so that all aggregated data can be
            obtained from a single table. So when displaying data for a
            particular location it will not be necessary to try the aggregate
            table, and if it's not there then try the data table. Rather just
            look at the aggregate table.

            Once this has run then a complete set of aggregate records should
            exists for this parameter_id and location for every time period from
            the first data item until the current time period.

            @ToDo: Add test cases to modules/unit_tests/s3db/disease.py
         """

        if not records:
            return

        # Test to see which date format we have based on how we were called
        if isinstance(records, basestring):
            from_json = True
            from dateutil.parser import parse
            records = json.loads(records)
        elif isinstance(records[0]["date"],
                        (datetime.date, datetime.datetime)):
            from_json = False
        else:
            from_json = True
            from dateutil.parser import parse

        db = current.db
        s3db = current.s3db
        atable = db.disease_stats_aggregate

        if not all:
            # Read the database to get all the relevant records
            # @ToDo: Complete this
            return
            dtable = s3db.disease_stats_data

        # For each location/parameter pair, create a time-aggregate summing all
        # the data so far

        now = current.request.now

        # Assemble raw data
        earliest_period = now.date()
        locations = {}
        parameters = []
        pappend = parameters.append
        for record in records:
            location_id = record["location_id"]
            if location_id not in locations:
                locations[location_id] = {}
            parameter_id = record["parameter_id"]
            if parameter_id not in parameters:
                pappend(parameter_id)
            if parameter_id not in locations[location_id]:
                locations[location_id][parameter_id] = {}
            if from_json:
                date = parse(record["date"]) # produces a datetime
                date = date.date()
            else:
                date = record["date"]
            if date < earliest_period:
                earliest_period = date
            locations[location_id][parameter_id][date] = record["value"]

        # Full range of dates
        # 1 per day from the start of the data to the present day
        from dateutil.rrule import rrule, DAILY
        dates = rrule(DAILY, dtstart=earliest_period, until=now)
        dates = [d.date() for d in dates]

        # Add the sums
        insert = atable.insert
        lfield = atable.location_id
        pfield = atable.parameter_id
        dfield = atable.date
        ifield = atable.id
        _q = (atable.agg_type == 1)
        for location_id in locations:
            location = locations[location_id]
            query = _q & (lfield == location_id)
            for parameter_id in location:
                parameter = location[parameter_id]
                q = query & (pfield == parameter_id)
                for d in dates:
                    values = []
                    vappend = values.append
                    for date in parameter:
                        if date <= d:
                            vappend(parameter[date])
                    values_sum = sum(values)
                    exists = db(q & (dfield == d)).select(ifield,
                                                          limitby=(0, 1))
                    if exists:
                        db(ifield == exists.first().id).update(sum = values_sum)
                    else:
                        insert(agg_type = 1, # Time
                               location_id = location_id,
                               parameter_id = parameter_id,
                               date = d,
                               sum = values_sum,
                               )

        # For each location/parameter pair, build a location-aggregate for all
        # ancestors, by level (immediate parents first).
        # Ensure that we don't duplicate builds
        # Do this for all dates between the changed date and the current date

        # Get all the ancestors
        # Read all the Paths
        # NB Currently we're assuming that all Paths have been built correctly
        gtable = db.gis_location
        ifield = gtable.id
        location_ids = locations.keys()
        paths = db(ifield.belongs(location_ids)).select(gtable.path)
        paths = [p.path.split("/") for p in paths]
        # Convert list of lists to flattened list & remove duplicates
        import itertools
        ancestors = tuple(itertools.chain.from_iterable(paths))
        # Remove locations which we already have data for
        ancestors = [a for a in ancestors if a not in location_ids]

        # Get all the children for each ancestor (immediate children not descendants)
        pfield = gtable.parent
        query = (gtable.deleted == False) & \
                (pfield.belongs(ancestors))
        all_children = db(query).select(ifield,
                                        pfield)

        # Read the levels
        rows = db(ifield.belongs(ancestors)).select(ifield,
                                                    gtable.level)
        L0 = []
        L0_append = L0.append
        L1 = []
        L1_append = L1.append
        L2 = []
        L2_append = L2.append
        L3 = []
        L3_append = L3.append
        L4 = []
        L4_append = L4.append
        for row in rows:
            if row.level == "L0":
                L0_append(row.id)
            elif row.level == "L1":
                L1_append(row.id)
            elif row.level == "L2":
                L2_append(row.id)
            elif row.level == "L3":
                L3_append(row.id)
            elif row.level == "L4":
                L4_append(row.id)

        async = current.s3task.async
        from gluon.serializers import json as jsons
        dates = jsons(dates)
        # Build the lowest level first
        for level in (L4, L3, L2, L1):
            for location_id in level:
                children = [c.id for c in all_children if c.parent == location_id]
                children = json.dumps(children)
                for parameter_id in parameters:
                    async("disease_stats_update_location_aggregates",
                          args = [location_id, children, parameter_id, dates],
                          timeout = 1800 # 30m
                          )

    # -------------------------------------------------------------------------
dataset/ETHPy150Open sahana/eden/modules/s3db/disease.py/DiseaseStatsModel.disease_stats_update_aggregates/original