{"action":"created","comment":{"url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/comments/156503582","pull_request_review_id":82683736,"id":156503582,"diff_hunk":"@@ -0,0 +1,633 @@\n+#!/usr/bin/env python\n+\"\"\"\n+If no arguments are passed in this application runs as a process that will\n+run once a day, looking at all of the ZPLSC-C instruments' latest raw data\n+and generate the 24-hour echograms for each instrument.  It will store the\n+echogram in a local directory specified in the configuration file.\n+\n+If arguments are passed in, the application will run once and generate the\n+echograms based on the passed arguments.\n+\n+Usage:\n+    zplcs_echogram_generator.py [<subsites>] [<deployments>] [<dates>] [--keep]\n+    zplcs_echogram_generator.py (-h | --help)\n+\n+Arguments:\n+    subsites      Single or a list of subsites of ZPLSC instruments (delimit lists by \"'s and space separated).\n+                  If omitted, all subsites from the config file are used and yesterday's echograms are generated.\n+    deployments   Single or a list of deployment numbers. (ex: 4 or \"4 5\")\n+    dates         Single or a list of dates of the desired 24 hour echograms.\n+                  (Ex: 2016-10-01 for one day or 2016-11 for the entire month or \"2016-10-01 2016-11\")\n+\n+Options:\n+    -h --help     Print this help message\n+    --keep        Keeps the temporary files after the echogram has been created.\n+\"\"\"\n+\n+import docopt\n+import yaml\n+import types\n+import shutil\n+import re\n+import calendar\n+import threading\n+import urllib\n+import urllib2\n+from os import mkdir, system\n+from os.path import exists, expanduser, join\n+from datetime import date, timedelta\n+\n+from mi.logging import log\n+from mi.dataset.parser.zplsc_c import ZplscCParser\n+from mi.dataset.dataset_parser import DataSetDriverConfigKeys\n+from mi.dataset.parser.common_regexes import DATE2_YYYY_MM_DD_REGEX, DATE2_YYYY_MM_REGEX\n+\n+__author__ = 'Rene Gelinas'\n+\n+ZPLSC_CONFIG_FILE = 'zplsc_echogram_config.yml'\n+\n+MODULE_NAME = 'mi.dataset.parser.zplsc_c'\n+CLASS_NAME = 'ZplscCRecoveredDataParticle'\n+CONFIG = {\n+    DataSetDriverConfigKeys.PARTICLE_MODULE: MODULE_NAME,\n+    DataSetDriverConfigKeys.PARTICLE_CLASS: CLASS_NAME\n+}\n+\n+RAW_DATA_URL = r'https://rawdata.oceanobservatories.org/files'\n+DCL_PATH = r'instrmts/dcl37'\n+DCL_PATH2 = r'instruments/dcl37'\n+DCL_PATH3 = r'instrmt/dcl37'\n+RECOVERED_DIR = r'#SUBSITE_LC#_zplsc_#SERIAL_NUM#_recovered_'\n+DATA_PATH = r'DATA'\n+RECOVERED_DATE_FMT = r'YYYY-MM-DD'\n+\n+SECONDS_IN_DAY = 86400\n+\n+RAW_FILE_EXT = r'.01A'\n+PNG_FILE_EXT = r'.png'\n+USER_HOME = r'~'\n+ECHOGRAM_DIRECTORY = r'~/ZPLSC_ECHOGRAMS'\n+TEMP_DIR = r'TEMP_ZPLSC_DATA'\n+\n+DATE_YYYY_MM_DD_REGEX_MATCHER = re.compile(DATE2_YYYY_MM_DD_REGEX + '$')\n+DATE_YYYY_MM_REGEX_MATCHER = re.compile(DATE2_YYYY_MM_REGEX + '$')\n+\n+SERIAL_NUM_DIR_RE = r'(ZPLSC_sn(\\d*))\\/?'\n+SERIAL_NUM_DIR_MATCHER = re.compile(SERIAL_NUM_DIR_RE)\n+\n+DEPLOYMENT_DIR_RE = r'R(\\d{5})\\/?.*'\n+DEPLOYMENT_DIR_MATCHER = re.compile(DEPLOYMENT_DIR_RE)\n+\n+DATE_DIR_RE = r'(\\d{4})(\\d{2}).*\\/?'\n+DATE_DIR_RE_MATCHER = re.compile(DATE_DIR_RE)\n+\n+HOUR_23_RAW_DATA_FILE_RE = r'(\\d{4})(\\d{2})23\\.?.*'\n+HOUR_23_RAW_DATA_FILE_RE_MATCHER = re.compile(HOUR_23_RAW_DATA_FILE_RE)\n+\n+\"\"\"\n+    Note: the format of the raw data URL is as follows:\n+        'https://rawdata.oceanobservatories.org/files/[SUBSITE]/[DEPLOYMENT]/[instrmt|instrmts|instruments]/dcl37/\n+        ZPLSC_sn[SERIAL_NUM]/ce01issm_zplsc_[SERIAL_NUM]_recovered_2016-05-26/DATA/[YYYYMM]/'\n+\"\"\"\n+\n+\n+class ZPLSCEchogramGenerator(object):\n+    \"\"\"\n+    The ZPLSC Echogram Generator class will execute one time when command line\n+    parameters are passed in, generating all 24-hour echograms based on the\n+    parameters.  When no command line parameters are passed in, it will run as\n+    a process, executing once every day, generating echograms for every ZPLSC\n+    instruments from the previous day.\n+    \"\"\"\n+\n+    def __init__(self, _subsites, _deployments=None, _echogram_dates=None, _keep_temp_files=False):\n+        self.subsites = []\n+        self.deployments = []\n+        self.echogram_dates = {}\n+        self.keep_temp_files = _keep_temp_files\n+        self.process_mode = False\n+        self.zplsc_24_datafile_prefix = ''\n+        self.serial_num = ''\n+        self.temp_directory = ''\n+        self.echogram_directory = ''\n+\n+        # Raise an exception if the any of the command line parameters are not valid.\n+        if not self.input_is_valid(_subsites, _deployments, _echogram_dates):\n+            raise ValueError\n+\n+    #\n+    # Static Methods\n+    #\n+    @staticmethod\n+    def rec_exception_callback(exception):\n+        \"\"\"\n+        Call back method to for exceptions\n+\n+        @param exception - Exception that occurred\n+        \"\"\"\n+\n+        log.info(\"Exception occurred: %s\", exception.message)\n+\n+    @staticmethod\n+    def get_dir_contents(url):\n+        \"\"\"\n+        From the URL path passed return the contents of the remote directory.\n+\n+        :param url: The URL of the directory for which the contents are returned.\n+        :return: dir_contents: The contents of the remote URL directory.\n+        \"\"\"\n+\n+        try:\n+            url_io = urllib2.urlopen(url)\n+            dir_contents = url_io.read().decode('utf-8')\n+\n+        except urllib2.HTTPError as ex:\n+            log.warning('URL does not exist: %s: %s', url, ex)\n+            raise\n+\n+        return dir_contents\n+\n+    @staticmethod\n+    def parse_dates(echogram_dates):\n+        \"\"\"\n+        Parse the date(s) passed in for proper format and convert them to a\n+        datetime object. Also, determine and set whether the entire month\n+        of echograms will be generated.\n+\n+        :param echogram_dates: List of d\n+        ates in string format.\n+        :return: parsed_dates: List of dates in the datetime object format.\n+        \"\"\"\n+\n+        parsed_dates = dict()\n+\n+        if echogram_dates is not None:\n+            for echogram_date in echogram_dates:\n+                date_regex = DATE_YYYY_MM_DD_REGEX_MATCHER.match(echogram_date)\n+                if date_regex:\n+                    year = int(date_regex.group(1))\n+                    month = int(date_regex.group(2))\n+                    day = int(date_regex.group(3))\n+\n+                    converted_date = date(year, month, day)\n+\n+                    # Indicate this date is not an entire month.\n+                    parsed_dates[converted_date] = False\n+\n+                else:\n+                    date_regex = DATE_YYYY_MM_REGEX_MATCHER.match(echogram_date)\n+\n+                    if date_regex:\n+                        year = int(date_regex.group(1))\n+                        month = int(date_regex.group(2))\n+                        day = 1\n+\n+                        converted_date = date(year, month, day)\n+\n+                        # Indicate this date is an entire month.\n+                        parsed_dates[converted_date] = True\n+\n+                    else:\n+                        log.error('Incorrect date format: %s: Correct format is YYYY-MM-DD or YYYY-MM', date)\n+                        parsed_dates = False\n+                        break\n+\n+        return parsed_dates\n+\n+    #\n+    # Member Methods\n+    #\n+    def input_is_valid(self, subsites, deployments, echogram_dates):\n+        \"\"\"\n+        This method validates the command line parameters entered.\n+\n+        :param subsites: The subsite(s) that the ZPLSC instrument(s) are attached.\n+        :param deployments: The command line deployment number of interest.\n+        :param echogram_dates: The command line dates of interest.\n+        :return valid_input:  Boolean indicating whether all the inputs validated.\n+        \"\"\"\n+\n+        valid_input = True\n+\n+        # Get the configuration parameters.\n+        zplsc_config = yaml.load(open(ZPLSC_CONFIG_FILE))\n+        zplsc_subsites = zplsc_config['zplsc_subsites']\n+\n+        # If no subsites were passed in, set the list of subsites to all in the config file.\n+        self.subsites = subsites\n+        if subsites is None:\n+            self.subsites = zplsc_subsites\n+            self.process_mode = True\n+\n+        # Validate the subsites in the list.\n+        for subsite in self.subsites:\n+            if subsite not in zplsc_subsites:\n+                log.error('Subsite %s is not in the list of subsites with ZPLSC instrumentation.', subsite)\n+                valid_input = False\n+                break\n+\n+        if valid_input and deployments is not None:\n+            self.deployments = []\n+            if not isinstance(deployments, types.ListType):\n+                deployments = [deployments]\n+\n+            for deployment in deployments:\n+                try:\n+                    self.deployments.append(int(deployment))\n+\n+                except ValueError as ex:\n+                    log.error('Invalid deployment number: %s: %s', deployment, ex.message)\n+                    valid_input = False\n+                    break\n+\n+        if valid_input:\n+            self.echogram_dates = self.parse_dates(echogram_dates)\n+            if self.echogram_dates is False:\n+                log.error('Invalid start date: %s', echogram_dates)\n+                valid_input = False\n+\n+        zplsc_echrogram_directory = zplsc_config['zplsc_echrogram_directory']\n+        if not zplsc_echrogram_directory:\n+            self.echogram_directory = ECHOGRAM_DIRECTORY\n+        else:\n+            self.echogram_directory = zplsc_echrogram_directory\n+\n+        return valid_input\n+\n+    def get_data_filenames(self, date_dirs_url, data_date):\n+        \"\"\"\n+        This method will take the data directory URL and data date and it will\n+        return the list of filenames generated on the data date.\n+\n+        :param date_dirs_url: The URL of the date directories on the raw data server\n+        :param data_date: The specific date of the data files.\n+        :return: filenames_list: The list of filenames for the data date passed in.\n+        \"\"\"\n+\n+        year = str(data_date.year)\n+        year2 = year[-2:]\n+        month = \"%02d\" % data_date.month\n+        day = \"%02d\" % data_date.day\n+        raw_datafile_prefix = year2 + month + day\n+        date_dir = year + month\n+\n+        # Get the URL for the ZPLSC raw data of the given instrument.\n+        raw_data_url = join(date_dirs_url, date_dir)\n+        try:\n+            all_files_list = self.get_dir_contents(raw_data_url)\n+        except urllib2.HTTPError:\n+            raise\n+\n+        # Generate the list of the 24 1-hour raw data files for the given date.\n+        filename_re = '(' + raw_datafile_prefix + '.*?' + RAW_FILE_EXT + ').*'\n+        pattern = re.compile(filename_re)\n+        filenames_list = pattern.findall(all_files_list)\n+\n+        return filenames_list, raw_data_url, raw_datafile_prefix\n+\n+    def aggregate_raw_data(self, date_dirs_url, data_date):\n+        \"\"\"\n+        This method will retrieve the 24 1-hour files residing at the URL\n+        passed in for the date passed in.  It will store the files locally,\n+        concatenate them to one file.  It will return the file name of the\n+        concatenated file and the file name of the echogram that will be\n+        generated.\n+\n+        :param date_dirs_url: The URL of the raw data server where the 24 1-hour files reside.\n+        :param data_date: The date of the raw data for the echogram to be generated.\n+        :return: zplsc_24_datafile: The 34-hour concatenated raw data file\n+                 zplsc_echogram_file: The file name for the echogram to be generated.\n+        \"\"\"\n+\n+        filenames_list, raw_data_url, raw_datafile_prefix = self.get_data_filenames(date_dirs_url, data_date)\n+\n+        if len(filenames_list) == 24:\n+            # Download the 24 1-hour raw data files to a temporary local directory.\n+            for raw_data_file in filenames_list:\n+                remote_raw_data_file = join(raw_data_url, raw_data_file)\n+                local_raw_data = join(self.temp_directory, raw_data_file)\n+                try:\n+                    urllib.urlretrieve(remote_raw_data_file, local_raw_data)\n+                except urllib.ContentTooShortError as ex:\n+                    log.error('Error retrieving: %s: %s', remote_raw_data_file, ex.message)\n+                    continue\n+\n+            # Concatenate the 24 1-hour raw data files to 1 24-hour raw data file and return the filename.\n+            zplsc_24_datafilename = self.zplsc_24_datafile_prefix + raw_datafile_prefix\n+            zplsc_24_datafile = join(self.temp_directory, zplsc_24_datafilename) + RAW_FILE_EXT\n+            system('cat ' + join(self.temp_directory, raw_datafile_prefix) + '*' + RAW_FILE_EXT + ' > ' +\n+                   zplsc_24_datafile)\n+\n+            # Generate the ZPLSC Echogram filename.\n+            zplsc_echogram_file = zplsc_24_datafile.replace(RAW_FILE_EXT, PNG_FILE_EXT)\n+        else:\n+            zplsc_24_datafile = ''\n+            zplsc_echogram_file = ''\n+\n+        return zplsc_24_datafile, zplsc_echogram_file\n+\n+    def get_latest_echogram_date(self, date_dirs_url, date_dirs_list):\n+        \"\"\"\n+        This method will return the list of raw data files, in the format - YYMMDDHHMM,\n+        of the latest, complete set of 24 1-hour raw data files.\n+\n+        :param date_dirs_url: The URL to the directory of the YYYYMM sub-directories.\n+        :param date_dirs_list: The list of YYYYMM sub-directories\n+        :return: latest_echogram_date: The date of the latest complete set of 24 1-hour raw data files.\n+        \"\"\"\n+\n+        latest_echogram_date = None\n+        latest_echogram_date_found = False\n+\n+        for date_dir in date_dirs_list:\n+            raw_data_files_url = join(date_dirs_url, date_dir[0]+date_dir[1])\n+\n+            # Get all the year/month/day/hour raw data files date for this year/month directory.\n+            try:\n+                raw_data_files_response = self.get_dir_contents(raw_data_files_url)\n+            except urllib2.HTTPError:\n+                continue\n+\n+            # Create a list of the last hour of each day of the month and sort them in reverse order\n+            hour_23_data_file_list = HOUR_23_RAW_DATA_FILE_RE_MATCHER.findall(raw_data_files_response)\n+            hour_23_data_file_list = sorted(hour_23_data_file_list, key=lambda x: x[1], reverse=True)\n+\n+            # For each day that has a file of the last hour of the day, check the number of files for that day.\n+            for data_file in hour_23_data_file_list:\n+                latest_echogram_date_re = data_file[0] + data_file[1] + r'\\d{2}\\.?.*'\n+                latest_echogram_date_re_matcher = re.compile(latest_echogram_date_re)\n+                latest_echogram_date_list = latest_echogram_date_re_matcher.findall(raw_data_files_response)\n+\n+                # If this day has 24 1-hour files, save it for the latest echogram date.\n+                if len(latest_echogram_date_list) == 24:\n+                    year = int(date_dir[0])\n+                    month = int(date_dir[1])\n+                    day = int(data_file[1])\n+                    latest_echogram_date = date(year, month, day)\n+\n+                    latest_echogram_date_found = True\n+                    break\n+\n+            if latest_echogram_date_found:\n+                break\n+\n+        return latest_echogram_date\n+\n+    def get_date_dirs(self, subsite, deployment):\n+        \"\"\"\n+        This method will generate the path to the directory of date directories\n+        in the format of YYYYMM.\n+\n+        :param subsite: The subsite of the ZPLSC instrument.\n+        :param deployment: The deployment number of the data of interest.\n+        :return: echogram_dates: The mapping of echogram dates to the entire month flag\n+                 date_dirs_url: The path to the date directories.\n+        \"\"\"\n+\n+        echogram_dates = dict()\n+        date_dirs_url = ''\n+\n+        # Generate the portion of the URL up to the DCL directory to get the all the instrument sub-directories.\n+        deployment_path = join(RAW_DATA_URL, subsite.upper(), 'R%05d' % deployment)\n+        dcl_path = join(deployment_path, DCL_PATH)\n+        try:\n+            instrument_dirs = self.get_dir_contents(dcl_path)\n+        except urllib2.HTTPError:\n+            log.info('Could not find \"%s\": searching for \"%s\"', DCL_PATH, DCL_PATH2)\n+\n+            # Check the alternate directory structure.\n+            dcl_path = join(deployment_path, DCL_PATH2)\n+            try:\n+                instrument_dirs = self.get_dir_contents(dcl_path)\n+            except urllib2.HTTPError:\n+                log.info('Could not find \"%s\": searching for \"%s\"', DCL_PATH2, DCL_PATH3)\n+\n+                # Check the alternate directory structure.\n+                dcl_path = join(deployment_path, DCL_PATH3)\n+                try:\n+                    instrument_dirs = self.get_dir_contents(dcl_path)\n+                except urllib2.HTTPError:\n+                    raise\n+\n+        # Generate the portion of the URL up to the ZPLSC Instrument serial number.\n+        serial_num_found = SERIAL_NUM_DIR_MATCHER.search(instrument_dirs)\n+        if serial_num_found is None:\n+            log.warning('Could not find ZPLSC data for subsite: %s and deployment: %s', subsite, deployment)\n+\n+        else:\n+            self.serial_num = serial_num_found.group(2)\n+            serial_num_url = join(dcl_path, SERIAL_NUM_DIR_MATCHER.search(instrument_dirs).group(1))\n+            try:\n+                sub_dirs = self.get_dir_contents(serial_num_url)\n+            except urllib2.HTTPError:\n+                raise\n+\n+            # Generate the portion of the URL that contains the recovered data path.\n+            recovered_path = RECOVERED_DIR.replace('#SUBSITE_LC#', subsite.lower())\n+            recovered_path = recovered_path.replace('#SERIAL_NUM#', self.serial_num)\n+            start_idx = sub_dirs.find(recovered_path)\n+\n+            # If this is the directory structure that has the recovered directory, add it to the URL.\n+            date_dirs_url = serial_num_url\n+            if start_idx != -1:\n+                end_idx = start_idx + len(recovered_path) + len(RECOVERED_DATE_FMT)\n+                recovered_path = join(sub_dirs[start_idx:end_idx], DATA_PATH)\n+\n+                # Create the raw data URL with the recovered path\n+                date_dirs_url = join(serial_num_url, recovered_path)\n+\n+            # If no dates were entered on the command line, get the entire  list of date directories.\n+            echogram_dates = self.echogram_dates\n+            if not echogram_dates:\n+                # Get all the year/month date subdirectories for this subsite and deployment.\n+                try:\n+                    date_dirs_response = self.get_dir_contents(date_dirs_url)\n+                except urllib2.HTTPError:\n+                    raise\n+\n+                # Generate the list of the date directories.\n+                echogram_dates = dict()\n+                date_dirs_list = DATE_DIR_RE_MATCHER.findall(date_dirs_response)\n+                date_dirs_list = sorted(date_dirs_list, key=lambda x: (x[0], x[1]), reverse=True)\n+\n+                # If in process mode, get the latest date that has 24 1-hour data files for echogram generation.\n+                if self.process_mode:\n+                    echogram_dates[self.get_latest_echogram_date(date_dirs_url, date_dirs_list)] = False\n+\n+                # Otherwise, get all the year/month date subdirectories for this subsite and deployment.\n+                else:\n+                    for date_dir in date_dirs_list:\n+                        year = int(date_dir[0])\n+                        month = int(date_dir[1])\n+\n+                        # Save the date and indicate that the entire month should be generated.\n+                        echogram_dates[date(year, month, 1)] = True\n+\n+        return echogram_dates, date_dirs_url\n+\n+    def get_deployment_dirs(self, subsite):\n+        \"\"\"\n+        This method will determine the deployment directories for the subsite\n+        passed in.\n+\n+        :param subsite: The subsite of the ZPLSC instrument.\n+        :return: deployments: The list of deployment directories.\n+        \"\"\"\n+\n+        deployments = [deployment for deployment in self.deployments]\n+        if not deployments:\n+            # Generate the portion of the URL up to the subsite directory.\n+            subsite_url = join(RAW_DATA_URL, subsite.upper())\n+            try:\n+                # Get the all the deployment sub-directories under the subsite directory.\n+                deployment_dirs = self.get_dir_contents(subsite_url)\n+            except urllib2.HTTPError:\n+                raise\n+\n+            # Generate the list of the 24 1-hour raw data files for the given date.\n+            deployment_list = DEPLOYMENT_DIR_MATCHER.findall(deployment_dirs)\n+            deployment_list.sort()\n+\n+            if self.process_mode:\n+                deployment_list = [deployment_list[-1]]\n+\n+            for deployment in deployment_list:\n+                try:\n+                    deployments.append(int(deployment))\n+\n+                except ValueError as ex:\n+                    log.error('Invalid deployment number: %s: %s', deployment, ex.message)\n+                    break\n+\n+        return deployments\n+\n+    def store_echogram(self, zplsc_echogram_file):\n+        \"\"\"\n+        This method will store the generated echogram in the location specified\n+        in the configuration file. It will then clean up the temporary folder.\n+\n+        :param zplsc_echogram_file: The generated ZPLSC echogram\n+        :return:\n+        \"\"\"\n+\n+        # Create the echogram directory, then copy the echogram to the local directory.\n+        self.echogram_directory = expanduser(self.echogram_directory)\n+        if not exists(self.echogram_directory):\n+            mkdir(self.echogram_directory)\n+\n+        system('cp ' + zplsc_echogram_file + ' ' + self.echogram_directory)\n+\n+        # Clean up the temporary files.\n+        system('rm -f ' + join(self.temp_directory, '*'))\n+\n+    def generate_zplsc_echograms(self):\n+        \"\"\"\n+        This method will get the subsites, deployments and dates from the\n+        command line or all of the subsites, deployments and dates for the\n+        daily process.  It will generate the echograms based on those inputs\n+        and upload the echograms to the raw data server.\n+\n+        :return:\n+        \"\"\"\n+\n+        # Create the temporary data file directory.\n+        self.temp_directory = join(expanduser(USER_HOME), TEMP_DIR)\n+        if not exists(self.temp_directory):\n+            mkdir(self.temp_directory)\n+\n+        # Create the echograms for the zplsc instruments of each subsite.\n+        for subsite in self.subsites:\n+            zplsc_24_subsite_prefix = subsite + '-'\n+\n+            try:\n+                deployments = self.get_deployment_dirs(subsite)\n+            except urllib2.HTTPError:\n+                continue\n+\n+            for deployment in deployments:\n+                zplsc_24_deployment_prefix = zplsc_24_subsite_prefix + 'R' + str(deployment) + '-'\n+\n+                try:\n+                    echogram_dates, date_dirs_url = self.get_date_dirs(subsite, deployment)\n+                except urllib2.HTTPError:\n+                    continue\n+\n+                for date_dir, entire_month in echogram_dates.items():\n+                    self.zplsc_24_datafile_prefix = zplsc_24_deployment_prefix + 'sn' + self.serial_num + '-'\n+\n+                    if entire_month:\n+                        number_of_days_in_the_month = calendar.monthrange(date_dir.year, date_dir.month)[1]\n+                        for day in range(number_of_days_in_the_month):\n+                            echogram_date = date_dir + timedelta(days=day)\n+\n+                            # Aggregate the 24 raw data files for the given instrument to 1 24-hour data file.\n+                            try:\n+                                zplsc_24_datafile, zplsc_echogram_file = self.aggregate_raw_data(date_dirs_url,\n+                                                                                                 echogram_date)\n+                                if not zplsc_24_datafile:\n+                                    continue\n+\n+                            except urllib2.HTTPError:\n+                                continue\n+\n+                            # Send the 24-hour raw data file to the zplsc C Series parser to generate the echogram.\n+                            with open(zplsc_24_datafile) as file_handle:\n+                                parser = ZplscCParser(CONFIG, file_handle, self.rec_exception_callback)\n+                                parser.create_echogram()\n+\n+                            self.store_echogram(zplsc_echogram_file)\n+                    else:\n+                        # Aggregate the 24 raw data files for the given instrument to 1 24-hour data file.\n+                        try:\n+                            zplsc_24_datafile, zplsc_echogram_file = self.aggregate_raw_data(date_dirs_url, date_dir)\n+\n+                            if not zplsc_24_datafile:\n+                                continue\n+\n+                        except urllib2.HTTPError:\n+                            continue\n+\n+                        # Send the 24-hour raw data file to the zplsc C Series parser to generate the echogram.\n+                        with open(zplsc_24_datafile) as file_handle:\n+                            parser = ZplscCParser(CONFIG, file_handle, self.rec_exception_callback)\n+                            parser.create_echogram()\n+\n+                        self.store_echogram(zplsc_echogram_file)\n+\n+        # Remove the temporary data file directory and its content.\n+        if not self.keep_temp_files:\n+            shutil.rmtree(self.temp_directory)\n+\n+        # If it's running as a daily process, wait 24 hours and re-run this method\n+        if self.process_mode:\n+            threading.Timer(SECONDS_IN_DAY, self.generate_zplsc_echograms).start()\n+\n+\n+def main():\n+    # Get the command line arguments\n+    options = docopt.docopt(__doc__)\n+    subsites = options['<subsites>']\n+    deployments = options['<deployments>']\n+    dates = options['<dates>']\n+    keep_temp_files = options['--keep']\n+\n+    if subsites is not None:","path":"mi/dataset/driver/zplsc_c/zplsc_echogram_generator.py","position":614,"original_position":614,"commit_id":"aa2a1e895e6f19f3293554439db626a91915be89","original_commit_id":"aa2a1e895e6f19f3293554439db626a91915be89","user":{"login":"danmergens","id":6914984,"avatar_url":"https://avatars1.githubusercontent.com/u/6914984?v=4","gravatar_id":"","url":"https://api.github.com/users/danmergens","html_url":"https://github.com/danmergens","followers_url":"https://api.github.com/users/danmergens/followers","following_url":"https://api.github.com/users/danmergens/following{/other_user}","gists_url":"https://api.github.com/users/danmergens/gists{/gist_id}","starred_url":"https://api.github.com/users/danmergens/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/danmergens/subscriptions","organizations_url":"https://api.github.com/users/danmergens/orgs","repos_url":"https://api.github.com/users/danmergens/repos","events_url":"https://api.github.com/users/danmergens/events{/privacy}","received_events_url":"https://api.github.com/users/danmergens/received_events","type":"User","site_admin":false},"body":"Use get.\r\n\r\n```python\r\nsubsites = options.get('<subsites>', ' ')\r\n```","created_at":"2017-12-12T21:41:08Z","updated_at":"2017-12-12T21:48:53Z","html_url":"https://github.com/oceanobservatories/mi-instrument/pull/36#discussion_r156503582","pull_request_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/comments/156503582"},"html":{"href":"https://github.com/oceanobservatories/mi-instrument/pull/36#discussion_r156503582"},"pull_request":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36"}}},"pull_request":{"url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36","id":157696158,"html_url":"https://github.com/oceanobservatories/mi-instrument/pull/36","diff_url":"https://github.com/oceanobservatories/mi-instrument/pull/36.diff","patch_url":"https://github.com/oceanobservatories/mi-instrument/pull/36.patch","issue_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/36","number":36,"state":"open","locked":false,"title":"Issue #12833 - Added support for offline process echogram generation.","user":{"login":"renegelinas","id":13528715,"avatar_url":"https://avatars2.githubusercontent.com/u/13528715?v=4","gravatar_id":"","url":"https://api.github.com/users/renegelinas","html_url":"https://github.com/renegelinas","followers_url":"https://api.github.com/users/renegelinas/followers","following_url":"https://api.github.com/users/renegelinas/following{/other_user}","gists_url":"https://api.github.com/users/renegelinas/gists{/gist_id}","starred_url":"https://api.github.com/users/renegelinas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/renegelinas/subscriptions","organizations_url":"https://api.github.com/users/renegelinas/orgs","repos_url":"https://api.github.com/users/renegelinas/repos","events_url":"https://api.github.com/users/renegelinas/events{/privacy}","received_events_url":"https://api.github.com/users/renegelinas/received_events","type":"User","site_admin":false},"body":"","created_at":"2017-12-11T23:10:27Z","updated_at":"2017-12-12T21:48:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2f7b1cdea952843fb8f5f3b21c1cf45c8f8826b9","assignee":null,"assignees":[],"requested_reviewers":[{"login":"vipullakhani","id":12614144,"avatar_url":"https://avatars2.githubusercontent.com/u/12614144?v=4","gravatar_id":"","url":"https://api.github.com/users/vipullakhani","html_url":"https://github.com/vipullakhani","followers_url":"https://api.github.com/users/vipullakhani/followers","following_url":"https://api.github.com/users/vipullakhani/following{/other_user}","gists_url":"https://api.github.com/users/vipullakhani/gists{/gist_id}","starred_url":"https://api.github.com/users/vipullakhani/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vipullakhani/subscriptions","organizations_url":"https://api.github.com/users/vipullakhani/orgs","repos_url":"https://api.github.com/users/vipullakhani/repos","events_url":"https://api.github.com/users/vipullakhani/events{/privacy}","received_events_url":"https://api.github.com/users/vipullakhani/received_events","type":"User","site_admin":false},{"login":"sgfoote","id":18015208,"avatar_url":"https://avatars1.githubusercontent.com/u/18015208?v=4","gravatar_id":"","url":"https://api.github.com/users/sgfoote","html_url":"https://github.com/sgfoote","followers_url":"https://api.github.com/users/sgfoote/followers","following_url":"https://api.github.com/users/sgfoote/following{/other_user}","gists_url":"https://api.github.com/users/sgfoote/gists{/gist_id}","starred_url":"https://api.github.com/users/sgfoote/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sgfoote/subscriptions","organizations_url":"https://api.github.com/users/sgfoote/orgs","repos_url":"https://api.github.com/users/sgfoote/repos","events_url":"https://api.github.com/users/sgfoote/events{/privacy}","received_events_url":"https://api.github.com/users/sgfoote/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36/commits","review_comments_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36/comments","review_comment_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/comments{/number}","comments_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/36/comments","statuses_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/statuses/aa2a1e895e6f19f3293554439db626a91915be89","head":{"label":"renegelinas:rm12833","ref":"rm12833","sha":"aa2a1e895e6f19f3293554439db626a91915be89","user":{"login":"renegelinas","id":13528715,"avatar_url":"https://avatars2.githubusercontent.com/u/13528715?v=4","gravatar_id":"","url":"https://api.github.com/users/renegelinas","html_url":"https://github.com/renegelinas","followers_url":"https://api.github.com/users/renegelinas/followers","following_url":"https://api.github.com/users/renegelinas/following{/other_user}","gists_url":"https://api.github.com/users/renegelinas/gists{/gist_id}","starred_url":"https://api.github.com/users/renegelinas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/renegelinas/subscriptions","organizations_url":"https://api.github.com/users/renegelinas/orgs","repos_url":"https://api.github.com/users/renegelinas/repos","events_url":"https://api.github.com/users/renegelinas/events{/privacy}","received_events_url":"https://api.github.com/users/renegelinas/received_events","type":"User","site_admin":false},"repo":{"id":40263087,"name":"mi-instrument","full_name":"renegelinas/mi-instrument","owner":{"login":"renegelinas","id":13528715,"avatar_url":"https://avatars2.githubusercontent.com/u/13528715?v=4","gravatar_id":"","url":"https://api.github.com/users/renegelinas","html_url":"https://github.com/renegelinas","followers_url":"https://api.github.com/users/renegelinas/followers","following_url":"https://api.github.com/users/renegelinas/following{/other_user}","gists_url":"https://api.github.com/users/renegelinas/gists{/gist_id}","starred_url":"https://api.github.com/users/renegelinas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/renegelinas/subscriptions","organizations_url":"https://api.github.com/users/renegelinas/orgs","repos_url":"https://api.github.com/users/renegelinas/repos","events_url":"https://api.github.com/users/renegelinas/events{/privacy}","received_events_url":"https://api.github.com/users/renegelinas/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/renegelinas/mi-instrument","description":null,"fork":true,"url":"https://api.github.com/repos/renegelinas/mi-instrument","forks_url":"https://api.github.com/repos/renegelinas/mi-instrument/forks","keys_url":"https://api.github.com/repos/renegelinas/mi-instrument/keys{/key_id}","collaborators_url":"https://api.github.com/repos/renegelinas/mi-instrument/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/renegelinas/mi-instrument/teams","hooks_url":"https://api.github.com/repos/renegelinas/mi-instrument/hooks","issue_events_url":"https://api.github.com/repos/renegelinas/mi-instrument/issues/events{/number}","events_url":"https://api.github.com/repos/renegelinas/mi-instrument/events","assignees_url":"https://api.github.com/repos/renegelinas/mi-instrument/assignees{/user}","branches_url":"https://api.github.com/repos/renegelinas/mi-instrument/branches{/branch}","tags_url":"https://api.github.com/repos/renegelinas/mi-instrument/tags","blobs_url":"https://api.github.com/repos/renegelinas/mi-instrument/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/renegelinas/mi-instrument/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/renegelinas/mi-instrument/git/refs{/sha}","trees_url":"https://api.github.com/repos/renegelinas/mi-instrument/git/trees{/sha}","statuses_url":"https://api.github.com/repos/renegelinas/mi-instrument/statuses/{sha}","languages_url":"https://api.github.com/repos/renegelinas/mi-instrument/languages","stargazers_url":"https://api.github.com/repos/renegelinas/mi-instrument/stargazers","contributors_url":"https://api.github.com/repos/renegelinas/mi-instrument/contributors","subscribers_url":"https://api.github.com/repos/renegelinas/mi-instrument/subscribers","subscription_url":"https://api.github.com/repos/renegelinas/mi-instrument/subscription","commits_url":"https://api.github.com/repos/renegelinas/mi-instrument/commits{/sha}","git_commits_url":"https://api.github.com/repos/renegelinas/mi-instrument/git/commits{/sha}","comments_url":"https://api.github.com/repos/renegelinas/mi-instrument/comments{/number}","issue_comment_url":"https://api.github.com/repos/renegelinas/mi-instrument/issues/comments{/number}","contents_url":"https://api.github.com/repos/renegelinas/mi-instrument/contents/{+path}","compare_url":"https://api.github.com/repos/renegelinas/mi-instrument/compare/{base}...{head}","merges_url":"https://api.github.com/repos/renegelinas/mi-instrument/merges","archive_url":"https://api.github.com/repos/renegelinas/mi-instrument/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/renegelinas/mi-instrument/downloads","issues_url":"https://api.github.com/repos/renegelinas/mi-instrument/issues{/number}","pulls_url":"https://api.github.com/repos/renegelinas/mi-instrument/pulls{/number}","milestones_url":"https://api.github.com/repos/renegelinas/mi-instrument/milestones{/number}","notifications_url":"https://api.github.com/repos/renegelinas/mi-instrument/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/renegelinas/mi-instrument/labels{/name}","releases_url":"https://api.github.com/repos/renegelinas/mi-instrument/releases{/id}","deployments_url":"https://api.github.com/repos/renegelinas/mi-instrument/deployments","created_at":"2015-08-05T18:48:45Z","updated_at":"2017-10-21T20:31:01Z","pushed_at":"2017-12-11T22:56:27Z","git_url":"git://github.com/renegelinas/mi-instrument.git","ssh_url":"git@github.com:renegelinas/mi-instrument.git","clone_url":"https://github.com/renegelinas/mi-instrument.git","svn_url":"https://github.com/renegelinas/mi-instrument","homepage":null,"size":144301,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"bsd-2-clause","name":"BSD 2-clause \"Simplified\" License","spdx_id":"BSD-2-Clause","url":"https://api.github.com/licenses/bsd-2-clause"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"oceanobservatories:master","ref":"master","sha":"440804291488ffbd6d3047435b416b281b38809d","user":{"login":"oceanobservatories","id":8750280,"avatar_url":"https://avatars0.githubusercontent.com/u/8750280?v=4","gravatar_id":"","url":"https://api.github.com/users/oceanobservatories","html_url":"https://github.com/oceanobservatories","followers_url":"https://api.github.com/users/oceanobservatories/followers","following_url":"https://api.github.com/users/oceanobservatories/following{/other_user}","gists_url":"https://api.github.com/users/oceanobservatories/gists{/gist_id}","starred_url":"https://api.github.com/users/oceanobservatories/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oceanobservatories/subscriptions","organizations_url":"https://api.github.com/users/oceanobservatories/orgs","repos_url":"https://api.github.com/users/oceanobservatories/repos","events_url":"https://api.github.com/users/oceanobservatories/events{/privacy}","received_events_url":"https://api.github.com/users/oceanobservatories/received_events","type":"Organization","site_admin":false},"repo":{"id":24165325,"name":"mi-instrument","full_name":"oceanobservatories/mi-instrument","owner":{"login":"oceanobservatories","id":8750280,"avatar_url":"https://avatars0.githubusercontent.com/u/8750280?v=4","gravatar_id":"","url":"https://api.github.com/users/oceanobservatories","html_url":"https://github.com/oceanobservatories","followers_url":"https://api.github.com/users/oceanobservatories/followers","following_url":"https://api.github.com/users/oceanobservatories/following{/other_user}","gists_url":"https://api.github.com/users/oceanobservatories/gists{/gist_id}","starred_url":"https://api.github.com/users/oceanobservatories/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oceanobservatories/subscriptions","organizations_url":"https://api.github.com/users/oceanobservatories/orgs","repos_url":"https://api.github.com/users/oceanobservatories/repos","events_url":"https://api.github.com/users/oceanobservatories/events{/privacy}","received_events_url":"https://api.github.com/users/oceanobservatories/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/oceanobservatories/mi-instrument","description":null,"fork":false,"url":"https://api.github.com/repos/oceanobservatories/mi-instrument","forks_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/forks","keys_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/keys{/key_id}","collaborators_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/teams","hooks_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/hooks","issue_events_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/events{/number}","events_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/events","assignees_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/assignees{/user}","branches_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/branches{/branch}","tags_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/tags","blobs_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/git/refs{/sha}","trees_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/git/trees{/sha}","statuses_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/statuses/{sha}","languages_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/languages","stargazers_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/stargazers","contributors_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/contributors","subscribers_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/subscribers","subscription_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/subscription","commits_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/commits{/sha}","git_commits_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/git/commits{/sha}","comments_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/comments{/number}","issue_comment_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/comments{/number}","contents_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/contents/{+path}","compare_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/compare/{base}...{head}","merges_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/merges","archive_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/downloads","issues_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues{/number}","pulls_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls{/number}","milestones_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/milestones{/number}","notifications_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/labels{/name}","releases_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/releases{/id}","deployments_url":"https://api.github.com/repos/oceanobservatories/mi-instrument/deployments","created_at":"2014-09-17T22:53:22Z","updated_at":"2016-11-10T01:32:37Z","pushed_at":"2017-12-11T23:10:28Z","git_url":"git://github.com/oceanobservatories/mi-instrument.git","ssh_url":"git@github.com:oceanobservatories/mi-instrument.git","clone_url":"https://github.com/oceanobservatories/mi-instrument.git","svn_url":"https://github.com/oceanobservatories/mi-instrument","homepage":null,"size":148702,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":24,"mirror_url":null,"archived":false,"open_issues_count":3,"license":{"key":"bsd-2-clause","name":"BSD 2-clause \"Simplified\" License","spdx_id":"BSD-2-Clause","url":"https://api.github.com/licenses/bsd-2-clause"},"forks":24,"open_issues":3,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36"},"html":{"href":"https://github.com/oceanobservatories/mi-instrument/pull/36"},"issue":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/36"},"comments":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/issues/36/comments"},"review_comments":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36/comments"},"review_comment":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/pulls/36/commits"},"statuses":{"href":"https://api.github.com/repos/oceanobservatories/mi-instrument/statuses/aa2a1e895e6f19f3293554439db626a91915be89"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"danmergens"}}