/* File:    $Id$
 * Revision: $Revision$
 * Author:   $Author$
 * Date:     $Date$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2012 The Royal Danish Library, the Danish State and
 * University Library, the National Library of France and the Austrian
 * National Library.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

package dk.netarkivet.harvester.scheduler;

import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.netarkivet.common.lifecycle.ComponentLifeCycle;
import dk.netarkivet.common.lifecycle.PeriodicTaskExecutor;
import dk.netarkivet.common.utils.NotificationsFactory;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.harvester.HarvesterSettings;
import dk.netarkivet.harvester.datamodel.HarvestDefinition;
import dk.netarkivet.harvester.datamodel.HarvestDefinitionDAO;

/**
 * Handles the generation of new jobs based on the harvest definitions in
 * persistent storage. The <code>HarvestJobGenerator</code> continuously scans
 * the harvest definition database for harvest which should be run now. If a HD
 * defines a harvest which should be run, a Harvest Job is created in the
 * harvest job database.
 */
public class HarvestJobGenerator implements ComponentLifeCycle {

    /** The set of HDs (or rather their OIDs) that are currently being
     * scheduled in a separate thread.
     * This set is a SynchronizedSet
     */
    protected static Set<Long> harvestDefinitionsBeingScheduled =
        Collections.synchronizedSet(new HashSet<Long>());
    /** The class logger. */
    private static final Log log =
        LogFactory.getLog(HarvestJobGenerator.class.getName());

    /** The executor used to schedule the generator jobs. */
    private PeriodicTaskExecutor genExec;
    
    /** The HarvestDefinitionDAO used by the HarvestJobGenerator. */
    private static final HarvestDefinitionDAO haDefinitionDAO =
        HarvestDefinitionDAO.getInstance();

    /**
     * Starts the job generation scheduler.
     */
    @Override
    public void start() {
        genExec = new PeriodicTaskExecutor(
                "JobGeneratorTask",
                new JobGeneratorTask(),
                0,
                Settings.getInt(HarvesterSettings.GENERATE_JOBS_PERIOD));
    }

    @Override
    public void shutdown() {
        if (genExec != null) {
            genExec.shutdown();
        }
    }

    /**
     * Contains the functionality for the individual JobGenerations.
     */
    static class JobGeneratorTask implements Runnable {

        @Override
        public synchronized void run() {
            generateJobs(new Date());
        }

        /**
         * Check if jobs should be generated for any ready harvest definitions
         * for the specified time.
         * @param timeToGenerateJobsFor Jobs will be generated which should be
         * run at this time.
         * Note: In a production system the provided time will normally be
         * current time, but during testing we need to simulated other
         * points-in-time
         */
        static void generateJobs(Date timeToGenerateJobsFor) {
            final Iterable<Long> readyHarvestDefinitions =
                haDefinitionDAO.getReadyHarvestDefinitions(
                        timeToGenerateJobsFor);
            for (final Long id : readyHarvestDefinitions) {
                // Make every HD run in its own thread, but at most once.
                if (harvestDefinitionsBeingScheduled.contains(id)) {
                    String harvestName = haDefinitionDAO.getHarvestName(id);
                    String errMsg = "Not creating jobs for harvestdefinition #"
                            + id + " (" + harvestName + ")"
                            + " as the previous scheduling is still running";
                    log.warn(errMsg);
                    NotificationsFactory.getInstance().errorEvent(errMsg);
                    continue;
                }

                final HarvestDefinition harvestDefinition =
                    haDefinitionDAO.read(id);

                harvestDefinitionsBeingScheduled.add(id);

                if (!harvestDefinition.runNow(timeToGenerateJobsFor)) {
                    log.trace("The harvestdefinition '"
                            + harvestDefinition.getName() 
                            + "' should not run now.");
                    log.trace("numEvents: " + harvestDefinition.getNumEvents());
                    continue;
                }

                new Thread("JobGeneratorTask-" + id) {
                    public void run() {
                        try {
                            int jobsMade = harvestDefinition.createJobs();
                            log.info("Created " + jobsMade
                                    + " jobs for harvest definition '"
                                    + harvestDefinition.getName() + "'");
                            haDefinitionDAO.update(harvestDefinition);
                        } catch (Throwable e) {
                            try {
                                HarvestDefinition hd = haDefinitionDAO.read(
                                        harvestDefinition.getOid());
                                hd.setActive(false);
                                haDefinitionDAO.update(hd);
                                String errMsg = "Exception while scheduling"
                                    + "harvestdefinition '" 
                                    + harvestDefinition.getName() + "'. The "
                                    + "harvestdefinition has been deactivated!";
                                log.warn(errMsg, e);
                                NotificationsFactory.getInstance().
                                errorEvent(errMsg, e);
                            } catch (Exception e1) {
                                String errMsg = "Exception while scheduling"
                                    + "harvestdefinition '" 
                                    + harvestDefinition.getName()
                                    + "'. The harvestdefinition couldn't be "
                                    +   "deactivated!";
                                log.warn(errMsg, e);
                                log.warn("Unable to deactivate", e1);
                                NotificationsFactory.getInstance().
                                errorEvent(errMsg, e);
                            }
                        } finally {
                            harvestDefinitionsBeingScheduled.
                            remove(id);
                            log.debug("Removed '" + harvestDefinition.getName()
                                    + "' from list of harvestdefinitions to be "
                                    + "scheduled. Harvestdefinitions still to "
                                    + "be scheduled: "
                                    + harvestDefinitionsBeingScheduled);
                        }
                    }
                }.start();
            }
        }
    }

    //Hack, used by test
    static void clearGeneratingJobs() {
        harvestDefinitionsBeingScheduled.clear();
    }
}

@@ -4507,20 +4507,16 @@
         
-    
 + id + %22
@@ -4539,20 +4539,16 @@
 e + %22)%22%0A
-    
         
@@ -4613,16 +4613,399 @@
 nning%22;%0A
+                    if (haDefinitionDAO.isSnapshot(id)) %7B%0A                        // Log only at level debug if the ID represents %0A                        // is a snapshot harvestdefinition, which are only run %0A                        // once anyway%0A                        log.debug(errMsg);%0A                    %7D else %7B // Log at level WARN, and send a notification, otherwise%0A    
         
@@ -5026,32 +5026,36 @@
 g.warn(errMsg);%0A
+    
                 
@@ -5105,32 +5105,54 @@
 rEvent(errMsg);%0A
+                    %7D%0A
                 

Fix for NAS-2005