// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.resource;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.LibvirtException;
import org.libvirt.StoragePool;
import org.libvirt.StoragePoolInfo.StoragePoolState;

import org.apache.cloudstack.managed.context.ManagedContextRunnable;

import com.cloud.utils.script.Script;

public class KVMHAMonitor extends KVMHABase implements Runnable {
    private static final Logger s_logger = Logger.getLogger(KVMHAMonitor.class);
    private Map<String, NfsStoragePool> _storagePool = new ConcurrentHashMap<String, NfsStoragePool>();

    private String _hostIP; /* private ip address */

    public KVMHAMonitor(NfsStoragePool pool, String host, String scriptPath) {
        if (pool != null) {
            this._storagePool.put(pool._poolUUID, pool);
        }
        this._hostIP = host;
        KVMHABase.s_heartBeatPath = scriptPath;
    }

    public void addStoragePool(NfsStoragePool pool) {
        synchronized (_storagePool) {
            this._storagePool.put(pool._poolUUID, pool);
        }
    }

    public void removeStoragePool(String uuid) {
        synchronized (_storagePool) {
            NfsStoragePool pool = this._storagePool.get(uuid);
            if (pool != null) {
                Script.runSimpleBashScript("umount " + pool._mountDestPath);
                this._storagePool.remove(uuid);
            }
        }
    }

    public List<NfsStoragePool> getStoragePools() {
        synchronized (_storagePool) {
            return new ArrayList<NfsStoragePool>(_storagePool.values());
        }
    }

    private class Monitor extends ManagedContextRunnable {

        @Override
        protected void runInContext() {
            synchronized (_storagePool) {
                for (String uuid : _storagePool.keySet()) {
                    NfsStoragePool primaryStoragePool = _storagePool.get(uuid);

                    // check for any that have been deregistered with libvirt and
                    // skip,remove them

                    StoragePool storage = null;
                    try {
                        Connect conn = LibvirtConnection.getConnection();
                        storage = conn.storagePoolLookupByUUIDString(uuid);
                        if (storage == null) {
                            s_logger.debug("Libvirt storage pool " + uuid + " not found, removing from HA list");
                            removeStoragePool(uuid);
                            continue;

                        } else if (storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) {
                            s_logger.debug("Libvirt storage pool " + uuid + " found, but not running, removing from HA list");

                            removeStoragePool(uuid);
                            continue;
                        }
                        s_logger.debug("Found NFS storage pool " + uuid + " in libvirt, continuing");

                    } catch (LibvirtException e) {
                        s_logger.debug("Failed to lookup libvirt storage pool " + uuid + " due to: " + e);

                        // we only want to remove pool if it's not found, not if libvirt
                        // connection fails
                        if (e.toString().contains("pool not found")) {
                            s_logger.debug("removing pool from HA monitor since it was deleted");
                            removeStoragePool(uuid);
                            continue;
                        }
                    }

                    String result = null;
                    for (int i = 0; i < 5; i++) {
                        Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger);
                        cmd.add("-i", primaryStoragePool._poolIp);
                        cmd.add("-p", primaryStoragePool._poolMountSourcePath);
                        cmd.add("-m", primaryStoragePool._mountDestPath);
                        cmd.add("-h", _hostIP);
                        result = cmd.execute();
                        if (result != null) {
                            s_logger.warn("write heartbeat failed: " + result + ", retry: " + i);
                        } else {
                            break;
                        }
                    }

                    if (result != null) {
                        s_logger.warn("write heartbeat failed: " + result + "; reboot the host");
                        Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger);
                        cmd.add("-i", primaryStoragePool._poolIp);
                        cmd.add("-p", primaryStoragePool._poolMountSourcePath);
                        cmd.add("-m", primaryStoragePool._mountDestPath);
                        cmd.add("-c");
                        result = cmd.execute();
                    }
                }
            }

        }
    }

    @Override
    public void run() {
        // s_logger.addAppender(new org.apache.log4j.ConsoleAppender(new
        // org.apache.log4j.PatternLayout(), "System.out"));
        while (true) {
            Thread monitorThread = new Thread(new Monitor());
            monitorThread.start();
            try {
                monitorThread.join();
            } catch (InterruptedException e) {

            }

            try {
                Thread.sleep(_heartBeatUpdateFreq);
            } catch (InterruptedException e) {

            }
        }
    }

}

@@ -862,24 +862,50 @@
 .ArrayList;%0A
+import java.util.HashSet;%0A
 import java.
@@ -937,16 +937,38 @@
 il.Map;%0A
+import java.util.Set;%0A
 import j
@@ -1716,37 +1716,32 @@
 ) %7B%0A            
-this.
 _storagePool.put
@@ -1782,21 +1782,16 @@
         
-this.
 _hostIP 
@@ -1949,37 +1949,32 @@
 ) %7B%0A            
-this.
 _storagePool.put
@@ -2135,21 +2135,16 @@
  pool = 
-this.
 _storage
@@ -2288,13 +2288,8 @@
     
-this.
 _sto
@@ -2674,32 +2674,98 @@
 _storagePool) %7B%0A
+                Set%3CString%3E removedPools = new HashSet%3CString%3E();%0A
                 
@@ -3427,35 +3427,34 @@
       remove
-StoragePool
+dPools.add
 (uuid);%0A    
@@ -3753,35 +3753,34 @@
       remove
-StoragePool
+dPools.add
 (uuid);%0A    
@@ -4437,27 +4437,26 @@
   remove
-StoragePool
+dPools.add
 (uuid);%0A
@@ -5893,32 +5893,224 @@
                %7D
+%0A%0A                if (!removedPools.isEmpty()) %7B%0A                    for (String uuid : removedPools) %7B%0A                        removeStoragePool(uuid);%0A                    %7D%0A                %7D
 %0A            %7D%0A%0A

Fix CID 1116267 Don't modify the set while the iterator is still in use.