cloud/google/compute/resources/instancegroup.go
func (r *InstanceGroup) Apply(actual, expected cloud.Resource, immutable *cluster.Cluster) (*cluster.Cluster, cloud.Resource, error) {
	logger.Debug("instanceGroup.Apply")
	applyResource := expected.(*InstanceGroup)
	isEqual, err := compare.IsEqual(actual.(*InstanceGroup), expected.(*InstanceGroup))
	if err != nil {
		return nil, nil, err
	}
	if isEqual {
		return immutable, applyResource, nil
	}

	masterIPPrivate := ""
	masterIPPublic := ""
	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		found := false
		for i := 0; i < MasterIPAttempts; i++ {
			masterTag := ""
			machineConfigs := immutable.MachineProviderConfigs()
			for _, machineConfig := range machineConfigs {
				serverPool := machineConfig.ServerPool
				if serverPool.Type == cluster.ServerPoolTypeMaster {
					masterTag = serverPool.Name
				}
			}
			if masterTag == "" {
				return nil, nil, fmt.Errorf("Unable to find master tag")
			}

			instanceGroupManager, err := Sdk.Service.InstanceGroupManagers.ListManagedInstances(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, strings.ToLower(masterTag)).Do()
			if err != nil {
				return nil, nil, err
			}

			if err != nil || len(instanceGroupManager.ManagedInstances) == 0 {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			parts := strings.Split(instanceGroupManager.ManagedInstances[0].Instance, "/")
			instance, err := Sdk.Service.Instances.Get(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, parts[len(parts)-1]).Do()
			if err != nil {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			for _, networkInterface := range instance.NetworkInterfaces {
				if networkInterface.Name == "nic0" {
					masterIPPrivate = networkInterface.NetworkIP
					for _, accessConfigs := range networkInterface.AccessConfigs {
						masterIPPublic = accessConfigs.NatIP
					}
				}
			}

			if masterIPPublic == "" {
				logger.Debug("Hanging for master IP..")
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			found = true
			providerConfig := immutable.ProviderConfig()
			providerConfig.Values.ItemMap["INJECTEDMASTER"] = fmt.Sprintf("%s:%s", masterIPPrivate, immutable.ProviderConfig().KubernetesAPI.Port)
			immutable.SetProviderConfig(providerConfig)
			break
		}
		if !found {
			return nil, nil, fmt.Errorf("Unable to find Master IP after defined wait")
		}
	}

	providerConfig := immutable.ProviderConfig()
	providerConfig.Values.ItemMap["INJECTEDPORT"] = immutable.ProviderConfig().KubernetesAPI.Port
	immutable.SetProviderConfig(providerConfig)

	scripts, err := script.BuildBootstrapScript(r.ServerPool.BootstrapScripts, immutable)
	if err != nil {
		return nil, nil, err
	}

	finalScripts := string(scripts)
	if err != nil {
		return nil, nil, err
	}

	tags := []string{}
	if r.ServerPool.Type == cluster.ServerPoolTypeMaster {
		if immutable.ProviderConfig().KubernetesAPI.Port == "443" {
			tags = append(tags, "https-server")
		}

		if immutable.ProviderConfig().KubernetesAPI.Port == "80" {
			tags = append(tags, "http-server")
		}

		tags = append(tags, "kubicorn-master")
	}

	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		tags = append(tags, "kubicorn-node")
	}

	prefix := "https://www.googleapis.com/compute/v1/projects/" + immutable.ProviderConfig().CloudId
	imageURL := "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/" + expected.(*InstanceGroup).Image

	templateInstance, err := Sdk.Service.InstanceTemplates.Get(immutable.ProviderConfig().CloudId, strings.ToLower(expected.(*InstanceGroup).Name)).Do()
	if err != nil {
		sshPublicKeyValue := fmt.Sprintf("%s:%s", immutable.ProviderConfig().SSH.User, string(immutable.ProviderConfig().SSH.PublicKeyData))

		templateInstance = &compute.InstanceTemplate{
			Name: strings.ToLower(expected.(*InstanceGroup).Name),
			Properties: &compute.InstanceProperties{
				MachineType: expected.(*InstanceGroup).Size,
				Disks: []*compute.AttachedDisk{
					{
						AutoDelete: true,
						Boot:       true,
						Type:       "PERSISTENT",
						InitializeParams: &compute.AttachedDiskInitializeParams{
							SourceImage: imageURL,
						},
					},
				},
				NetworkInterfaces: []*compute.NetworkInterface{
					{
						AccessConfigs: []*compute.AccessConfig{
							{
								Type: "ONE_TO_ONE_NAT",
								Name: "External NAT",
							},
						},
						Network: prefix + "/global/networks/default",
					},
				},
				ServiceAccounts: []*compute.ServiceAccount{
					{
						Email: "default",
						Scopes: []string{
							compute.DevstorageFullControlScope,
							compute.ComputeScope,
						},
					},
				},
				Metadata: &compute.Metadata{
					Kind: "compute#metadata",
					Items: []*compute.MetadataItems{
						{
							Key:   "ssh-keys",
							Value: &sshPublicKeyValue,
						},
						{
							Key:   "startup-script",
							Value: &finalScripts,
						},
					},
				},
				Tags: &compute.Tags{
					Items: tags,
				},
			},
		}

		_, err = Sdk.Service.InstanceTemplates.Insert(immutable.ProviderConfig().CloudId, templateInstance).Do()
		if err != nil {
			return nil, nil, err
		}
	}

	_, err = Sdk.Service.InstanceGroupManagers.Get(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, strings.ToLower(expected.(*InstanceGroup).Name)).Do()
	if err != nil {
		instanceGroupManager := &compute.InstanceGroupManager{
			Name:             templateInstance.Name,
			BaseInstanceName: templateInstance.Name,
			InstanceTemplate: prefix + "/global/instanceTemplates/" + templateInstance.Name,
			TargetSize:       int64(expected.(*InstanceGroup).Count),
		}

		for i := 0; i < MasterIPAttempts; i++ {
			logger.Debug("Creating instance group manager")
			_, err = Sdk.Service.InstanceGroupManagers.Insert(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, instanceGroupManager).Do()
			if err == nil {
				break
			}

			logger.Debug("Waiting for instance template to be ready.")
			time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
		}

		logger.Success("Created instance group manager [%s]", templateInstance.Name)
	}

	newResource := &InstanceGroup{
		Shared: Shared{
			Name: r.ServerPool.Name,
			//CloudID: id,
		},
		Image:            expected.(*InstanceGroup).Image,
		Size:             expected.(*InstanceGroup).Size,
		Location:         expected.(*InstanceGroup).Location,
		Count:            expected.(*InstanceGroup).Count,
		BootstrapScripts: expected.(*InstanceGroup).BootstrapScripts,
	}

	providerConfig = immutable.ProviderConfig()
	providerConfig.KubernetesAPI.Endpoint = masterIPPublic
	immutable.SetProviderConfig(providerConfig)

	renderedCluster, err := r.immutableRender(newResource, immutable)
	if err != nil {
		return nil, nil, err
	}
	return renderedCluster, newResource, nil
}
func (r *InstanceGroup) Apply(actual, expected cloud.Resource, immutable *cluster.Cluster) (*cluster.Cluster, cloud.Resource, error) {
	logger.Debug("instanceGroup.Apply")
	applyResource := expected.(*InstanceGroup)
	isEqual, err := compare.IsEqual(actual.(*InstanceGroup), expected.(*InstanceGroup))
	if err != nil {
		return nil, nil, err
	}
	if isEqual {
		return immutable, applyResource, nil
	}

	masterIPPrivate := ""
	masterIPPublic := ""
	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		found := false
		for i := 0; i < MasterIPAttempts; i++ {
			masterTag := ""
			machineConfigs := immutable.MachineProviderConfigs()
			for _, machineConfig := range machineConfigs {
				serverPool := machineConfig.ServerPool
				if serverPool.Type == cluster.ServerPoolTypeMaster {
					masterTag = serverPool.Name
				}
			}
			if masterTag == "" {
				return nil, nil, fmt.Errorf("Unable to find master tag")
			}

			instanceGroupManager, err := Sdk.Service.InstanceGroupManagers.ListManagedInstances(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, strings.ToLower(masterTag)).Do()
			if err != nil {
				return nil, nil, err
			}

			if err != nil || len(instanceGroupManager.ManagedInstances) == 0 {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			parts := strings.Split(instanceGroupManager.ManagedInstances[0].Instance, "/")
			instance, err := Sdk.Service.Instances.Get(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, parts[len(parts)-1]).Do()
			if err != nil {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			for _, networkInterface := range instance.NetworkInterfaces {
				if networkInterface.Name == "nic0" {
					masterIPPrivate = networkInterface.NetworkIP
					for _, accessConfigs := range networkInterface.AccessConfigs {
						masterIPPublic = accessConfigs.NatIP
					}
				}
			}

			if masterIPPublic == "" {
				logger.Debug("Hanging for master IP..")
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}

			found = true
			providerConfig := immutable.ProviderConfig()
			providerConfig.Values.ItemMap["INJECTEDMASTER"] = fmt.Sprintf("%s:%s", masterIPPrivate, immutable.ProviderConfig().KubernetesAPI.Port)
			immutable.SetProviderConfig(providerConfig)
			break
		}
		if !found {
			return nil, nil, fmt.Errorf("Unable to find Master IP after defined wait")
		}
	}

	providerConfig := immutable.ProviderConfig()
	providerConfig.Values.ItemMap["INJECTEDPORT"] = immutable.ProviderConfig().KubernetesAPI.Port
	immutable.SetProviderConfig(providerConfig)

	scripts, err := script.BuildBootstrapScript(r.ServerPool.BootstrapScripts, immutable)
	if err != nil {
		return nil, nil, err
	}

	finalScripts := string(scripts)
	if err != nil {
		return nil, nil, err
	}

	tags := []string{}
	if r.ServerPool.Type == cluster.ServerPoolTypeMaster {
		if immutable.ProviderConfig().KubernetesAPI.Port == "443" {
			tags = append(tags, "https-server")
		}

		if immutable.ProviderConfig().KubernetesAPI.Port == "80" {
			tags = append(tags, "http-server")
		}

		tags = append(tags, "kubicorn-master")
	}

	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		tags = append(tags, "kubicorn-node")
	}

	prefix := "https://www.googleapis.com/compute/v1/projects/" + immutable.ProviderConfig().CloudId
	imageURL := "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/" + expected.(*InstanceGroup).Image

	templateInstance, err := Sdk.Service.InstanceTemplates.Get(immutable.ProviderConfig().CloudId, strings.ToLower(expected.(*InstanceGroup).Name)).Do()
	if err != nil {
		sshPublicKeyValue := fmt.Sprintf("%s:%s", immutable.ProviderConfig().SSH.User, string(immutable.ProviderConfig().SSH.PublicKeyData))

		templateInstance = &compute.InstanceTemplate{
			Name: strings.ToLower(expected.(*InstanceGroup).Name),
			Properties: &compute.InstanceProperties{
				MachineType: expected.(*InstanceGroup).Size,
				Disks: []*compute.AttachedDisk{
					{
						AutoDelete: true,
						Boot:       true,
						Type:       "PERSISTENT",
						InitializeParams: &compute.AttachedDiskInitializeParams{
							SourceImage: imageURL,
						},
					},
				},
				NetworkInterfaces: []*compute.NetworkInterface{
					{
						AccessConfigs: []*compute.AccessConfig{
							{
								Type: "ONE_TO_ONE_NAT",
								Name: "External NAT",
							},
						},
						Network: prefix + "/global/networks/default",
					},
				},
				ServiceAccounts: []*compute.ServiceAccount{
					{
						Email: "default",
						Scopes: []string{
							compute.DevstorageFullControlScope,
							compute.ComputeScope,
						},
					},
				},
				Metadata: &compute.Metadata{
					Kind: "compute#metadata",
					Items: []*compute.MetadataItems{
						{
							Key:   "ssh-keys",
							Value: &sshPublicKeyValue,
						},
						{
							Key:   "startup-script",
							Value: &finalScripts,
						},
					},
				},
				Tags: &compute.Tags{
					Items: tags,
				},
			},
		}

		_, err = Sdk.Service.InstanceTemplates.Insert(immutable.ProviderConfig().CloudId, templateInstance).Do()
		if err != nil {
			return nil, nil, err
		}
	}

	_, err = Sdk.Service.InstanceGroupManagers.Get(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, strings.ToLower(expected.(*InstanceGroup).Name)).Do()
	if err != nil {
		instanceGroupManager := &compute.InstanceGroupManager{
			Name:             templateInstance.Name,
			BaseInstanceName: templateInstance.Name,
			InstanceTemplate: prefix + "/global/instanceTemplates/" + templateInstance.Name,
			TargetSize:       int64(expected.(*InstanceGroup).Count),
		}

		for i := 0; i < MasterIPAttempts; i++ {
			logger.Debug("Creating instance group manager")
			_, err = Sdk.Service.InstanceGroupManagers.Insert(immutable.ProviderConfig().CloudId, expected.(*InstanceGroup).Location, instanceGroupManager).Do()
			if err == nil {
				break
			}

			logger.Debug("Waiting for instance template to be ready.")
			time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
		}

		logger.Success("Created instance group manager [%s]", templateInstance.Name)
	}

	newResource := &InstanceGroup{
		Shared: Shared{
			Name: r.ServerPool.Name,
			//CloudID: id,
		},
		Image:            expected.(*InstanceGroup).Image,
		Size:             expected.(*InstanceGroup).Size,
		Location:         expected.(*InstanceGroup).Location,
		Count:            expected.(*InstanceGroup).Count,
		BootstrapScripts: expected.(*InstanceGroup).BootstrapScripts,
	}

	providerConfig = immutable.ProviderConfig()
	providerConfig.KubernetesAPI.Endpoint = masterIPPublic
	immutable.SetProviderConfig(providerConfig)

	renderedCluster, err := r.immutableRender(newResource, immutable)
	if err != nil {
		return nil, nil, err
	}
	return renderedCluster, newResource, nil
}
// Apply is used to create the expected resources on the cloud provider.