Merge pull request #917 from allmightyspiff/issues588

Issues588 - hardware wait for ready

Author: allmightyspiff

SoftLayer/CLI/__init__.py

@@ -6,5 +6,10 @@
     :license: MIT, see LICENSE for more details.
 """
 # pylint: disable=w0401, invalid-name
+import logging
 
 from SoftLayer.CLI.helpers import *  # NOQA
+
+logger = logging.getLogger()
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.INFO)

SoftLayer/CLI/hardware/ready.py

@@ -0,0 +1,25 @@
+"""Check if a virtual server is ready."""
+# :license: MIT, see LICENSE for more details.
+
+import click
+
+import SoftLayer
+from SoftLayer.CLI import environment
+from SoftLayer.CLI import exceptions
+from SoftLayer.CLI import helpers
+
+
+@click.command()
+@click.argument('identifier')
+@click.option('--wait', default=0, show_default=True, type=click.INT, help="Seconds to wait")
+@environment.pass_env
+def cli(env, identifier, wait):
+    """Check if a virtual server is ready."""
+
+    compute = SoftLayer.HardwareManager(env.client)
+    compute_id = helpers.resolve_id(compute.resolve_ids, identifier, 'hardware')
+    ready = compute.wait_for_ready(compute_id, wait)
+    if ready:
+        env.fout("READY")
+    else:
+        raise exceptions.CLIAbort("Instance %s not ready" % compute_id)

SoftLayer/CLI/routes.py

@@ -230,6 +230,7 @@
     ('hardware:credentials', 'SoftLayer.CLI.hardware.credentials:cli'),
     ('hardware:update-firmware', 'SoftLayer.CLI.hardware.update_firmware:cli'),
     ('hardware:rescue', 'SoftLayer.CLI.hardware.power:rescue'),
+    ('hardware:ready', 'SoftLayer.CLI.hardware.ready:cli'),
 
     ('securitygroup', 'SoftLayer.CLI.securitygroup'),
     ('securitygroup:list', 'SoftLayer.CLI.securitygroup.list:cli'),

SoftLayer/CLI/virt/ready.py

@@ -11,11 +11,7 @@
 
 @click.command()
 @click.argument('identifier')
-@click.option('--wait',
-              default=0,
-              show_default=True,
-              type=click.INT,
-              help="Name of the image")
+@click.option('--wait', default=0, show_default=True, type=click.INT, help="Seconds to wait")
 @environment.pass_env
 def cli(env, identifier, wait):
     """Check if a virtual server is ready."""

SoftLayer/decoration.py

@@ -9,8 +9,17 @@
 from random import randint
 from time import sleep
 
+from SoftLayer import exceptions
 
-def retry(ex, tries=4, delay=5, backoff=2, logger=None):
+RETRIABLE = (
+    exceptions.ServerError,
+    exceptions.ApplicationError,
+    exceptions.RemoteSystemError,
+    exceptions.TransportError
+)
+
+
+def retry(ex=RETRIABLE, tries=4, delay=5, backoff=2, logger=None):
     """Retry calling the decorated function using an exponential backoff.
 
     http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/

SoftLayer/exceptions.py

@@ -27,14 +27,10 @@ def __init__(self, fault_code, fault_string, *args):
         self.reason = self.faultString = fault_string
 
     def __repr__(self):
-        return '<%s(%s): %s>' % (self.__class__.__name__,
-                                 self.faultCode,
-                                 self.faultString)
+        return '<%s(%s): %s>' % (self.__class__.__name__, self.faultCode, self.faultString)
 
     def __str__(self):
-        return '%s(%s): %s' % (self.__class__.__name__,
-                               self.faultCode,
-                               self.faultString)
+        return '%s(%s): %s' % (self.__class__.__name__, self.faultCode, self.faultString)
 
 
 class ParseError(SoftLayerAPIError):
@@ -78,12 +74,12 @@ class SpecViolation(ServerError):
     pass
 
 
-class MethodNotFound(ServerError):
+class MethodNotFound(SoftLayerAPIError):
     """Method name not found."""
     pass
 
 
-class InvalidMethodParameters(ServerError):
+class InvalidMethodParameters(SoftLayerAPIError):
     """Invalid method paramters."""
     pass
 

SoftLayer/managers/hardware.py

@@ -7,10 +7,10 @@
 """
 import logging
 import socket
+import time
 
 import SoftLayer
 from SoftLayer.decoration import retry
-from SoftLayer import exceptions
 from SoftLayer.managers import ordering
 from SoftLayer import utils
 
@@ -88,7 +88,7 @@ def cancel_hardware(self, hardware_id, reason='unneeded', comment='',
                                 immediate, False, cancel_reason, comment,
                                 id=billing_id)
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def list_hardware(self, tags=None, cpus=None, memory=None, hostname=None,
                       domain=None, datacenter=None, nic_speed=None,
                       public_ip=None, private_ip=None, **kwargs):
@@ -176,7 +176,7 @@ def list_hardware(self, tags=None, cpus=None, memory=None, hostname=None,
         kwargs['filter'] = _filter.to_dict()
         return self.account.getHardware(**kwargs)
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def get_hardware(self, hardware_id, **kwargs):
         """Get details about a hardware device.
 
@@ -343,7 +343,7 @@ def get_cancellation_reasons(self):
             'moving': 'Moving to competitor',
         }
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def get_create_options(self):
         """Returns valid options for ordering hardware."""
 
@@ -404,7 +404,7 @@ def get_create_options(self):
             'extras': extras,
         }
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def _get_package(self):
         """Get the package related to simple hardware ordering."""
         mask = '''
@@ -584,8 +584,33 @@ def update_firmware(self,
         """
 
         return self.hardware.createFirmwareUpdateTransaction(
-            bool(ipmi), bool(raid_controller), bool(bios), bool(hard_drive),
-            id=hardware_id)
+            bool(ipmi), bool(raid_controller), bool(bios), bool(hard_drive), id=hardware_id)
+
+    def wait_for_ready(self, instance_id, limit=14400, delay=10, pending=False):
+        """Determine if a Server is ready.
+
+        A server is ready when no transactions are running on it.
+
+        :param int instance_id: The instance ID with the pending transaction
+        :param int limit: The maximum amount of seconds to wait.
+        :param int delay: The number of seconds to sleep before checks. Defaults to 10.
+        """
+        now = time.time()
+        until = now + limit
+        mask = "mask[id, lastOperatingSystemReload[id], activeTransaction, provisionDate]"
+        instance = self.get_hardware(instance_id, mask=mask)
+        while now <= until:
+            if utils.is_ready(instance, pending):
+                return True
+            transaction = utils.lookup(instance, 'activeTransaction', 'transactionStatus', 'friendlyName')
+            snooze = min(delay, until - now)
+            LOGGER.info("%s - %d not ready. Auto retry in %ds", transaction, instance_id, snooze)
+            time.sleep(snooze)
+            instance = self.get_hardware(instance_id, mask=mask)
+            now = time.time()
+
+        LOGGER.info("Waiting for %d expired.", instance_id)
+        return False
 
 
 def _get_extra_price_id(items, key_name, hourly, location):

SoftLayer/managers/vs.py

@@ -6,9 +6,7 @@
     :license: MIT, see LICENSE for more details.
 """
 import datetime
-import itertools
 import logging
-import random
 import socket
 import time
 import warnings
@@ -58,7 +56,7 @@ def __init__(self, client, ordering_manager=None):
         else:
             self.ordering_manager = ordering_manager
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def list_instances(self, hourly=True, monthly=True, tags=None, cpus=None,
                        memory=None, hostname=None, domain=None,
                        local_disk=None, datacenter=None, nic_speed=None,
@@ -162,7 +160,7 @@ def list_instances(self, hourly=True, monthly=True, tags=None, cpus=None,
         func = getattr(self.account, call)
         return func(**kwargs)
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def get_instance(self, instance_id, **kwargs):
         """Get details about a virtual server instance.
 
@@ -237,7 +235,7 @@ def get_instance(self, instance_id, **kwargs):
 
         return self.guest.getObject(id=instance_id, **kwargs)
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def get_create_options(self):
         """Retrieves the available options for creating a VS.
 
@@ -414,7 +412,7 @@ def _generate_create_dict(
 
         return data
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def wait_for_transaction(self, instance_id, limit, delay=10):
         """Waits on a VS transaction for the specified amount of time.
 
@@ -428,7 +426,7 @@ def wait_for_transaction(self, instance_id, limit, delay=10):
 
         return self.wait_for_ready(instance_id, limit, delay=delay, pending=True)
 
-    def wait_for_ready(self, instance_id, limit, delay=10, pending=False):
+    def wait_for_ready(self, instance_id, limit=3600, delay=10, pending=False):
         """Determine if a VS is ready and available.
 
         In some cases though, that can mean that no transactions are running.
@@ -439,7 +437,7 @@ def wait_for_ready(self, instance_id, limit, delay=10, pending=False):
         cancellations.
 
         :param int instance_id: The instance ID with the pending transaction
-        :param int limit: The maximum amount of time to wait.
+        :param int limit: The maximum amount of seconds to wait.
         :param int delay: The number of seconds to sleep before checks. Defaults to 10.
         :param bool pending: Wait for pending transactions not related to
                              provisioning or reloads such as monitoring.
@@ -449,43 +447,21 @@ def wait_for_ready(self, instance_id, limit, delay=10, pending=False):
             # Will return once vsi 12345 is ready, or after 10 checks
             ready = mgr.wait_for_ready(12345, 10)
         """
-        until = time.time() + limit
-        for new_instance in itertools.repeat(instance_id):
-            mask = """id,
-                      lastOperatingSystemReload.id,
-                      activeTransaction.id,provisionDate"""
-            try:
-                instance = self.get_instance(new_instance, mask=mask)
-                last_reload = utils.lookup(instance, 'lastOperatingSystemReload', 'id')
-                active_transaction = utils.lookup(instance, 'activeTransaction', 'id')
-
-                reloading = all((
-                    active_transaction,
-                    last_reload,
-                    last_reload == active_transaction,
-                ))
-
-                # only check for outstanding transactions if requested
-                outstanding = False
-                if pending:
-                    outstanding = active_transaction
-
-                # return True if the instance has finished provisioning
-                # and isn't currently reloading the OS.
-                if all([instance.get('provisionDate'),
-                        not reloading,
-                        not outstanding]):
-                    return True
-                LOGGER.info("%s not ready.", str(instance_id))
-            except exceptions.SoftLayerAPIError as exception:
-                delay = (delay * 2) + random.randint(0, 9)
-                LOGGER.info('Exception: %s', str(exception))
-
+        now = time.time()
+        until = now + limit
+        mask = "mask[id, lastOperatingSystemReload[id], activeTransaction, provisionDate]"
+
+        while now <= until:
+            instance = self.get_instance(instance_id, mask=mask)
+            if utils.is_ready(instance, pending):
+                return True
+            transaction = utils.lookup(instance, 'activeTransaction', 'transactionStatus', 'friendlyName')
+            snooze = min(delay, until - now)
+            LOGGER.info("%s - %d not ready. Auto retry in %ds", transaction, instance_id, snooze)
+            time.sleep(snooze)
             now = time.time()
-            if now >= until:
-                return False
-            LOGGER.info('Auto retry in %s seconds', str(min(delay, until - now)))
-            time.sleep(min(delay, until - now))
+
+        LOGGER.info("Waiting for %d expired.", instance_id)
         return False
 
     def verify_create_instance(self, **kwargs):
@@ -581,7 +557,7 @@ def create_instance(self, **kwargs):
             self.set_tags(tags, guest_id=inst['id'])
         return inst
 
-    @retry(exceptions.SoftLayerAPIError, logger=LOGGER)
+    @retry(logger=LOGGER)
     def set_tags(self, tags, guest_id):
         """Sets tags on a guest with a retry decorator
 

SoftLayer/transports.py

@@ -174,7 +174,7 @@ def __call__(self, request):
             verify = self.verify
 
         LOGGER.debug("=== REQUEST ===")
-        LOGGER.info('POST %s', url)
+        LOGGER.debug('POST %s', url)
         LOGGER.debug(request.transport_headers)
         LOGGER.debug(payload)
 
@@ -302,7 +302,7 @@ def __call__(self, request):
             verify = self.verify
 
         LOGGER.debug("=== REQUEST ===")
-        LOGGER.info(url)
+        LOGGER.debug(url)
         LOGGER.debug(request.transport_headers)
         LOGGER.debug(raw_body)
         try:

SoftLayer/utils.py

@@ -185,3 +185,27 @@ def tzname(self, _):
 
     def dst(self, _):
         return datetime.timedelta(0)
+
+
+def is_ready(instance, pending=False):
+    """Returns True if instance is ready to be used
+
+    :param Object instance: Hardware or Virt with transaction data retrieved from the API
+    :param bool pending: Wait for ALL transactions to finish?
+    :returns bool:
+    """
+
+    last_reload = lookup(instance, 'lastOperatingSystemReload', 'id')
+    active_transaction = lookup(instance, 'activeTransaction', 'id')
+
+    reloading = all((
+        active_transaction,
+        last_reload,
+        last_reload == active_transaction,
+    ))
+    outstanding = False
+    if pending:
+        outstanding = active_transaction
+    if instance.get('provisionDate') and not reloading and not outstanding:
+        return True
+    return False