'test_cluster_create' systest flakes with 'TimeoutError' #451

_____________________________ test_cluster_create ______________________________

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7f4d5ef43ee0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>
deadline = 30, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.

        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.

        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.

        Returns:
            Any: the return value of the target function.

        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None

        last_exc = None

        for sleep in sleep_generator:
            try:
>               return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

    def _done_or_raise(self, retry=DEFAULT_RETRY):
        """Check if the future is done and raise if it's not."""
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

        if not self.done(**kwargs):
>           raise _OperationNotComplete()
E           google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:89: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
timeout = 30, retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.

        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return

        retry_ = self._retry.with_deadline(timeout)

        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
>           retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>

    @functools.wraps(func)
    def retry_wrapped_func(*args, **kwargs):
        """A wrapper that calls target function with retry."""
        target = functools.partial(func, *args, **kwargs)
        sleep_generator = exponential_sleep_generator(
            self._initial, self._maximum, multiplier=self._multiplier
        )
>       return retry_target(
            target,
            self._predicate,
            sleep_generator,
            self._deadline,
            on_error=on_error,
        )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7f4d5ef43ee0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>
deadline = 30, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.

        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.

        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.

        Returns:
            Any: the return value of the target function.

        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None

        last_exc = None

        for sleep in sleep_generator:
            try:
                return target()

            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)

            now = datetime_helpers.utcnow()

            if deadline_datetime is not None:
                if deadline_datetime <= now:
>                   raise exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ) from last_exc
E                   google.api_core.exceptions.RetryError: Deadline of 30.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>), last exception:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:205: RetryError

During handling of the above exception, another exception occurred:

admin_instance_populated = <google.cloud.bigtable.instance.Instance object at 0x7f4d5eef10a0>
admin_instance_id = 'g-c-p-1634667466967', skip_on_emulator = None

    def test_cluster_create(
        admin_instance_populated, admin_instance_id, skip_on_emulator,
    ):
        alt_cluster_id = f"{admin_instance_id}-c2"
        alt_location_id = "us-central1-f"
        serve_nodes = 2

        cluster_2 = admin_instance_populated.cluster(
            alt_cluster_id,
            location_id=alt_location_id,
            serve_nodes=serve_nodes,
            default_storage_type=(enums.StorageType.SSD),
        )
        operation = cluster_2.create()
>       operation.result(timeout=30)  # Ensure the operation completes.

tests/system/test_instance_admin.py:576:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:132: in result
    self._blocking_poll(timeout=timeout, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
timeout = 30, retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.

        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return

        retry_ = self._retry.with_deadline(timeout)

        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
            retry_(self._done_or_raise)(**kwargs)
        except exceptions.RetryError:
>           raise concurrent.futures.TimeoutError(
                "Operation did not complete within the designated " "timeout."
            )
E           concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:112: TimeoutError