[BigTable] Improve resiliency with BigTable

Description

Issue Component(s) on Oct 19, 2023, 3:36:57 PM: BIG_QUERY | PROVISIONING

We need to make our connections to BigTable more resilient.

After a lot of requests to our endpoint of record_statistics, Google started to return:

google.api_core.exceptions.ServiceUnavailable: 503 Authentication backend unavailable

https://cloudlogging.app.goo.gl/JonFnW4vHtrgPzQ99

Full error log:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 165, in error_remapped_callable
    return _StreamingResponseIterator(
  File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 91, in __init__
    self._stored_first_result = next(self._wrapped)
  File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 541, in __next__
    return self._next()
  File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_grpc.py", line 97, in wrap_next
    return _wrapped(*_args, **_kwargs)
  File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 967, in _next
    raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
	status = StatusCode.UNAVAILABLE
	details = "Authentication backend unavailable."
	debug_error_string = "UNKNOWN:Error received from peer ipv4:172.253.119.95:443 {grpc_message:"Authentication backend unavailable.", grpc_status:14, created_time:"2023-10-19T15:00:55.624643338+00:00"}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/src/app/./carol_nosql/services/record_statistics_service.py", line 159, in get_record_statistics
    rows = self.table.read_rows(row_set=row_set)
  File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 637, in read_rows
    return PartialRowsData(data_client.read_rows, request_pb, retry)
  File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable/row_data.py", line 160, in __init__
    self.response_iterator = read_method(request, timeout=self.retry._deadline + 1)
  File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable_v2/services/bigtable/client.py", line 555, in read_rows
    response = rpc(
  File "/usr/local/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py", line 131, in __call__
    return wrapped_func(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/google/api_core/timeout.py", line 120, in func_with_timeout
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 169, in error_remapped_callable
    raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.ServiceUnavailable: 503 Authentication backend unavailable.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 2528, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1825, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python3.10/site-packages/flask_cors/extension.py", line 165, in wrapped_function
    return cors_after_request(app.make_response(f(*args, **kwargs)))
  File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_flask.py", line 130, in _nr_wrapper_Flask_handle_exception_
    return FunctionTraceWrapper(wrapped, name=name)(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/newrelic/api/function_trace.py", line 166, in literal_wrapper
    return wrapped(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1823, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1799, in dispatch_request
    return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
  File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_flask.py", line 64, in _nr_wrapper_handler_
    return wrapped(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/connexion/decorators/decorator.py", line 68, in wrapper
    response = function(request)
  File "/usr/local/lib/python3.10/site-packages/connexion/security/security_handler_factory.py", line 388, in wrapper
    return function(request)
  File "/usr/local/lib/python3.10/site-packages/connexion/decorators/uri_parsing.py", line 149, in wrapper
    response = function(request)
  File "/usr/local/lib/python3.10/site-packages/connexion/decorators/validation.py", line 399, in wrapper
    return function(request)
  File "/usr/local/lib/python3.10/site-packages/connexion/decorators/parameter.py", line 120, in wrapper
    return function(**kwargs)
  File "/usr/src/app/./carol_nosql/controllers/record_statistics_controller.py", line 33, in get_record_statistics
    record_statistics, total_rows = BigTableRecordStatistics().get_record_statistics(environment_id=environment_id,
  File "/usr/src/app/./carol_nosql/services/record_statistics_service.py", line 164, in get_record_statistics
    raise RuntimeError(msg)
RuntimeError: Error reading rows in BigTable range from 9250887694c945388043172301339cc3#STAGING#c2abc4647d0b429d84d808caf4ba1578_pfperff#DAILY#2023-10-12 to 9250887694c945388043172301339cc3#STAGING#c2abc4647d0b429d84d808caf4ba1578_pfperff#DAILY#2023-10-19: 503 Authentication backend unavailable."