[BigTable] Improve resiliency with BigTable
Description
Issue Component(s) on Oct 19, 2023, 3:36:57 PM: BIG_QUERY | PROVISIONING
We need to make our connections to BigTable more resilient.
After a lot of requests to our endpoint of record_statistics, Google started to return:
google.api_core.exceptions.ServiceUnavailable: 503 Authentication backend unavailable
https://cloudlogging.app.goo.gl/JonFnW4vHtrgPzQ99
Full error log:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 165, in error_remapped_callable
return _StreamingResponseIterator(
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 91, in __init__
self._stored_first_result = next(self._wrapped)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 541, in __next__
return self._next()
File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_grpc.py", line 97, in wrap_next
return _wrapped(*_args, **_kwargs)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 967, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Authentication backend unavailable."
debug_error_string = "UNKNOWN:Error received from peer ipv4:172.253.119.95:443 {grpc_message:"Authentication backend unavailable.", grpc_status:14, created_time:"2023-10-19T15:00:55.624643338+00:00"}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/src/app/./carol_nosql/services/record_statistics_service.py", line 159, in get_record_statistics
rows = self.table.read_rows(row_set=row_set)
File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 637, in read_rows
return PartialRowsData(data_client.read_rows, request_pb, retry)
File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable/row_data.py", line 160, in __init__
self.response_iterator = read_method(request, timeout=self.retry._deadline + 1)
File "/usr/local/lib/python3.10/site-packages/google/cloud/bigtable_v2/services/bigtable/client.py", line 555, in read_rows
response = rpc(
File "/usr/local/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py", line 131, in __call__
return wrapped_func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/timeout.py", line 120, in func_with_timeout
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 169, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.ServiceUnavailable: 503 Authentication backend unavailable.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 2528, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1825, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.10/site-packages/flask_cors/extension.py", line 165, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_flask.py", line 130, in _nr_wrapper_Flask_handle_exception_
return FunctionTraceWrapper(wrapped, name=name)(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/newrelic/api/function_trace.py", line 166, in literal_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1823, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.10/site-packages/flask/app.py", line 1799, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/framework_flask.py", line 64, in _nr_wrapper_handler_
return wrapped(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/connexion/decorators/decorator.py", line 68, in wrapper
response = function(request)
File "/usr/local/lib/python3.10/site-packages/connexion/security/security_handler_factory.py", line 388, in wrapper
return function(request)
File "/usr/local/lib/python3.10/site-packages/connexion/decorators/uri_parsing.py", line 149, in wrapper
response = function(request)
File "/usr/local/lib/python3.10/site-packages/connexion/decorators/validation.py", line 399, in wrapper
return function(request)
File "/usr/local/lib/python3.10/site-packages/connexion/decorators/parameter.py", line 120, in wrapper
return function(**kwargs)
File "/usr/src/app/./carol_nosql/controllers/record_statistics_controller.py", line 33, in get_record_statistics
record_statistics, total_rows = BigTableRecordStatistics().get_record_statistics(environment_id=environment_id,
File "/usr/src/app/./carol_nosql/services/record_statistics_service.py", line 164, in get_record_statistics
raise RuntimeError(msg)
RuntimeError: Error reading rows in BigTable range from 9250887694c945388043172301339cc3#STAGING#c2abc4647d0b429d84d808caf4ba1578_pfperff#DAILY#2023-10-12 to 9250887694c945388043172301339cc3#STAGING#c2abc4647d0b429d84d808caf4ba1578_pfperff#DAILY#2023-10-19: 503 Authentication backend unavailable."