asyncpg.exceptions.ConnectionFailureError: connection failure during authentication

[marswong]

I'm running a FastAPI server connecting to Supabase through asyncpg on my VPS, while I got lots of below errors sometimes:

Traceback (most recent call last):
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
    return await self.app(scope, receive, send)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/applications.py", line 1134, in __call__
    await super().__call__(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/applications.py", line 113, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 186, in __call__
    raise exc
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 164, in __call__
    await self.app(scope, receive, _send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 93, in __call__
    await self.simple_response(scope, receive, send, request_headers=headers)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 144, in simple_response
    await self.app(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
    await self.app(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/routing.py", line 716, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/routing.py", line 736, in app
    await route.handle(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/routing.py", line 290, in handle
    await self.app(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 125, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 111, in app
    response = await f(request)
               ^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 391, in app
    raw_response = await run_endpoint_function(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 290, in run_endpoint_function
    return await dependant.call(**values)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/app/api/routes/chat.py", line 29, in chat_completion
    sbx = await sandbox_service.get_by_app_id(app_id)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/app/services/sandbox_service.py", line 303, in get_by_app_id
    result = await session.exec(select(SandboxModel).where(SandboxModel.app_id == app_id))
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlmodel/ext/asyncio/session.py", line 98, in exec
    result = await greenlet_spawn(
             ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 201, in greenlet_spawn
    result = context.throw(*sys.exc_info())
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlmodel/orm/session.py", line 83, in exec
    results = super().execute(
              ^^^^^^^^^^^^^^^^
File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/orm/state_changes.py", line 137, in _go                                                                                                                                                                                           [65/1901]
    ret_value = fn(self, *arg, **kw)
                ^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/orm/session.py", line 1187, in _connection_for_bind
    conn = bind.connect()
           ^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 3277, in connect
    return self._connection_cls(self)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 143, in __init__
    self._dbapi_connection = engine.raw_connection()
                             ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 3301, in raw_connection
    return self.pool.connect()
           ^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 447, in connect
    return _ConnectionFairy._checkout(self)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 1264, in _checkout
    fairy = _ConnectionRecord.checkout(pool)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 711, in checkout
    rec = pool._do_get()
          ^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/impl.py", line 306, in _do_get
    return self._create_connection()
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 388, in _create_connection
    return _ConnectionRecord(self)
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 673, in __init__
    self.__connect()
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 899, in __connect
    with util.safe_reraise():
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/util/langhelpers.py", line 224, in __exit__
    raise exc_value.with_traceback(exc_tb)
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/pool/base.py", line 895, in __connect
    self.dbapi_connection = connection = pool._invoke_creator(self)
                                         ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/engine/create.py", line 661, in connect
    return dialect.connect(*cargs, **cparams)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/engine/default.py", line 629, in connect
    return self.loaded_dbapi.connect(*cargs, **cparams)  # type: ignore[no-any-return]  # NOQA: E501
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 955, in connect
    await_only(creator_fn(*arg, **kw)),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
    return current.parent.switch(awaitable)  # type: ignore[no-any-return,attr-defined] # noqa: E501
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
    value = await result
            ^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2443, in connect
    return await connect_utils._connect(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1218, in _connect
    conn = await _connect_addr(
           ^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1054, in _connect_addr
    return await __connect_addr(params, True, *args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mars/Projects/x/backend/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1102, in __connect_addr
    await connected
asyncpg.exceptions.ConnectionFailureError: connection failure during authentication

Could someone tell me the reason for this error, is it due to unstable network or something else? Thanks!