KeyError when clearing Connector._cache

[iamlikeme]

Bug Description

KeyError raised at this line:
https://github.com/GoogleCloudPlatform/cloud-sql-python-connector/blob/21a38d602beb980d80cf4f414773c1cd2b3bdd7d/google/cloud/sql/connector/connector.py#L421C9-L421C79

Example code (or command)

import asyncio
import functools
from typing import Any

import asyncpg
import asyncpg.connection
import google.auth
import google.oauth2.service_account
from google.cloud.sql.connector import Connector, create_async_connector

from shared.settings import settings

_GCLOUD_CONNECTORS: dict[int, tuple[Connector, str | None]] = {}


async def _gcloud_async_creator() -> asyncpg.Connection[asyncpg.Record]:
    return await _gcloud_connect(
        dsn=settings.database.gcloud_dsn,
        user=settings.database.user,
        database=settings.database.name,
    )


@functools.wraps(asyncpg.connection.connect)
async def _gcloud_connect(
    dsn: str,
    *,
    database: str | None = None,
    user: str | None = None,
    **kwargs: Any,
) -> asyncpg.Connection[asyncpg.Record]:
    loop = asyncio.get_running_loop()
    loop_id = id(loop)
    if loop_id not in _GCLOUD_CONNECTORS:
        credentials, _ = google.auth.default()
        user_id = (
            credentials.service_account_email
            if isinstance(credentials, google.oauth2.service_account.Credentials)
            else None
        )
        _GCLOUD_CONNECTORS[loop_id] = (
            await create_async_connector(credentials=credentials),
            gcloud_db_user(user_id) if user_id else None,
        )

    connector, credential_user = _GCLOUD_CONNECTORS[loop_id]

    if credential_user is not None:
        user = credential_user

    return await connector.connect_async(
        dsn,
        "asyncpg",
        db=database,
        user=user,
        enable_iam_auth=True,
        **kwargs,
    )


def gcloud_db_user(user_id: str) -> str:
    """
    Return the Google Cloud DB user name for the given Google Cloud user.
    """
    return user_id.removesuffix(".gserviceaccount.com")

Stacktrace

KeyError: ('xyz:europe-west3:main', True)
  ...
  File "sqlalchemy/ext/asyncio/session.py", line 463, in execute
    result = await greenlet_spawn(
  File "sqlalchemy/util/_concurrency_py3k.py", line 201, in greenlet_spawn
    result = context.throw(*sys.exc_info())
  File "kron/db.py", line 193, in execute
    result = super().execute(statement, *args, **kwargs)
  File "sqlalchemy/orm/session.py", line 2365, in execute
    return self._execute_internal(
  File "sqlalchemy/orm/session.py", line 2241, in _execute_internal
    conn = self._connection_for_bind(bind)
  File "sqlalchemy/orm/session.py", line 2110, in _connection_for_bind
    return trans._connection_for_bind(engine, execution_options)
  File "<string>", line 2, in _connection_for_bind
  File "sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "sqlalchemy/orm/session.py", line 1189, in _connection_for_bind
    conn = bind.connect()
  File "sqlalchemy/engine/base.py", line 3273, in connect
    return self._connection_cls(self)
  File "sqlalchemy/engine/base.py", line 145, in __init__
    self._dbapi_connection = engine.raw_connection()
  File "sqlalchemy/engine/base.py", line 3297, in raw_connection
    return self.pool.connect()
  File "sqlalchemy/pool/base.py", line 449, in connect
    return _ConnectionFairy._checkout(self)
  File "sqlalchemy/pool/base.py", line 1264, in _checkout
    fairy = _ConnectionRecord.checkout(pool)
  File "sqlalchemy/pool/base.py", line 713, in checkout
    rec = pool._do_get()
  File "sqlalchemy/pool/impl.py", line 179, in _do_get
    with util.safe_reraise():
  File "sqlalchemy/util/langhelpers.py", line 224, in __exit__
    raise exc_value.with_traceback(exc_tb)
  File "sqlalchemy/pool/impl.py", line 177, in _do_get
    return self._create_connection()
  File "sqlalchemy/pool/base.py", line 390, in _create_connection
    return _ConnectionRecord(self)
  File "sqlalchemy/pool/base.py", line 675, in __init__
    self.__connect()
  File "sqlalchemy/pool/base.py", line 901, in __connect
    with util.safe_reraise():
  File "sqlalchemy/util/langhelpers.py", line 224, in __exit__
    raise exc_value.with_traceback(exc_tb)
  File "sqlalchemy/pool/base.py", line 897, in __connect
    self.dbapi_connection = connection = pool._invoke_creator(self)
  File "sqlalchemy/pool/base.py", line 362, in <lambda>
    return lambda rec: creator_fn()
  File "sqlalchemy/ext/asyncio/engine.py", line 115, in creator
    return sync_engine.dialect.dbapi.connect(  # type: ignore
  File "sqlalchemy/dialects/postgresql/asyncpg.py", line 961, in connect
    await_only(creator_fn(*arg, **kw)),
  File "sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
    return current.parent.switch(awaitable)  # type: ignore[no-any-return,attr-defined] # noqa: E501
  File "sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
    value = await result
  File "kron/db.py", line 435, in _gcloud_async_creator
    return await _gcloud_connect(
  File "kron/db.py", line 423, in _gcloud_connect
    return await connector.connect_async(
  File "/srv/venv/apiserver/kron-9TtSrW0h-py3.13/lib/python3.13/site-packages/google/cloud/sql/connector/connector.py", line 366, in connect_async
    await self._remove_cached(str(conn_name), enable_iam_auth)
  File "/srv/venv/apiserver/kron-9TtSrW0h-py3.13/lib/python3.13/site-packages/google/cloud/sql/connector/connector.py", line 421, in _remove_cached
    cache = self._cache.pop((instance_connection_string, enable_iam_auth))

Steps to reproduce?

I've observed just one occurrence of the issue so far and it seemed pretty random. Unfortunately I'm not able to reproduce.

Environment

1. OS type and version: Debian GNU/Linux 12 (bookworm)
2. Python version: 3.13.4
3. Cloud SQL Python Connector version: 1.18.2

Additional Details

No response

[kgala2]

Hi @iamlikeme,

Thanks for sharing the transient KeyError when clearing Connector._cache, I will keep this issue open to capture more instances of KeyError and also investigate it from my end