/track end point is erroring out due to cache timeout.

[Pradeepallanki]

Hi,

When we use /track endpoint to collect the events, events are getting errored out due to a cache timeout.

Event source is already been created and details are loading fine in UI.

curl command:

curl -X 'POST'
'http://localhost:8686/track?profile_less=false'
-H 'accept: application/json'
-H 'Content-Type: application/json'
-d '{
"source": {
"id": "56d9b9b3-fa52-4c2a-8fce-776fa28a576c"
},
"session": {
"id": "session-id"
},
"profile": {

"id": "id123"
},
"events": [
{
"type": "purchase-order",
"properties": {
"product": "Nike shoes",
"quantity": 1
}
},
{
"type": "page-view"
}
]
} '

Stack trace:

2025-04-07 05:21:40,687: WARNING: TIMEOUT for cache load_event_source_via_cache in 1.8739: (tracardi.service.decorators.async_cache:async_cache.py:71)
2025-04-07 05:22:04,525: WARNING: TIMEOUT for cache load_event_to_profile in 4.5198: (tracardi.service.decorators.async_cache:async_cache.py:71)
2025-04-07 05:22:09,269: ERROR: Error when processing source=EventSource(id='56d9b9b3-fa52-4c2a-8fce-776fa28a576c', name='smartbuy', production=False, running=False, type=['rest'], bridge=NamedEntity(id='778ded05-4ff3-4e08-9a86-72c0195fa95d', name='REST API Bridge'), timestamp=datetime.datetime(2025, 4, 7, 5, 18, 32), description='No description provided', channel='', enabled=True, transitional=False, tags=['rest', 'api'], groups=[], permanent_profile_id=False, requires_consent=False, manual=None, locked=False, synchronize_profiles=True, config={'restrict_to': 'none', 'restriction': '', 'static_profile_id': True}) session=DefaultEntity(id='session-id', metadata=None) metadata=EventPayloadMetadata(time=Time(insert=datetime.datetime(2025, 4, 7, 5, 21, 38, 87642, tzinfo=zoneinfo.ZoneInfo(key='UTC')), create=datetime.datetime(2025, 4, 7, 5, 21, 38, 87642, tzinfo=zoneinfo.ZoneInfo(key='UTC')), update=None), ip=None, status=None) profile=PrimaryEntity(id='id123', primary_id=None, metadata=None, ids=None) app=None device=None context={} properties={} request={'headers': {'host': 'localhost:8686', 'connection': 'keep-alive', 'content-length': '334', 'sec-ch-ua-platform': '"macOS"', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', 'accept': 'application/json', 'sec-ch-ua': '"Chromium";v="134", "Not:A-Brand";v="24", "Google Chrome";v="134"', 'content-type': 'application/json', 'sec-ch-ua-mobile': '?0', 'origin': 'http://localhost:8686', 'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'cors', 'sec-fetch-dest': 'empty', 'referer': 'http://localhost:8686/docs', 'accept-encoding': 'gzip, deflate, br, zstd', 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'}} events=[EventPayload(id='fdfeb72b-ca00-4478-a948-bbb658141128', time=Time(insert=datetime.datetime(2025, 4, 7, 5, 21, 38, 324161, tzinfo=zoneinfo.ZoneInfo(key='UTC')), create=datetime.datetime(2025, 4, 7, 5, 21, 38, 324161, tzinfo=zoneinfo.ZoneInfo(key='UTC')), update=None), type='purchase-order', properties={'product': 'Nike shoes', 'quantity': 1}, options={}, context={}, tags=[], validation=None, reshaping=None, error=None), EventPayload(id='b524b50b-19d1-4e15-b090-6e57710617e9', time=Time(insert=datetime.datetime(2025, 4, 7, 5, 21, 38, 409082, tzinfo=zoneinfo.ZoneInfo(key='UTC')), create=datetime.datetime(2025, 4, 7, 5, 21, 38, 409082, tzinfo=zoneinfo.ZoneInfo(key='UTC')), update=None), type='page-view', properties={}, options={}, context={}, tags=[], validation=None, reshaping=None, error=None)] options={} profile_less=False debug=False. Details: : (app.api.track.event_server_endpoint:log_handler.py:23)
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/tracardi/service/decorators/async_cache.py", line 48, in _run
return await func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/tracardi/service/cache/event_to_profile_mapping.py", line 15, in load_event_to_profile
mappings, total = await event_to_profile_dao.load_event_to_profile_mapping_by_type(event_type_id, enabled_only=True)
File "/usr/local/lib/python3.10/site-packages/tracardi/service/storage/mysql/interface/event_to_profile_mapping.py", line 38, in load_event_to_profile_mapping_by_type
records = await etpms.load_by_type(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/storage/mysql/service/event_to_profile_service.py", line 40, in load_by_type
return await self._select_in_deployment_mode(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/storage/mysql/service/table_service.py", line 52, in _select_in_deployment_mode
result = await resource.select(table,
File "/usr/local/lib/python3.10/site-packages/tracardi/service/storage/mysql/query_service.py", line 117, in select
return MySqlQueryResult(await self.session.execute(query))
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/ext/asyncio/session.py", line 455, in execute
result = await greenlet_spawn(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 190, in greenlet_spawn
result = context.throw(*sys.exc_info())
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 2308, in execute
return self._execute_internal(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 2190, in _execute_internal
result: Result[Any] = compile_state_cls.orm_execute_statement(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/context.py", line 293, in orm_execute_statement
result = conn.execute(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1416, in execute
return meth(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py", line 516, in _execute_on_connection
return connection._execute_clauseelement(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1639, in _execute_clauseelement
ret = self._execute_context(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1848, in _execute_context
return self._exec_single_context(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1988, in _exec_single_context
self._handle_dbapi_exception(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2346, in _handle_dbapi_exception
raise exc_info[1].with_traceback(exc_info[2])
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1969, in exec_single_context
self.dialect.do_execute(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 922, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/aiomysql.py", line 93, in execute
return self.await(self._execute_async(operation, parameters))
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 125, in await_only
return current.driver.switch(awaitable) # type: ignore[no-any-return]
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 185, in greenlet_spawn
value = await result
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/aiomysql.py", line 102, in _execute_async
result = await self._cursor.execute(operation, parameters)
File "/usr/local/lib/python3.10/site-packages/aiomysql/cursors.py", line 239, in execute
await self._query(query)
File "/usr/local/lib/python3.10/site-packages/aiomysql/cursors.py", line 457, in _query
await conn.query(q)
File "/usr/local/lib/python3.10/site-packages/aiomysql/connection.py", line 469, in query
await self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python3.10/site-packages/aiomysql/connection.py", line 672, in _read_query_result
await result.read()
File "/usr/local/lib/python3.10/site-packages/aiomysql/connection.py", line 1153, in read
first_packet = await self.connection._read_packet()
File "/usr/local/lib/python3.10/site-packages/aiomysql/connection.py", line 598, in _read_packet
packet_header = await self._read_bytes(4)
File "/usr/local/lib/python3.10/site-packages/aiomysql/connection.py", line 646, in _read_bytes
data = await self._reader.readexactly(num_bytes)
File "/usr/local/lib/python3.10/asyncio/streams.py", line 708, in readexactly
await self._wait_for_data('readexactly')
File "/usr/local/lib/python3.10/asyncio/streams.py", line 501, in _wait_for_data
await self._waiter
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/app/app/api/track/event_server_endpoint.py", line 54, in _track
return await track_event(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/track_event.py", line 26, in track_event
return await tr.track_event(tracker_payload, tracking_start)
File "/usr/local/lib/python3.10/site-packages/tracardi/service/tracker.py", line 100, in track_event
return await os_tracker(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/tracking/tracker.py", line 52, in os_tracker
flat_profile, session, flat_events, tracker_payload = await compute_data(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/tracking/track_data_computation.py", line 90, in compute_data
flat_profile, session, flat_events, tracker_payload = await _compute(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/tracking/track_data_computation.py", line 64, in _compute
flat_events, session, flat_profile = await compute_events(
File "/usr/local/lib/python3.10/site-packages/tracardi/service/tracking/event_data_computation.py", line 245, in compute_events
custom_event_to_profile_mapping_schemas = await load_event_to_profile(event_type_id=flat_event['type'])
File "/usr/local/lib/python3.10/site-packages/tracardi/service/decorators/async_cache.py", line 124, in wrapper
return await self._run_and_fill_cache(key, func, args, kwargs)
File "/usr/local/lib/python3.10/site-packages/tracardi/service/decorators/async_cache.py", line 89, in _run_and_fill_cache
result = await self._run_function(key, func, args, kwargs, timeout)
File "/usr/local/lib/python3.10/site-packages/tracardi/service/decorators/async_cache.py", line 75, in _run_function
raise e
File "/usr/local/lib/python3.10/site-packages/tracardi/service/decorators/async_cache.py", line 66, in _run_function
result = await asyncio.wait_for(
File "/usr/local/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError
Stack (most recent call last):
File "/usr/local/bin/uvicorn", line 8, in
sys.exit(main())
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1130, in call
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/uvicorn/main.py", line 412, in main
run(
File "/usr/local/lib/python3.10/site-packages/uvicorn/main.py", line 579, in run
server.run()
File "/usr/local/lib/python3.10/site-packages/uvicorn/server.py", line 65, in run
return asyncio.run(self.serve(sockets=sockets))
File "/usr/local/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 149, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/usr/local/lib/python3.10/site-packages/starlette/middleware/cors.py", line 93, in call
await self.simple_response(scope, receive, send, request_headers=headers)
File "/usr/local/lib/python3.10/site-packages/starlette/middleware/cors.py", line 144, in simple_response
await self.app(scope, receive, send)
File "/app/app/middleware/context.py", line 72, in call
await self.app(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in call
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 715, in call
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 735, in app
await route.handle(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/usr/local/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
return await dependant.call(**values)
File "/app/app/api/track/event_server_endpoint.py", line 95, in track
result = await _track(tracker_payload,
File "/app/app/api/track/event_server_endpoint.py", line 82, in _track
logger.error(f"Error when processing {tracker_payload}. Details: {message}")
File "/usr/local/lib/python3.10/site-packages/tracardi/exceptions/log_handler.py", line 23, in error
super().error(msg, *args, **kwargs)