diff --git a/python/instrumentation/openinference-instrumentation-agno/pyproject.toml b/python/instrumentation/openinference-instrumentation-agno/pyproject.toml index ad326371f..ecbcf766b 100644 --- a/python/instrumentation/openinference-instrumentation-agno/pyproject.toml +++ b/python/instrumentation/openinference-instrumentation-agno/pyproject.toml @@ -42,7 +42,7 @@ test = [ "opentelemetry-sdk", "pytest-recording", "openai", - "ddgs", + "duckduckgo-search", "yfinance", ] diff --git a/python/instrumentation/openinference-instrumentation-beeai/pyproject.toml b/python/instrumentation/openinference-instrumentation-beeai/pyproject.toml index 65de79e30..554e1a3c9 100644 --- a/python/instrumentation/openinference-instrumentation-beeai/pyproject.toml +++ b/python/instrumentation/openinference-instrumentation-beeai/pyproject.toml @@ -38,7 +38,11 @@ instruments = [ test = [ "beeai-framework >= 0.1.36", "opentelemetry-sdk", - "opentelemetry-exporter-otlp" + "opentelemetry-exporter-otlp", + "pytest", + "pytest-vcr", + "pytest-asyncio", + "vcrpy" ] [project.entry-points.opentelemetry_instrumentor] diff --git a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/base.py b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/base.py index cc2acd97f..fef565806 100644 --- a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/base.py +++ b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/base.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Optional if TYPE_CHECKING: from beeai_framework.context import RunContextFinishEvent, RunContextStartEvent @@ -18,7 +18,9 @@ class Processor: kind: ClassVar[OpenInferenceSpanKindValues] = OpenInferenceSpanKindValues.UNKNOWN - def __init__(self, event: "RunContextStartEvent", meta: "EventMeta"): + def __init__( + self, event: "RunContextStartEvent", meta: "EventMeta", span_name: Optional[str] = None + ): from beeai_framework.context import RunContext assert isinstance(meta.creator, RunContext) @@ -27,7 +29,7 @@ def __init__(self, event: "RunContextStartEvent", meta: "EventMeta"): assert meta.trace is not None self.run_id = meta.trace.run_id - self.span = SpanWrapper(name=target_cls.__name__, kind=type(self).kind) + self.span = SpanWrapper(name=span_name or target_cls.__name__, kind=type(self).kind) self.span.started_at = meta.created_at self.span.attributes.update( { diff --git a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/embedding.py b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/embedding.py index 6214fe50e..b59e66aba 100644 --- a/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/embedding.py +++ b/python/instrumentation/openinference-instrumentation-beeai/src/openinference/instrumentation/beeai/processors/embedding.py @@ -12,6 +12,7 @@ from beeai_framework.context import RunContext from typing_extensions import override +from openinference.instrumentation import safe_json_dumps from openinference.instrumentation.beeai.processors.base import Processor from openinference.semconv.trace import ( EmbeddingAttributes, @@ -19,12 +20,15 @@ SpanAttributes, ) +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +_EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" + class EmbeddingModelProcessor(Processor): kind: ClassVar[OpenInferenceSpanKindValues] = OpenInferenceSpanKindValues.EMBEDDING def __init__(self, event: "RunContextStartEvent", meta: "EventMeta"): - super().__init__(event, meta) + super().__init__(event, meta, span_name="CreateEmbeddings") assert isinstance(meta.creator, RunContext) assert isinstance(meta.creator.instance, EmbeddingModel) @@ -34,6 +38,7 @@ def __init__(self, event: "RunContextStartEvent", meta: "EventMeta"): { SpanAttributes.EMBEDDING_MODEL_NAME: llm.model_id, SpanAttributes.LLM_PROVIDER: llm.provider_id, + SpanAttributes.LLM_SYSTEM: "beeai", } ) @@ -45,10 +50,22 @@ async def update( ) -> None: await super().update(event, meta) + # Add event to the span but don't create child spans self.span.add_event(f"{meta.name} ({meta.path})", timestamp=meta.created_at) - self.span.child(meta.name, event=(event, meta)) if isinstance(event, EmbeddingModelStartEvent): + # Extract invocation parameters + invocation_params = {} + if hasattr(event.input, "__dict__"): + input_dict = vars(event.input) + # Remove the actual text values from invocation parameters + invocation_params = {k: v for k, v in input_dict.items() if k != "values"} + if invocation_params: + self.span.set_attribute( + _EMBEDDING_INVOCATION_PARAMETERS, + safe_json_dumps(invocation_params), + ) + for idx, txt in enumerate(event.input.values): self.span.set_attribute( f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{idx}.{EmbeddingAttributes.EMBEDDING_TEXT}", @@ -56,9 +73,12 @@ async def update( ) elif isinstance(event, EmbeddingModelSuccessEvent): for idx, embedding in enumerate(event.value.embeddings): + # Ensure the embedding vector is a list, not a tuple + # Always convert to list to handle tuples from BeeAI framework + vector = list(embedding) self.span.set_attribute( f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{idx}.{EmbeddingAttributes.EMBEDDING_VECTOR}", - embedding, + vector, ) if event.value.usage: diff --git a/python/instrumentation/openinference-instrumentation-beeai/tests/README.md b/python/instrumentation/openinference-instrumentation-beeai/tests/README.md new file mode 100644 index 000000000..a3573f59b --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-beeai/tests/README.md @@ -0,0 +1,27 @@ +# BeeAI Instrumentation Tests + +## Re-recording VCR Cassettes + +When tests fail due to outdated VCR cassettes (e.g., API authentication errors or changed responses), follow these steps to re-record: + +### Prerequisites +1. Ensure `OPENAI_API_KEY` is set in your environment with a valid API key +2. The `passenv = OPENAI_API_KEY` directive must be present in the root `tox.ini` file + +### Steps to Re-record + +1. Delete the existing cassette file: +```bash +rm tests/cassettes/test_openai_embeddings.yaml +``` + +2. Run the tests with VCR in record mode using tox: +```bash +OPENAI_API_KEY=$OPENAI_API_KEY uvx --with tox-uv tox -r -e py313-ci-beeai -- tests/test_instrumentor.py::test_openai_embeddings -xvs --vcr-record=once +``` + +### Important Notes +- The test reads `OPENAI_API_KEY` from the environment, falling back to "sk-test" if not set +- VCR will cache responses including authentication errors (401), so always delete the cassette before re-recording +- The `--vcr-record=once` flag ensures the cassette is only recorded when it doesn't exist +- Use `-r` flag with tox to ensure a clean environment when re-recording diff --git a/python/instrumentation/openinference-instrumentation-beeai/tests/cassettes/test_openai_embeddings.yaml b/python/instrumentation/openinference-instrumentation-beeai/tests/cassettes/test_openai_embeddings.yaml new file mode 100644 index 000000000..427a7f974 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-beeai/tests/cassettes/test_openai_embeddings.yaml @@ -0,0 +1,31 @@ +interactions: +- request: + body: '{"input":["Hello world","Test embedding"],"model":"text-embedding-3-small","encoding_format":"base64"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": + \"embedding\",\n \"index\": 0,\n \"embedding\": \"KDgIu0UOSb2amKs872kAPQ+SOb0FSti8yJDtvPsBdz2zn9K8atpyvP4MfTx1n/W8WBCnvNiYCL2JodM8EchoPHWFj70Ws0s8f1lyPG8MSD1k1ak8W6kRvCGjd7ztF4i86WLUPIw6PrsPS8e8X+rGPESA5DrPP2S9exi9PHBTOr0/wg28gjlPO50xljtsLOs6Ya7aPDNvJjzZh0W8a/Y7vCpddLwreb28bbrPPMLSFj3maBG9az2uPFArgb1dbSU9hk9bPSgLfD0j2wm9di3au33c0DwomeA9k3eZu3ymIb38SOk7YwBTPeCo17wKYOQ8HEb5PFSzqDxUQY08y3BKPO8ijjphZ+g7mNQXvSK/wDxWvi68zlAnPZS+C7sNegA9y/4uvW1z3buCgEG9cqWyvNAUO73WjYI69bW7OB1iQr3d9Qa9h92/PHuKWL16Q2a9r9C4vEHNkzzhCxO9k3eZOj6mxLsXiKK8FO83POsMgjr9q6S8r9A4PbR0KT2957M7iBPvuzFkID2gnXQ97yIOvH+g5Dyz5sS89SfXPDZPg7yrSBE8/B1APFjJNLlB+Dy9EVbNvIiyljsFA+a9ssp7vHHQWzyUkf+8eVSpO8d2Bz3jz6Y9bCxrvXrijTypPYu9OteqPEwEMjtwKJG86dRvvVJQbb0ayde8wDksvM6XGT3Eliq9ze3rO8iQ7TyGT9s8t1SGPPUn17uX/0C9kltQPAsKEr0LNbu83wANvcfoIj0hMVw9uPxQveOINDuFGSy9iVphu4UZLL0OXAq7UjYHPdzZvbyJL7g7SKczPSP1bz0DsW298kd6PL7W8Dxoz+y7wMcQPXWFD70uyzW8lNjxO4zIojwQrgI97dAVvQxRhLwLNTu8+T1jO8Ps/Lo9t4c7LoRDvUvo6DsEoo08duZnPAbYvLx6Q2a9khRevLC/dTzFJI88RcdWvcv+Lrxujya9FDYqPdrqgD2DDqY70KKfPEGGIb1J3WK8Cf+LO7R0qTz7AXe75UzIPNQqR71L6Gg9PFTMPLp5crya3508nzwcvU2Slj1ujyY8euINvIpLAT125me8HvAmO8U+9bx25me8uV8MvTt/db2DnAo8VPqaPUCxSry2fy+9kWyTPRAgnrxvDMi84QsTucMZCb0cRvm7U97RPLyxhDyk4Iy7OMwkvTMoNL0QZ5A9btaYPdVxuTv+DP08DnZwPOsm6DsJcae7loIfPSeqIzz7LoM8/NZNPLJpozyfgw69Ep2/OtQqx7slnx09+E6mvGFnaLz+DH0725JLvdhr/DnQop88PXAVPTAdLj1hrlq8UjYHvb1ZT72NKXs8O3/1PLlfjDwgQh89Q5GnO1jJNDxQ5A49neojOtqjDrxLQJ47Fd50PfSZcryv0Li84O9Jvdqjjry6B9e8NXqsvDReY7yL88s68BFLuxmTKDya3x29aF1RPcZaPrzrDII9ZuCvvH8/DDr8HcA8TUskvKXPSbyd6iO96HMXPQcO7Lz4Tia8fSNDvPOqNT0lLYK9rcWyuqDkZj0CUJW8/TmJPDIM67ppwIw81XE5PZttAj15xkS8nTEWPFPeUTyw7AE8GiGNvF4V8Dyx2768M7aYPf05CbyFp5A8O391PGuEILyPYQ09SGDBPBNhU73O3os9wMeQPI9hDb3xLZQ7Fvo9PXvRyrzoLKW7DFEEvSxOlLslnx29LhIovCccP716Q+Y8R9JcvZYQhLuU2HE7Ya7aPGe1BrznEFw99Dgau9VGELzjz6a7xT51PAn/C72COU+8euINPHTb4brLKVi8/gz9vHBTOj0z/Yq8o8RDPTusgbzCi6S8ePHtO7QtN7zhxCA8xy8VvCwHIj3siSO9pJmaOyfxFTuQwuW6sOyBPKWIVzyC8ly8DlwKvPGfr7yDVRg9MauSvMTBU7pajcg8A7FtvMA5rLzGE8w6d0kjurj8UD3hUoU7AlCVvKdMa71Fx9Y6Oh4dPVvwg7v5r/47FQsBPZhGM7xpwAy9MlPdvBZsWTze5MO7/yhGPbBeHbx/59Y8URq+u03ZiLwPBFW8ows2PIOcirxImoA8QnVePBEP2zqwBmi6QnVePUcZT7wfbcg8Dc4lPLUCDjyqczq8EK4Cu4zIorxGnK276RtivcmsNj2orya9O8bnPNqjjjxb8IO9cFM6vVWi5bv8HUA9Ik2lPO544Lzbkks8KbUpvQKXBz359vA8Bw7svEvOArxtuk+8E7kIvf7h0zuU2HE8kltQO6tIET398pa8p5PdPDziMLyoaDQ9a/Y7PROM/Lw/e5s62JiIvRgwbbx5f1K8qGi0vIsedTyDDqa6wkQyvc/4cbxE8v+7siIxPTO2mDxopEM8fmo1PGaZvbsFvHO8dYUPvGSON71e+wm8LlkavKTgDL0AjAG8IEKfPGJYiL3yjmw8Uu+UPBvloLs3sFs8yaw2PNpcHLuE/WK8mBsKPd7kw7t5f1K7hERVvYeWTT1jK3w8sU1avORdi7vB4fY7/I/bu0juJb0InFC8Qi5sO0+dnDxqaFe80w7+vNmHxbscRvm8J/EVPKeT3bwPBFW7gvJcPOZoEbwMJPg7DGvqO0SA5Dsg+6w99tGEvQ35zjp6mxu9KDiIvCIGs7yiqHo9aM/sPLnRp7wj24k6N4UyvWGu2jtOgdO8O391PDDWO73V/507THZNvIfdP7z3pPg3U2w2PcxFobzJ8yi9xszZvMbM2TzS2M68ZBwcvWZSSzwtPdE6gZGEvE7IRTy7ahK8ir0cO6n2mDxLQB68dnTMvC6EQzz75xA9nzycvDHyBL3AgJ48dfcqPMzTBb2amCu8pfpyucFv2zxIp7M7aXkau4eWzTxVouW7joy2vACMgbx7GL26a/Y7PDZPg7tlxOa8y7c8PIEDoDzRSuo8qK8mPH0jw7w2TwM8lJH/PCt5PTzU49S7mt8dPeF9rjujxMO85ZO6vPyPW7xntQY9MB2uuylut7wpJ0W86wwCvO+UKT2Cx7M8S0AeOg1AwTzhxCA8aes1OxULAT3Elqq8aTIovRsQyryrAR+9RDlyvK774bwaguW8QYahvKcyhTvAgB47otWGPHZ0zDx87RM8aM/su8LSlrx7GL28a6/JvBtXvDxNrHy9SZZwPei6iTy0Lbc8tarYPM3CwrzjzyY86HMXPBxzBT3GWr48RkR4vGws67zkBda6TZKWurbGIb0pJ0U8InhOPFqNyDtXZvk7a69JvY1WhzxKa8e8DfnOvP+2KrwVJec7/lPvvAxRBL3aXBw90/QXvF8xOb1LzoK8AIwBPTKaT7zoc5c7MfKEu4zIoryMOj68t1SGvdED+Dvi+s88BM22OQM/UjwfbUi7e9HKuxO5CLxu1pg8fO0TPYGRhDxINZg8qixIvJiNJbw2Ine75oJ3O/d5z7uhuT289JnyvAm4GT0DP1I76WLUPDkC1Lw5SUY9hRmsPOrFjzyeIFM8Jo7aO8Eo6btlxGY8k3eZPM170DzMjBO8RxnPPEwEsjvGhee88S2UvIvzy7xNrPw7Nk+DvKLvbD3ueGC8GUw2vHsYvbxAati7dnRMvEDc8zxk1am7HHOFuvsBdzvB4fa6ZyciPe0XiLsnHD8932Hlu9X/HbwSnT+9HRvQu514CL3bS9k7pwX5PAlxJ70cLBO96n4dPGwsazzdVl88TZKWPPaKEruORUS7ntngvGJYiLxmUks8mgrHPJ0xFruSzeu8+oRVPIjMfDpB+Dy8kMJlvZuH6DwVJWe8dNthvNtL2TtGnC07Bh8vvBlMNjv3ec882JgIPVxR3DqERNW8YoMxvFpGVrwPS8e7eX9SvaFyyzv0OBq8ecZEvRxGeTx++Bk8OpA4vWV9dDyLrFm8wShpO73nM7xsEgW8CbgZPBSoxbz36+o8jA+VPIKAwTzYmIi8b8XVPL3aAD2cFc06nUt8u9c1Tb061yq8/NbNu4j5CL3i+k+8KJlgPCg4iDyWyRE9rkJUuyr8m7wiv8A71UaQPIT94rxAsco6yoGNO/qEVTxMds28euINvHeQFTz0OBq9oqh6O1T6Gjo4Lf27qNrPPJ4g07vzqrU8dcwBvAlxJz3fuZq8cFM6PCt5vTxYEKc7YL+duoXSuTt5VCk9chdOvBmTKDwkg9Q7DUDBuSWfHbzQoh+9kd6uO4pLAb3fAA27GdoavVsbrTvzHNG82JiIvEJ13jtpeRq9niDTO120lzxG4588kSUhvWbgr7wmAPa840FCvCGj97x3uz67QCNmPb9kVbx5VCm9/7YqPAvDn7swj0k8tRz0vJ1L/DwrMss8q48DO7bxSrzHdge8CbgZPFJQ7bvq8Li7Pl/SO6qeYz3Fa4G8frGnOzDWO7wieE69GslXPNqjjrzt0BU8Qud5PAicUDy0dCm7tsYhvZ88HL1dGYA9pJmaPOELk7zBKOk825JLPLuVu7yJWmG8yuJlPGws67uVOy28o31RPKZdLj3Qoh87dGnGvD3R7bzzY8O8HHOFPAm4GbzpqUY8chdOvFVbcz1/59a7bh2LPC+gDLxmmT27dnTMPI8aG7zXNc28w+z8PPEtlLzR6RG9rcUyPIkvOL1VMEq7oUciuuPPJr2Y1Be8/284PczThTwieE48nFy/PLpOybxJ3eI7GjtzOhO5CD2G7gI9NTO6PMtwyjrbS1m8p5PdvGMr/DzxWD09ikuBvcbM2TwuEqi8rvthvAt8rTycFU081OPUOxsQSrzq8Li8hyQyvMzThbtnJ6I8me59PIWnkLsVfRw62LJuO4oEjzvyR/q7uEPDO7uVuzuyaSO7Z24UPQ8E1TsZk6g8l//Auxfperw/wg28U5ffPB9tyDsUNiq7qizIuj23hzv4BzQ8NNB+OzXsR7tDA0O9aKTDPP8oxjuH3T89L7pyO1HTy7rQzcg8uEPDPLzL6rmyaaO81f8dvYNVmLxgIHY8RQ5JvG9+YzyvFyu9yoGNPQHtWTyq5dU7VPoaPZKGeTwRyOi8TZKWPMRPODz2ipK8CzU7vZ14CD2Z7n28xoVnvBo787xZcX88JBE5POVMyDuKBI89nUt8POrwuDroLCW7iejFuvsB97uumgm8W6mRu9VxOTz+mmG8W6kRvacyhTu6eXI97dAVvNlAU7wQrgI6ENkrO7OfUruYjaW8xT51ugJQFbzFa4G8QT8vvfQ4mrzHdoe6evxzOzP9CjtZcf+7dwIxPXvRSjxPDzi8VkyTOxKdP7xdbaU8kbMFPUlPfjyYRrM8BFubPNanaDyqnuO8Z24UvG9+47s84jA8o33ROyg4CLy+dRi8nBVNuhJyFry7IyC9TvPuPL1Zzzz0OJq8MNa7vGKDMTyq5dW8az0uuuxCMT0D+N886vC4PHAokTxWd7w8gZGEPC32XrsbV7w8HdTdu02SFjxbqZE8cqWyPB1iwryDVRg70h9BPLyxhDzZzje8062lPJtAdrsH9IW8tw2Uu/Q4mruv0Di8+hK6vNsEZzvbS9m7jIGwOxxzBTq+vAo8PqZEPbbGobzGhec76wwCu8qb8zwln506MasSPTgt/bxJJNW8obk9vfZDILu8sQS8W6kRvK5CVLvYUZa7KbWpuTO2GLoXiCK9SmvHPBh337p1hQ+8neqjvESA5DvQop+8RPL/OsmsNjwLfK08YwDTPOL6T7vLtzy8ZBwcPOF9Lr2IzPy86WJUvSpDjjsaydc8NBdxvGnrNTyv0Dg8v6tHPOyJIz2LrNm8Az9SPBohDbwjlBc9FrPLu9bu2rxhPD88vIT4vFSzqDyamKs75ZO6ujSlVbwQgXY8lWbWvCC0urwbEEq82Gv8PEy9P7xa1Lo8jxqbPIAuSTxYECc6dZ91vEqyObwMsty8SJqAPOo3qzwYd9884QsTPb/yubwmjlo8AlAVPU+dHDyO0yi8ZcTmu3bm5zt1zAG8+MBBO+mpxrx+arU8vdoAPEx2TTu6wGS89kOgu7p58jz+4dO7n663vJdxXLxOgVM7z7H/u5rfHTyq5dW8PJu+PG7w/rsbVzy8f+fWO5N3GT1PVqo8WMk0vQ8EVTr755A8rm19O9n54Dx8Xy+9PdHtO7iKNT39q6S8NXosvHe7vryX/8A8rpoJPQ1Awbwpbjc8WtQ6vODvSb3hUgW8eKp7PBO5CD4pbre86RvivNBbrTy3DZQ85UxIuz3RbbzrDAI8UOSOuwKXB7z3Mt26N/fNvK1TF71JlvC7N/dNvNCinzyO06i7ows2vfGfL7y/q8e8Z26UPEZE+LwQZxC8PYr7O46MNjxPD7i85q+DvK1+QLzqNys8cdBbPYNVGDwK7sg6UjYHPc7eCzxiWIi7cUL3PGaZvbx+arW8rvvhvKPEQzzTDn68vrwKPFjJND3K4mW8dT4dvHBTOrxRqCI8rvthuyCJETmhcku8iksBvfDKWD2yyns7xWsBPHSwODxlqgC8TdkIPRohDT3wg2Y9IImRu66077sHrRM72wRnPML9vzw2Inc8WDtQudX/nbw6ZQ891wokvFYFIT1XkwW8I9uJPIjMfDyALsk8MWSgPKn2GD0tPdG8ugfXOXFvg7zMpvk8U5dfvKghQryVZlY8zIyTPP3yljycXD88rrTvOwIJozwTYVO8tvHKvPn28DvHdgc7uPzQPKFyS7zxWL06gZEEPbbxSr2WyZE8o33RO1vwgzyU2HE8obm9PGdulDt+arU8KeDSu4K6ALy+j347/yhGvV0mM72a3507QnXeOu9pgLzmaBG7xMHTPM5QJz2WEIS9HI3rvJwVzbxSNoe618OxO7i1Xj3H6KI8UP50PNoVqrz7oB490TCEuvUnVz1Dkac81XE5PTsNWryUkf+7hghpvCPbiTwgiRE8rkLUOxZsWTyvFys8dYWPvN8ADTvd9YY8Q0q1Op+ut7w+7TY8QYYhvAhVXryczto7SDUYPRXedLyWEAS9XFFcvcXdnLy4Q0O820tZPAetk7zi+s87UyVEPHriDb0swC+7Kvwbu1iCwrwZ2hq9S4eQOyhS7juoIUK7mt8dvFg7UDyyynu8FmzZOjMotLzB4fa70FstPYvzS71YybS8w+z8POcQ3DxQjFm8FKjFO3qbGzvP+PE8VLMoPP9vOL2s8Nu8XvsJPXlUKbyX/8C8pc/JPPaKkrwK7si7623auyD7LL2kmRo8oqh6vHX3KrtwKJG89SfXuwJq+7xpMqi8ZgtZvRV9HLy9Lia82hWqvDCPybzhfS486wwCOy8BZTy0Lbc8E4x8vPncCjye2WC7J/GVvDDWOzwMUQQ9la3IOxSoxTtIpzM9ZaqAPei6ibty7KQ8quXVuDjMpDsh0IM78/Enu3ztEz1eFXC8qK8mvO3qezza6oA7WBCnvIvzy7yWEIQ70tjOPBULAT0uWZo8q7osPeiNfbtOOmE8BKKNu8xFITyO06i7eQ03vA5cCrw7Ddq8LWj6u3KlMrxxiWm8GUw2PaFHIj3wyli8D5I5OxAgHjpiyiM8otUGPZsmkLy1Ag68bh2LvG1z3bs4Wok7q4+DPK6aiTyCuoC7oo4UvQ35Trzs+748Z4j6vAruyDvVRhC9xlo+PVSzqDyEi8e69tEEPcUkjzu+vAq8MB2uPPXgZLww1ru8ZuCvPO9pAD26efK7cJqsPPugnrywXp28QnVevOrFj7vUKsc7ypvzOym1Kbux2z480/QXvMN64bwFvHM65slpPexCMbuq5dU6gjnPPGJYiDwYBcQ8q7qsvAbYvDza6gA92JgIvJtA9ryLZWe6l//APJjUl7wvoAw8GHffPBfPFD3nENw7FKjFvJSRf7xdtBe72710PPrLR7rnENy8JBE5O3N6CTxYEKe8QNzzPOOItLzR6RE9hdK5PDt/9bh7ilg7wtIWPJbJkTz1tbu8I9sJO2e1hrzBKOk8qp7jvJTYcTy5X4w820tZPYZP27yDnIo85ZO6vJ1LfLzEwdM8Ae1ZPPJHerwr69g8kAnYPNW4Kzvl2qw8fz+MPF77CTzET7g8gC7Ju4zIorsptSk8wW/bvGZSSzw7xmc82flgPDjMJL2vF6s8gQOgvJ+Djrx8Xy87N7DbvP6a4bz75xC9bbpPvDP9irrg78m8rVMXPV94q7y28cq5Ae3ZPDvG57zAOSy7iLIWO2nrtTy1qti8WbhxPB3U3bzVcbm6G56uvDYi9zw0XuM7Pu22u8DHkLxAI+Y7+y6DOxnamjzibOs71qdou2ohZbw17Me7\"\n + \ },\n {\n \"object\": \"embedding\",\n \"index\": 1,\n \"embedding\": + \"O6+uu4hrIL08vRg9GKoevU+2T7xG4h67YTxSPFxBijvC1pA7Sa2dPJZimjzaG2W9ePOHPF4FlLx35Z08OTUFPHN4dLwpc4w89ALkvEXbKT3AEge85dsgvBBJIjtF2ym8tgjrPNzEBL3VhWc8xmWZPBTYKj3KWWy8vDkePYZWwbwr9Ko5DePtO4WZLDz9u6o8MO/yPIddNj2PCBO90pgJPIIz+DviEKI64VONvG4zzDxAsWu8K/SqPFUC7bz2F8O6roUPvfSdGT3eo3i9digJvBJ567zm6Yo8AbD9O2XL2jr1Uzm9Y7Z7O67xTryv/7i856afOu9sZrwzBNI6zLE2PeC41zz+eL+8W/A0OwCHqTz5mOG6QPvLPPB6UD3JMJi7bn2sOtMLvjx2lEg980XPPI4Bnjq6kH49eRVnvAxwOb10f2m8xAbaO6fonDnUfvK8IHdavDy9GDxcrUm9nLUsu3hYUjr9uyq9mZnYO4IYjr1FJYq8RuKevH2CED1U4I08OTUFveCxYr2dvCG93TDEuw3IgzzqJ747w90FPAIB0zsgLfo7crtfvBJ56zsd78Y6iSg1vcpZbDwH4TC9rTS6vDrymbzeN7k8x2wOPQCHqbwBsH09yllsvVksKzqWYpq927Yauwgyhr1TiEM9Xy7ovJRUsLsrWXW9NBI8vRokSL2Tl5u8cnF/u9YnEr30TMS7GiRIPSowoT2o7xG9hlbBujbWRb0jk647Bto7vfmYYTtSy668bAMDvEjwCLwMwQ49XSD+vFxBijuKeQq9ZFgmPWGNJ7o0XBw9sxuNOzgukLt+q2S9p+GnPJmZWL0A0Ym8ke77vFakl7vzjy89WzoVveRA6zvOxhU93u1YPQlb2jxycX+85NSrPIIYDj2wUI68DHC5vH/8Ob2VW6U7soBXvDRcHLzkiku8Jl6tvAOcCLoiO+S69x44vZGJMbyHrou8QVMWvQK38jxrsi29nnk2PfozFzxbn1+8upD+vG/w4LttwJc89J2ZPB32O730lqS77UOSPcbK4zuqsxu7heMMvS1n3zznph89aJZZPSLPpLqf0QC8GGA+PVUC7bonr4K7XwwJPaGwdLuDQeK8DMEOO824K73WjNw7Gys9PQ3Ig7zDkyU8QUyhvI2wyLpA+0s9HIMHOgeX0Lx+iYU8l3CEPRwyMr1PShA9CDKGPaCOFb3iWoK9fe7PPKNZFD3z2Q+9goRNPAFEPr2MqdO8zsaVvAieRbyESNe8+CUtPHsqxjxyBUC8u55ovFLSI72GVkG9Z9lEvaof2zv/yZQ8wkLQvKzj5LyL7L661c/HPDCDM71VAm08/ni/OpHuezvEBlo8PsuCvJWlBb3kJYG6p5dHPSz7nzsLs6Q8J68CPI900rvuACe9TTymPNIEyTvZXtC8VEXYvP5xSr2PD4i6ppDSvGS98Dwj3Y69Sc98uuitlLzALXG7yjeNvDuvrry+/Sc7H04Gum0s1zxfDAm9/P6VPMtg4bwpegG9U484PWObkbydKOG8QLHrvKx3JTx11zO9zbgrvQUdJzyuO6882Q17vBG8Vj0Ct/K7zPsWO3JWlTyX1c45eV9HvBactDwuCQq96Mj+PGwDgzwhyK87vUeIPZ1yQTw1z1A81YXnPDk1hTxMNbE8y6rBvGlTbr3oyH48jWboujNOMj2wUI68U9mYvK/4Q7wMJtm8iSHAPPiKd7xbOpW8nCHsvG4zTDygqX88Y7Z7vHPJSTwauIi93MQEvSxFADxyBUA8yu0svAGOnrw+ywI9bXa3vBOA4Ds6FPm64LHivMWoBD227QC9rTQ6PRiqnr2PCBM8ylL3uhFy9juCM3g8y/QhPdqvJbyG6gE9tgjrPGlT7jsx9uc8/P6VvKx3pTw3J5u7izYfPbqQ/rydKOG84LhXPD3EDTz5LCK9A1kdu0nPfDxh14c8oES1Oy0CFTwq5kA87gAnvAqsrzz2q4M8Xy5oPIIzeLzl2yA9fj8lPMAt8Tvutsa8d+UdPADRCbyVEcU8bSzXPGwDAz30lqQ8DCbZvLluHz38rcC8eiNRPK//OLoDCEg9b86BvCEZBb07AAS8JsP3O7LKN70obBe8gFQEvY63PT3nVco8vUATPA9CrTyiUh896y4zPAKVE70X7Ym9BWcHvZHTkbyItQA83qP4O4XjjDsIMga8dNC+O7syKTuyyre7J6+CuhftCT3M+5a7jUQJvRD/wbuMPRQ8ZMTlvD6BojwSees8AD3JvNKYiTt3B328YH89u8gpIz3VGSi8cPdVPU5l+ru1S1a7NtZFO/NFTzvlR2C7/bQ1PDgukDwJ75q9U4hDvNaMXD0KYk87AgHTu0IQqzvZFPC8BMVcvFmRdTw/qna8jbBIvaU//bz7pku99ExEvN9FI716txG9tZW2PI1Eib2Ekre7alrjvFPZGD3Ic4O9OvIZvRPKQD3+cco9u3yJPByecTwpeoG6nGvMvEpxJz0znwc9eRVnu858NT123qg6ZMRlPRcP6bzgTBi9aDEPvECPDD1K1nG8B0Z7up82SzvXLgc9cf7KvL2sUrxBTCG91BIzPD3EjbyvSRm9Q8ZKu5RUML3Q7+k91H7yvC/GnrwMJtm7+um2vNsi2rzpIMm89hdDvFT7d7xdIP47gFQEPUVHaT3sPB28OhT5PJk0jjwiO+S7ZL1wuu1e/LyY3EM9TwCwPCE0bzuR05G8pR2evPceOLtF2yk9xJoavSDBOjwK9g+9NmoGvLFybTxL3Wa8PXqtvLSOwTwU2Kq7/WrVOU9s77kN4+064EyYvL1HiLxbplQ851XKvLAGLr1rYVi93YGZvPs6DL14orK8oQFKvJGJMT1lFTu9tQH2vAIB07xDxkq8Eg0sPW7HDDtQB6U8shSYOtBAPzxZkfU8Q8ZKvFT7d7sMJtk8znXAOy8y3ryu8c68l4vuu4AKpDzPgyq9KGyXPMcirry4Z6o8ExQhO6FLKjxSyy46WzqVvHGZALxxSKu8iXKVPNRjCD04LhA9eKmnvPNFTz2CfVg8yZzXvAbaOz2BwEO6pYldvSbD9zzuACe872zmvNkN+7td/p48wMgmPY1maLwuvym88+CEOv8uXzuD1aI8aVNuPJ0GgjwSw0s8WW+WO97tWDyEkre8cPdVvY36KLvUfvI8gzptvLXmCzvTd/28+zoMPcNJRbs33To9nShhvEMXIL1dal68fTgwPMGFuztIXMg8/bS1PHjzBzzz+248u3wJvUFTFj2DOu28VPt3vBTYqrxsAwO97V78vIgaS7zfRaM7A1IovNKYCbpJtBK803f9PAzc+LujDzS9S93mvOVHYDvSmIk4VQnivD0p2DwfToY7c8JUvKGw9DsXD2k8pdO9vJ7DFj1oKho9w92FvM58tbtZLKu8xcNuuoddtjyVW6U8loR5OzJHvbvTC745AbD9PBZLX7wUIos7KNhWu/KIujzYV9s8xFC6vMv0IbzK7ay8KjAhvau6kDwj3Q69aqRDPeLGwbxMNTG8fHsbvcRQOrwQ/8G8IAsbvN+qbbwJW1o7b86BvJKQJjw2jGU9mZlYvSI75Lp7vga8gREZvNYnkrxEHhU8mOO4uxacNL0hEhA9RUfpvDKRnbyPCBO9Xy5oOovsvjz4ive80EA/PLw5nrxafYC9tUtWu4p5ijvvbOa8L+j9u8tg4bz1Wq45PCJjvQzBDrzTVZ47zzlKPMfRWLwtZ188h6cWvXfsEj3iECK8COglvQbaOzzDk6U8WjOgvCHIL73L9CG7TkMbvQfhsLwsRYA8YdeHPZP8Zbz78Cu93TDEvB79sDwZZ7M5mJLjO/AOEbyS9fC7yHMDvIzzMz2rJtC7zGdWvNy9DzxmZpC8FY5KO+nP87xr/A28n9GAvFEOGrz/fzS7qybQvFBz5LuL7D48PcQNPAzc+Luyyje8q7oQu6A9wLzjFxc8pisIvIMfA7yqH9u8e76GPEpqMj0xQEi7xLx5u7qXcz0jky48/8kUPT+Ilzw6Xlm8iLUAvCgiNzyJchW6ktqGPNYnkryNRIm8c8LUPPceOL3r5NK7dBofPVhoIT3LPgK71ta8OxPKQLt35R28fokFvavVerxEHpW8WuJKPJnqrTyR7vu7k/zlvHcHfTwMJlm8ANEJPWXL2rtsuSK9zzLVPM24q7x7vgY8vfYyvTZqhjy64VO7jwgTvQNZHb16HFw8DS1Ove5KBz14DnK8FSkAvWpa47yDi0I7RdspPMASBzy9Rwg9DNx4u4p5Cr1qWmO91ozcvFC9xDdhhjK9He/GPJxrzDyCM3i7C7MkPTRcHLxIpqi8DNz4u+itlLzf+0K8mxp3PJZiGryR05G81y4HuyE077xSy668qybQu1VTwrj7Ogy7cU8gPYypUzz9tDW9+p9WPabasrwgC5s8zb8gvSKFRDzIcwO9g4vCvG7p6zyjDzQ9Mke9u6IIvzsr9Ko8ZMTlvJeLbjxGmD65t2C1OzguED1YaCG8e76GOS/ofT0w7/K7ii8qPJmgTTxp7qM83qN4vKaQ0jxZb5a7h8l1u2B/vTwvxh499ALkPB3vRjwssT874zn2OtR+8rwNNEO9ExsWvEWRSbw5oUQ8A1kdvKYriDzC1hA6JVc4PQK38jstApW8Ce+aOJxrTLz428w82q+lPJiSY73welC95UfgvKJSHztyBcC8c8lJPeemHz1VU8I8nXLBPAGOHj3TVZ67RB6Vu/z+lTtQB6W7G3ySPJjcQzn0nRk9ExQhPChsF7yHyfU8L8aePNCKHzzL9CG99J2ZO94+rjy2CGs6aEz5PJy1LD1PShC9HJ5xvIZWQb1ZLKs8ITTvPGObkTw4LhC8VhDXuwaQ2zxAsWs85CWBvFSWLb0zBNK8qybQu5v4lzyQzBw9LQKVO71Akzzz2Q891y4HPavc7zxARSy922w6va5CJL3I38I8f0YaPLzvvbxVAm08CaW6PFrpPz0RvFa6MYooveTUqzw8bMM8WZH1O87h/zz4ive8PXqtPAlb2rmsLUW8bcAXPIhkK7ws+5+8SruHvARgkrxfDAk8W6ZUPNKYCb14WFI9bulrOVp9gLzAyCY8bn2sPIxY/jxVnaI8Z9nEvGr1GDpsb8I85NQrvQdG+zyxeWK7Q8ZKvIxY/jwUIgs90O9pvGXL2jwHTXC8nbwhPd+PAzxkvfC8z4OqulXngrzrLrM7s4fMPEefMzzs60c9sXliPJHTEb2dckE9uis0PQpiT71t4nY8g9UiPb8Lkrsbl3w8wMgmuxXfnzwgwTo9jbDIO1BRhbtH6ZM8JspsPKtwMDx48we9PjdCPCESEDyopTE85EBrvfiK97yBEZm8znw1vJZimrrorZS8zuH/vLFy7Txv8OC84loCPDiaTzpxSKs8h102vH9NDz3DScU8Fw/pPPiRbDwtblS9DS1Ou0rWcTwPjI08iNDqvGS9cLuJcpU7XWreO1cXzLxVAm08OvIZPDOfhzyhAco4mOM4PJjjuLx4WNI8RSUKvB5HkbsiO+Q7WB7BvJ1yQbyeL9a8ordpuazjZDtJGd0736rtPLt8iTuf0QC8XgWUPMNJxTsoIjc9c8JUvDwiY7yxeWK79QnZPFHEuTtJGd078cslvQ+MDbu9R4i8QlqLvFiygTvgArg8eiPRO88yVbxLJ0c8UL3EO5JGRr1mHLC8DcgDvYWZrDyIZKu8NX77uyaojToED7288vT5PD16LT0bfBI9xagEPKbasjtCEKs8b86Bu+5KhzxA+0u8eKknve9sZr287z092yJauxwyMj0mw3e856afPFmR9To7ry49I0JZu3TQPjzJMJi8CVtavLeqFb2K5Uk8LGDqOzDv8ruswYW8/y5fPebpCrz8Y+A8GiRIPO+9O7wNfiO9eKKyO7hnqjy2CGu8vfYyugnvmjzfj4O8sLxNuliyAbxGmL48gcDDvO1DEj1NPCY86+TSPG3i9ruxDaO74822PBmxE70mw/e8H04GvP9/tDxraM06gs4tPXFPoDsnrwK91YVnPJ/RADxnbQU88MSwPK47rzyTTbs71ieSu2thWLxK1vG8QLHrvMgpI73+eD88OeukO9xzLzxgyR275ZHAvAxwObyS9XC72OsbvM2/oDy2CGs8nbwhPKtwsD2xDSO8BdPGvOOD1jr3Hri83/TNPN/7wrzmmLU7NRmxvIAKJD2lid28SydHOxJ567tLeBy8QEUsvYADrzwbfBK8upD+vBye8Tzhv0y7jw+IPLbtALwtZ1+82Q37PPP7brxly1q8tZU2vK00Orx+q+Q8O6+uPCxg6ruGVkG7N5Nau6fhJzy3YLU601UePMVXL70xiqg8G3WdOwK38jygqX+6yz4CvCv0KjzmmDW8pdO9vNnyEDztQ5K8C7qZukD7SzxjmxE8X8Iovdsi2jyLNh+9IX7PPOvk0jwLupm8FZU/O+FTDbymKwg9NmoGvLqX8zzO4f88jgGevP81VD0Yqp68oI6VvCuj1bw6qLk7RSWKPINBYjyb+Jc7tqOgPNCKn7xK1nG9OvmOuoSSt7uQzBw9BiScu851QL2ewxa8td8WO78mfDxH6ZO8lJ4QPHebPT202KE8WXYLu0fpE72xcu08RZFJPJL18LytNLq7YTzSPNobZTxo4Dk8Cv0EPHUhFLwwOdO7ehzcPEFMITyn6Bw9L+j9u1kltrw2jGU9YMmdvFmR9boO6mK92KG7vFp9ADshEpC6HJ5xPEVHabzfRaM6ByuRvN0wRDtPtk88dpTIvINB4jx/Rpq76WopvWhMeTzTwd04pYldPOafqjxkvXA8w0nFuRMUoTw2agY8kThcumjguTvl4pU64loCvbajoLt0f2m7ITRvPAXTRrtKuwc6z+j0vIxYfjmKeYo8pGAJPVUC7buxcu089quDuew1KD39IPW8beJ2PHMTKrx5Fee8XbS+vIagobu3YLU7lx8vPG3AF7yYLRk9bLmiOxEGt7t3Ud08royEumJKvLyUVLA8zivgPEt4nLxwkgs9iNBqPCbK7LzCQlC6Fwh0O885Sjtp7qO8IjtkPH2CELxp56482mVFuyTkgzxycf+8SrsHPYTcF7sDWR295NSrvPWkDj2FT0y8yTCYvKu6ELzHIq67YTxSPc7GFT0bl/y8jfqovGTE5byJIUC7VxfMvGlTbjywBq688culOenP87lzEyq91RkovfLSmrhgfz086dboOwlb2rnHbA49BpBbvNDvabxkWCY9xxu5OjoUeTxU+3e8+jOXOwA9ST3Bzxs7xJqaPLFy7bzlkUA9G5f8uj+Ilzy1Afa8maBNvC98vrwbfBK8iLUAPLZSSzxpU268D/jMu87h/7i8pV08dSEUvSAt+jteBZQ8f0YavfSdGbs1yNu7+zqMvOafqrw7ry48RIPfOyxg6jyjD7S7yeY3PAeX0Lx35R08jWboO9PBXT1+iYW6vrNHPIERmbwH4bA8FSkAvU9KELxdtL476y4zOp55tryit2m8wBIHvECx67vC1hA815NRPL8m/LxvzoG8ofpUO2dtBbybGvc8KSksOmcjpTyGVsG8zLG2PFp9AD0wgzO8gs4tPC8y3jvZFHA8zLG2vB+6xbyItQC8HaXmPDQLxzl9pG86gx8DvCF+T73TVR67DX4jvWiW2byK5Uk79hfDPDRcHDwV3x88/y5fPKMPtDow73I88A6RvDV++7zMsbY7jUSJPGcjJbzaG+U65k7VPBejKbxU4I294802PJo7A72dKGE84LjXO3jzB7ykYAk9ITTvPOpxnrwCt/I8LgmKvBftiTyDQWK8m643PAO+Z7zKN428CDKGvCTkgzl5X8e7VQLtPEbiHjxKINI8oI6VvAQWMjwI6CU8348DPVHEObz4b428SPCIPIXjDLp+P6U8xyKuPIkotTzaG+U8W5/fOw1+I7x9pG89jF/zu+ZOVbzwetC81c/Hu1LSI70DCMi8w90FPSVXOLpzeHQ8xAbaO5GCvDwgCxu85UfgvHgO8jz/f7Q7qyZQvNle0Du0IgI9QPvLvEQ5/7zsNSg87UMSPfl2grwfTga9ItYZvXocXLs2jOW5CFRlPKbasrxIXMi6H7rFPM0JgbwTG5a8dH9pvJfVTrw9MM28YMkdPTxzODyMWP47Nozlu5jcw7zOK+C8u57ovDqoOby5bh+9cUgrOmxvQr3r5NI7+XYCvFnbVbyXi247CvYPu9wpz7xh1wc7\"\n + \ }\n ],\n \"model\": \"text-embedding-3-small\",\n \"usage\": {\n \"prompt_tokens\": + 4,\n \"total_tokens\": 4\n }\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: '{"input":["Hello world","Test embedding"],"model":"text-embedding-3-small","encoding_format":"base64"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: '' + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/python/instrumentation/openinference-instrumentation-beeai/tests/conftest.py b/python/instrumentation/openinference-instrumentation-beeai/tests/conftest.py new file mode 100644 index 000000000..a9ffc4543 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-beeai/tests/conftest.py @@ -0,0 +1,36 @@ +from typing import Generator + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from openinference.instrumentation.beeai import BeeAIInstrumentor + + +@pytest.fixture(scope="session") +def in_memory_span_exporter() -> InMemorySpanExporter: + return InMemorySpanExporter() + + +@pytest.fixture(scope="session") +def tracer_provider( + in_memory_span_exporter: InMemorySpanExporter, +) -> trace_api.TracerProvider: + tracer_provider = trace_sdk.TracerProvider() + span_processor = SimpleSpanProcessor(span_exporter=in_memory_span_exporter) + tracer_provider.add_span_processor(span_processor=span_processor) + return tracer_provider + + +@pytest.fixture(autouse=True) +def instrument( + tracer_provider: trace_api.TracerProvider, + in_memory_span_exporter: InMemorySpanExporter, +) -> Generator[None, None, None]: + BeeAIInstrumentor().instrument(tracer_provider=tracer_provider) + in_memory_span_exporter.clear() + yield + BeeAIInstrumentor().uninstrument() + in_memory_span_exporter.clear() diff --git a/python/instrumentation/openinference-instrumentation-beeai/tests/test_dummy.py b/python/instrumentation/openinference-instrumentation-beeai/tests/test_dummy.py deleted file mode 100644 index ddc252f2a..000000000 --- a/python/instrumentation/openinference-instrumentation-beeai/tests/test_dummy.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy() -> None: - assert 1 == 1 diff --git a/python/instrumentation/openinference-instrumentation-beeai/tests/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-beeai/tests/test_instrumentor.py new file mode 100644 index 000000000..9c0c4062a --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-beeai/tests/test_instrumentor.py @@ -0,0 +1,109 @@ +import json +import os +from typing import Mapping, cast + +import pytest +from beeai_framework.adapters.openai import OpenAIEmbeddingModel +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.util.types import AttributeValue + +from openinference.semconv.trace import ( + EmbeddingAttributes, + OpenInferenceSpanKindValues, + SpanAttributes, +) + + +@pytest.mark.vcr( + decode_compressed_response=True, + before_record_request=lambda _: _.headers.clear() or _, + before_record_response=lambda _: {**_, "headers": {}}, +) +@pytest.mark.asyncio +async def test_openai_embeddings(in_memory_span_exporter: InMemorySpanExporter) -> None: + """Test that BeeAI OpenAI embeddings are properly traced.""" + # API key from environment - only used when re-recording the cassette + # When using the cassette, the key is not needed + api_key = os.getenv("OPENAI_API_KEY", "sk-test") + + # Create an embedding model + embedding_model = OpenAIEmbeddingModel( + model_id="text-embedding-3-small", + api_key=api_key, + ) + + # Create embeddings for test texts + texts = ["Hello world", "Test embedding"] + + # Run the embedding request + response = await embedding_model.create(texts) + + # Verify we got embeddings back + assert response is not None + assert response.embeddings is not None + assert len(response.embeddings) == 2 + + # Get the spans + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + + # Get the single span + openinference_span = spans[0] + assert openinference_span is not None + + # Verify span attributes + attributes = dict(cast(Mapping[str, AttributeValue], openinference_span.attributes)) + + # Check basic attributes as per spec + assert ( + attributes.get(SpanAttributes.OPENINFERENCE_SPAN_KIND) + == OpenInferenceSpanKindValues.EMBEDDING.value + ) + assert attributes.get(SpanAttributes.EMBEDDING_MODEL_NAME) == "text-embedding-3-small" + assert attributes.get(SpanAttributes.LLM_SYSTEM) == "beeai" + assert attributes.get(SpanAttributes.LLM_PROVIDER) == "openai" + + # Check embedding texts + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "Hello world" + ) + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.1.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "Test embedding" + ) + + # Check embedding vectors exist and have correct structure + vector_0 = attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) + vector_1 = attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.1.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) + + assert vector_0 is not None + assert vector_1 is not None + # Vectors are tuples in the cassette, check exact length from recorded data + assert isinstance(vector_0, (list, tuple)) + assert isinstance(vector_1, (list, tuple)) + assert len(vector_0) == 1536 # text-embedding-3-small dimension + assert len(vector_1) == 1536 # text-embedding-3-small dimension + # Check first few values are correct floats from cassette + assert vector_0[0] == pytest.approx(-0.002078542485833168) + assert vector_0[1] == pytest.approx(-0.04908587411046028) + assert vector_1[0] == pytest.approx(-0.005330947693437338) + assert vector_1[1] == pytest.approx(-0.03916504979133606) + + # Check invocation parameters + invocation_params = attributes.get("embedding.invocation_parameters") + assert isinstance(invocation_params, str) + assert json.loads(invocation_params) == {"abort_signal": None, "max_retries": 0} + + # Check token counts + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_TOTAL) == 4 + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_PROMPT) == 4 + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION) == 0 diff --git a/python/instrumentation/openinference-instrumentation-haystack/src/openinference/instrumentation/haystack/_wrappers.py b/python/instrumentation/openinference-instrumentation-haystack/src/openinference/instrumentation/haystack/_wrappers.py index 69109e0a0..2e9fe1e54 100644 --- a/python/instrumentation/openinference-instrumentation-haystack/src/openinference/instrumentation/haystack/_wrappers.py +++ b/python/instrumentation/openinference-instrumentation-haystack/src/openinference/instrumentation/haystack/_wrappers.py @@ -1,3 +1,5 @@ +import base64 +import struct from enum import Enum, auto from inspect import BoundArguments, Parameter, signature from typing import ( @@ -20,7 +22,10 @@ from opentelemetry import trace as trace_api from typing_extensions import TypeGuard, assert_never -from openinference.instrumentation import get_attributes_from_context, safe_json_dumps +from openinference.instrumentation import ( + get_attributes_from_context, + safe_json_dumps, +) from openinference.semconv.trace import ( DocumentAttributes, EmbeddingAttributes, @@ -95,16 +100,17 @@ def __call__( component = instance component_class_name = _get_component_class_name(component) + component_type = _get_component_type(component) bound_arguments = _get_bound_arguments(wrapped, *args, **kwargs) arguments = bound_arguments.arguments with self._tracer.start_as_current_span( - name=_get_component_span_name(component_class_name) + name=_get_component_span_name(component_class_name, component_type) ) as span: span.set_attributes( {**dict(get_attributes_from_context()), **dict(_get_input_attributes(arguments))} ) - if (component_type := _get_component_type(component)) is ComponentType.GENERATOR: + if component_type is ComponentType.GENERATOR: span.set_attributes( { **dict(_get_span_kind_attributes(LLM)), @@ -112,12 +118,14 @@ def __call__( } ) elif component_type is ComponentType.EMBEDDER: - span.set_attributes( - { - **dict(_get_span_kind_attributes(EMBEDDING)), - **dict(_get_embedding_model_attributes(component)), - } - ) + embedding_attrs = { + **dict(_get_span_kind_attributes(EMBEDDING)), + **dict(_get_embedding_model_attributes(component)), + **dict(_get_embedding_invocation_parameters(arguments)), + } + # Use the component class name as the LLM system + embedding_attrs[LLM_SYSTEM] = component_class_name + span.set_attributes(embedding_attrs) elif component_type is ComponentType.RANKER: span.set_attributes( { @@ -243,10 +251,12 @@ def _get_component_class_name(component: "Component") -> str: return str(component.__class__.__name__) -def _get_component_span_name(component_class_name: str) -> str: +def _get_component_span_name(component_class_name: str, component_type: ComponentType) -> str: """ Gets the name of the span for a component. """ + if component_type is ComponentType.EMBEDDER: + return "CreateEmbeddings" return f"{component_class_name}.run" @@ -309,27 +319,52 @@ def _has_generator_output_type(run_method: Callable[..., Any]) -> bool: """ Uses heuristics to infer if a component has a generator-like `run` method. """ + from typing import get_args, get_origin + from haystack.dataclasses.chat_message import ChatMessage if (output_types := _get_run_method_output_types(run_method)) is None or ( replies := output_types.get("replies") ) is None: return False - return replies == List[ChatMessage] or replies == List[str] + + # Check if it's List[ChatMessage] or List[str], handling both List and list + origin = get_origin(replies) + args = get_args(replies) + if origin is list and args: + return args[0] is ChatMessage or args[0] is str + return False def _has_ranker_io_types(run_method: Callable[..., Any]) -> bool: """ Uses heuristics to infer if a component has a ranker-like `run` method. """ + from typing import get_args, get_origin + from haystack import Document if (input_types := _get_run_method_input_types(run_method)) is None or ( output_types := _get_run_method_output_types(run_method) ) is None: return False - has_documents_parameter = input_types.get("documents") == List[Document] - outputs_list_of_documents = output_types.get("documents") == List[Document] + + # Check input type + input_doc_type = input_types.get("documents") + has_documents_parameter = False + if input_doc_type is not None: + origin = get_origin(input_doc_type) + args = get_args(input_doc_type) + has_documents_parameter = (origin is list) and bool(args) and (args[0] is Document) + + # Check output type + output_doc_type = output_types.get("documents") + outputs_list_of_documents = False + if output_doc_type is not None: + origin = get_origin(output_doc_type) + args = get_args(output_doc_type) + outputs_list_of_documents = bool(origin is list and args and args[0] is Document) + return has_documents_parameter and outputs_list_of_documents @@ -340,6 +375,8 @@ def _has_retriever_io_types(run_method: Callable[..., Any]) -> bool: This is used to find unusual retrievers such as `SerperDevWebSearch`. See: https://github.com/deepset-ai/haystack/blob/21c507331c98c76aed88cd8046373dfa2a3590e7/haystack/components/websearch/serper_dev.py#L93 """ + from typing import get_args, get_origin + from haystack import Document if (input_types := _get_run_method_input_types(run_method)) is None or ( @@ -347,7 +384,13 @@ def _has_retriever_io_types(run_method: Callable[..., Any]) -> bool: ) is None: return False has_documents_parameter = "documents" in input_types - outputs_list_of_documents = output_types.get("documents") == List[Document] + doc_type = output_types.get("documents") + # Check if it's a list of Documents, handling both List[Document] and list[Document] + outputs_list_of_documents = False + if doc_type is not None: + origin = get_origin(doc_type) + args = get_args(doc_type) + outputs_list_of_documents = bool(origin is list and args and args[0] is Document) return not has_documents_parameter and outputs_list_of_documents @@ -617,6 +660,47 @@ def _get_embedding_model_attributes(component: "Component") -> Iterator[Tuple[st yield EMBEDDING_MODEL_NAME, model +def _get_embedding_invocation_parameters(arguments: Mapping[str, Any]) -> Iterator[Tuple[str, Any]]: + """ + Extract invocation parameters from embedder arguments (excluding documents/text). + """ + # Exclude input data from invocation parameters + params = ( + {k: v for k, v in arguments.items() if k not in ("documents", "text", "texts")} + if arguments + else {} + ) + # Always yield invocation parameters, even if empty + yield EMBEDDING_INVOCATION_PARAMETERS, safe_json_dumps(params) + + +def _decode_embedding_vector(raw_vector: Any) -> Optional[List[float]]: + """ + Decodes an embedding vector which can be a list of floats or base64-encoded string. + Returns None if the vector cannot be decoded. + """ + if not raw_vector: + return None + + # Check if it's already a list/tuple of numbers + if isinstance(raw_vector, (list, tuple)) and raw_vector: + if isinstance(raw_vector[0], (int, float)): + return list(raw_vector) + elif isinstance(raw_vector, str) and raw_vector: + # Base64-encoded vector + try: + # Decode base64 to float32 array + decoded = base64.b64decode(raw_vector) + # Unpack as float32 values + num_floats = len(decoded) // 4 + return list(struct.unpack(f"{num_floats}f", decoded)) + except Exception: + # If decoding fails, return None + return None + + return None + + def _get_embedding_attributes( arguments: Mapping[str, Any], response: Mapping[str, Any] ) -> Iterator[Tuple[str, Any]]: @@ -628,18 +712,24 @@ def _get_embedding_attributes( ): for doc_index, doc in enumerate(documents): yield f"{EMBEDDING_EMBEDDINGS}.{doc_index}.{EMBEDDING_TEXT}", doc.content - yield ( - f"{EMBEDDING_EMBEDDINGS}.{doc_index}.{EMBEDDING_VECTOR}", - list(doc.embedding), - ) - elif _is_vector(embedding := response.get("embedding")) and isinstance( + + vector = _decode_embedding_vector(doc.embedding) + if vector: + yield ( + f"{EMBEDDING_EMBEDDINGS}.{doc_index}.{EMBEDDING_VECTOR}", + vector, + ) + elif (embedding := response.get("embedding")) is not None and isinstance( text := arguments.get("text"), str ): yield f"{EMBEDDING_EMBEDDINGS}.0.{EMBEDDING_TEXT}", text - yield ( - f"{EMBEDDING_EMBEDDINGS}.0.{EMBEDDING_VECTOR}", - list(embedding), - ) + + vector = _decode_embedding_vector(embedding) + if vector: + yield ( + f"{EMBEDDING_EMBEDDINGS}.0.{EMBEDDING_VECTOR}", + vector, + ) def _is_embedding_doc(maybe_doc: Any) -> bool: @@ -652,7 +742,7 @@ def _is_embedding_doc(maybe_doc: Any) -> bool: return ( isinstance(maybe_doc, Document) and isinstance(maybe_doc.content, str) - and _is_vector(maybe_doc.embedding) + and maybe_doc.embedding is not None ) @@ -669,9 +759,16 @@ def _is_vector( value: Any, ) -> TypeGuard[Sequence[Union[int, float]]]: """ - Checks for sequences of numbers. + Checks for sequences of numbers, including numpy arrays. """ + # Check for numpy arrays + if hasattr(value, "__array__") and hasattr(value, "dtype"): + # It's likely a numpy array + import numpy as np + + return isinstance(value, np.ndarray) and value.dtype.kind in ["f", "i"] + # Check for regular sequences is_sequence_of_numbers = isinstance(value, Sequence) and all( map(lambda x: isinstance(x, (int, float)), value) ) @@ -721,6 +818,8 @@ def _get_bound_arguments(function: Callable[..., Any], *args: Any, **kwargs: Any DOCUMENT_SCORE = DocumentAttributes.DOCUMENT_SCORE DOCUMENT_METADATA = DocumentAttributes.DOCUMENT_METADATA EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" EMBEDDING_MODEL_NAME = SpanAttributes.EMBEDDING_MODEL_NAME EMBEDDING_TEXT = EmbeddingAttributes.EMBEDDING_TEXT EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR @@ -732,6 +831,7 @@ def _get_bound_arguments(function: Callable[..., Any], *args: Any, **kwargs: Any LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES LLM_PROMPTS = SpanAttributes.LLM_PROMPTS LLM_PROMPT_TEMPLATE = SpanAttributes.LLM_PROMPT_TEMPLATE +LLM_SYSTEM = SpanAttributes.LLM_SYSTEM LLM_PROMPT_TEMPLATE_VARIABLES = SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES LLM_PROMPT_TEMPLATE_VERSION = SpanAttributes.LLM_PROMPT_TEMPLATE_VERSION LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION diff --git a/python/instrumentation/openinference-instrumentation-haystack/tests/README.md b/python/instrumentation/openinference-instrumentation-haystack/tests/README.md new file mode 100644 index 000000000..ba4b30ba1 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-haystack/tests/README.md @@ -0,0 +1,27 @@ +# Haystack Instrumentation Tests + +## Re-recording VCR Cassettes + +When tests fail due to outdated VCR cassettes (e.g., API authentication errors or changed responses), follow these steps to re-record: + +### Prerequisites +1. Ensure `OPENAI_API_KEY` is set in your environment with a valid API key +2. The `passenv = OPENAI_API_KEY` directive must be present in the root `tox.ini` file + +### Steps to Re-record + +1. Delete the existing cassette file: +```bash +rm tests/openinference/haystack/cassettes/test_openai_document_embedder_embedding_span_has_expected_attributes.yaml +``` + +2. Run the test with VCR in record mode using tox: +```bash +OPENAI_API_KEY=$OPENAI_API_KEY uvx --with tox-uv tox -r -e py313-ci-haystack -- tests/openinference/haystack/test_instrumentor.py::test_openai_document_embedder_embedding_span_has_expected_attributes -xvs --vcr-record=once +``` + +### Important Notes +- The test reads `OPENAI_API_KEY` from the environment, falling back to "sk-test" if not set +- VCR will cache responses including authentication errors (401), so always delete the cassette before re-recording +- The `--vcr-record=once` flag ensures the cassette is only recorded when it doesn't exist +- Use `-r` flag with tox to ensure a clean environment when re-recording \ No newline at end of file diff --git a/python/instrumentation/openinference-instrumentation-haystack/tests/openinference/haystack/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-haystack/tests/openinference/haystack/test_instrumentor.py index 77a17ee1a..c6b26a487 100644 --- a/python/instrumentation/openinference-instrumentation-haystack/tests/openinference/haystack/test_instrumentor.py +++ b/python/instrumentation/openinference-instrumentation-haystack/tests/openinference/haystack/test_instrumentor.py @@ -716,7 +716,7 @@ def test_openai_document_embedder_embedding_span_has_expected_attributes( spans = in_memory_span_exporter.get_finished_spans() assert len(spans) == 2 span = spans[0] - assert span.name == "OpenAIDocumentEmbedder.run" + assert span.name == "CreateEmbeddings" assert span.status.is_ok assert not span.events attributes = dict(span.attributes or {}) @@ -748,6 +748,8 @@ def test_openai_document_embedder_embedding_span_has_expected_attributes( == "France won the World Cup in 2018." ) assert _is_vector(attributes.pop(f"{EMBEDDING_EMBEDDINGS}.1.{EMBEDDING_VECTOR}")) + assert attributes.pop(LLM_SYSTEM) == "OpenAIDocumentEmbedder" # Component class name + assert isinstance(attributes.pop(EMBEDDING_INVOCATION_PARAMETERS), str) assert not attributes @@ -898,6 +900,9 @@ def _is_vector( EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS EMBEDDING_MODEL_NAME = SpanAttributes.EMBEDDING_MODEL_NAME EMBEDDING_TEXT = EmbeddingAttributes.EMBEDDING_TEXT +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" +LLM_SYSTEM = SpanAttributes.LLM_SYSTEM EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE INPUT_VALUE = SpanAttributes.INPUT_VALUE diff --git a/python/instrumentation/openinference-instrumentation-litellm/src/openinference/instrumentation/litellm/__init__.py b/python/instrumentation/openinference-instrumentation-litellm/src/openinference/instrumentation/litellm/__init__.py index 4113b7169..12b89943f 100644 --- a/python/instrumentation/openinference-instrumentation-litellm/src/openinference/instrumentation/litellm/__init__.py +++ b/python/instrumentation/openinference-instrumentation-litellm/src/openinference/instrumentation/litellm/__init__.py @@ -1,4 +1,3 @@ -import json from enum import Enum from functools import wraps from typing import ( @@ -48,6 +47,9 @@ ToolCallAttributes, ) +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +_EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" + # Skip capture KEYS_TO_REDACT = ["api_key", "messages"] @@ -226,10 +228,39 @@ def _instrument_func_type_embedding(span: trace_api.Span, kwargs: Dict[str, Any] SpanAttributes.OPENINFERENCE_SPAN_KIND, OpenInferenceSpanKindValues.EMBEDDING.value, ) + _set_span_attribute(span, SpanAttributes.LLM_SYSTEM, "litellm") _set_span_attribute( span, SpanAttributes.EMBEDDING_MODEL_NAME, kwargs.get("model", "unknown_model") ) - _set_span_attribute(span, EmbeddingAttributes.EMBEDDING_TEXT, str(kwargs.get("input"))) + + # Extract invocation parameters (everything except input) + invocation_params = {k: v for k, v in kwargs.items() if k != "input"} + if invocation_params: + _set_span_attribute( + span, _EMBEDDING_INVOCATION_PARAMETERS, safe_json_dumps(invocation_params) + ) + + # Extract text from embedding input + embedding_input = kwargs.get("input") + if embedding_input is not None: + if isinstance(embedding_input, str): + # Single string input + _set_span_attribute( + span, + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}", + embedding_input, + ) + elif isinstance(embedding_input, list) and embedding_input: + # Check if it's a list of strings (not tokens) + if all(isinstance(item, str) for item in embedding_input): + # List of strings + for index, text in enumerate(embedding_input): + _set_span_attribute( + span, + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_TEXT}", + text, + ) + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(kwargs.get("input"))) @@ -262,12 +293,27 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: elif isinstance(result, EmbeddingResponse): if result_data := result.data: - first_embedding = result_data[0] - _set_span_attribute( - span, - EmbeddingAttributes.EMBEDDING_VECTOR, - json.dumps(first_embedding.get("embedding", [])), - ) + # Extract embedding vectors directly + for index, embedding_item in enumerate(result_data): + # LiteLLM returns dicts with 'embedding' key + raw_vector = ( + embedding_item.get("embedding") if hasattr(embedding_item, "get") else None + ) + if not raw_vector: + continue + + vector = None + # Check if it's a list of floats + if isinstance(raw_vector, (list, tuple)) and raw_vector: + if isinstance(raw_vector[0], (int, float)): + vector = tuple(raw_vector) + + if vector: + _set_span_attribute( + span, + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_VECTOR}", + vector, + ) elif isinstance(result, ImageResponse): if result.data and len(result.data) > 0: if img_data := result.data[0]: @@ -591,7 +637,7 @@ def _embedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingResponse: if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): return self.original_litellm_funcs["embedding"](*args, **kwargs) # type:ignore with self._tracer.start_as_current_span( - name="embedding", attributes=dict(get_attributes_from_context()) + name="CreateEmbeddings", attributes=dict(get_attributes_from_context()) ) as span: _instrument_func_type_embedding(span, kwargs) result = self.original_litellm_funcs["embedding"](*args, **kwargs) @@ -603,7 +649,7 @@ async def _aembedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingRespo if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): return self.original_litellm_funcs["aembedding"](*args, **kwargs) # type:ignore with self._tracer.start_as_current_span( - name="aembedding", attributes=dict(get_attributes_from_context()) + name="CreateEmbeddings", attributes=dict(get_attributes_from_context()) ) as span: _instrument_func_type_embedding(span, kwargs) result = await self.original_litellm_funcs["aembedding"](*args, **kwargs) diff --git a/python/instrumentation/openinference-instrumentation-litellm/tests/conftest.py b/python/instrumentation/openinference-instrumentation-litellm/tests/conftest.py index b8db92e28..c67911ad6 100644 --- a/python/instrumentation/openinference-instrumentation-litellm/tests/conftest.py +++ b/python/instrumentation/openinference-instrumentation-litellm/tests/conftest.py @@ -1,4 +1,4 @@ -from typing import Iterator +from typing import Generator, Iterator import pytest from opentelemetry.sdk.trace import TracerProvider @@ -22,6 +22,15 @@ def tracer_provider( return tracer_provider +@pytest.fixture +def setup_litellm_instrumentation( + tracer_provider: TracerProvider, +) -> Generator[None, None, None]: + LiteLLMInstrumentor().instrument(tracer_provider=tracer_provider) + yield + LiteLLMInstrumentor().uninstrument() + + @pytest.fixture(autouse=True) def uninstrument() -> Iterator[None]: yield diff --git a/python/instrumentation/openinference-instrumentation-litellm/tests/test_batch_embedding.py b/python/instrumentation/openinference-instrumentation-litellm/tests/test_batch_embedding.py new file mode 100644 index 000000000..7162f6663 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-litellm/tests/test_batch_embedding.py @@ -0,0 +1,156 @@ +"""Test batch embedding functionality for LiteLLM instrumentation.""" + +from typing import Any, cast +from unittest.mock import patch + +import litellm +from litellm import OpenAIChatCompletion # type: ignore[attr-defined] +from litellm.types.utils import EmbeddingResponse, Usage +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from openinference.semconv.trace import EmbeddingAttributes, SpanAttributes + + +def test_batch_embedding( + in_memory_span_exporter: InMemorySpanExporter, + setup_litellm_instrumentation: Any, +) -> None: + """Test that batch embeddings (multiple inputs) are properly instrumented.""" + in_memory_span_exporter.clear() + + # Mock response with multiple embeddings matching the input + mock_response_embedding = EmbeddingResponse( + model="text-embedding-ada-002", + data=[ + {"embedding": [0.1, 0.2, 0.3], "index": 0, "object": "embedding"}, + {"embedding": [0.4, 0.5, 0.6], "index": 1, "object": "embedding"}, + {"embedding": [0.7, 0.8, 0.9], "index": 2, "object": "embedding"}, + ], + object="list", + usage=Usage(prompt_tokens=18, completion_tokens=3, total_tokens=21), + ) + + input_texts = ["hello", "world", "test"] + + with patch.object(OpenAIChatCompletion, "embedding", return_value=mock_response_embedding): + litellm.embedding(model="text-embedding-ada-002", input=input_texts) + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == "CreateEmbeddings" + attributes = dict(cast(Any, span.attributes)) + + # Check model name + assert attributes.get(SpanAttributes.EMBEDDING_MODEL_NAME) == "text-embedding-ada-002" + + # Check each input text is recorded + for i, text in enumerate(input_texts): + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{i}.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == text + ) + + # Check each output vector is recorded + expected_vectors = [ + (0.1, 0.2, 0.3), + (0.4, 0.5, 0.6), + (0.7, 0.8, 0.9), + ] + for i, vector in enumerate(expected_vectors): + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{i}.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) + == vector + ) + + # Check token counts + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_PROMPT) == 18 + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION) == 3 + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_TOTAL) == 21 + + assert span.status.status_code == StatusCode.OK + + +def test_single_string_embedding( + in_memory_span_exporter: InMemorySpanExporter, + setup_litellm_instrumentation: Any, +) -> None: + """Test that single string input (not in list) is properly instrumented.""" + in_memory_span_exporter.clear() + + mock_response_embedding = EmbeddingResponse( + model="text-embedding-ada-002", + data=[ + {"embedding": [0.1, 0.2, 0.3], "index": 0, "object": "embedding"}, + ], + object="list", + usage=Usage(prompt_tokens=6, completion_tokens=1, total_tokens=7), + ) + + with patch.object(OpenAIChatCompletion, "embedding", return_value=mock_response_embedding): + litellm.embedding(model="text-embedding-ada-002", input="hello world") + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + attributes = dict(cast(Any, span.attributes)) + + # Single string should still be recorded at index 0 + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "hello world" + ) + assert attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) == (0.1, 0.2, 0.3) + + +def test_token_ids_embedding_no_text_attributes( + in_memory_span_exporter: InMemorySpanExporter, + setup_litellm_instrumentation: Any, +) -> None: + """Test that token IDs (integers) as input do NOT produce text attributes.""" + in_memory_span_exporter.clear() + + mock_response_embedding = EmbeddingResponse( + model="text-embedding-ada-002", + data=[ + {"embedding": [0.1, 0.2, 0.3], "index": 0, "object": "embedding"}, + ], + object="list", + usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4), + ) + + # Input as token IDs (integers) instead of text + token_ids = [15339, 1917, 123] # Example token IDs + + with patch.object(OpenAIChatCompletion, "embedding", return_value=mock_response_embedding): + litellm.embedding(model="text-embedding-ada-002", input=token_ids) + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + attributes = dict(cast(Any, span.attributes)) + + # Token IDs should NOT produce text attributes + assert ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + not in attributes + ), "Token IDs should not produce text attributes" + + # But vectors should still be recorded + assert attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) == (0.1, 0.2, 0.3) + + # Model name and token counts should still be present + assert attributes.get(SpanAttributes.EMBEDDING_MODEL_NAME) == "text-embedding-ada-002" + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_PROMPT) == 3 + assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_TOTAL) == 4 diff --git a/python/instrumentation/openinference-instrumentation-litellm/tests/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-litellm/tests/test_instrumentor.py index 15b2a2262..3be2cf2cb 100644 --- a/python/instrumentation/openinference-instrumentation-litellm/tests/test_instrumentor.py +++ b/python/instrumentation/openinference-instrumentation-litellm/tests/test_instrumentor.py @@ -821,12 +821,25 @@ def test_embedding( spans = in_memory_span_exporter.get_finished_spans() assert len(spans) == 1 span = spans[0] - assert span.name == "embedding" + assert span.name == "CreateEmbeddings" attributes = dict(cast(Mapping[str, AttributeValue], span.attributes)) assert attributes.get(SpanAttributes.EMBEDDING_MODEL_NAME) == "text-embedding-ada-002" assert attributes.get(SpanAttributes.INPUT_VALUE) == str(["good morning from litellm"]) + assert attributes.get(SpanAttributes.LLM_SYSTEM) == "litellm" + # TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv + assert ( + attributes.get("embedding.invocation_parameters") == '{"model": "text-embedding-ada-002"}' + ) - assert attributes.get(EmbeddingAttributes.EMBEDDING_VECTOR) == str([0.1, 0.2, 0.3]) + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "good morning from litellm" + ) + assert attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) == (0.1, 0.2, 0.3) assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_PROMPT) == 6 assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION) == 1 assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_TOTAL) == 6 @@ -858,9 +871,18 @@ def test_embedding_with_invalid_model_triggers_exception_event( assert len(spans) == 1, "Expected one span to be recorded" span = spans[0] - assert span.name == "embedding" + assert span.name == "CreateEmbeddings" assert span.status.status_code == StatusCode.ERROR + attributes = dict(cast(Mapping[str, AttributeValue], span.attributes)) + # Check that embedding text is still recorded even on error (recorded in request phase) + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "good morning from litellm" + ) + exception_events = [e for e in span.events if e.name == "exception"] assert len(exception_events) == 1, "Expected one exception event to be recorded" @@ -914,12 +936,25 @@ async def test_aembedding( spans = in_memory_span_exporter.get_finished_spans() assert len(spans) == 1 span = spans[0] - assert span.name == "aembedding" + assert span.name == "CreateEmbeddings" attributes = dict(cast(Mapping[str, AttributeValue], span.attributes)) assert attributes.get(SpanAttributes.EMBEDDING_MODEL_NAME) == "text-embedding-ada-002" assert attributes.get(SpanAttributes.INPUT_VALUE) == str(["good morning from litellm"]) + assert attributes.get(SpanAttributes.LLM_SYSTEM) == "litellm" + # TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv + assert ( + attributes.get("embedding.invocation_parameters") == '{"model": "text-embedding-ada-002"}' + ) - assert attributes.get(EmbeddingAttributes.EMBEDDING_VECTOR) == str([0.1, 0.2, 0.3]) + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "good morning from litellm" + ) + assert attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" + ) == (0.1, 0.2, 0.3) assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_PROMPT) == 6 assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION) == 1 assert attributes.get(SpanAttributes.LLM_TOKEN_COUNT_TOTAL) == 6 @@ -951,9 +986,18 @@ async def test_aembedding_with_invalid_model_triggers_exception_event( assert len(spans) == 1, "Expected one span to be recorded" span = spans[0] - assert span.name == "aembedding" + assert span.name == "CreateEmbeddings" assert span.status.status_code == StatusCode.ERROR + # Check that embedding text is still recorded even on error (recorded in request phase) + attributes = dict(cast(Mapping[str, AttributeValue], span.attributes)) + assert ( + attributes.get( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + == "good morning from litellm" + ) + exception_events = [e for e in span.events if e.name == "exception"] assert len(exception_events) == 1, "Expected one exception event to be recorded" diff --git a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request.py b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request.py index 176d2701a..407138fbd 100644 --- a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request.py +++ b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request.py @@ -302,8 +302,12 @@ def __call__( return wrapped(*args, **kwargs) try: cast_to, request_parameters = _parse_request_args(args) - # E.g. cast_to = openai.types.chat.ChatCompletion => span_name = "ChatCompletion" - span_name: str = cast_to.__name__.split(".")[-1] + # Use consistent span names: "CreateEmbeddings" for embeddings, class name for others + if cast_to is self._openai.types.CreateEmbeddingResponse: + span_name = "CreateEmbeddings" + else: + # E.g. cast_to = openai.types.chat.ChatCompletion => span_name = "ChatCompletion" + span_name = cast_to.__name__.split(".")[-1] except Exception: logger.exception("Failed to parse request args") return wrapped(*args, **kwargs) @@ -359,8 +363,12 @@ async def __call__( return await wrapped(*args, **kwargs) try: cast_to, request_parameters = _parse_request_args(args) - # E.g. cast_to = openai.types.chat.ChatCompletion => span_name = "ChatCompletion" - span_name: str = cast_to.__name__.split(".")[-1] + # Use consistent span names: "CreateEmbeddings" for embeddings, class name for others + if cast_to is self._openai.types.CreateEmbeddingResponse: + span_name = "CreateEmbeddings" + else: + # E.g. cast_to = openai.types.chat.ChatCompletion => span_name = "ChatCompletion" + span_name = cast_to.__name__.split(".")[-1] except Exception: logger.exception("Failed to parse request args") return await wrapped(*args, **kwargs) diff --git a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request_attributes_extractor.py b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request_attributes_extractor.py index 74b051484..116753403 100644 --- a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request_attributes_extractor.py +++ b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request_attributes_extractor.py @@ -22,6 +22,7 @@ from openinference.instrumentation.openai._attributes._responses_api import _ResponsesApiAttributes from openinference.instrumentation.openai._utils import _get_openai_version from openinference.semconv.trace import ( + EmbeddingAttributes, ImageAttributes, MessageAttributes, MessageContentAttributes, @@ -29,6 +30,9 @@ ToolCallAttributes, ) +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +_EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" + if TYPE_CHECKING: from openai.types import Completion, CreateEmbeddingResponse from openai.types.chat import ChatCompletion @@ -212,6 +216,16 @@ def _get_attributes_from_completion_create_param( invocation_params.pop("prompt", None) yield SpanAttributes.LLM_INVOCATION_PARAMETERS, safe_json_dumps(invocation_params) + model_prompt = params.get("prompt") + if isinstance(model_prompt, str): + yield SpanAttributes.LLM_PROMPTS, [model_prompt] + elif ( + isinstance(model_prompt, list) + and model_prompt + and all(isinstance(item, str) for item in model_prompt) + ): + yield SpanAttributes.LLM_PROMPTS, model_prompt + def _get_attributes_from_embedding_create_param( params: Mapping[str, Any], @@ -222,7 +236,26 @@ def _get_attributes_from_embedding_create_param( return invocation_params = dict(params) invocation_params.pop("input", None) - yield SpanAttributes.LLM_INVOCATION_PARAMETERS, safe_json_dumps(invocation_params) + yield _EMBEDDING_INVOCATION_PARAMETERS, safe_json_dumps(invocation_params) + + # Extract text from embedding input - only records text, not token IDs + embedding_input = params.get("input") + if embedding_input is not None: + if isinstance(embedding_input, str): + # Single string input + yield ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_TEXT}", + embedding_input, + ) + elif isinstance(embedding_input, list) and embedding_input: + # Check if it's a list of strings (not tokens) + if all(isinstance(item, str) for item in embedding_input): + # List of strings + for index, text in enumerate(embedding_input): + yield ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_TEXT}", + text, + ) T = TypeVar("T", bound=type) diff --git a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_response_attributes_extractor.py b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_response_attributes_extractor.py index 152caef25..16544306a 100644 --- a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_response_attributes_extractor.py +++ b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_response_attributes_extractor.py @@ -1,8 +1,6 @@ from __future__ import annotations -import base64 import logging -from importlib import import_module from types import ModuleType from typing import ( TYPE_CHECKING, @@ -10,8 +8,6 @@ Iterable, Iterator, Mapping, - Optional, - Sequence, Tuple, Type, ) @@ -19,7 +15,7 @@ from opentelemetry.util.types import AttributeValue from openinference.instrumentation.openai._attributes._responses_api import _ResponsesApiAttributes -from openinference.instrumentation.openai._utils import _get_openai_version, _get_texts +from openinference.instrumentation.openai._utils import _get_openai_version from openinference.semconv.trace import ( EmbeddingAttributes, MessageAttributes, @@ -37,11 +33,6 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -try: - _NUMPY: Optional[ModuleType] = import_module("numpy") -except ImportError: - _NUMPY = None - class _ResponseAttributesExtractor: __slots__ = ( @@ -79,12 +70,10 @@ def get_attributes_from_response( elif isinstance(response, self._create_embedding_response_type): yield from self._get_attributes_from_create_embedding_response( response=response, - request_parameters=request_parameters, ) elif isinstance(response, self._completion_type): yield from self._get_attributes_from_completion( completion=response, - request_parameters=request_parameters, ) def _get_attributes_from_responses_response( @@ -116,26 +105,16 @@ def _get_attributes_from_chat_completion( def _get_attributes_from_completion( self, completion: "Completion", - request_parameters: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/completion.py#L13 # noqa: E501 if model := getattr(completion, "model", None): yield SpanAttributes.LLM_MODEL_NAME, model if usage := getattr(completion, "usage", None): yield from self._get_attributes_from_completion_usage(usage) - if model_prompt := request_parameters.get("prompt"): - # FIXME: this step should move to request attributes extractor if decoding is not necessary.# noqa: E501 - # prompt: Required[Union[str, List[str], List[int], List[List[int]], None]] - # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/completion_create_params.py#L38 - # FIXME: tokens (List[int], List[List[int]]) can't be decoded reliably because model - # names are not reliable (across OpenAI and Azure). - if prompts := list(_get_texts(model_prompt, model)): - yield SpanAttributes.LLM_PROMPTS, prompts def _get_attributes_from_create_embedding_response( self, response: "CreateEmbeddingResponse", - request_parameters: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/create_embedding_response.py#L20 # noqa: E501 if usage := getattr(response, "usage", None): @@ -144,48 +123,37 @@ def _get_attributes_from_create_embedding_response( if model := getattr(response, "model"): yield f"{SpanAttributes.EMBEDDING_MODEL_NAME}", model if (data := getattr(response, "data", None)) and isinstance(data, Iterable): - for embedding in data: - if (index := getattr(embedding, "index", None)) is None: + # Extract embedding vectors directly + for index, embedding_item in enumerate(data): + raw_vector = getattr(embedding_item, "embedding", None) + if not raw_vector: continue - for key, value in self._get_attributes_from_embedding(embedding): - yield f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{key}", value - embedding_input = request_parameters.get("input") - for index, text in enumerate(_get_texts(embedding_input, model)): - # FIXME: this step should move to request attributes extractor if decoding is not necessary.# noqa: E501 - # input: Required[Union[str, List[str], List[int], List[List[int]]]] - # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/embedding_create_params.py#L12 - # FIXME: tokens (List[int], List[List[int]]) can't be decoded reliably because model - # names are not reliable (across OpenAI and Azure). - yield ( - ( - f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}." - f"{EmbeddingAttributes.EMBEDDING_TEXT}" - ), - text, - ) - - def _get_attributes_from_embedding( - self, - embedding: object, - ) -> Iterator[Tuple[str, AttributeValue]]: - # openai.types.Embedding - # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/embedding.py#L11 # noqa: E501 - if not (_vector := getattr(embedding, "embedding", None)): - return - if isinstance(_vector, Sequence) and len(_vector) and isinstance(_vector[0], float): - vector = list(_vector) - yield f"{EmbeddingAttributes.EMBEDDING_VECTOR}", vector - elif isinstance(_vector, str) and _vector and _NUMPY: - # FIXME: this step should be removed if decoding is not necessary. - try: - # See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/resources/embeddings.py#L100 # noqa: E501 - vector = _NUMPY.frombuffer(base64.b64decode(_vector), dtype="float32").tolist() - except Exception: - logger.exception("Failed to decode embedding") - pass - else: - yield f"{EmbeddingAttributes.EMBEDDING_VECTOR}", vector + vector = None + # Check if it's a list of floats + if isinstance(raw_vector, (list, tuple)) and raw_vector: + if isinstance(raw_vector[0], (int, float)): + vector = list(raw_vector) + elif isinstance(raw_vector, str) and raw_vector: + # Base64-encoded vector (when encoding_format="base64") + try: + import base64 + import struct + + # Decode base64 to float32 array + decoded = base64.b64decode(raw_vector) + # Unpack as float32 values + num_floats = len(decoded) // 4 + vector = list(struct.unpack(f"{num_floats}f", decoded)) + except Exception: + # If decoding fails, skip this vector + continue + + if vector: + yield ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_VECTOR}", + vector, + ) def _get_attributes_from_chat_completion_message( self, diff --git a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_utils.py b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_utils.py index dd02674bf..c42a9bf66 100644 --- a/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_utils.py +++ b/python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_utils.py @@ -5,14 +5,12 @@ from typing import ( Any, Iterator, - List, Mapping, NamedTuple, Optional, Protocol, Sequence, Tuple, - Union, cast, ) @@ -110,26 +108,3 @@ def _finish_tracing( ) except Exception: logger.exception("Failed to finish tracing") - - -def _get_texts( - model_input: Optional[Union[str, List[str], List[int], List[List[int]]]], - model: Optional[str], -) -> Iterator[str]: - if not model_input: - return - if isinstance(model_input, str): - text = model_input - yield text - return - if not isinstance(model_input, Sequence): - return - if any(not isinstance(item, str) for item in model_input): - # FIXME: We can't decode tokens (List[int]) reliably because the model name is not reliable, - # e.g. for text-embedding-ada-002 (cl100k_base), OpenAI returns "text-embedding-ada-002-v2", - # and Azure returns "ada", which refers to a different model (r50k_base). We could use the - # request model name instead, but that doesn't work for Azure because Azure uses the - # deployment name (which differs from the model name). - return - for text in cast(List[str], model_input): - yield text diff --git a/python/instrumentation/openinference-instrumentation-openai/tests/openinference/instrumentation/openai/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-openai/tests/openinference/instrumentation/openai/test_instrumentor.py index d3070b44c..660671afe 100644 --- a/python/instrumentation/openinference-instrumentation-openai/tests/openinference/instrumentation/openai/test_instrumentor.py +++ b/python/instrumentation/openinference-instrumentation-openai/tests/openinference/instrumentation/openai/test_instrumentor.py @@ -388,10 +388,11 @@ async def task() -> None: ) assert isinstance(attributes.pop(INPUT_VALUE, None), str) assert isinstance(attributes.pop(INPUT_MIME_TYPE, None), str) + # Prompts are recorded in request phase, so present regardless of status + assert list(cast(Sequence[str], attributes.pop(LLM_PROMPTS, None))) == prompt if status_code == 200: assert isinstance(attributes.pop(OUTPUT_VALUE, None), str) assert isinstance(attributes.pop(OUTPUT_MIME_TYPE, None), str) - assert list(cast(Sequence[str], attributes.pop(LLM_PROMPTS, None))) == prompt if not is_stream: # Usage is not available for streaming in general. assert attributes.pop(LLM_TOKEN_COUNT_TOTAL, None) == completion_usage["total_tokens"] @@ -448,7 +449,15 @@ def test_embeddings( "prompt_tokens": random.randint(10, 100), "total_tokens": random.randint(10, 100), } - output_embeddings = [("AACAPwAAAEA=", (1.0, 2.0)), ((2.0, 3.0), (2.0, 3.0))] + # Match output structure to input structure + num_inputs = len(input_text) if isinstance(input_text, list) else 1 + # First element is for API response, second is for verification + output_embeddings: list[tuple[Any, tuple[float, ...]]] + if encoding_format == "base64": + output_embeddings = [("AACAPwAAAEA=", (1.0, 2.0)) for _ in range(num_inputs)] + else: + # API returns list, verification expects tuple + output_embeddings = [([1.0 + i, 2.0 + i], (1.0 + i, 2.0 + i)) for i in range(num_inputs)] url = urljoin(base_url, "embeddings") respx_mock.post(url).mock( return_value=Response( @@ -506,19 +515,20 @@ async def task() -> None: ) assert attributes.pop(LLM_SYSTEM, None) == LLM_SYSTEM_OPENAI assert ( - json.loads(cast(str, attributes.pop(LLM_INVOCATION_PARAMETERS, None))) + json.loads(cast(str, attributes.pop(EMBEDDING_INVOCATION_PARAMETERS, None))) == invocation_parameters ) assert isinstance(attributes.pop(INPUT_VALUE, None), str) assert isinstance(attributes.pop(INPUT_MIME_TYPE, None), str) + # Text attributes are recorded in request phase, so they're present regardless of status + for i, text in enumerate(input_text if isinstance(input_text, list) else [input_text]): + assert attributes.pop(f"{EMBEDDING_EMBEDDINGS}.{i}.{EMBEDDING_TEXT}", None) == text if status_code == 200: assert isinstance(attributes.pop(OUTPUT_VALUE, None), str) assert isinstance(attributes.pop(OUTPUT_MIME_TYPE, None), str) assert attributes.pop(EMBEDDING_MODEL_NAME, None) == embedding_model_name assert attributes.pop(LLM_TOKEN_COUNT_TOTAL, None) == embedding_usage["total_tokens"] assert attributes.pop(LLM_TOKEN_COUNT_PROMPT, None) == embedding_usage["prompt_tokens"] - for i, text in enumerate(input_text if isinstance(input_text, list) else [input_text]): - assert attributes.pop(f"{EMBEDDING_EMBEDDINGS}.{i}.{EMBEDDING_TEXT}", None) == text for i, embedding in enumerate(output_embeddings): assert ( attributes.pop(f"{EMBEDDING_EMBEDDINGS}.{i}.{EMBEDDING_VECTOR}", None) @@ -1755,6 +1765,8 @@ def tool_call_function_arguments(prefix: str, i: int, j: int) -> str: TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS EMBEDDING_MODEL_NAME = SpanAttributes.EMBEDDING_MODEL_NAME +# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv +EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR EMBEDDING_TEXT = EmbeddingAttributes.EMBEDDING_TEXT SESSION_ID = SpanAttributes.SESSION_ID diff --git a/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py index 63ea69658..f77535842 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py @@ -105,7 +105,7 @@ def anthropic_api_key(monkeypatch: pytest.MonkeyPatch) -> str: class TestInstrumentor: def test_entrypoint_for_opentelemetry_instrument(self) -> None: - (instrumentor_entrypoint,) = entry_points( + (instrumentor_entrypoint,) = entry_points( # type: ignore[no-untyped-call] group="opentelemetry_instrumentor", name="smolagents" ) instrumentor = instrumentor_entrypoint.load()() diff --git a/python/openinference-instrumentation/src/openinference/instrumentation/config.py b/python/openinference-instrumentation/src/openinference/instrumentation/config.py index 3bc1cf134..23d127d81 100644 --- a/python/openinference-instrumentation/src/openinference/instrumentation/config.py +++ b/python/openinference-instrumentation/src/openinference/instrumentation/config.py @@ -1,5 +1,5 @@ import os -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, fields from types import TracebackType from typing import ( Any, @@ -64,44 +64,24 @@ def __aexit__( detach(self._token) -OPENINFERENCE_HIDE_LLM_INVOCATION_PARAMETERS = "OPENINFERENCE_HIDE_LLM_INVOCATION_PARAMETERS" -OPENINFERENCE_HIDE_INPUTS = "OPENINFERENCE_HIDE_INPUTS" -# Hides input value & messages -OPENINFERENCE_HIDE_OUTPUTS = "OPENINFERENCE_HIDE_OUTPUTS" -# Hides output value & messages -OPENINFERENCE_HIDE_INPUT_MESSAGES = "OPENINFERENCE_HIDE_INPUT_MESSAGES" -# Hides all input messages -OPENINFERENCE_HIDE_OUTPUT_MESSAGES = "OPENINFERENCE_HIDE_OUTPUT_MESSAGES" -# Hides all output messages -OPENINFERENCE_HIDE_INPUT_IMAGES = "OPENINFERENCE_HIDE_INPUT_IMAGES" -# Hides images from input messages -OPENINFERENCE_HIDE_INPUT_TEXT = "OPENINFERENCE_HIDE_INPUT_TEXT" -# Hides text from input messages -OPENINFERENCE_HIDE_OUTPUT_TEXT = "OPENINFERENCE_HIDE_OUTPUT_TEXT" -# Hides text from output messages -OPENINFERENCE_HIDE_EMBEDDING_VECTORS = "OPENINFERENCE_HIDE_EMBEDDING_VECTORS" -# Hides embedding vectors -OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH = "OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH" -# Limits characters of a base64 encoding of an image -OPENINFERENCE_HIDE_PROMPTS = "OPENINFERENCE_HIDE_PROMPTS" -# Hides LLM prompts +# Replacement value for hidden/redacted sensitive data REDACTED_VALUE = "__REDACTED__" -# When a value is hidden, it will be replaced by this redacted value -DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS = False -DEFAULT_HIDE_PROMPTS = False -DEFAULT_HIDE_INPUTS = False -DEFAULT_HIDE_OUTPUTS = False - -DEFAULT_HIDE_INPUT_MESSAGES = False -DEFAULT_HIDE_OUTPUT_MESSAGES = False - -DEFAULT_HIDE_INPUT_IMAGES = False -DEFAULT_HIDE_INPUT_TEXT = False -DEFAULT_HIDE_OUTPUT_TEXT = False - -DEFAULT_HIDE_EMBEDDING_VECTORS = False -DEFAULT_BASE64_IMAGE_MAX_LENGTH = 32_000 +# Default values for trace configuration, keyed by field names in TraceConfig +_TRACE_CONFIG_DEFAULTS = { + "hide_llm_invocation_parameters": False, + "hide_inputs": False, + "hide_outputs": False, + "hide_input_messages": False, + "hide_output_messages": False, + "hide_input_images": False, + "hide_input_text": False, + "hide_output_text": False, + "hide_embeddings_vectors": False, + "hide_embeddings_text": False, + "hide_prompts": False, + "base64_image_max_length": 32_000, # 32KB +} @dataclass(frozen=True) @@ -113,108 +93,43 @@ class TraceConfig: encoded images to reduce payloads. For those attributes not passed, this object tries to read from designated - environment variables and, if not found, has default values that maximize - observability. + environment variables (computed as f"OPENINFERENCE_{field_name.upper()}") and, + if not found, falls back to default values that maximize observability. """ - hide_llm_invocation_parameters: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_LLM_INVOCATION_PARAMETERS, - "default_value": DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS, - }, - ) - hide_inputs: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_INPUTS, - "default_value": DEFAULT_HIDE_INPUTS, - }, - ) - """Hides input value & messages""" - hide_outputs: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_OUTPUTS, - "default_value": DEFAULT_HIDE_OUTPUTS, - }, - ) - """Hides output value & messages""" - hide_input_messages: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_INPUT_MESSAGES, - "default_value": DEFAULT_HIDE_INPUT_MESSAGES, - }, - ) - """Hides all input messages""" - hide_output_messages: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_OUTPUT_MESSAGES, - "default_value": DEFAULT_HIDE_OUTPUT_MESSAGES, - }, - ) - """Hides all output messages""" - hide_input_images: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_INPUT_IMAGES, - "default_value": DEFAULT_HIDE_INPUT_IMAGES, - }, - ) - """Hides images from input messages""" - hide_input_text: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_INPUT_TEXT, - "default_value": DEFAULT_HIDE_INPUT_TEXT, - }, - ) - """Hides text from input messages""" - hide_output_text: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_OUTPUT_TEXT, - "default_value": DEFAULT_HIDE_OUTPUT_TEXT, - }, - ) - """Hides text from output messages""" - hide_embedding_vectors: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_EMBEDDING_VECTORS, - "default_value": DEFAULT_HIDE_EMBEDDING_VECTORS, - }, - ) - """Hides embedding vectors""" - hide_prompts: Optional[bool] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_HIDE_PROMPTS, - "default_value": DEFAULT_HIDE_PROMPTS, - }, - ) - """Hides LLM prompts""" - base64_image_max_length: Optional[int] = field( - default=None, - metadata={ - "env_var": OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, - "default_value": DEFAULT_BASE64_IMAGE_MAX_LENGTH, - }, - ) - """Limits characters of a base64 encoding of an image""" + hide_llm_invocation_parameters: Optional[bool] = None + """Removes llm.invocation_parameters attribute entirely from spans""" + hide_inputs: Optional[bool] = None + """Replaces input.value with REDACTED_VALUE and removes input.mime_type""" + hide_outputs: Optional[bool] = None + """Replaces output.value with REDACTED_VALUE and removes output.mime_type""" + hide_input_messages: Optional[bool] = None + """Removes all llm.input_messages attributes entirely from spans""" + hide_output_messages: Optional[bool] = None + """Removes all llm.output_messages attributes entirely from spans""" + hide_input_images: Optional[bool] = None + """Removes image URLs from llm.input_messages message content blocks""" + hide_input_text: Optional[bool] = None + """Replaces text content in llm.input_messages with REDACTED_VALUE""" + hide_output_text: Optional[bool] = None + """Replaces text content in llm.output_messages with REDACTED_VALUE""" + hide_embeddings_vectors: Optional[bool] = None + """Replaces embedding.embeddings.*.embedding.vector values with REDACTED_VALUE""" + hide_embeddings_text: Optional[bool] = None + """Replaces embedding.embeddings.*.embedding.text values with REDACTED_VALUE""" + hide_prompts: Optional[bool] = None + """Replaces llm.prompts values with REDACTED_VALUE""" + base64_image_max_length: Optional[int] = None + """Truncates base64-encoded images to this length, replacing excess with REDACTED_VALUE""" def __post_init__(self) -> None: for f in fields(self): expected_type = get_args(f.type)[0] - # Optional is Union[T,NoneType]. get_args()returns (T, NoneType). - # We collect the first type self._parse_value( f.name, expected_type, - f.metadata["env_var"], - f.metadata["default_value"], + f"OPENINFERENCE_{f.name.upper()}", + _TRACE_CONFIG_DEFAULTS[f.name], ) def mask( @@ -283,11 +198,17 @@ def mask( ): value = REDACTED_VALUE elif ( - self.hide_embedding_vectors + self.hide_embeddings_vectors and SpanAttributes.EMBEDDING_EMBEDDINGS in key and EmbeddingAttributes.EMBEDDING_VECTOR in key ): - return None + value = REDACTED_VALUE + elif ( + self.hide_embeddings_text + and SpanAttributes.EMBEDDING_EMBEDDINGS in key + and EmbeddingAttributes.EMBEDDING_TEXT in key + ): + value = REDACTED_VALUE return value() if callable(value) else value def _parse_value( diff --git a/python/openinference-instrumentation/tests/test_config.py b/python/openinference-instrumentation/tests/test_config.py index 62b33e542..91e44541d 100644 --- a/python/openinference-instrumentation/tests/test_config.py +++ b/python/openinference-instrumentation/tests/test_config.py @@ -14,25 +14,7 @@ from openinference.instrumentation import OITracer, TraceConfig from openinference.instrumentation._spans import _IMPORTANT_ATTRIBUTES # type:ignore[attr-defined] from openinference.instrumentation.config import ( - DEFAULT_BASE64_IMAGE_MAX_LENGTH, - DEFAULT_HIDE_INPUT_IMAGES, - DEFAULT_HIDE_INPUT_MESSAGES, - DEFAULT_HIDE_INPUT_TEXT, - DEFAULT_HIDE_INPUTS, - DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS, - DEFAULT_HIDE_OUTPUT_MESSAGES, - DEFAULT_HIDE_OUTPUT_TEXT, - DEFAULT_HIDE_OUTPUTS, - DEFAULT_HIDE_PROMPTS, - OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, - OPENINFERENCE_HIDE_INPUT_IMAGES, - OPENINFERENCE_HIDE_INPUT_MESSAGES, - OPENINFERENCE_HIDE_INPUT_TEXT, - OPENINFERENCE_HIDE_INPUTS, - OPENINFERENCE_HIDE_OUTPUT_MESSAGES, - OPENINFERENCE_HIDE_OUTPUT_TEXT, - OPENINFERENCE_HIDE_OUTPUTS, - OPENINFERENCE_HIDE_PROMPTS, + _TRACE_CONFIG_DEFAULTS, REDACTED_VALUE, ) from openinference.semconv.trace import SpanAttributes @@ -40,16 +22,19 @@ def test_default_settings() -> None: config = TraceConfig() - assert config.hide_llm_invocation_parameters == DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS - assert config.hide_inputs == DEFAULT_HIDE_INPUTS - assert config.hide_outputs == DEFAULT_HIDE_OUTPUTS - assert config.hide_input_messages == DEFAULT_HIDE_INPUT_MESSAGES - assert config.hide_output_messages == DEFAULT_HIDE_OUTPUT_MESSAGES - assert config.hide_input_images == DEFAULT_HIDE_INPUT_IMAGES - assert config.hide_input_text == DEFAULT_HIDE_INPUT_TEXT - assert config.hide_output_text == DEFAULT_HIDE_OUTPUT_TEXT - assert config.hide_prompts == DEFAULT_HIDE_PROMPTS - assert config.base64_image_max_length == DEFAULT_BASE64_IMAGE_MAX_LENGTH + assert ( + config.hide_llm_invocation_parameters + == _TRACE_CONFIG_DEFAULTS["hide_llm_invocation_parameters"] + ) + assert config.hide_inputs == _TRACE_CONFIG_DEFAULTS["hide_inputs"] + assert config.hide_outputs == _TRACE_CONFIG_DEFAULTS["hide_outputs"] + assert config.hide_input_messages == _TRACE_CONFIG_DEFAULTS["hide_input_messages"] + assert config.hide_output_messages == _TRACE_CONFIG_DEFAULTS["hide_output_messages"] + assert config.hide_input_images == _TRACE_CONFIG_DEFAULTS["hide_input_images"] + assert config.hide_input_text == _TRACE_CONFIG_DEFAULTS["hide_input_text"] + assert config.hide_output_text == _TRACE_CONFIG_DEFAULTS["hide_output_text"] + assert config.hide_prompts == _TRACE_CONFIG_DEFAULTS["hide_prompts"] + assert config.base64_image_max_length == _TRACE_CONFIG_DEFAULTS["base64_image_max_length"] def test_oi_tracer( @@ -135,27 +120,27 @@ def test_settings_from_env_vars_and_code( monkeypatch: pytest.MonkeyPatch, ) -> None: # First part of the test verifies that environment variables are read correctly - monkeypatch.setenv(OPENINFERENCE_HIDE_INPUTS, str(hide_inputs)) - monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUTS, str(hide_outputs)) - monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_MESSAGES, str(hide_input_messages)) - monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_MESSAGES, str(hide_output_messages)) - monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_IMAGES, str(hide_input_images)) - monkeypatch.setenv(OPENINFERENCE_HIDE_PROMPTS, str(hide_prompts)) - monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_TEXT, str(hide_input_text)) - monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_TEXT, str(hide_output_text)) - monkeypatch.setenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, str(base64_image_max_length)) + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUTS", str(hide_inputs)) + monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUTS", str(hide_outputs)) + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUT_MESSAGES", str(hide_input_messages)) + monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUT_MESSAGES", str(hide_output_messages)) + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUT_IMAGES", str(hide_input_images)) + monkeypatch.setenv("OPENINFERENCE_HIDE_PROMPTS", str(hide_prompts)) + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUT_TEXT", str(hide_input_text)) + monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUT_TEXT", str(hide_output_text)) + monkeypatch.setenv("OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH", str(base64_image_max_length)) config = TraceConfig() - assert config.hide_inputs is parse_bool_from_env(OPENINFERENCE_HIDE_INPUTS) - assert config.hide_outputs is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUTS) - assert config.hide_input_messages is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_MESSAGES) - assert config.hide_output_messages is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUT_MESSAGES) - assert config.hide_input_images is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_IMAGES) - assert config.hide_input_text is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_TEXT) - assert config.hide_output_text is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUT_TEXT) - assert config.hide_prompts is parse_bool_from_env(OPENINFERENCE_HIDE_PROMPTS) + assert config.hide_inputs is parse_bool_from_env("OPENINFERENCE_HIDE_INPUTS") + assert config.hide_outputs is parse_bool_from_env("OPENINFERENCE_HIDE_OUTPUTS") + assert config.hide_input_messages is parse_bool_from_env("OPENINFERENCE_HIDE_INPUT_MESSAGES") + assert config.hide_output_messages is parse_bool_from_env("OPENINFERENCE_HIDE_OUTPUT_MESSAGES") + assert config.hide_input_images is parse_bool_from_env("OPENINFERENCE_HIDE_INPUT_IMAGES") + assert config.hide_input_text is parse_bool_from_env("OPENINFERENCE_HIDE_INPUT_TEXT") + assert config.hide_output_text is parse_bool_from_env("OPENINFERENCE_HIDE_OUTPUT_TEXT") + assert config.hide_prompts is parse_bool_from_env("OPENINFERENCE_HIDE_PROMPTS") assert config.base64_image_max_length == int( - os.getenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, default=-1) + os.getenv("OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH", default=-1) ) # This next part of the text verifies that the code specified values overwrite diff --git a/python/openinference-semantic-conventions/src/openinference/semconv/trace/__init__.py b/python/openinference-semantic-conventions/src/openinference/semconv/trace/__init__.py index 13ef59067..c384a4251 100644 --- a/python/openinference-semantic-conventions/src/openinference/semconv/trace/__init__.py +++ b/python/openinference-semantic-conventions/src/openinference/semconv/trace/__init__.py @@ -24,6 +24,10 @@ class SpanAttributes: """ The name of the embedding model. """ + EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters" + """ + Invocation parameters passed to the embedding model or API, such as the model name, encoding format, etc. + """ LLM_FUNCTION_CALL = "llm.function_call" """ diff --git a/python/openinference-semantic-conventions/tests/openinference/semconv/test_attributes.py b/python/openinference-semantic-conventions/tests/openinference/semconv/test_attributes.py index 99b670c75..0013a700c 100644 --- a/python/openinference-semantic-conventions/tests/openinference/semconv/test_attributes.py +++ b/python/openinference-semantic-conventions/tests/openinference/semconv/test_attributes.py @@ -84,6 +84,7 @@ def test_nesting(self) -> None: }, "embedding": { "embeddings": SpanAttributes.EMBEDDING_EMBEDDINGS, + "invocation_parameters": SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS, "model_name": SpanAttributes.EMBEDDING_MODEL_NAME, }, "graph": { diff --git a/spec/configuration.md b/spec/configuration.md index 81b0f7ec2..200863c16 100644 --- a/spec/configuration.md +++ b/spec/configuration.md @@ -8,17 +8,18 @@ The possible settings are: | Environment Variable Name | Effect | Type | Default | |----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|------|---------| -| OPENINFERENCE_HIDE_LLM_INVOCATION_PARAMETERS | Hides LLM invocation parameters (independent of input/output hiding) | bool | False | -| OPENINFERENCE_HIDE_INPUTS | Hides input.value and all input messages (input messages are hidden if either HIDE_INPUTS OR HIDE_INPUT_MESSAGES is true) | bool | False | -| OPENINFERENCE_HIDE_OUTPUTS | Hides output.value and all output messages (output messages are hidden if either HIDE_OUTPUTS OR HIDE_OUTPUT_MESSAGES is true) | bool | False | -| OPENINFERENCE_HIDE_INPUT_MESSAGES | Hides all input messages (independent of HIDE_INPUTS) | bool | False | -| OPENINFERENCE_HIDE_OUTPUT_MESSAGES | Hides all output messages (independent of HIDE_OUTPUTS) | bool | False | -| OPENINFERENCE_HIDE_INPUT_IMAGES | Hides images from input messages (only applies when input messages are not already hidden) | bool | False | -| OPENINFERENCE_HIDE_INPUT_TEXT | Hides text from input messages (only applies when input messages are not already hidden) | bool | False | -| OPENINFERENCE_HIDE_PROMPTS | Hides LLM prompts | bool | False | -| OPENINFERENCE_HIDE_OUTPUT_TEXT | Hides text from output messages (only applies when output messages are not already hidden) | bool | False | -| OPENINFERENCE_HIDE_EMBEDDING_VECTORS | Hides embedding vectors | bool | False | -| OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH | Limits characters of a base64 encoding of an image | int | 32,000 | +| OPENINFERENCE_HIDE_LLM_INVOCATION_PARAMETERS | Removes llm.invocation_parameters attribute entirely from spans | bool | False | +| OPENINFERENCE_HIDE_INPUTS | Replaces input.value with `"__REDACTED__"` and removes input.mime_type | bool | False | +| OPENINFERENCE_HIDE_OUTPUTS | Replaces output.value with `"__REDACTED__"` and removes output.mime_type | bool | False | +| OPENINFERENCE_HIDE_INPUT_MESSAGES | Removes all llm.input_messages attributes entirely from spans | bool | False | +| OPENINFERENCE_HIDE_OUTPUT_MESSAGES | Removes all llm.output_messages attributes entirely from spans | bool | False | +| OPENINFERENCE_HIDE_INPUT_IMAGES | Removes image URLs from llm.input_messages message content blocks | bool | False | +| OPENINFERENCE_HIDE_INPUT_TEXT | Replaces text content in llm.input_messages with `"__REDACTED__"` | bool | False | +| OPENINFERENCE_HIDE_OUTPUT_TEXT | Replaces text content in llm.output_messages with `"__REDACTED__"` | bool | False | +| OPENINFERENCE_HIDE_EMBEDDINGS_VECTORS | Replaces embedding.embeddings.*.embedding.vector values with `"__REDACTED__"` | bool | False | +| OPENINFERENCE_HIDE_EMBEDDINGS_TEXT | Replaces embedding.embeddings.*.embedding.text values with `"__REDACTED__"` | bool | False | +| OPENINFERENCE_HIDE_PROMPTS | Replaces llm.prompts values with `"__REDACTED__"` | bool | False | +| OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH | Truncates base64-encoded images to this length, replacing excess with `"__REDACTED__"` | int | 32,000 | ## Redacted Content @@ -49,9 +50,10 @@ If you are working in Python, and want to set up a configuration different than hide_input_images=..., hide_input_text=..., hide_output_text=..., - hide_embedding_vectors=..., - base64_image_max_length=..., + hide_embeddings_vectors=..., + hide_embeddings_text=..., hide_prompts=..., + base64_image_max_length=..., ) from openinference.instrumentation.openai import OpenAIInstrumentor diff --git a/spec/embedding_spans.md b/spec/embedding_spans.md index 61a926edf..1849e31b3 100644 --- a/spec/embedding_spans.md +++ b/spec/embedding_spans.md @@ -1,68 +1,142 @@ # Embedding Spans -Embedding spans capture operations that generate vector embeddings from text, images, or other inputs. +Embedding spans capture operations that convert text or token IDs into dense float vectors for semantic search, clustering, and similarity comparison. + +## Span Name + +The span name MUST be `"CreateEmbeddings"` for embedding operations. ## Required Attributes All embedding spans MUST include: - `openinference.span.kind`: Set to `"EMBEDDING"` -- `llm.system`: The AI system/product (e.g., "openai", "cohere") ## Common Attributes Embedding spans typically include: -- `llm.model_name`: The specific embedding model used (e.g., "text-embedding-3-small") -- `llm.invocation_parameters`: JSON string of parameters sent to the model -- `input.value`: The raw input as a JSON string (may contain text, array of texts, or tokens) + +- `llm.system`: The AI system/product (e.g., "openai", "litellm") +- `embedding.model_name`: Name of the embedding model (e.g., "text-embedding-3-small") +- `embedding.embeddings`: Nested structure for embedding objects in batch operations +- `embedding.invocation_parameters`: JSON string of parameters sent to the model (excluding input) +- `input.value`: The raw input as a JSON string (text strings or token ID arrays) - `input.mime_type`: Usually "application/json" -- `output.value`: The raw output (often base64-encoded vectors or array of vectors) +- `output.value`: The raw output (embedding vectors as JSON or base64-encoded) - `output.mime_type`: Usually "application/json" -- `embedding.model_name`: Name of the embedding model (may duplicate `llm.model_name`) -- `embedding.text`: The text being embedded (when hiding is not enabled) -- `embedding.vector`: The resulting embedding vector (when hiding is not enabled) -- `embedding.embeddings`: List of embedding objects for batch operations + +### Text Attributes + +The `embedding.embeddings.N.embedding.text` attributes are populated ONLY when the input is already text (strings). These attributes are recorded during the request phase to ensure availability even on errors. + +Token IDs (pre-tokenized integer arrays) are NOT decoded to text because: +- **Cross-provider incompatibility**: Same token IDs represent different text across tokenizers (OpenAI uses cl100k_base, Ollama uses BERT/WordPiece/etc.) +- **Runtime impossibility**: OpenAI-compatible APIs may serve any model with unknown tokenizers +- **Heavy dependencies**: Supporting all tokenizers would require libraries beyond tiktoken (which only supports OpenAI) + +### Vector Attributes + +The `embedding.embeddings.N.embedding.vector` attributes MUST contain float arrays, regardless of the API response format: + +1. **Float response format**: Store vectors directly as float arrays +2. **Base64 response format**: MUST decode base64-encoded strings to float arrays before recording + - Base64 encoding is ~25% more compact in transmission but must be decoded for consistency + - Example: "AACAPwAAAEA=" → [1.5, 2.0] ## Privacy Considerations -When `OPENINFERENCE_HIDE_EMBEDDING_VECTORS` is set to true: -- The `embedding.vector` attribute will contain `"__REDACTED__"` +When `OPENINFERENCE_HIDE_EMBEDDINGS_VECTORS` is set to true: +- The `embedding.embeddings.N.embedding.vector` attribute will contain `"__REDACTED__"` - The actual vector data will not be included in traces -When `OPENINFERENCE_HIDE_INPUT_TEXT` is set to true: -- The `embedding.text` attribute will contain `"__REDACTED__"` +When `OPENINFERENCE_HIDE_EMBEDDINGS_TEXT` is set to true: +- The `embedding.embeddings.N.embedding.text` attribute will contain `"__REDACTED__"` - The input text will not be included in traces -## Example +## Input/Output Structure + +The response structure matches the input structure: +- Single input (text or token array) → `data[0]` with one embedding +- Array of N inputs → `data[0..N-1]` with N embeddings + +Input formats (cannot mix text and tokens in one request): +- Single text: `"hello world"` → single embedding +- Text array: `["hello", "world"]` → array of embeddings +- Single token array: `[15339, 1917]` → single embedding +- Token array of arrays: `[[15339, 1917], [991, 1345]]` → array of embeddings + +## Examples -A span for generating embeddings with OpenAI: +### Text Input (Recorded in Traces) + +A span for generating embeddings from text: ```json { - "name": "CreateEmbeddingResponse", + "name": "CreateEmbeddings", "span_kind": "SPAN_KIND_INTERNAL", "attributes": { "openinference.span.kind": "EMBEDDING", "llm.system": "openai", - "llm.model_name": "text-embedding-3-small", - "input.value": "{\"input\": \"hello world\", \"model\": \"text-embedding-3-small\", \"encoding_format\": \"base64\"}", + "embedding.model_name": "text-embedding-3-small", + "embedding.invocation_parameters": "{\"model\": \"text-embedding-3-small\", \"encoding_format\": \"float\"}", + "input.value": "{\"input\": \"hello world\", \"model\": \"text-embedding-3-small\", \"encoding_format\": \"float\"}", "input.mime_type": "application/json", - "llm.invocation_parameters": "{\"model\": \"text-embedding-3-small\", \"encoding_format\": \"base64\"}", + "output.value": "{\"data\": [{\"embedding\": [0.1, 0.2, 0.3], \"index\": 0}], \"model\": \"text-embedding-3-small\", \"usage\": {\"prompt_tokens\": 2, \"total_tokens\": 2}}", + "output.mime_type": "application/json", + "embedding.embeddings.0.embedding.text": "hello world", + "embedding.embeddings.0.embedding.vector": [0.1, 0.2, 0.3], + "llm.token_count.prompt": 2, + "llm.token_count.total": 2 + } +} +``` + +### Token Input (No Text Attributes) + +When input consists of pre-tokenized integer arrays, text attributes are NOT recorded: + +```json +{ + "name": "CreateEmbeddings", + "span_kind": "SPAN_KIND_INTERNAL", + "attributes": { + "openinference.span.kind": "EMBEDDING", + "llm.system": "openai", "embedding.model_name": "text-embedding-3-small", - "embedding.text": "hello world", - "embedding.vector": "[0.1, 0.2, 0.3, ...]" + "embedding.invocation_parameters": "{\"model\": \"text-embedding-3-small\", \"encoding_format\": \"float\"}", + "input.value": "{\"input\": [15339, 1917], \"model\": \"text-embedding-3-small\", \"encoding_format\": \"float\"}", + "input.mime_type": "application/json", + "output.value": "{\"data\": [{\"embedding\": [0.1, 0.2, 0.3], \"index\": 0}], \"model\": \"text-embedding-3-small\", \"usage\": {\"prompt_tokens\": 2, \"total_tokens\": 2}}", + "output.mime_type": "application/json", + "embedding.embeddings.0.embedding.vector": [0.1, 0.2, 0.3], + "llm.token_count.prompt": 2, + "llm.token_count.total": 2 } } ``` -For batch embedding operations, the embeddings are flattened: +### Batch Text Input (Multiple Embeddings) + +A span for generating embeddings from multiple text inputs: ```json { + "name": "CreateEmbeddings", + "span_kind": "SPAN_KIND_INTERNAL", "attributes": { - "embedding.embeddings.0.embedding.text": "first text", - "embedding.embeddings.0.embedding.vector": "[0.1, 0.2, ...]", - "embedding.embeddings.1.embedding.text": "second text", - "embedding.embeddings.1.embedding.vector": "[0.3, 0.4, ...]" + "openinference.span.kind": "EMBEDDING", + "llm.system": "litellm", + "embedding.model_name": "text-embedding-ada-002", + "embedding.invocation_parameters": "{\"model\": \"text-embedding-ada-002\"}", + "input.value": "[\"hello\", \"world\", \"test\"]", + "embedding.embeddings.0.embedding.text": "hello", + "embedding.embeddings.0.embedding.vector": [0.1, 0.2, 0.3], + "embedding.embeddings.1.embedding.text": "world", + "embedding.embeddings.1.embedding.vector": [0.4, 0.5, 0.6], + "embedding.embeddings.2.embedding.text": "test", + "embedding.embeddings.2.embedding.vector": [0.7, 0.8, 0.9], + "llm.token_count.prompt": 3, + "llm.token_count.total": 3 } } -``` \ No newline at end of file +``` diff --git a/spec/llm_spans.md b/spec/llm_spans.md index 83dbba75c..3e4353ef7 100644 --- a/spec/llm_spans.md +++ b/spec/llm_spans.md @@ -2,6 +2,10 @@ LLM spans capture the API parameters sent to a LLM provider such as OpenAI or Cohere. +## Span Name + +The span name MUST be `"ChatCompletion"` for chat completion operations. + ## Required Attributes All LLM spans MUST include: