From 32756ed864849082f9eefc63e7d1fd8ec999d0b3 Mon Sep 17 00:00:00 2001 From: Xander Song Date: Mon, 28 Oct 2024 16:07:10 -0700 Subject: [PATCH] feat(anthropic): add llm provider and system attributes to anthropic instrumentation (#1084) --- .../instrumentation/anthropic/_wrappers.py | 22 +++++++++++++++ .../anthropic/test_instrumentor.py | 28 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/python/instrumentation/openinference-instrumentation-anthropic/src/openinference/instrumentation/anthropic/_wrappers.py b/python/instrumentation/openinference-instrumentation-anthropic/src/openinference/instrumentation/anthropic/_wrappers.py index f13b95490..fefb2d564 100644 --- a/python/instrumentation/openinference-instrumentation-anthropic/src/openinference/instrumentation/anthropic/_wrappers.py +++ b/python/instrumentation/openinference-instrumentation-anthropic/src/openinference/instrumentation/anthropic/_wrappers.py @@ -16,6 +16,8 @@ DocumentAttributes, EmbeddingAttributes, MessageAttributes, + OpenInferenceLLMProviderValues, + OpenInferenceLLMSystemValues, OpenInferenceMimeTypeValues, OpenInferenceSpanKindValues, SpanAttributes, @@ -88,6 +90,8 @@ def __call__( span.set_attributes( { **dict(_get_llm_model(arguments)), + **dict(_get_llm_provider()), + **dict(_get_llm_system()), OPENINFERENCE_SPAN_KIND: LLM, LLM_PROMPTS: [llm_prompt], INPUT_VALUE: safe_json_dumps(arguments), @@ -146,6 +150,8 @@ async def __call__( span.set_attributes( { **dict(_get_llm_model(arguments)), + **dict(_get_llm_provider()), + **dict(_get_llm_system()), OPENINFERENCE_SPAN_KIND: LLM, LLM_PROMPTS: [llm_prompt], INPUT_VALUE: safe_json_dumps(arguments), @@ -204,6 +210,8 @@ def __call__( span.set_attributes( { **dict(_get_llm_model(arguments)), + **dict(_get_llm_provider()), + **dict(_get_llm_system()), OPENINFERENCE_SPAN_KIND: LLM, **dict(_get_input_messages(llm_input_messages)), LLM_INVOCATION_PARAMETERS: safe_json_dumps(invocation_parameters), @@ -263,6 +271,8 @@ async def __call__( span.set_attributes( { + **dict(_get_llm_provider()), + **dict(_get_llm_system()), **dict(_get_llm_model(arguments)), OPENINFERENCE_SPAN_KIND: LLM, **dict(_get_input_messages(llm_input_messages)), @@ -295,6 +305,14 @@ async def __call__( return response +def _get_llm_provider() -> Iterator[Tuple[str, Any]]: + yield LLM_PROVIDER, LLM_PROVIDER_ANTHROPIC + + +def _get_llm_system() -> Iterator[Tuple[str, Any]]: + yield LLM_SYSTEM, LLM_SYSTEM_ANTHROPIC + + def _get_llm_model(arguments: Mapping[str, Any]) -> Iterator[Tuple[str, Any]]: if model_name := arguments.get("model"): yield LLM_MODEL_NAME, model_name @@ -450,3 +468,7 @@ def _validate_invocation_parameter(parameter: Any) -> bool: TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME USER_ID = SpanAttributes.USER_ID +LLM_PROVIDER = SpanAttributes.LLM_PROVIDER +LLM_SYSTEM = SpanAttributes.LLM_SYSTEM +LLM_PROVIDER_ANTHROPIC = OpenInferenceLLMProviderValues.ANTHROPIC.value +LLM_SYSTEM_ANTHROPIC = OpenInferenceLLMSystemValues.ANTHROPIC.value diff --git a/python/instrumentation/openinference-instrumentation-anthropic/tests/openinference/anthropic/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-anthropic/tests/openinference/anthropic/test_instrumentor.py index 178cbf1e9..f77834cb7 100644 --- a/python/instrumentation/openinference-instrumentation-anthropic/tests/openinference/anthropic/test_instrumentor.py +++ b/python/instrumentation/openinference-instrumentation-anthropic/tests/openinference/anthropic/test_instrumentor.py @@ -31,6 +31,8 @@ DocumentAttributes, EmbeddingAttributes, MessageAttributes, + OpenInferenceLLMProviderValues, + OpenInferenceLLMSystemValues, OpenInferenceMimeTypeValues, OpenInferenceSpanKindValues, SpanAttributes, @@ -147,6 +149,8 @@ def test_anthropic_instrumentation_completions_streaming( print(attributes) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert isinstance(attributes.pop(INPUT_VALUE), str) assert attributes.pop(INPUT_MIME_TYPE) == JSON assert isinstance(attributes.pop(OUTPUT_VALUE), str) @@ -159,6 +163,7 @@ def test_anthropic_instrumentation_completions_streaming( invocation_params = {"model": "claude-2.1", "max_tokens_to_sample": 1000, "stream": True} assert json.loads(inv_params) == invocation_params assert attributes.pop(LLM_OUTPUT_MESSAGES) == " Light scatters blue." + assert not attributes @pytest.mark.asyncio @@ -196,6 +201,8 @@ async def test_anthropic_instrumentation_async_completions_streaming( print(attributes) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert isinstance(attributes.pop(INPUT_VALUE), str) assert attributes.pop(INPUT_MIME_TYPE) == JSON assert isinstance(attributes.pop(OUTPUT_VALUE), str) @@ -208,6 +215,7 @@ async def test_anthropic_instrumentation_async_completions_streaming( invocation_params = {"model": "claude-2.1", "max_tokens_to_sample": 1000, "stream": True} assert json.loads(inv_params) == invocation_params assert attributes.pop(LLM_OUTPUT_MESSAGES) == " Light scatters blue." + assert not attributes @pytest.mark.vcr( @@ -242,6 +250,8 @@ def test_anthropic_instrumentation_completions( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert isinstance(attributes.pop(INPUT_VALUE), str) assert attributes.pop(INPUT_MIME_TYPE) == JSON assert isinstance(attributes.pop(OUTPUT_VALUE), str) @@ -286,6 +296,8 @@ def test_anthropic_instrumentation_messages( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" assert isinstance( @@ -343,6 +355,8 @@ def test_anthropic_instrumentation_messages_streaming( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" assert isinstance( @@ -405,6 +419,8 @@ async def test_anthropic_instrumentation_async_messages_streaming( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" assert isinstance( @@ -462,6 +478,8 @@ async def test_anthropic_instrumentation_async_completions( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert isinstance(attributes.pop(INPUT_VALUE), str) assert attributes.pop(INPUT_MIME_TYPE) == JSON assert isinstance(attributes.pop(OUTPUT_VALUE), str) @@ -506,6 +524,8 @@ async def test_anthropic_instrumentation_async_messages( attributes = dict(spans[0].attributes or {}) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" assert isinstance( @@ -625,6 +645,8 @@ def test_anthropic_instrumentation_multiple_tool_calling( assert isinstance(attributes.pop(OUTPUT_VALUE), str) assert isinstance(attributes.pop(OUTPUT_MIME_TYPE), str) assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert not attributes @@ -728,6 +750,8 @@ def test_anthropic_instrumentation_multiple_tool_calling_streaming( assert isinstance(attributes.pop(OUTPUT_VALUE), str) assert attributes.pop(OUTPUT_MIME_TYPE) == "application/json" assert attributes.pop(OPENINFERENCE_SPAN_KIND) == "LLM" + assert attributes.pop(LLM_PROVIDER) == LLM_PROVIDER_ANTHROPIC + assert attributes.pop(LLM_SYSTEM) == LLM_SYSTEM_ANTHROPIC assert not attributes @@ -993,3 +1017,7 @@ def test_oitracer( LLM_PROMPT_TEMPLATE = SpanAttributes.LLM_PROMPT_TEMPLATE LLM_PROMPT_TEMPLATE_VARIABLES = SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES USER_ID = SpanAttributes.USER_ID +LLM_PROVIDER = SpanAttributes.LLM_PROVIDER +LLM_SYSTEM = SpanAttributes.LLM_SYSTEM +LLM_PROVIDER_ANTHROPIC = OpenInferenceLLMProviderValues.ANTHROPIC.value +LLM_SYSTEM_ANTHROPIC = OpenInferenceLLMSystemValues.ANTHROPIC.value