Merge pull request #1011 from parea-ai/PAI-1403-flag-eval-function-tr…

…acing

Pai 1403 flag eval function tracing

parea/constants.py

@@ -16,6 +16,7 @@ def str2bool(v):
 
 
 TURN_OFF_PAREA_LOGGING = str2bool(os.getenv("TURN_OFF_PAREA_LOGGING", False))
+TURN_OFF_PAREA_EVAL_LOGGING = str2bool(os.getenv("TURN_OFF_PAREA_EVAL_LOGGING", False))
 
 PAREA_OS_ENV_EXPERIMENT_UUID = "_PAREA_EXPERIMENT_UUID"
 PAREA_DVC_DIR = ".parea"

parea/utils/trace_utils.py

@@ -12,7 +12,7 @@
 from functools import wraps
 from random import random
 
-from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID
+from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID, TURN_OFF_PAREA_EVAL_LOGGING
 from parea.helpers import gen_trace_id, is_logging_disabled, timezone_aware_now
 from parea.parea_logger import parea_logger
 from parea.schemas import EvaluationResult
@@ -315,7 +315,10 @@ def call_eval_funcs_then_log(trace_id: str, eval_funcs: List[Callable] = None):
         scores = []
         for func in eval_funcs:
             try:
-                score = trace()(func)(data)
+                if TURN_OFF_PAREA_EVAL_LOGGING:
+                    score = func(data)
+                else:
+                    score = trace()(func)(data)
                 if isinstance(score, EvaluationResult):
                     scores.append(score)
                 elif isinstance(score, list):

parea/wrapper/utils.py

@@ -8,7 +8,7 @@
 import tiktoken
 from openai import __version__ as openai_version
 
-from parea.constants import ALL_NON_AZURE_MODELS_INFO, AZURE_MODEL_INFO
+from parea.constants import ALL_NON_AZURE_MODELS_INFO, AZURE_MODEL_INFO, TURN_OFF_PAREA_EVAL_LOGGING
 from parea.parea_logger import parea_logger
 from parea.schemas.log import LLMInputs, Message, ModelParams, Role
 from parea.schemas.models import UpdateTraceScenario
@@ -21,11 +21,13 @@
 
 
 # https://gist.github.com/JettJones/c236494013f22723c1822126df944b12
-def skip_decorator_if_func_in_stack(*funcs_to_check: Callable) -> Callable:
+def skip_decorator_if_func_in_stack_for_evals(*funcs_to_check: Callable) -> Callable:
     def decorator_wrapper(decorator: Callable) -> Callable:
         def new_decorator(self, func: Callable) -> Callable:  # Include self
             @wraps(func)
             def wrapper(*args, **kwargs):
+                if not TURN_OFF_PAREA_EVAL_LOGGING:
+                    return decorator(self, func)(*args, **kwargs)  # Include self
                 frame = sys._getframe().f_back
                 caller_names = ""
                 while frame:

parea/wrapper/wrapper.py

@@ -14,7 +14,7 @@
 from parea.helpers import is_logging_disabled, timezone_aware_now
 from parea.schemas.models import TraceLog, UpdateLog, UpdateTraceScenario
 from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, logger_update_record, trace_context, trace_data
-from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack
+from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack_for_evals
 
 logger = logging.getLogger()
 
@@ -130,6 +130,7 @@ def _init_trace(self, kwargs) -> Tuple[str, datetime, contextvars.Token]:
 
         return trace_id, start_time, token
 
+    @skip_decorator_if_func_in_stack_for_evals(call_eval_funcs_then_log, _make_evaluations)
     def async_decorator(self, orig_func: Callable) -> Callable:
         @functools.wraps(orig_func)
         async def wrapper(*args, **kwargs):
@@ -161,6 +162,7 @@ async def wrapper(*args, **kwargs):
 
         return wrapper
 
+    @skip_decorator_if_func_in_stack_for_evals(call_eval_funcs_then_log, _make_evaluations)
     def sync_decorator(self, orig_func: Callable) -> Callable:
         @functools.wraps(orig_func)
         def wrapper(*args, **kwargs):

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.189"
+version = "0.2.190"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]