diff --git a/parea/constants.py b/parea/constants.py index 71b6cef3..6f87536d 100644 --- a/parea/constants.py +++ b/parea/constants.py @@ -16,6 +16,7 @@ def str2bool(v): TURN_OFF_PAREA_LOGGING = str2bool(os.getenv("TURN_OFF_PAREA_LOGGING", False)) +TURN_OFF_PAREA_EVAL_LOGGING = str2bool(os.getenv("TURN_OFF_PAREA_EVAL_LOGGING", False)) PAREA_OS_ENV_EXPERIMENT_UUID = "_PAREA_EXPERIMENT_UUID" PAREA_DVC_DIR = ".parea" diff --git a/parea/utils/trace_utils.py b/parea/utils/trace_utils.py index 0d8af82b..63433b3f 100644 --- a/parea/utils/trace_utils.py +++ b/parea/utils/trace_utils.py @@ -12,7 +12,7 @@ from functools import wraps from random import random -from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID +from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID, TURN_OFF_PAREA_EVAL_LOGGING from parea.helpers import gen_trace_id, is_logging_disabled, timezone_aware_now from parea.parea_logger import parea_logger from parea.schemas import EvaluationResult @@ -315,7 +315,10 @@ def call_eval_funcs_then_log(trace_id: str, eval_funcs: List[Callable] = None): scores = [] for func in eval_funcs: try: - score = trace()(func)(data) + if TURN_OFF_PAREA_EVAL_LOGGING: + score = func(data) + else: + score = trace()(func)(data) if isinstance(score, EvaluationResult): scores.append(score) elif isinstance(score, list): diff --git a/parea/wrapper/utils.py b/parea/wrapper/utils.py index 0326b312..b334f5e3 100644 --- a/parea/wrapper/utils.py +++ b/parea/wrapper/utils.py @@ -8,7 +8,7 @@ import tiktoken from openai import __version__ as openai_version -from parea.constants import ALL_NON_AZURE_MODELS_INFO, AZURE_MODEL_INFO +from parea.constants import ALL_NON_AZURE_MODELS_INFO, AZURE_MODEL_INFO, TURN_OFF_PAREA_EVAL_LOGGING from parea.parea_logger import parea_logger from parea.schemas.log import LLMInputs, Message, ModelParams, Role from parea.schemas.models import UpdateTraceScenario @@ -21,11 +21,13 @@ # https://gist.github.com/JettJones/c236494013f22723c1822126df944b12 -def skip_decorator_if_func_in_stack(*funcs_to_check: Callable) -> Callable: +def skip_decorator_if_func_in_stack_for_evals(*funcs_to_check: Callable) -> Callable: def decorator_wrapper(decorator: Callable) -> Callable: def new_decorator(self, func: Callable) -> Callable: # Include self @wraps(func) def wrapper(*args, **kwargs): + if not TURN_OFF_PAREA_EVAL_LOGGING: + return decorator(self, func)(*args, **kwargs) # Include self frame = sys._getframe().f_back caller_names = "" while frame: diff --git a/parea/wrapper/wrapper.py b/parea/wrapper/wrapper.py index 5804e1cf..a9ee3bab 100644 --- a/parea/wrapper/wrapper.py +++ b/parea/wrapper/wrapper.py @@ -14,7 +14,7 @@ from parea.helpers import is_logging_disabled, timezone_aware_now from parea.schemas.models import TraceLog, UpdateLog, UpdateTraceScenario from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, logger_update_record, trace_context, trace_data -from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack +from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack_for_evals logger = logging.getLogger() @@ -130,6 +130,7 @@ def _init_trace(self, kwargs) -> Tuple[str, datetime, contextvars.Token]: return trace_id, start_time, token + @skip_decorator_if_func_in_stack_for_evals(call_eval_funcs_then_log, _make_evaluations) def async_decorator(self, orig_func: Callable) -> Callable: @functools.wraps(orig_func) async def wrapper(*args, **kwargs): @@ -161,6 +162,7 @@ async def wrapper(*args, **kwargs): return wrapper + @skip_decorator_if_func_in_stack_for_evals(call_eval_funcs_then_log, _make_evaluations) def sync_decorator(self, orig_func: Callable) -> Callable: @functools.wraps(orig_func) def wrapper(*args, **kwargs): diff --git a/pyproject.toml b/pyproject.toml index ae48a881..66800fe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "parea-ai" packages = [{ include = "parea" }] -version = "0.2.189" +version = "0.2.190" description = "Parea python sdk" readme = "README.md" authors = ["joel-parea-ai "]