From 1265346714ae9364f2350d199d213cfb104ed3a0 Mon Sep 17 00:00:00 2001 From: "Richard Edgar (Microsoft)" Date: Mon, 29 Apr 2024 10:40:06 -0400 Subject: [PATCH] Make sure that cache is unique to endpoint/deployment --- guidance/models/_azureai_studio.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/guidance/models/_azureai_studio.py b/guidance/models/_azureai_studio.py index 97b2f31b1..e90f2430f 100644 --- a/guidance/models/_azureai_studio.py +++ b/guidance/models/_azureai_studio.py @@ -25,9 +25,13 @@ def __init__( self._deployment = azureai_model_deployment self._api_key = azureai_studio_key + # There is a cache... better make sure it's specific + # to the endpoint and deployment + deployment_id = self._hash_prompt(self._endpoint + self._deployment) + path = ( pathlib.Path(platformdirs.user_cache_dir("guidance")) - / "azureaistudio.tokens" + / f"azureaistudio.tokens.{deployment_id}" ) self.cache = dc.Cache(path) @@ -44,7 +48,7 @@ def _generator(self, prompt, temperature: float): # and quite possibly belongs in a library function or superclass # That said, I'm not _completely sure that there aren't subtle # differences between the various versions - + # find the role tags pos = 0 role_end = b"<|im_end|>"