Skip to content

Commit

Permalink
Make sure that cache is unique to endpoint/deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
riedgar-ms committed Apr 29, 2024
1 parent 25ecccf commit 1265346
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions guidance/models/_azureai_studio.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ def __init__(
self._deployment = azureai_model_deployment
self._api_key = azureai_studio_key

# There is a cache... better make sure it's specific
# to the endpoint and deployment
deployment_id = self._hash_prompt(self._endpoint + self._deployment)

path = (
pathlib.Path(platformdirs.user_cache_dir("guidance"))
/ "azureaistudio.tokens"
/ f"azureaistudio.tokens.{deployment_id}"
)
self.cache = dc.Cache(path)

Expand All @@ -44,7 +48,7 @@ def _generator(self, prompt, temperature: float):
# and quite possibly belongs in a library function or superclass
# That said, I'm not _completely sure that there aren't subtle
# differences between the various versions

# find the role tags
pos = 0
role_end = b"<|im_end|>"
Expand Down

0 comments on commit 1265346

Please sign in to comment.