From fecb88d35b94f12883e482d39f27e35397ab5087 Mon Sep 17 00:00:00 2001 From: = Enea_Gore Date: Thu, 16 Jan 2025 06:22:47 +0100 Subject: [PATCH] store feedback quickly --- athena/athena/endpoints.py | 2 +- modules/text/module_text_llm/embeddings.index | Bin 0 -> 6189 bytes modules/text/module_text_llm/indices.json | 47 ++++++++++++++++++ .../module_text_llm/__main__.py | 2 +- .../module_text_llm/feedback_storage.py | 0 .../icl_rag_approach/generate_suggestions.py | 10 ++-- .../module_text_llm/index_storage.py | 18 +++++-- 7 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 modules/text/module_text_llm/embeddings.index create mode 100644 modules/text/module_text_llm/indices.json create mode 100644 modules/text/module_text_llm/module_text_llm/feedback_storage.py diff --git a/athena/athena/endpoints.py b/athena/athena/endpoints.py index 4aff85cd8..6d259a2a2 100644 --- a/athena/athena/endpoints.py +++ b/athena/athena/endpoints.py @@ -235,7 +235,7 @@ def feedback_consumer(func: Union[ feedback_type = inspect.signature(func).parameters["feedbacks"].annotation.__args__[0] module_config_type = inspect.signature(func).parameters["module_config"].annotation if "module_config" in inspect.signature(func).parameters else None - @app.post("/feed_feedbacks", responses=module_responses) + @app.post("/feedbacks", responses=module_responses) @authenticated @with_meta async def wrapper( diff --git a/modules/text/module_text_llm/embeddings.index b/modules/text/module_text_llm/embeddings.index new file mode 100644 index 0000000000000000000000000000000000000000..866f242253066a636eca2a3c3338d65865d14f24 GIT binary patch literal 6189 zcmXw72UyNs*e{hrDGeh^NQ2Cjdd__)lAS%W_nwhap`w&hA}bZ-1u=lP#=kKgLO%(tUXO&uLwo&Wzb`@diQ_or_4Kj;POYr3w1 z{8By}hxZ3w+PV%O;grp{nr-0qO1Hz08>8joDHinBSSM z!#8ZPkXBzdaHrTHjMOF|lM7k&cT@SXqApLG(OHIs_TX+#{lGZ-5hMl{V5i9XJf-+M z>XvnrC&uP#m!L-Ws#=C|ce`s3^jfSrbFKjRgBY3K<|_6NZpi!F6yt#MWc==-FC)hk zh`3DPvnH>@l2}7oJfQ?u8V<%!5kA=Nd<@HV3zcVVH;RTsx3KA#^Vpw*spyf|j^v;0H@ zeAnv|Cf-iKrrP1$JTe2%?l^~3Q|A09l)o-}iu>=h0OL`=U|-&J9OLDJ*^yNkIi@Gw z&68bilf`CcY{NGjOT@TsL*%Yw7QA9mV_ARQ40yY{J4_?jOD5`vI;NiH| zpj&nx-Yq@MI@s64ZL25nmktZ%1-V&fiq&j^<7R9>uPLUknF)ISpYiAbJ88IOer|zZ zXm0q5O&F3J#ZS$?10c)K3nwXl@tAp_*djuzN5FC3^Q@aB`Pym!_S z=u8@&@s^Nyv^G9D*Iw4`-H!LF`T@gkn4oUjD%nWq5KAdu%2M3zVCsS-7}8r;;*1n5 z-0uiQ6)k!6`%H}Wx57mOM}YJ79Xzx5Z_#bq5Ei-(8SM(%HrxYWdRWR!nhR*MiOVLF z+G%K(u**mdSLZgF=dSvIZ$9mU;|3e?R`Os7D6h%S|Gucs%Bd$nb(ftR=<>4p--O?X zS2(tDOFq$cr@A{Be_jhyHt)fJqkGg2$ooll@F_kOHuj4^s|MRxU9&kd`g^?GbJ7ob zMjgaY@y+=feLL~>KqHq84@KCG$CNHdnyClMV# z9Ukl&4YxDnpycav=$Fz|DlDP7MhAlr(9C=#Y~R{MdQQE9y}Kmg@BFKD_U8QE($?5u zmp$B{I9Bpg`h1q%OpQB4fp6MP$e7g|{r6R%ZQXp__(99syOrVi)J$w0u^vzDwSxOe zDNy>QCq&1;q`Ri#gCZ~fJMjgmIy~KH$dBK3;XBWb6SZ%@#|$~l6#GhMyM zG-D9F;{z@%>;o&ktmLlHwm2hA!~0&>XGVuwGs0v|{?jJfx0645JSaLRDmRrws&wHs zPBU0svo}1sQWul=WmA1$<2UPB+N&*wvlG*|LgTgWe577myftqw?p|jMu7_=SMvwE_ zI`vIuuU6y05}L~O*Pe@xe{1n434LXf=V0F3)J;0Hol+HH;9?l$6MmNO@e<+6Jtp240$U{_=QA>}Ybyt= ztcu|C`!(TKxkVmyPPLm5eWo85Xv_%@(bIh@{z{5wx5LwEe;aeE8{u+2G1#pKpD}AG z(%og+%yXK!$IWm)vj?18l8eRLx2ir8cj#b3w*xq)&pO4cs5r-` z$whqTVkv1CWyi0rd29 zYKf1!*FdKhQQ}0~5;ps8DdyJ;N4g&#wPX0axHr|$NbS2&&op?vLJ+?b23WJ{-yNjl zx>uSu^1Puv&^&|ciT>O+{Mh*gY^8f=>~SUtZng@C*Uf#lGA+@ow-y$(QWRFn$ zzwovb%p6b*tYwmhu#OpS`AD^_B0R0Zqs((fR`z3TQ{GAG2!zzjT6Zwwgdc)5fuK1= zQ+p?=?o4wi;bt2&{L=s^{>O*BGMPPVCki8ICX(FPbggp#9Ja{T^jNg zd1$vg0}H$S)uw4i!|yN0VMX&QMp&e9o6}rs>0aPc1FL(`Z+aOdmss;!HX)*WkSSm7 z?1;4@_U2OmB|Q`0?%yD~y1MeF|2D$MAFZTAr`xD>k?Kk0=GwxsS3Dy;2e0E3r6ueZ z2gkJMejV1~nR7jP{dKRv@52nFI|=$s^!~So;w{!Psw-4pO96VG;z>?5RM<({X|1SP z=}CCr2a1A!3$-VTU%K*o9@KwGpH^48o1}sNu{_1`G@rUs%?y3ZRqqZ$Mp`VK`MHi& z^yH8o8H{z8Sz^7zop|(_YcTz?Hy`@e9ZT#VF||*GBic~eU2Ziqvu^|RXn?U@K z0B!w5U*eCwm|`d}a`0&s)Ghw)yWKDhNn9*S}2VRN(|SMTZP z@tt-$(vxb#OJB`|;fHI2(#6xi?`N-@t^t~3;q>N$c4I#S{=B%5`8c%Tx^j%kU31w!I34ss&OMU?jd9t{{@G+TQEP}pE&O(G_3a(+OC?Vb!^m1 zF0NUq&MB(`YhaT}$FzlAR>7iy_kccM;StV_y{5Q<=iYOxz6b4%%(7?=q}4%xC+VKM zR>Db)2h!Di#E2weYJX4$tU|2Y`G5*DPul z0D2p<;FIGoVZ)tZ*OD8eQ%H_h>AIh1H-OSchnwhd1EJfo{L0FkAkhoHgKN3$K zz?U~_XDX?7KzPK6V_;c;i_%X>Tq+0DXbJTbFCpzGCrp6;{aj(MPcAC2a>%?n=>lUZ zyKhIDBl0!ZXlC;S`6~J1X#|8Va^o4Z3XyOC?8{p7KlcntYo~~pI_2ieG#ezf+u#6DNh?=Lr(l8a*IPJE{1A5zjG! zv5Yi2tkF-wpewft?*n1R_A>E2{X8fgaIyUh(m?M7`3qi|%oP7%vmgiT>Uxri^^WrW zm>PsJ+hN$&D@xNast2mqdd@#63jA~=c_bEeWe60U4!{~#HF4(^z{BsYxl7?P82UAl zeGA{HC0qvOh2n)Zv^;GO$^*@q^_^Xu=?qr6{TXQku`^*gp8WfPjmq84h>Jjd25BU5 zF|IwoGMLe`LLuZ>E9rmCo2%I>9cYf-gW{ReW(wCo{5g#_cUCDc1f(HJ3tYqN9p;02 zrrFjX;ve5m`17n8#`O3Ho!h7J_PqW{s7Y<>qV8xDBL{IOJRxpVmt`C zFUXW5OD>CuN83pIm_g-~WpJ$OB#<6Osts23l;TVITk?NGJy$A^OZ5ZSfKsI0gde4z zlIoyUydFQ~i{fO+f4UP)=A9Cck5uN2EAXiHwX|c^{)M<391P~sPCGF2hfrB%0Po*a zu>va&5n+CsoG%ZdRd73aWNCx6|BPl%?C2jSNOKUzHB|Zq75`A5kh~X6oTMv17;S;m zfyeRpe{uMEd#G6bD^Cz^v97=7YYz0P#DDEfko*ahj7^0`?c%}ZM>$Muay1G}yGnYi1>YWW$zR57>agHGE2CWWR^2ot68F4G^=s{d_bqu2(q4L_x_9S3T z{Ys>r#eq5XIbpq4`MvZ%QyJki&hmF5{Sl43dd|ivG5MnNtHu(yMljNAjOLMYicQs; zY`%p*lAmI94t6T3vzXH6r_#VwTgk`^Ang?80XoWiV^V;XTS=(82rEWzDrR3q}(A<=y$nHzWS+n zIczv@)ol)_*DDOCd%%I?J3R@5wRO7=!UaFtk;V!Z$~#@HTmah!>LBz|OdUZ$hcJ)Mu;M>QA3&#FJ*eIc2AuL07IR?lmV7j}59 zvpH$CB-~qPiH7(M%2%Ht|7}h=#%C>glw8s-Vnt$@ni)YD54AgZkk2*5X%%nKpi!Rq zo!^&of#W#j$wKW~Wqx$AsM%kj*@h7Oxue4V)?}}<1 z&dhA7E#BTh{w_*ad&PXy1~J1wTOnid0osj3G3Mz+Haw&mzh=}1NKXmpi;|J%fSAQy zr1B&$)-XYsMRO4?YQ@(OdQo@9C&wVoiRlf5(rF3@R3AvAbNX95tzs4o8r2Vfjd-Oc z&8>VblBdM!4#t%GJz#tN8UW!tTp29L@9f42Bhv(FTt@yJ5^7C>oc!e&{LoaA9_N&u zD4(PBn}+aQxHxu~O2d-AgW?H8l`j@lI}!BLgZFup0bUJj%UTnXvB7d19x`ZxP_uR3 zrM0BY0WbM9$5oBvkn$tMcIDt5_8(hrdrlC?N|hS?h@hBCQw`#AL~Zv_F|MlAmKbXiAR{Gc_2LU6V>hjm1KnOEH-K-`TfQw_*J zq$pjiq4(nQ=OLi-mY|&O!*Sx zDl8e|1LU6-))0mzBITu8(v)a?K9%`79wBY?3ugHrLH~#QRenbp+lRkcBfwjGR_oLE z5L5+*YmKAFqwnH5G>@6E+4rf!V6M1hY)d`Le-3Naxkpbusk9pPOpgMMw^p#SiC#JN8dp5$zfUJZLrQnqWn8k8n!o0Ea_wzOijACUFzBH_g?^2m-5 zv$BaGO`)aSnX;*$Kz@~x=2O`O&FxR598G0xOx<1iD)JJ<)450<#nWbv38UVyxnrku z%8j)N7To}6*fG*3l-ULW<%r^Ji5`TdCMhq5ly5_0zkgYJ=X9W~8De_)qq|89KL6Rj zDi>pv;Rwo)v7q$C7_TE3P#y_EPP-txun8w`29zVfE1wutxuu%p%fB{JmZXoP)@e}r hl7@XeAaSpYRK9|8EJhqJD9=TjKc@200k_Yx{{g2?ehUBq literal 0 HcmV?d00001 diff --git a/modules/text/module_text_llm/indices.json b/modules/text/module_text_llm/indices.json new file mode 100644 index 000000000..b7b4f99f1 --- /dev/null +++ b/modules/text/module_text_llm/indices.json @@ -0,0 +1,47 @@ +{ + "0": { + "exercise_id": 3002, + "submission_id": 787782, + "feedbacks": [ + { + "id": 6620, + "title": "Assessment of the Difference between Coupling/Cohesion", + "description": "You correctly explained the difference between coupling and cohesion, well done!", + "credits": 1.0, + "structured_grading_instruction_id": 6053, + "is_graded": null, + "meta": {}, + "exercise_id": 3002, + "submission_id": 787782, + "index_start": 0, + "index_end": 722 + }, + { + "id": 6621, + "title": "Assessment of the Example", + "description": "Great example, well done!", + "credits": 1.0, + "structured_grading_instruction_id": 6058, + "is_graded": null, + "meta": {}, + "exercise_id": 3002, + "submission_id": 787782, + "index_start": 909, + "index_end": 1299 + }, + { + "id": 6622, + "title": "Assessment of the Explanation why Coupling/Cohesion are important", + "description": "You correctly explained why coupling and cohesion are important, well done!", + "credits": 1.0, + "structured_grading_instruction_id": 6056, + "is_graded": null, + "meta": {}, + "exercise_id": 3002, + "submission_id": 787782, + "index_start": null, + "index_end": null + } + ] + } +} \ No newline at end of file diff --git a/modules/text/module_text_llm/module_text_llm/__main__.py b/modules/text/module_text_llm/module_text_llm/__main__.py index 135edcb12..d1dad801d 100644 --- a/modules/text/module_text_llm/module_text_llm/__main__.py +++ b/modules/text/module_text_llm/module_text_llm/__main__.py @@ -32,7 +32,7 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba submission_id = submission.id exercise_id = exercise.id embedded_submission = embed_text(submission.text) - store_embedding_index(exercise_id, submission_id) + store_embedding_index(exercise_id, submission_id, feedbacks) save_embedding(embedded_submission) @feedback_provider diff --git a/modules/text/module_text_llm/module_text_llm/feedback_storage.py b/modules/text/module_text_llm/module_text_llm/feedback_storage.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/text/module_text_llm/module_text_llm/icl_rag_approach/generate_suggestions.py b/modules/text/module_text_llm/module_text_llm/icl_rag_approach/generate_suggestions.py index 7ade1f23f..376fbe221 100644 --- a/modules/text/module_text_llm/module_text_llm/icl_rag_approach/generate_suggestions.py +++ b/modules/text/module_text_llm/module_text_llm/icl_rag_approach/generate_suggestions.py @@ -11,7 +11,7 @@ from llm_core.utils.predict_and_parse import predict_and_parse from module_text_llm.helpers.utils import add_sentence_numbers, get_index_range_from_line_range, format_grading_instructions from module_text_llm.icl_rag_approach.prompt_generate_suggestions import AssessmentModel -from module_text_llm.index_storage import retrieve_embedding_index +from module_text_llm.index_storage import retrieve_embedding_index, retrieve_feedbacks from module_text_llm.storage_embeddings import query_embedding from module_text_llm.generate_embeddings import embed_text from athena.text import get_stored_feedback @@ -47,10 +47,12 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi for index in list_of_indices[0]: if index != -1: exercise_id, submission_id = retrieve_embedding_index(list_of_indices) - stored_feedback = list(get_stored_feedback(exercise_id, submission_id)) + stored_feedback = retrieve_feedbacks(index) # -> List[Feedback] + # stored_feedback = list(get_stored_feedback(exercise_id, submission_id)) logger.info("Stored feedback:") - for feedback_item in stored_feedback: - logger.info("- %s", feedback_item) + if stored_feedback is not None: + for feedback_item in stored_feedback: + logger.info("- %s", feedback_item) logger.info("Stored submission:") rag_context.append({"submission": submission.text, "feedback": stored_feedback}) diff --git a/modules/text/module_text_llm/module_text_llm/index_storage.py b/modules/text/module_text_llm/module_text_llm/index_storage.py index c7cf0651d..f72e55c10 100644 --- a/modules/text/module_text_llm/module_text_llm/index_storage.py +++ b/modules/text/module_text_llm/module_text_llm/index_storage.py @@ -1,6 +1,5 @@ import json import os - INDEX_FILE = "indices.json" def load_indices(): @@ -10,15 +9,15 @@ def load_indices(): return json.load(f) else: return {} - -def store_embedding_index(exercise_id, submission_id): + +def store_embedding_index(exercise_id, submission_id,feedbacks): """ Store a new submission and exercise ID with an auto-incrementing index. """ indices = load_indices() next_index = len(indices) - indices[next_index] = { "exercise_id": exercise_id, - "submission_id": submission_id + "submission_id": submission_id, + "feedbacks": [feedback.dict() for feedback in feedbacks] } with open(INDEX_FILE, 'w', encoding="utf-8") as f: @@ -34,3 +33,12 @@ def retrieve_embedding_index(index): return indices[index]["exercise_id"], indices[index]["submission_id"] return None, None + +def retrieve_feedbacks(index): + index = str(index) + indices = load_indices() + + if index in indices: + return indices[index]["feedbacks"] + + return None \ No newline at end of file