diff --git a/README.md b/README.md index bb90428c7..a5e943e58 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,5 @@ ## What is Athena? Athena is an advanced system designed to assist educators by providing (semi-)automated assessments for various types of academic exercises. Through its integration with learning management systems (LMS), Athena offers an efficient and innovative way to evaluate students' work in large courses. The system has been expanded from its original focus on textual exercises to now include support for programming exercises and has plans for future support of additional exercise types such as UML modeling and mathematics. + +**Documentation:** [ls1intum.github.io/Athena/](https://ls1intum.github.io/Athena) diff --git a/assessment_module_manager/modules.docker.ini b/assessment_module_manager/modules.docker.ini index 5fc804e45..618f7d6bc 100644 --- a/assessment_module_manager/modules.docker.ini +++ b/assessment_module_manager/modules.docker.ini @@ -45,4 +45,5 @@ url = http://module-modeling-llm:5008 type = modeling supports_evaluation = false supports_non_graded_feedback_requests = false -supports_graded_feedback_requests = true \ No newline at end of file +supports_graded_feedback_requests = true + diff --git a/assessment_module_manager/modules.ini b/assessment_module_manager/modules.ini index 5cd2cd259..e601a5e08 100644 --- a/assessment_module_manager/modules.ini +++ b/assessment_module_manager/modules.ini @@ -45,4 +45,5 @@ url = http://localhost:5008 type = modeling supports_evaluation = false supports_non_graded_feedback_requests = false -supports_graded_feedback_requests = true \ No newline at end of file +supports_graded_feedback_requests = true + diff --git a/docs/Makefile b/docs/Makefile index 3473b9c8d..5193b8812 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,6 +3,24 @@ # You can set these variables from the command line, and also # from the environment for the first two. +SPHINXAUTOBUILD = sphinx-autobuild +ALLSPHINXLIVEOPTS = $(ALLSPHINXOPTS) -q \ + --port 0 \ + --host 0.0.0.0 \ + --open-browser \ + --delay 1 \ + --ignore "*.swp" \ + --ignore "*.pdf" \ + --ignore "*.log" \ + --ignore "*.out" \ + --ignore "*.toc" \ + --ignore "*.aux" \ + --ignore "*.idx" \ + --ignore "*.ind" \ + --ignore "*.ilg" \ + --ignore "*.tex" \ + --watch source + SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . @@ -13,11 +31,14 @@ help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + $(SPHINXAUTOBUILD) -b html $(ALLSPHINXLIVEOPTS) $(SOURCEDIR) $(BUILDDIR) + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)." .PHONY: help Makefile +# .PHONY: livehtml # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 604c9fac0..0b1a3aebc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,13 @@ Athena will use the information it is given and provide the automatic suggestion overview/athena overview/playground +.. toctree:: + :caption: User Guide + :includehidden: + :maxdepth: 2 + + user_guide/index + .. toctree:: :caption: Setup :includehidden: diff --git a/docs/requirements.txt b/docs/requirements.txt index 545fd6945..15f9b37f9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ Sphinx==6.2.1 -sphinx-rtd-theme==1.2.0 +sphinx-rtd-theme==2.0.0 sphinx-autobuild==2021.3.14 docutils==0.19 sphinxcontrib-bibtex==2.5.0 diff --git a/docs/run/docker.rst b/docs/run/docker.rst index c68e24c01..a3c2c0cc1 100644 --- a/docs/run/docker.rst +++ b/docs/run/docker.rst @@ -1,3 +1,5 @@ +.. _run_docker: + From Docker =========================================== diff --git a/docs/run/local.rst b/docs/run/local.rst index 701967641..3b6862017 100644 --- a/docs/run/local.rst +++ b/docs/run/local.rst @@ -1,3 +1,5 @@ +.. _run_local: + From the Command Line =========================================== diff --git a/docs/run/playground.rst b/docs/run/playground.rst index 8ac41afb1..cc3d7a74d 100644 --- a/docs/run/playground.rst +++ b/docs/run/playground.rst @@ -1,3 +1,5 @@ +.. _run_playground: + Run the Playground =========================================== diff --git a/docs/setup/install.rst b/docs/setup/install.rst index a23ed11ea..fdb94e839 100644 --- a/docs/setup/install.rst +++ b/docs/setup/install.rst @@ -1,3 +1,5 @@ +.. _setup_install: + Python and Poetry Setup =========================================== diff --git a/docs/user_guide/conduct_experiment.rst b/docs/user_guide/conduct_experiment.rst new file mode 100644 index 000000000..3bc29f4aa --- /dev/null +++ b/docs/user_guide/conduct_experiment.rst @@ -0,0 +1,52 @@ +.. _conduct_experiment_guide: + +============================= +Conducting an Experiment +============================= + +To conduct an experiment in the Athena Playground, follow these steps: + +1. **Define Experiment:** + - Scroll to the Evaluation Mode section. + - In "Define Experiments", choose execution modes, exercise types, and manage training and evaluation data. + - Alternatively, import an experiment configuration using the "Import" button. + - When done, press "Define Experiment". + - Export the experiment configuration using the "Export" button for future reference. + + .. figure:: ../images/playground/evaluation_mode/define_experiment.png + :width: 500px + :alt: Define Experiment Interface of the Athena Playground + + Evaluation Mode: Define Experiment Interface of the Athena Playground + +2. **Configure Modules:** + - Select and configure the modules you wish to include in your experiment. + - Ensure each module is set up with appropriate parameters for effective comparison. + - Import module configurations using the "Import" button, if needed. + - Export the module configurations using the "Export" button for future reference. + + .. figure:: ../images/playground/evaluation_mode/configure_modules.png + :width: 500px + :alt: Configure Modules Interface of the Athena Playground + + Evaluation Mode: Configure Modules Interface of the Athena Playground + +3. **Conduct Experiment:** + - Press "Start Experiment" to begin the experiment. + - The steps performed include sending submissions, sending feedback for training submissions, generating feedback suggestions, and running automatic evaluations. + - If training submissions are provided, you will need to manually continue the experiment by pressing "Continue". + - If automatic evaluations is enabled, for instance LLM-as-a-judge for text exercises, you will also need to manually confirm it. + - Export and import the experiment results as needed using the "Export" and "Import" buttons, respectively. + + .. figure:: ../images/playground/evaluation_mode/conduct_experiment_text.png + :width: 500px + :alt: Conduct Experiment Interface for a Text Exercise of the Athena Playground + + Evaluation Mode: Conduct Experiment Interface for a Text Exercise of the Athena Playground + +4. **Annotate Feedback Suggestions:** + - Annotate the generated feedback suggestions with "Accept" or "Reject" as a tutor would. + +5. **Export Results:** + - At the end of the experiment, or at any time during the experiment, export the results using the "Export" button. + - Make sure that you also exported the experiment configuration and module configurations to have a complete record of the experiment. diff --git a/docs/user_guide/evaluation_data_format.rst b/docs/user_guide/evaluation_data_format.rst new file mode 100644 index 000000000..1894aa8e8 --- /dev/null +++ b/docs/user_guide/evaluation_data_format.rst @@ -0,0 +1,255 @@ +.. _evaluation_data_format: + +=========================== +Evaluation Data Format +=========================== + +This document describes the structure of the evaluation data format used in Athena. It covers various exercise types, their submissions, and feedback types. The following sections provide detailed explanations and examples. Please refer to the example data provided in the `playground/data/example/` directory for examples, if the structure is unclear contact the Athena team. + +Exercise Data Structure +======================= + +Each exercise is represented as a JSON object with the following fields: + +- **id**: Unique identifier for the exercise. +- **course_id**: Identifier for the course to which the exercise belongs. +- **title**: Title of the exercise. +- **type**: Type of the exercise (e.g., "programming", "text", "modeling"). +- **max_points**: Maximum points achievable for the exercise. +- **bonus_points**: Bonus points achievable for the exercise. +- **problem_statement**: Detailed description of the problem to be solved. +- **grading_instructions**: Instructions on how the exercise should be graded (can also be `null` in favor of structured grading instructions, i.e. `grading_criteria`). +- **grading_criteria**: Structured grading instructions on how the exercise should be graded (are not required) +- **solution_repository_uri**: URI for the solution repository (for programming exercises). +- **template_repository_uri**: URI for the template repository (for programming exercises). +- **tests_repository_uri**: URI for the tests repository (for programming exercises). +- **meta**: Additional metadata for the exercise. +- **submissions**: List of submissions for the exercise. + +Example Exercise Data +===================== + +1. **Programming Exercise** + +.. code-block:: json + + { + "id": 1, + "course_id": 101, + "title": "Sorting Algorithms", + "type": "programming", + "max_points": 15, + "bonus_points": 2, + "problem_statement": "Implement different sorting algorithms and choose them based on runtime conditions...", + "grading_instructions": "1. **QuickSort.java** - 5 points\n2. **MergeSort.java** - 5 points\n3. **HeapSort.java** - 5 points...", + "programming_language": "java", + "solution_repository_uri": "{{exerciseDataUrl}}/solution.zip", + "template_repository_uri": "{{exerciseDataUrl}}/template.zip", + "tests_repository_uri": "{{exerciseDataUrl}}/tests.zip", + "meta": {}, + "submissions": [ + { + "id": 101, + "repository_uri": "{{exerciseDataUrl}}/submissions/101.zip", + "meta": {}, + "feedbacks": [ + { + "id": 201, + "title": "File QuickSort.java at line 22", + "description": "The sort method is not implemented correctly.", + "file_path": "QuickSort.java", + "line_start": 22, + "line_end": null, + "credits": -3.0, + "meta": {} + } + ] + }, + { + "id": 102, + "repository_uri": "{{exerciseDataUrl}}/submissions/102.zip", + "meta": {} + } + ] + } + +2. **Text Exercise with General Grading Instructions** + +.. code-block:: json + + { + "id": 2, + "course_id": 102, + "title": "Describe Your Favorite Book", + "type": "text", + "max_points": 10, + "bonus_points": 1, + "problem_statement": "Write a brief essay about your favorite book and why you like it.", + "grading_instructions": "Full points if the essay is well-structured and provides clear reasons for liking the book.", + "example_solution": "My favorite book is 'To Kill a Mockingbird' because...", + "meta": {}, + "submissions": [ + { + "id": 201, + "text": "My favorite book is '1984' by George Orwell because...", + "meta": {}, + "feedbacks": [ + { + "id": 301, + "title": "Content Feedback", + "description": "Good job! However, you could elaborate more on the themes of the book.", + "index_start": null, + "index_end": null, + "credits": 8.0, + "meta": {} + } + ] + }, + { + "id": 202, + "text": "I like 'Pride and Prejudice' because...", + "meta": {} + } + ] + } + +3. **Text Exercise with Structured Grading Instructions** + +.. code-block:: json + + { + "id": 3, + "course_id": 103, + "title": "Gene Prediction Strategies", + "type": "text", + "max_points": 10, + "bonus_points": 0, + "problem_statement": "What are the three strategies for gene prediction? Give an example for each.", + "example_solution": "The three strategies for gene prediction are content-based, site-based, and comparative...", + "meta": {}, + "submissions": [ + { + "id": 301, + "text": "Three strategies for gene prediction are:...", + "language": "ENGLISH", + "meta": {} + }, + { + "id": 302, + "text": "Gene prediction strategies include:...", + "language": "ENGLISH", + "meta": {} + } + ], + "grading_criteria": [ + { + "id": 10, + "title": "Content-based Strategy", + "structured_grading_instructions": [ + { + "id": 19, + "credits": 3.3, + "feedback": "Correct identification of content-based strategy.", + "grading_scale": "Correct Identification", + "instruction_description": "Identification of content-based strategy." + }, + { + "id": 20, + "credits": 0.0, + "feedback": "Incorrect or no identification of content-based strategy.", + "grading_scale": "Incorrect Identification", + "instruction_description": "Incorrect or no identification." + } + ] + }, + { + "id": 11, + "title": "Content-based Example", + "structured_grading_instructions": [ + { + "id": 21, + "credits": 3.3, + "feedback": "Correct example for content-based strategy.", + "grading_scale": "Correct Example", + "instruction_description": "Example for content-based strategy (e.g., ORFs, codon usage)." + }, + { + "id": 22, + "credits": 0.0, + "feedback": "Incorrect or no example for content-based strategy.", + "grading_scale": "Incorrect Example", + "instruction_description": "Incorrect or no example." + } + ] + } + ] + } + +4. **Modeling Exercise** + +.. code-block:: json + + { + "id": 4, + "course_id": 104, + "title": "Create a UML Diagram", + "type": "modeling", + "max_points": 20, + "bonus_points": 0, + "problem_statement": "Create a UML class diagram for a library management system.", + "grading_instructions": "1 point for each correct class and relationship.", + "example_solution": "{}", + "meta": {}, + "submissions": [ + { + "id": 401, + "text": "UML Diagram for Library Management System", + "model": "{\"version\":\"2.0\",\"type\":\"UML\",\"elements\":{},\"relationships\":{}}", + "meta": {} + }, + { + "id": 402, + "text": "Another UML Diagram", + "model": "{\"version\":\"2.0\",\"type\":\"UML\",\"elements\":{},\"relationships\":{}}", + "meta": {} + } + ] + } + +Data Fields Explanation +======================= + +- **Submissions**: Each submission is an object containing: + - **id**: Unique identifier for the submission. + - **repository_uri** (for programming exercises): URI for the submission repository. + - **text** (for text exercises): The text content of the submission. + - **model** (for modeling exercises): The serialized model data. + - **language** (for text exercises): Language of the submission. + - **meta**: Additional metadata for the submission. + - **feedbacks**: List of feedback objects associated with the submission. + +- **Feedbacks**: Each feedback object contains: + - **id**: Unique identifier for the feedback. + - **title**: Title of the feedback. + - **description**: Detailed feedback description. + - **file_path** (for programming exercises): Path to the file the feedback is related to. + - **line_start** (for programming exercises): Start line number of the feedback. + - **line_end** (for programming exercises): End line number of the feedback. + - **index_start** (for text exercises): Start index of the feedback. + - **index_end** (for text exercises): End index of the feedback. + - **credits**: Points awarded or deducted based on the feedback. + - **meta**: Additional metadata for the feedback. + +- **Grading Criteria** (for structured grading instructions): Each grading criterion contains: + - **id**: Unique identifier for the grading criterion. + - **title**: Title of the grading criterion. + - **structured_grading_instructions**: List of structured grading instructions associated with the criterion. + +- **Structured Grading Instructions**: Each structured grading instruction contains: + - **id**: Unique identifier for the grading instruction. + - **credits**: Points awarded for the instruction. + - **feedback**: Feedback provided based on the instruction. + - **grading_scale**: Scale used for grading (e.g., "Correct Identification"). + - **instruction_description**: Detailed description of the grading instruction. + +This structure ensures that the evaluation data is well-organized and easy to understand for both automated systems and human evaluators. Each exercise type has specific fields tailored to its requirements, making the data format flexible and comprehensive. diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst new file mode 100644 index 000000000..ab2cccffa --- /dev/null +++ b/docs/user_guide/index.rst @@ -0,0 +1,14 @@ +.. _user_guide: + +============================ +Athena Playground User Guide +============================ + +This user guide provides a comprehensive walkthrough for instructors to effectively use the Athena Playground for conducting evaluations. Follow the steps outlined in the sections below to set up and conduct experiments. + +.. toctree:: + :maxdepth: 2 + + setup/setup + evaluation_data_format + conduct_experiment diff --git a/docs/user_guide/setup/base_info_header.png b/docs/user_guide/setup/base_info_header.png new file mode 100644 index 000000000..371457dd3 Binary files /dev/null and b/docs/user_guide/setup/base_info_header.png differ diff --git a/docs/user_guide/setup/custom_evaluation_data.png b/docs/user_guide/setup/custom_evaluation_data.png new file mode 100644 index 000000000..5e9cf6937 Binary files /dev/null and b/docs/user_guide/setup/custom_evaluation_data.png differ diff --git a/docs/user_guide/setup/evaluation_data.png b/docs/user_guide/setup/evaluation_data.png new file mode 100644 index 000000000..4828bbbc6 Binary files /dev/null and b/docs/user_guide/setup/evaluation_data.png differ diff --git a/docs/user_guide/setup/setup.rst b/docs/user_guide/setup/setup.rst new file mode 100644 index 000000000..03b398809 --- /dev/null +++ b/docs/user_guide/setup/setup.rst @@ -0,0 +1,90 @@ +.. _setup_guide: + +Setting Up the Playground +======================== + +Before conducting experiments, ensure the Athena Playground is correctly set up and running. + +There are multiple ways of accessing Athena through the Athena Playground: + +1. **Test-server setup:** + - Everything is taken care of, i.e., hosting the Athena Playground and the Athena Modules. + - The cost for model inference is carried by our organization, if you do extensive testing, please let us know. + - You can upload, download, and remove your own evaluation data under a directory name. + - Keep in mind that the **team can technically access the data on the test-server**. If encrypted/secured evaluation data is needed, you must request it. + - Request the Athena Playground secret from the team or find it on the team's Confluence. + - **Playground access:** https://athenetest1-03.ase.cit.tum.de/playground + +2. **Local setup:** + - **Pro:** We cannot access your data this way. + - **Con:** You setup the everything locally, configure the environment, and carry the cost of the models. + - Set up the Athena Assessment Module Manager and all Athena Modules you want to evaluate locally. (See :ref:`setup_install`, then see :ref:`run_local` or see :ref:`run_docker`) + - In `.env.example`, you will find example environment variables that need to be set in `.env` (duplicate the file, rename it to `.env`, then edit). + - Run the Athena Playground. (See :ref:`run_playground`). + + + +Connect to Athena Instance through the Playground +-------------------------- + +1. Open the playground + - Test-server setup: https://athenetest1-03.ase.cit.tum.de/playground + - Local setup: http://localhost:3000 +2. Up top you see the **Base Info Header** containing all configuration + .. figure:: ./base_info_header.png + :width: 800px + :alt: Base Info Header Interface of the Athena Playground + + Base Info Header Interface of the Athena Playground (local setup) +3. For the test-server setup: **Enter the secret using the provided credentials** +4. Check health of the services + - Assessment Module Manage has to be green + - All modules for evaluation have to be green + +Loading the Datasets +----------------------- + +In the base info header you have the option to select a dataset: + +1. **Example Data:** If you just want to experiment with the modules using the provided example data for testing purposes. +2. **Evaluation Data:** If you want to use existing or your own evaluation data. + + +Default Evaluation Data +^^^^^^^^^^^^^^^^^^^^^^^ + +When selecting `Evaluation Data` you can see the available exercises in `data/evaluation/` locally or remotely on the test-server (none if there is no data configured). + +.. figure:: ./evaluation_data.png + :width: 800px + :alt: Evaluation Data Interface of the Athena Playground + + Evaluation Data Interface of the Athena Playground + +Custom Evaluation Data +^^^^^^^^^^^^^^^^^^^^^^^ + +You can add your own evaluation data like this: + +1. Select **Evaluation Data** +2. Enter a directory name for the custom evaluation data, e.g. `my_awesome_evaluation_data` +3. Press **Import** and select the data you want to import (See :ref:`evaluation_data_format_guide`) + - Select one or multiple `.json` files (in case of non-programming exercises) + - Alternatively: Select a `.zip` that contains `.json` files and the associated code repositories. **Important:** they have to be on the top level! +4. Verify that it lists "`Avaliable exercises:`" as expected +5. Optional: Press **Export** to export the data if needed (The playground does not modify it, currently) +6. Optional: Press **Delete** to delete the data again (Warning: Some data might still exist in the Athena database if it was sent to a module) + +.. figure:: ./custom_evaluation_data.png + :width: 800px + :alt: Custom Evaluation Data Interface of the Athena Playground + + Custom Evaluation Data, `my_awesome_evaluation_data`, in the Athena Playground + +Choose Testing Mode +------------------- + +- Select between **Module Requests** and **Evaluation Mode** for different testing experiences. + +.. tip:: + Module Requests are for testing individual module responses, while Evaluation Mode is for comprehensive experiments. diff --git a/playground/package-lock.json b/playground/package-lock.json index 42910ff23..8a5bd8bf7 100644 --- a/playground/package-lock.json +++ b/playground/package-lock.json @@ -19,6 +19,7 @@ "allotment": "1.19.3", "archiver": "6.0.1", "autoprefixer": "10.4.16", + "formidable": "^3.5.1", "jszip": "3.10.1", "monaco-editor": "0.44.0", "next": "14.1.1", @@ -36,14 +37,17 @@ "rehype-raw": "7.0.0", "tailwind-merge": "1.14.0", "tailwindcss": "3.3.3", + "unzipper": "^0.12.1", "uuid": "9.0.1" }, "devDependencies": { "@types/archiver": "5.3.4", + "@types/formidable": "^3.4.5", "@types/node": "20.8.7", "@types/react": "18.2.31", "@types/react-dom": "18.2.14", "@types/react-modal": "3.16.2", + "@types/unzipper": "^0.10.9", "@types/uuid": "9.0.6", "axios": "1.6.0", "eslint": "8.52.0", @@ -1465,6 +1469,15 @@ "@types/ms": "*" } }, + "node_modules/@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/hast": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.2.tgz", @@ -1562,6 +1575,15 @@ "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.1.tgz", "integrity": "sha512-ue/hDUpPjC85m+PM9OQDMZr3LywT+CT6mPsQq8OJtCLiERkGRcQUFvu9XASF5XWqyZFXbf15lvb3JFJ4dRLWPg==" }, + "node_modules/@types/unzipper": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.9.tgz", + "integrity": "sha512-vHbmFZAw8emNAOVkHVbS3qBnbr0x/qHQZ+ei1HE7Oy6Tyrptl+jpqnOX+BF5owcu/HZLOV0nJK+K9sjs1Ox2JA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", @@ -2036,6 +2058,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, "node_modules/ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", @@ -2220,6 +2247,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2850,6 +2882,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -2902,6 +2943,46 @@ "tslib": "^2.0.3" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.616", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.616.tgz", @@ -3733,6 +3814,19 @@ "node": ">= 6" } }, + "node_modules/formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -3745,6 +3839,19 @@ "url": "https://github.com/sponsors/rawify" } }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4186,6 +4293,14 @@ "tslib": "^2.0.3" } }, + "node_modules/hexoid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", + "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", + "engines": { + "node": ">=8" + } + }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -4848,6 +4963,17 @@ "json5": "lib/cli.js" } }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jsonpointer": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", @@ -5873,6 +5999,11 @@ "tslib": "^2.0.3" } }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" + }, "node_modules/node-releases": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", @@ -7998,6 +8129,14 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/unload": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/unload/-/unload-2.2.0.tgz", @@ -8007,6 +8146,18 @@ "detect-node": "^2.0.4" } }, + "node_modules/unzipper": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.1.tgz", + "integrity": "sha512-wjYe5XddA387WIAZbEMWOT0U8kw9yf1MdfLHccJer1y7a80t3DqVv0SHOAWV5NDBD2TUPj/pFYmK9tCeY6l9UQ==", + "dependencies": { + "bluebird": "~3.4.1", + "duplexer2": "~0.1.4", + "fs-extra": "^11.2.0", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", diff --git a/playground/package.json b/playground/package.json index fd913ee75..c15b46272 100644 --- a/playground/package.json +++ b/playground/package.json @@ -13,18 +13,19 @@ "export:artemis:4-link-programming-repositories": "node ./scripts/artemis/4_link_programming_repositories.mjs" }, "dependencies": { - "uuid": "9.0.1", "@blueprintjs/core": "5.5.1", "@ls1intum/apollon": "3.3.5", + "@monaco-editor/react": "4.6.0", "@radix-ui/react-collapsible": "1.0.3", "@rjsf/core": "5.13.2", "@rjsf/utils": "5.13.2", "@rjsf/validator-ajv8": "5.13.2", "@tailwindcss/typography": "0.5.10", - "jszip": "3.10.1", - "archiver": "6.0.1", "allotment": "1.19.3", - "@monaco-editor/react": "4.6.0", + "archiver": "6.0.1", + "autoprefixer": "10.4.16", + "formidable": "^3.5.1", + "jszip": "3.10.1", "monaco-editor": "0.44.0", "next": "14.1.1", "react": "18.2.0", @@ -38,18 +39,20 @@ "react-reverse-portal": "2.1.1", "react-textarea-autosize": "8.5.3", "rehype-raw": "7.0.0", - "postcss": "8.4.31", - "autoprefixer": "10.4.16", + "tailwind-merge": "1.14.0", "tailwindcss": "3.3.3", - "tailwind-merge": "1.14.0" + "unzipper": "^0.12.1", + "uuid": "9.0.1" }, "devDependencies": { - "@types/uuid": "9.0.6", - "@types/node": "20.8.7", "@types/archiver": "5.3.4", + "@types/formidable": "^3.4.5", + "@types/node": "20.8.7", "@types/react": "18.2.31", "@types/react-dom": "18.2.14", "@types/react-modal": "3.16.2", + "@types/unzipper": "^0.10.9", + "@types/uuid": "9.0.6", "axios": "1.6.0", "eslint": "8.52.0", "eslint-config-next": "13.5.6", diff --git a/playground/src/components/selectors/data_mode_select.tsx b/playground/src/components/selectors/data_mode_select.tsx index b21558888..6d7f5f551 100644 --- a/playground/src/components/selectors/data_mode_select.tsx +++ b/playground/src/components/selectors/data_mode_select.tsx @@ -1,14 +1,73 @@ import type { DataMode } from "@/model/data_mode"; +import { useQueryClient } from "react-query"; +import useExercises from "@/hooks/playground/exercises"; +import useImportEvaluationData from "@/hooks/playground/evaluation_data/import"; +import useExportEvaluationData from "@/hooks/playground/evaluation_data/export"; +import useDeleteEvaluationData from "@/hooks/playground/evaluation_data/delete"; + +function sanitizeDirectoryName(directory: string): string { + return directory.replace(/[^a-zA-Z0-9_-]/g, ""); +} export default function DataModeSelect({ dataMode, - onChangeDataMode, + onChangeDataMode }: { dataMode: DataMode; onChangeDataMode: (dataMode: DataMode) => void; }) { + const queryClient = useQueryClient(); + const evaluationKey = "evaluation-"; + const evaluationDataKey = dataMode.startsWith(evaluationKey) ? dataMode.slice(evaluationKey.length) : undefined; + + const { data: exerciseData, error: exerciseError, isLoading: isLoadingExercises } = useExercises() + const exerciseTypes = new Map(); + exerciseData?.forEach((exercise) => { + exerciseTypes.set(exercise.type, (exerciseTypes.get(exercise.type) || 0) + 1); + }); + const exerciseStatistics = Array.from(exerciseTypes.entries()) + .map(([type, count]) => `${count} ${type}`) + .join(", ") || "No exercises available"; + + const { mutate, isLoading: isLoadingImport } = useImportEvaluationData({ + onSuccess: () => { + alert("Import successful!"); + queryClient.invalidateQueries("exercises"); + }, + onError: () => { + alert("Import failed!"); + } + }); + const { mutate: exportData, isLoading: isLoadingExport } = useExportEvaluationData({ + onSuccess: (blob) => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.style.display = "none"; + a.href = url; + a.download = `${dataMode}.zip`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + } + }); + + const { mutate: deleteData, isLoading: isLoadingDelete } = useDeleteEvaluationData({ + onSuccess: () => queryClient.invalidateQueries("exercises"), + }); + + const handleImport = (files: FileList | null) => { + if (!files) return; + + const formData = new FormData(); + Array.from(files).forEach((file) => { + formData.append("file", file); + }); + + mutate({ dataMode, formData }); + }; + return ( -