Added a script to verify test data against the autograder.

.ci/check_style.sh

@@ -15,7 +15,7 @@ function main() {
 
     local error_count=0
 
-    python3 -m autograder.cli.style "${BASE_DIR}"
+    python3 -m autograder.cli.style "${BASE_DIR}/autograder" "${BASE_DIR}/tests"
     ((error_count += $?))
 
     if [[ ${error_count} -gt 0 ]] ; then

.ci/verify_test_api_requests.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+"""
+Verify that the API test data (responses) are correct by sending them to an online
+autograding server and verifying the responses.
+This script should also be run in the autograding server's CI.
+"""
+
+import glob
+import importlib
+import json
+import os
+import sys
+import traceback
+
+import autograder.api.config
+
+THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+ROOT_DIR = os.path.join(THIS_DIR, '..')
+TEST_DATA_DIR = os.path.join(ROOT_DIR, 'tests', 'api', 'data')
+
+# Add in the tests path.
+sys.path.append(ROOT_DIR)
+
+import tests.api.test_api
+
+def verify_test_case(cli_arguments, path):
+    import_module_name, arguments, expected = tests.api.test_api.get_api_test_info(path)
+
+    for key, value in vars(cli_arguments).items():
+        if ((value is not None) or (value)):
+            arguments[key] = value
+
+    api_module = importlib.import_module(import_module_name)
+
+    actual = api_module.send(arguments)
+
+    if (actual != expected):
+        expected_json = json.dumps(expected, indent = 4)
+        actual_json = json.dumps(actual, indent = 4)
+
+        print("ERROR: Test case does not have expected content: '%s'." % (path))
+        print(tests.api.test_api.FORMAT_STR % (expected_json, actual_json))
+
+        return 1
+
+    return 0
+
+def run(arguments):
+    error_count = 0
+
+    for path in glob.glob(os.path.join(TEST_DATA_DIR, '**', 'test_*.json'), recursive = True):
+        try:
+            error_count += verify_test_case(arguments, path)
+        except Exception as ex:
+            error_count += 1
+            print("Error verifying test '%s'." % (path))
+            traceback.print_exception(ex)
+
+    if (error_count > 0):
+        print("Found %d API test case issues." % (error_count))
+    else:
+        print("Found no API test case issues.")
+
+    return error_count
+
+def main():
+    parser = autograder.api.config.get_argument_parser(
+        description = 'Verify test API data against an autograder server.')
+
+    return run(parser.parse_args())
+
+if __name__ == '__main__':
+    sys.exit(main())

autograder/api/common.py

@@ -58,7 +58,8 @@ def send_api_request(endpoint, server = None, verbose = False, data = {}, files
     for path in files:
         filename = os.path.basename(path)
         if (filename in post_files):
-            raise autograder.api.error.APIError("Cannot submit duplicate filenames ('%s')." % (filename))
+            raise autograder.api.error.APIError("Cannot submit duplicate filenames ('%s')." % (
+                filename))
 
         post_files[filename] = open(path, 'rb')
 
@@ -74,9 +75,9 @@ def send_api_request(endpoint, server = None, verbose = False, data = {}, files
     try:
         response = raw_response.json()
     except Exception as ex:
-        raise autograder.api.error.APIError("Autograder response does not contain valid JSON." +
-            " Contact a server admin with the following. Response:\n---\n%s\n---" % (
-            raw_response.text)) from ex
+        raise autograder.api.error.APIError("Autograder response does not contain valid JSON."
+            + " Contact a server admin with the following. Response:\n---\n%s\n---" % (
+                raw_response.text)) from ex
 
     if (verbose):
         print("\nAutograder Reponse:\n---\n%s\n---\n" % (json.dumps(response, indent = 4)))

autograder/api/config.py

@@ -36,7 +36,8 @@ def parse_api_config(config, params,
         additional_optional_keys = ['verbose'],
         exit_on_error = False):
     """
-    Given a tiered config and api parameters, return a dict that can be directly sierialized and sent to the autograder.
+    Given a tiered config and api parameters,
+    return a dict that can be directly sierialized and sent to the autograder.
     Any hashed params will be hashed.
     If |exit_on_error| is true sys.exit() will be called on an error,
     otherwise an error will be raised on an error.
@@ -62,7 +63,8 @@ def _parse_api_config(config, params, additional_required_keys, additional_optio
     for param in params:
         if (param.config_key not in config):
             if (param.required):
-                raise autograder.api.error.APIError(f"Required parameter '{param.config_key}' not set.")
+                raise autograder.api.error.APIError(
+                    f"Required parameter '{param.config_key}' not set.")
 
             continue
 
@@ -165,7 +167,8 @@ def get_argument_parser(
 
     parser.add_argument('-v', '--verbose', dest = 'verbose',
         action = 'store_true', default = False,
-        help = 'Output detailed information about the API request and response (default: %(default)s).')
+        help = 'Output detailed information about the API request and response'
+            + " (default: %(default)s).")
 
     for param in params:
         parser.add_argument(f'--{param.config_key}', dest = param.config_key,

autograder/api/error.py

@@ -1,3 +1,2 @@
 class APIError(Exception):
     pass
-

setup.py

@@ -33,7 +33,10 @@ def get_description():
         'Programming Language :: Python :: 3.10',
     ],
 
-    packages = setuptools.find_packages(),
+    packages = setuptools.find_packages(
+        include = ["autograder*"],
+        exclude = ["test"],
+    ),
 
     install_requires = [
         'flake8>=6.0.0',

tests/api/data/test_user_get_base.json

@@ -1,5 +1,5 @@
 {
-    "api-method": "autograder.api.user.get.send",
+    "endpoint": "user/get",
     "arguments": {
         "target-email": "[email protected]"
     },

tests/api/data/test_user_get_missing.json

@@ -1,5 +1,5 @@
 {
-    "api-method": "autograder.api.user.get.send",
+    "endpoint": "user/get",
     "arguments": {
         "target-email": "[email protected]"
     },

tests/api/test_api.py

@@ -5,10 +5,6 @@
 import unittest
 import sys
 
-import autograder.api.history
-import autograder.api.peek
-import autograder.assignment
-import autograder.question
 import tests.api.server
 
 THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
@@ -61,26 +57,30 @@ def _discover_api_tests():
             raise ValueError("Failed to parse test case '%s'." % (path)) from ex
 
 def _add_api_test(path):
+    test_name = os.path.splitext(os.path.basename(path))[0]
+    setattr(APITest, test_name, _get_api_test_method(path))
+
+def get_api_test_info(path):
     with open(path, 'r') as file:
         data = json.load(file)
 
-    test_name = os.path.splitext(os.path.basename(path))[0]
-    setattr(APITest, test_name, _get_api_test_method(data))
-
-def _get_api_test_method(data):
-    import_module_name = '.'.join(data['api-method'].split('.')[0:-1])
+    import_module_name = '.'.join(['autograder', 'api'] + data['endpoint'].split('/'))
     expected = data['output']
-    arguments = data.get('arguments', {})
+
+    arguments = BASE_ARGUMENTS.copy()
+    for key, value in data.get('arguments', {}).items():
+        arguments[key] = value
+
+    return import_module_name, arguments, expected
+
+def _get_api_test_method(path):
+    import_module_name, arguments, expected = get_api_test_info(path)
 
     def __method(self):
         api_module = importlib.import_module(import_module_name)
 
-        args = BASE_ARGUMENTS.copy()
-        for key, value in arguments.items():
-            args[key] = value
-
         self._next_response_queue.put(expected)
-        actual = api_module.send(args)
+        actual = api_module.send(arguments)
 
         self.assertDictEqual(actual, expected)