diff --git a/.gitignore b/.gitignore index 27edbe03..99c0d65e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ main /test.* /config.yml - /testing_requests /EduOJBackend /backend diff --git a/app/controller/controller_test.go b/app/controller/controller_test.go index e8d804ef..20a4075a 100644 --- a/app/controller/controller_test.go +++ b/app/controller/controller_test.go @@ -144,6 +144,8 @@ func runFailTests(t *testing.T, tests []failTest, groupName string) { httpResp := makeResp(req) resp := response.Response{} mustJsonDecode(httpResp, &resp) + t.Logf("Expected Status Code: %d", test.statusCode) + t.Logf("Actual Status Code: %d", httpResp.StatusCode) assert.Equal(t, test.statusCode, httpResp.StatusCode) assert.Equal(t, test.resp, resp) }) diff --git a/app/controller/problem.go b/app/controller/problem.go index 4745d5d8..aa570703 100644 --- a/app/controller/problem.go +++ b/app/controller/problem.go @@ -397,12 +397,11 @@ func CreateTestCase(c echo.Context) error { InputFileName: inputFile.Filename, OutputFileName: outputFile.Filename, } - if err := base.DB.Model(&problem).Association("TestCases").Append(&testCase); err != nil { panic(errors.Wrap(err, "could not create test case")) } - - utils.MustPutObject(inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID)) + // upload to minio + utils.MustPutInputFile(*req.Sanitize, inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID)) utils.MustPutObject(outputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/output/%d.out", problem.ID, testCase.ID)) return c.JSON(http.StatusCreated, response.CreateTestCaseResponse{ @@ -478,7 +477,6 @@ func UpdateTestCase(c echo.Context) error { if err, ok := utils.BindAndValidate(&req, c); !ok { return err } - inputFile, err := c.FormFile("input_file") if err != nil && err != http.ErrMissingFile && err.Error() != "request Content-Type isn't multipart/form-data" { panic(errors.Wrap(err, "could not read input file")) @@ -489,7 +487,7 @@ func UpdateTestCase(c echo.Context) error { } if inputFile != nil { - utils.MustPutObject(inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID)) + utils.MustPutInputFile(*req.Sanitize, inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID)) testCase.InputFileName = inputFile.Filename } if outputFile != nil { diff --git a/app/controller/problem_test.go b/app/controller/problem_test.go index 9ca2e9b3..7601353c 100644 --- a/app/controller/problem_test.go +++ b/app/controller/problem_test.go @@ -1036,6 +1036,7 @@ func TestCreateProblem(t *testing.T) { assert.Equal(t, test.req.TimeLimit, databaseProblem.TimeLimit) assert.Equal(t, strings.Split(test.req.LanguageAllowed, ","), []string(databaseProblem.LanguageAllowed)) assert.Equal(t, test.req.CompareScriptName, databaseProblem.CompareScriptName) + //assert.Equal(t, *test.req.Sanitize, databaseProblem.Sanitize) assert.Equal(t, *test.req.Public, databaseProblem.Public) assert.Equal(t, *test.req.Privacy, databaseProblem.Privacy) // response == database @@ -1055,6 +1056,8 @@ func TestCreateProblem(t *testing.T) { storageContent := getObjectContent(t, "problems", fmt.Sprintf("%d/attachment", databaseProblem.ID)) expectedContent, err := ioutil.ReadAll(test.attachment.reader) assert.NoError(t, err) + t.Logf("Expected Content: %+v", expectedContent) + t.Logf("Storage Content: %+v", storageContent) assert.Equal(t, expectedContent, storageContent) assert.Equal(t, test.attachment.fileName, databaseProblem.AttachmentFileName) } else { @@ -1734,8 +1737,9 @@ func TestCreateTestCase(t *testing.T) { newFileContent("input_file", "test_create_test_case_non_existing_problem.in", inputTextBase64), newFileContent("output_file", "test_create_test_case_non_existing_problem.out", outputTextBase64), }, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), reqOptions: []reqOption{ applyAdminUser, @@ -1750,8 +1754,9 @@ func TestCreateTestCase(t *testing.T) { req: addFieldContentSlice([]reqContent{ newFileContent("output_file", "test_create_test_case_lack_input_file.out", outputTextBase64), }, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), reqOptions: []reqOption{ headerOption{ @@ -1768,8 +1773,9 @@ func TestCreateTestCase(t *testing.T) { req: addFieldContentSlice([]reqContent{ newFileContent("input_file", "test_create_test_case_lack_output_file.in", inputTextBase64), }, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), reqOptions: []reqOption{ headerOption{ @@ -1784,8 +1790,9 @@ func TestCreateTestCase(t *testing.T) { method: "POST", path: base.Echo.Reverse("problem.createTestCase", problem.ID), req: addFieldContentSlice([]reqContent{}, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), reqOptions: []reqOption{ headerOption{ @@ -1803,8 +1810,9 @@ func TestCreateTestCase(t *testing.T) { newFileContent("input_file", "test_create_test_case_permission_denied.in", inputTextBase64), newFileContent("output_file", "test_create_test_case_permission_denied.out", outputTextBase64), }, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), reqOptions: []reqOption{ applyNormalUser, @@ -1822,8 +1830,9 @@ func TestCreateTestCase(t *testing.T) { newFileContent("input_file", "test_create_test_case_success.in", inputTextBase64), newFileContent("output_file", "test_create_test_case_success.out", outputTextBase64), }, map[string]string{ - "score": "100", - "sample": "true", + "score": "100", + "sample": "true", + "sanitize": "false", }), headerOption{ "Set-User-For-Test": {fmt.Sprintf("%d", user.ID)}, }) @@ -2066,14 +2075,16 @@ func TestUpdateTestCase(t *testing.T) { problem := createProblemForTest(t, "update_test_case", 0, nil, user) boolTrue := true + boolFalse := false failTests := []failTest{ { name: "NonExistingProblem", method: "PUT", path: base.Echo.Reverse("problem.updateTestCase", -1, 1), req: request.UpdateTestCaseRequest{ - Score: 100, - Sample: &boolTrue, + Score: 100, + Sample: &boolTrue, + Sanitize: &boolFalse, }, reqOptions: []reqOption{ applyAdminUser, @@ -2086,8 +2097,9 @@ func TestUpdateTestCase(t *testing.T) { method: "PUT", path: base.Echo.Reverse("problem.updateTestCase", problem.ID, -1), req: request.UpdateTestCaseRequest{ - Score: 100, - Sample: &boolTrue, + Score: 100, + Sample: &boolTrue, + Sanitize: &boolFalse, }, reqOptions: []reqOption{ headerOption{ @@ -2106,8 +2118,9 @@ func TestUpdateTestCase(t *testing.T) { method: "PUT", path: base.Echo.Reverse("problem.updateTestCase", problem.ID, 1), req: request.UpdateTestCaseRequest{ - Score: 100, - Sample: &boolTrue, + Score: 100, + Sample: &boolTrue, + Sanitize: &boolFalse, }, reqOptions: []reqOption{ applyNormalUser, @@ -2128,7 +2141,7 @@ func TestUpdateTestCase(t *testing.T) { { name: "SuccessWithoutUpdatingFile", originalData: testCaseData{ - Score: 0, + Score: 100, Sample: false, InputFile: newFileContent("input_file", "test_update_test_case_1.in", inputTextBase64), OutputFile: newFileContent("output_file", "test_update_test_case_1.out", outputTextBase64), @@ -2229,10 +2242,12 @@ func TestUpdateTestCase(t *testing.T) { if test.updatedData.OutputFile != nil { reqContentSlice = append(reqContentSlice, test.updatedData.OutputFile) } + sanitizeValue := false req := makeReq(t, "PUT", base.Echo.Reverse("problem.updateTestCase", problem.ID, testCase.ID), addFieldContentSlice( reqContentSlice, map[string]string{ - "score": fmt.Sprintf("%d", test.updatedData.Score), - "sample": fmt.Sprintf("%t", test.updatedData.Sample), + "score": fmt.Sprintf("%d", test.updatedData.Score), + "sample": fmt.Sprintf("%t", test.updatedData.Sample), + "sanitize": fmt.Sprintf("%t", sanitizeValue), }), headerOption{ "Set-User-For-Test": {fmt.Sprintf("%d", user.ID)}, }) diff --git a/app/request/problem.go b/app/request/problem.go index 71adb84c..9f6a40a9 100644 --- a/app/request/problem.go +++ b/app/request/problem.go @@ -7,6 +7,8 @@ type CreateProblemRequest struct { Public *bool `json:"public" form:"public" query:"public" validate:"required"` Privacy *bool `json:"privacy" form:"privacy" query:"privacy" validate:"required"` + //Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"` + MemoryLimit uint64 `json:"memory_limit" form:"memory_limit" query:"memory_limit" validate:"required"` // Byte TimeLimit uint `json:"time_limit" form:"time_limit" query:"time_limit" validate:"required"` // ms LanguageAllowed string `json:"language_allowed" form:"language_allowed" query:"language_allowed" validate:"required,max=255"` // E.g. cpp,c,java,python @@ -36,8 +38,9 @@ type DeleteProblemRequest struct { } type CreateTestCaseRequest struct { - Score uint `json:"score" form:"score" query:"score"` // 0 for 平均分配 - Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"` + Score uint `json:"score" form:"score" query:"score"` + Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"` + Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"` // input_file(required) // output_file(required) } @@ -49,8 +52,9 @@ type GetTestCaseOutputFileRequest struct { } type UpdateTestCaseRequest struct { - Score uint `json:"score" form:"score" query:"score"` // 0 for 平均分配 - Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"` + Score uint `json:"score" form:"score" query:"score"` + Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"` + Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"` // input_file(optional) // output_file(optional) } diff --git a/base/utils/helpers.go b/base/utils/helpers.go index 5fa2b101..79f05515 100644 --- a/base/utils/helpers.go +++ b/base/utils/helpers.go @@ -1,9 +1,13 @@ package utils import ( + "bufio" "context" + "fmt" "mime/multipart" "net/http" + "os" + "strings" "github.com/EduOJ/backend/app/response" "github.com/EduOJ/backend/base" @@ -58,6 +62,61 @@ func MustPutObject(object *multipart.FileHeader, ctx context.Context, bucket str } } +func MustPutInputFile(sanitize bool, object *multipart.FileHeader, ctx context.Context, bucket string, path string) { + src, err := object.Open() + if err != nil { + panic(err) + } + defer src.Close() + + var fileSize int64 + + if sanitize { + scanner := bufio.NewScanner(src) + tempFile, err := os.CreateTemp("", "tempFile*.txt") + if err != nil { + panic(err) + } + writer := bufio.NewWriter(tempFile) + for scanner.Scan() { + line := strings.ReplaceAll(scanner.Text(), "\r\n", "\n") // replace '\r\n' to '\n' + + if !strings.HasSuffix(line, "\n") { + line += "\n" + } + _, err := fmt.Fprint(writer, line) + if err != nil { + panic(err) + } + } + if err := scanner.Err(); err != nil { + panic(err) + } + if err := writer.Flush(); err != nil { + panic(err) + } + + fileInfo, err := tempFile.Stat() + if err != nil { + panic(err) + } + fileSize = fileInfo.Size() + + src, err = os.Open(tempFile.Name()) + if err != nil { + panic(err) + } + defer src.Close() + } else { + fileSize = object.Size + } + + _, err = base.Storage.PutObject(ctx, bucket, path, src, fileSize, minio.PutObjectOptions{}) + if err != nil { + panic(errors.Wrap(err, "could write file to s3 storage.")) + } +} + func MustGetObject(c echo.Context, bucket string, path string) *minio.Object { object, err := base.Storage.GetObject(c.Request().Context(), bucket, path, minio.GetObjectOptions{}) if err != nil { diff --git a/base/validator/translations/zh/zh.go b/base/validator/translations/zh/zh.go index dcf35b17..77cfa735 100644 --- a/base/validator/translations/zh/zh.go +++ b/base/validator/translations/zh/zh.go @@ -52,6 +52,7 @@ var FieldTranslations = map[string]string{ "Tried": "选取尝试过题目", "Passed": "选取通过题目", "Token": "验证码", + "Sanitize": "是否清洗数据", } // RegisterDefaultTranslations registers a set of default translations