From fef17be9fd0cc89a38e9fcf0db47263aa090bca3 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 12:14:21 +0700 Subject: [PATCH 01/30] Fix embed access --- db/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrations.go b/db/migrations.go index 6156d266..9fb3b7dc 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -4,7 +4,7 @@ import ( "database/sql" "io/fs" "log" - "path/filepath" + "path" "regexp" "sort" @@ -44,7 +44,7 @@ func (db *DB) migrate() error { // Do migrations one by one for _, name := range versions { - sqlFile := filepath.Join(assetsSql, name+".sql") + sqlFile := path.Join(assetsSql, name+".sql") file, err := fs.ReadFile(embed.Content, sqlFile) if err != nil { return errors.Wrapf(err, "File %s", sqlFile) From a050bc2e85381ba8f288f200c291baabf025ed94 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 12:14:35 +0700 Subject: [PATCH 02/30] Run go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5a2f8e29..ce595bdd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.0 github.com/pkg/errors v0.9.1 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e - golang.org/x/net v0.7.0 + golang.org/x/net v0.7.0 // indirect golang.org/x/text v0.7.0 golang.org/x/tools v0.1.12 google.golang.org/appengine v1.6.5 // indirect From d4a196a1dcc27c95c2976de766f6c717e28d3436 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 12:26:54 +0700 Subject: [PATCH 03/30] Script with database wipe --- scripts/windows/production_test.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 scripts/windows/production_test.ps1 diff --git a/scripts/windows/production_test.ps1 b/scripts/windows/production_test.ps1 new file mode 100644 index 00000000..cda232ef --- /dev/null +++ b/scripts/windows/production_test.ps1 @@ -0,0 +1,8 @@ +$ErrorActionPreference = "Stop" +Remove-Item kjudge.db* + +& "scripts\windows\production_build.ps1" + +Invoke-Expression ".\kjudge $args" + +# Run pwsh -c scripts/windows/production_test.ps1 --sandbox=raw to test this script From f7ac3b55785c71d12b54ca553573027d922061bd Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 14:04:14 +0700 Subject: [PATCH 04/30] Add instruction for production_test.ps1 --- scripts/windows/production_test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/windows/production_test.ps1 b/scripts/windows/production_test.ps1 index cda232ef..8d9fafe2 100644 --- a/scripts/windows/production_test.ps1 +++ b/scripts/windows/production_test.ps1 @@ -5,4 +5,4 @@ Remove-Item kjudge.db* Invoke-Expression ".\kjudge $args" -# Run pwsh -c scripts/windows/production_test.ps1 --sandbox=raw to test this script +# pwsh -c scripts/windows/production_test.ps1 --sandbox=raw to run this test script From a73e79ed7cf109b1fdf499adf1fc28e42c3d5592 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 16:20:50 +0700 Subject: [PATCH 05/30] Remove ineffective break statement --- worker/score.go | 1 - 1 file changed, 1 deletion(-) diff --git a/worker/score.go b/worker/score.go index 36406c55..cb7aafcd 100644 --- a/worker/score.go +++ b/worker/score.go @@ -190,7 +190,6 @@ func (s *ScoreContext) CompareScores(subs []*models.Submission) *models.ProblemR if which == nil { which = sub maxScore = score - break } case models.ScoringModeLast: which = sub From e4735d0c02daa6170ac52ff08d4b2e6b296e1abc Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 17:18:33 +0700 Subject: [PATCH 06/30] Move modules --- cmd/kjudge/main.go | 15 ++++------ worker/queue.go | 4 +-- worker/run.go | 37 +++++++++++++------------ worker/{ => sandbox}/isolate/meta.go | 0 worker/{ => sandbox}/isolate/sandbox.go | 12 ++++---- worker/{sandbox.go => sandbox/main.go} | 2 +- worker/{ => sandbox}/raw/sandbox.go | 13 ++++----- 7 files changed, 39 insertions(+), 44 deletions(-) rename worker/{ => sandbox}/isolate/meta.go (100%) rename worker/{ => sandbox}/isolate/sandbox.go (90%) rename worker/{sandbox.go => sandbox/main.go} (99%) rename worker/{ => sandbox}/raw/sandbox.go (86%) diff --git a/cmd/kjudge/main.go b/cmd/kjudge/main.go index f1051a44..e03f13b2 100644 --- a/cmd/kjudge/main.go +++ b/cmd/kjudge/main.go @@ -12,8 +12,6 @@ import ( _ "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server" "github.com/natsukagami/kjudge/worker" - "github.com/natsukagami/kjudge/worker/isolate" - "github.com/natsukagami/kjudge/worker/raw" ) var ( @@ -34,15 +32,12 @@ func main() { } defer db.Close() - var sandbox worker.Sandbox - switch *sandboxImpl { - case "raw": + if *sandboxImpl == "raw" { log.Println("'raw' sandbox selected. WE ARE NOT RESPONSIBLE FOR ANY BREAKAGE CAUSED BY FOREIGN CODE.") - sandbox = &raw.Sandbox{} - case "isolate": - sandbox = isolate.New() - default: - log.Fatalf("Sandbox %s doesn't exists or not yet implemented.", *sandboxImpl) + } + sandbox, err := worker.NewSandbox(*sandboxImpl) + if err != nil { + log.Fatalf("%v", err) } opts := []server.Opt{} diff --git a/worker/queue.go b/worker/queue.go index 10f157da..423aba47 100644 --- a/worker/queue.go +++ b/worker/queue.go @@ -7,20 +7,20 @@ import ( "github.com/mattn/go-sqlite3" "github.com/natsukagami/kjudge/db" "github.com/natsukagami/kjudge/models" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) // Queue implements a queue that runs each job one by one. type Queue struct { DB *db.DB - Sandbox Sandbox + Sandbox sandbox.Sandbox } // Start starts the queue. It is blocking, so might wanna "go run" it. func (q *Queue) Start() { // Register the update callback toUpdate := q.startHook() - for { // Get the newest job job, err := models.FirstJob(q.DB) diff --git a/worker/run.go b/worker/run.go index bb2e0b5c..268da662 100644 --- a/worker/run.go +++ b/worker/run.go @@ -10,6 +10,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/natsukagami/kjudge/models" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -67,12 +68,12 @@ func (r *RunContext) CompiledSource() (bool, []byte) { } // RunInput creates a SandboxInput for running the submission's source. -func (r *RunContext) RunInput(source []byte) (*SandboxInput, error) { +func (r *RunContext) RunInput(source []byte) (*sandbox.SandboxInput, error) { command, args, err := RunCommand(r.Sub.Language) if err != nil { return nil, err } - return &SandboxInput{ + return &sandbox.SandboxInput{ Command: command, Args: args, Files: nil, @@ -86,11 +87,11 @@ func (r *RunContext) RunInput(source []byte) (*SandboxInput, error) { // CompareInput creates a SandboxInput for running the comparator. // Also returns whether we have diff-based or comparator-based input. -func (r *RunContext) CompareInput(submissionOutput []byte) (input *SandboxInput, useComparator bool, err error) { +func (r *RunContext) CompareInput(submissionOutput []byte) (input *sandbox.SandboxInput, useComparator bool, err error) { file, err := models.GetFileWithName(r.DB, r.Problem.ID, "compare") if errors.Is(err, sql.ErrNoRows) { // Use a simple diff - return &SandboxInput{ + return &sandbox.SandboxInput{ Command: "/usr/bin/diff", Args: []string{"-wqts", "output", "expected"}, Files: map[string][]byte{"output": submissionOutput, "expected": r.Test.Output}, @@ -102,31 +103,31 @@ func (r *RunContext) CompareInput(submissionOutput []byte) (input *SandboxInput, return nil, false, err } // Use the given comparator. - return &SandboxInput{ + return &sandbox.SandboxInput{ Command: "code", Args: []string{"input", "expected", "output"}, Files: map[string][]byte{"input": r.Test.Input, "expected": r.Test.Output, "output": submissionOutput}, TimeLimit: 20 * time.Second, - MemoryLimit: (2 << 20), // 1 GB + MemoryLimit: (1 << 20), // 1 GB CompiledSource: file.Content, }, true, nil } -func RunSingleCommand(sandbox Sandbox, r *RunContext, source []byte) (output *SandboxOutput, err error) { +func RunSingleCommand(s sandbox.Sandbox, r *RunContext, source []byte) (output *sandbox.SandboxOutput, err error) { // First, use the sandbox to run the submission itself. input, err := r.RunInput(source) if err != nil { return nil, err } - output, err = sandbox.Run(input) + output, err = s.Run(input) if err != nil { return nil, errors.WithStack(err) } return output, nil } -func RunMultipleCommands(sandbox Sandbox, r *RunContext, source []byte, stages []string) (output *SandboxOutput, err error) { +func RunMultipleCommands(s sandbox.Sandbox, r *RunContext, source []byte, stages []string) (output *sandbox.SandboxOutput, err error) { command, args, err := RunCommand(r.Sub.Language) if err != nil { return nil, err @@ -140,7 +141,7 @@ func RunMultipleCommands(sandbox Sandbox, r *RunContext, source []byte, stages [ } stageArgs := strings.Split(stage, " ") - sandboxInput := &SandboxInput{ + sandboxInput := &sandbox.SandboxInput{ Command: command, Args: append(stageArgs, args...), Files: nil, @@ -151,7 +152,7 @@ func RunMultipleCommands(sandbox Sandbox, r *RunContext, source []byte, stages [ Input: input, } - output, err = sandbox.Run(sandboxInput) + output, err = s.Run(sandboxInput) if err != nil { return nil, err } @@ -166,7 +167,7 @@ func RunMultipleCommands(sandbox Sandbox, r *RunContext, source []byte, stages [ } // Run runs a RunContext. -func Run(sandbox Sandbox, r *RunContext) error { +func Run(s sandbox.Sandbox, r *RunContext) error { compiled, source := r.CompiledSource() if !compiled { // Add a compilation job and re-add ourselves. @@ -180,11 +181,11 @@ func Run(sandbox Sandbox, r *RunContext) error { log.Printf("[WORKER] Running submission %v on [test `%v`, group `%v`]\n", r.Sub.ID, r.Test.Name, r.TestGroup.Name) - var output *SandboxOutput + var output *sandbox.SandboxOutput file, err := models.GetFileWithName(r.DB, r.Problem.ID, ".stages") if errors.Is(err, sql.ErrNoRows) { // Problem type is not Chained Type, run a single command - output, err = RunSingleCommand(sandbox, r, source) + output, err = RunSingleCommand(s, r, source) if err != nil { return err } @@ -193,7 +194,7 @@ func Run(sandbox Sandbox, r *RunContext) error { } else { // Problem Type is Chained Type, we need to run mutiple commands with arguments from .stages (file) stages := strings.Split(string(file.Content), "\n") - output, err = RunMultipleCommands(sandbox, r, source, stages) + output, err = RunMultipleCommands(s, r, source, stages) if err != nil { return err } @@ -214,7 +215,7 @@ func Run(sandbox Sandbox, r *RunContext) error { if err != nil { return err } - output, err = sandbox.Run(input) + output, err = s.Run(input) if err != nil { return err } @@ -229,7 +230,7 @@ func Run(sandbox Sandbox, r *RunContext) error { } // Parse the comparator's output and reflect it into `result`. -func parseComparatorOutput(s *SandboxOutput, result *models.TestResult, useComparator bool) error { +func parseComparatorOutput(s *sandbox.SandboxOutput, result *models.TestResult, useComparator bool) error { if useComparator { // Paste the comparator's output to result result.Verdict = strings.TrimSpace(string(s.Stderr)) @@ -258,7 +259,7 @@ func parseComparatorOutput(s *SandboxOutput, result *models.TestResult, useCompa } // Parse the sandbox output into a TestResult. -func parseSandboxOutput(s *SandboxOutput, r *RunContext) *models.TestResult { +func parseSandboxOutput(s *sandbox.SandboxOutput, r *RunContext) *models.TestResult { score := 1.0 if !s.Success { score = 0.0 diff --git a/worker/isolate/meta.go b/worker/sandbox/isolate/meta.go similarity index 100% rename from worker/isolate/meta.go rename to worker/sandbox/isolate/meta.go diff --git a/worker/isolate/sandbox.go b/worker/sandbox/isolate/sandbox.go similarity index 90% rename from worker/isolate/sandbox.go rename to worker/sandbox/isolate/sandbox.go index 4010adb5..7aa804c1 100644 --- a/worker/isolate/sandbox.go +++ b/worker/sandbox/isolate/sandbox.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -34,7 +34,7 @@ type Sandbox struct { private struct{} // Makes the sandbox not simply constructible } -var _ worker.Sandbox = (*Sandbox)(nil) +var _ sandbox.Sandbox = (*Sandbox)(nil) // Panics on not having "isolate" accessible. func mustHaveIsolate() { @@ -55,7 +55,7 @@ func New() *Sandbox { } // Run implements Sandbox.Run. -func (s *Sandbox) Run(input *worker.SandboxInput) (*worker.SandboxOutput, error) { +func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { // Init the sandbox defer s.cleanup() dirBytes, err := exec.Command(isolateCommand, "--init", "--cg").Output() @@ -86,7 +86,7 @@ func (s *Sandbox) Run(input *worker.SandboxInput) (*worker.SandboxOutput, error) } // Parse the meta file - output := &worker.SandboxOutput{ + output := &sandbox.SandboxOutput{ Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), } @@ -97,7 +97,7 @@ func (s *Sandbox) Run(input *worker.SandboxInput) (*worker.SandboxOutput, error) return output, nil } -func parseMetaFile(path string, output *worker.SandboxOutput) error { +func parseMetaFile(path string, output *sandbox.SandboxOutput) error { meta, err := ReadMetaFile(path) if err != nil { return err @@ -114,7 +114,7 @@ func parseMetaFile(path string, output *worker.SandboxOutput) error { } // Build the command for isolate --run. -func buildCmd(dir, metaFile string, input *worker.SandboxInput) *exec.Cmd { +func buildCmd(dir, metaFile string, input *sandbox.SandboxInput) *exec.Cmd { // Calculate stuff timeLimit := float64(input.TimeLimit) / float64(time.Second) diff --git a/worker/sandbox.go b/worker/sandbox/main.go similarity index 99% rename from worker/sandbox.go rename to worker/sandbox/main.go index 9336dd1b..87bc66a7 100644 --- a/worker/sandbox.go +++ b/worker/sandbox/main.go @@ -1,4 +1,4 @@ -package worker +package sandbox import ( "os" diff --git a/worker/raw/sandbox.go b/worker/sandbox/raw/sandbox.go similarity index 86% rename from worker/raw/sandbox.go rename to worker/sandbox/raw/sandbox.go index a8270afa..ee1e24b7 100644 --- a/worker/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -18,16 +18,16 @@ import ( "strings" "time" - "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/sandbox" ) // Sandbox implements worker.Sandbox. type Sandbox struct{} -var _ worker.Sandbox = (*Sandbox)(nil) +var _ sandbox.Sandbox = (*Sandbox)(nil) // Run implements Sandbox.Run -func (s *Sandbox) Run(input *worker.SandboxInput) (*worker.SandboxOutput, error) { +func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { dir := os.TempDir() log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) @@ -41,7 +41,7 @@ func (s *Sandbox) Run(input *worker.SandboxInput) (*worker.SandboxOutput, error) // - MEMORY LIMITS ARE NOT SET. It always reports a memory usage of 0 (it cannot measure them). // - THE PROGRAM DOES NOT MESS WITH THE COMPUTER. LMAO // - The folder will be thrown away later. -func (s *Sandbox) RunFrom(cwd string, input *worker.SandboxInput) (*worker.SandboxOutput, error) { +func (s *Sandbox) RunFrom(cwd string, input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { if err := input.CopyTo(cwd); err != nil { return nil, err } @@ -73,7 +73,7 @@ func (s *Sandbox) RunFrom(cwd string, input *worker.SandboxInput) (*worker.Sandb case <-time.After(input.TimeLimit): cancel() <-done - return &worker.SandboxOutput{ + return &sandbox.SandboxOutput{ Success: false, MemoryUsed: 0, RunningTime: input.TimeLimit, @@ -83,7 +83,7 @@ func (s *Sandbox) RunFrom(cwd string, input *worker.SandboxInput) (*worker.Sandb }, nil case commandErr := <-done: runningTime := time.Since(startTime) - return &worker.SandboxOutput{ + return &sandbox.SandboxOutput{ Success: commandErr == nil, MemoryUsed: 0, RunningTime: runningTime, @@ -93,5 +93,4 @@ func (s *Sandbox) RunFrom(cwd string, input *worker.SandboxInput) (*worker.Sandb }, nil } - } From 72753dc35dd88a4ef3c1b77564cabc17b8b3f604 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 17:19:55 +0700 Subject: [PATCH 07/30] function to get sandbox by name --- worker/sandbox.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 worker/sandbox.go diff --git a/worker/sandbox.go b/worker/sandbox.go new file mode 100644 index 00000000..c79ca1ff --- /dev/null +++ b/worker/sandbox.go @@ -0,0 +1,19 @@ +package worker + +import ( + "github.com/natsukagami/kjudge/worker/sandbox" + "github.com/natsukagami/kjudge/worker/sandbox/isolate" + "github.com/natsukagami/kjudge/worker/sandbox/raw" + "github.com/pkg/errors" +) + +func NewSandbox(name string) (sandbox.Sandbox, error) { + switch name { + case "raw": + return &raw.Sandbox{}, nil + case "isolate": + return isolate.New(), nil + default: + return nil, errors.Errorf("Sandbox %s doesn't exists or not yet implemented.", name) + } +} From 5dbd48e41aed6fafa3a25d9568cb4bc29d1899c2 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 22:02:17 +0700 Subject: [PATCH 08/30] WIP Code --- cmd/stress/main.go | 38 +++++++++++++++++++++++++++++++++++++ test/performance/problem.go | 31 ++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 cmd/stress/main.go create mode 100644 test/performance/problem.go diff --git a/cmd/stress/main.go b/cmd/stress/main.go new file mode 100644 index 00000000..5c2299f1 --- /dev/null +++ b/cmd/stress/main.go @@ -0,0 +1,38 @@ +// Compares provided sandboxes +package main + +import ( + "flag" + "log" + + "github.com/natsukagami/kjudge/db" + "github.com/natsukagami/kjudge/worker" +) + +var ( + dbfile = flag.String("file", "kjudge-test.db", "Path to database file.") + outputfile = flag.String("output", "kjudge-test.json", "Path to output file.") + sandboxImpls = flag.Args() + batchCount = flag.Int("count", 10, "Number of iterations.") + batchSize = flag.Int("count", 30, "Size of testset.") +) + +func main() { + flag.Parse() + + db, err := db.New(*dbfile) + if err != nil { + log.Fatalf("%+v", err) + } + defer db.Close() + + for i := 1; i <= *batchCount; i++ { + log.Printf("Running testset number %v", i) + for _, sbxName := range sandboxImpls { + sandbox, err := worker.NewSandbox(sbxName) + if err != nil { + log.Printf("%v", err) + } + } + } +} diff --git a/test/performance/problem.go b/test/performance/problem.go new file mode 100644 index 00000000..052d5e4e --- /dev/null +++ b/test/performance/problem.go @@ -0,0 +1,31 @@ +// Package performance provides performance testing +package performance + +import ( + "github.com/natsukagami/kjudge/db" + "github.com/natsukagami/kjudge/models" +) + +func generateProblem(db db.DBContext, contestID int, displayName string, name string) *models.Problem{ + return &models.Problem { + ContestID: contestID, + DisplayName: displayName, + ID: 0, // Auto generate ID + MaxSubmissionsCount: 0, // No limit + MemoryLimit: 1 << 20, // 1GB + Name: name, + PenaltyPolicy: "none", + ScoringMode: "best", + SecondsBetweenSubmissions: 0, + TimeLimit: 5, + } +} + +// Generates O(1), multitest problem to compare sandbox spawn time +func spawnTimeProblem(db db.DBContext, contestID int) error { + generateProblem() +} + +func GenerateContest() { + +} From 429bdb088ba1e3ef9112a0a9bac2ef518e12ea79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Minh=20Nh=E1=BA=ADt=20Nguy=E1=BB=85n?= <86871862+minhnhatnoe@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:56:56 +0700 Subject: [PATCH 09/30] WIP --- cmd/stress/main.go | 12 ++++---- test/performance/problem.go | 55 +++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/cmd/stress/main.go b/cmd/stress/main.go index 5c2299f1..27cebf6e 100644 --- a/cmd/stress/main.go +++ b/cmd/stress/main.go @@ -10,16 +10,16 @@ import ( ) var ( - dbfile = flag.String("file", "kjudge-test.db", "Path to database file.") - outputfile = flag.String("output", "kjudge-test.json", "Path to output file.") - sandboxImpls = flag.Args() - batchCount = flag.Int("count", 10, "Number of iterations.") - batchSize = flag.Int("count", 30, "Size of testset.") + dbfile = flag.String("file", "kjudge-test.db", "Path to database file.") + outputfile = flag.String("output", "kjudge-test.json", "Path to output file.") + sandboxImpls = flag.Args() + batchCount = flag.Int("count", 10, "Number of iterations.") + batchSize = flag.Int("count", 30, "Size of testset.") ) func main() { flag.Parse() - + db, err := db.New(*dbfile) if err != nil { log.Fatalf("%+v", err) diff --git a/test/performance/problem.go b/test/performance/problem.go index 052d5e4e..79e880a1 100644 --- a/test/performance/problem.go +++ b/test/performance/problem.go @@ -2,28 +2,53 @@ package performance import ( + "fmt" + "math/rand" + "github.com/natsukagami/kjudge/db" "github.com/natsukagami/kjudge/models" ) -func generateProblem(db db.DBContext, contestID int, displayName string, name string) *models.Problem{ - return &models.Problem { - ContestID: contestID, - DisplayName: displayName, - ID: 0, // Auto generate ID - MaxSubmissionsCount: 0, // No limit - MemoryLimit: 1 << 20, // 1GB - Name: name, - PenaltyPolicy: "none", - ScoringMode: "best", - SecondsBetweenSubmissions: 0, - TimeLimit: 5, +func generateProblem(db db.DBContext, contestID int, displayName string, name string) *models.Problem { + return &models.Problem{ + ContestID: contestID, + DisplayName: displayName, + ID: 0, // Auto generate ID + MaxSubmissionsCount: 0, // No limit + MemoryLimit: 1 << 20, // 1GB + Name: name, + PenaltyPolicy: "none", + ScoringMode: "best", + SecondsBetweenSubmissions: 0, + TimeLimit: 5, } } -// Generates O(1), multitest problem to compare sandbox spawn time -func spawnTimeProblem(db db.DBContext, contestID int) error { - generateProblem() +type PerfTest struct { + Input []byte + Output []byte +} + +type PerfTestSet struct { + Name string + TestCount int + TestGenerator func(*rand.Rand) *PerfTest +} + +// Generates O(1), multitest problem to compare sandbox spawn time. +// The problem used is input one number, then output double of that number +func spawnTimeProblem() *PerfTestSet { + return &PerfTestSet{ + Name: "SPAWN", + TestCount: 1000, + TestGenerator: func(r *rand.Rand) *PerfTest { + value := r.Int31() / 2 + return &PerfTest{ + Input: []byte(fmt.Sprintf("%v", value)), + Output: []byte(fmt.Sprintf("%v", value*2)), + } + }, + } } func GenerateContest() { From 5ed8d18d3d00a12e779f7b85c93d5eafbe498596 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Mon, 5 Jun 2023 22:49:29 +0700 Subject: [PATCH 10/30] WIP, still --- test/perf_test/calculate.go | 0 test/perf_test/input.go | 29 ++++++++++++++++++++++ test/perf_test/memory.go | 0 test/perf_test/output.go | 0 test/{performance => perf_test}/problem.go | 25 ++++--------------- test/perf_test/spawn.go | 26 +++++++++++++++++++ test/perf_test/tle.go | 0 7 files changed, 60 insertions(+), 20 deletions(-) create mode 100644 test/perf_test/calculate.go create mode 100644 test/perf_test/input.go create mode 100644 test/perf_test/memory.go create mode 100644 test/perf_test/output.go rename test/{performance => perf_test}/problem.go (60%) create mode 100644 test/perf_test/spawn.go create mode 100644 test/perf_test/tle.go diff --git a/test/perf_test/calculate.go b/test/perf_test/calculate.go new file mode 100644 index 00000000..e69de29b diff --git a/test/perf_test/input.go b/test/perf_test/input.go new file mode 100644 index 00000000..e54bb604 --- /dev/null +++ b/test/perf_test/input.go @@ -0,0 +1,29 @@ +package perf_test + +import ( + "fmt" + "math/rand" +) + +// 50MB input to compare disk read time. O(1) memory. +// Problem: Given a string, print it's length +func BigInputProblem() *PerfTestSet { + maxSize := 50000000 // 50MB + return &PerfTestSet{ + Name: "INPUT", + ExpectedTime: 2000, // TODO + TestGenerator: func(r *rand.Rand) *PerfTest { + strSize := maxSize - r.Intn(10) + input := make([]byte, strSize) + for i := range input { + input[i] = byte(r.Intn(26) + 'A') + } + return &PerfTest{ + Input: input, + Output: []byte(fmt.Sprintf("%v", strSize)), + } + }, + TestCode: []byte(""), // TODO + ExpectedAC: true, + } +} diff --git a/test/perf_test/memory.go b/test/perf_test/memory.go new file mode 100644 index 00000000..e69de29b diff --git a/test/perf_test/output.go b/test/perf_test/output.go new file mode 100644 index 00000000..e69de29b diff --git a/test/performance/problem.go b/test/perf_test/problem.go similarity index 60% rename from test/performance/problem.go rename to test/perf_test/problem.go index 79e880a1..7dbe9ad9 100644 --- a/test/performance/problem.go +++ b/test/perf_test/problem.go @@ -1,8 +1,7 @@ -// Package performance provides performance testing -package performance +// Package perf_test provides performance testing +package perf_test import ( - "fmt" "math/rand" "github.com/natsukagami/kjudge/db" @@ -31,24 +30,10 @@ type PerfTest struct { type PerfTestSet struct { Name string - TestCount int + ExpectedTime int // Expected running time of each test in ms TestGenerator func(*rand.Rand) *PerfTest -} - -// Generates O(1), multitest problem to compare sandbox spawn time. -// The problem used is input one number, then output double of that number -func spawnTimeProblem() *PerfTestSet { - return &PerfTestSet{ - Name: "SPAWN", - TestCount: 1000, - TestGenerator: func(r *rand.Rand) *PerfTest { - value := r.Int31() / 2 - return &PerfTest{ - Input: []byte(fmt.Sprintf("%v", value)), - Output: []byte(fmt.Sprintf("%v", value*2)), - } - }, - } + TestCode []byte // Solution to tested problem + ExpectedAC bool } func GenerateContest() { diff --git a/test/perf_test/spawn.go b/test/perf_test/spawn.go new file mode 100644 index 00000000..983cbae3 --- /dev/null +++ b/test/perf_test/spawn.go @@ -0,0 +1,26 @@ +package perf_test + +import ( + "fmt" + "math/rand" +) + +// O(1) problem to compare sandbox spawn time. +// Problem: Input one number, then output the double of that number +func SpawnTimeProblem() *PerfTestSet { + // maxValue * 2 must not cause integer overflow + maxValue := 1 << 30 + return &PerfTestSet{ + Name: "SPAWN", + ExpectedTime: 1, + TestGenerator: func(r *rand.Rand) *PerfTest { + value := r.Intn(maxValue) + return &PerfTest{ + Input: []byte(fmt.Sprintf("%v", value)), + Output: []byte(fmt.Sprintf("%v", value*2)), + } + }, + TestCode: []byte(""), // TODO + ExpectedAC: false, + } +} diff --git a/test/perf_test/tle.go b/test/perf_test/tle.go new file mode 100644 index 00000000..e69de29b From 46bb7705585a2f61baf0f8c52f62242a35b6e4a8 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:05:54 +0700 Subject: [PATCH 11/30] Finalize input and spawn problem --- test/perf_test/input.go | 15 +++++++++++++-- test/perf_test/problem.go | 23 +---------------------- test/perf_test/spawn.go | 10 ++++++++-- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/test/perf_test/input.go b/test/perf_test/input.go index e54bb604..b1007089 100644 --- a/test/perf_test/input.go +++ b/test/perf_test/input.go @@ -5,6 +5,18 @@ import ( "math/rand" ) +const bigInputCode = +`#include +int main(){ + int r = 0; + while (char c = get_char()){ + if ('a' <= c && c <= 'b') r++; + else break; + } + printf("%i", r); +} +` + // 50MB input to compare disk read time. O(1) memory. // Problem: Given a string, print it's length func BigInputProblem() *PerfTestSet { @@ -23,7 +35,6 @@ func BigInputProblem() *PerfTestSet { Output: []byte(fmt.Sprintf("%v", strSize)), } }, - TestCode: []byte(""), // TODO - ExpectedAC: true, + TestCode: []byte(bigInputCode), } } diff --git a/test/perf_test/problem.go b/test/perf_test/problem.go index 7dbe9ad9..62b4f10f 100644 --- a/test/perf_test/problem.go +++ b/test/perf_test/problem.go @@ -1,27 +1,7 @@ // Package perf_test provides performance testing package perf_test -import ( - "math/rand" - - "github.com/natsukagami/kjudge/db" - "github.com/natsukagami/kjudge/models" -) - -func generateProblem(db db.DBContext, contestID int, displayName string, name string) *models.Problem { - return &models.Problem{ - ContestID: contestID, - DisplayName: displayName, - ID: 0, // Auto generate ID - MaxSubmissionsCount: 0, // No limit - MemoryLimit: 1 << 20, // 1GB - Name: name, - PenaltyPolicy: "none", - ScoringMode: "best", - SecondsBetweenSubmissions: 0, - TimeLimit: 5, - } -} +import "math/rand" type PerfTest struct { Input []byte @@ -33,7 +13,6 @@ type PerfTestSet struct { ExpectedTime int // Expected running time of each test in ms TestGenerator func(*rand.Rand) *PerfTest TestCode []byte // Solution to tested problem - ExpectedAC bool } func GenerateContest() { diff --git a/test/perf_test/spawn.go b/test/perf_test/spawn.go index 983cbae3..5ad4904c 100644 --- a/test/perf_test/spawn.go +++ b/test/perf_test/spawn.go @@ -4,6 +4,13 @@ import ( "fmt" "math/rand" ) +const spawnTimeCode = +`#include +int main(){ + int a; scanf("%i", &a); + printf("%i", a*2); +} +` // O(1) problem to compare sandbox spawn time. // Problem: Input one number, then output the double of that number @@ -20,7 +27,6 @@ func SpawnTimeProblem() *PerfTestSet { Output: []byte(fmt.Sprintf("%v", value*2)), } }, - TestCode: []byte(""), // TODO - ExpectedAC: false, + TestCode: []byte(spawnTimeCode), // TODO } } From eb5e3694725090694fde416a16ea1a0467e680d0 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 11:16:28 +0700 Subject: [PATCH 12/30] Add perf testset to DB --- test/perf_test/calculate.go | 0 test/perf_test/input.go | 16 +++---- test/perf_test/memory.go | 0 test/perf_test/output.go | 0 test/perf_test/problem.go | 88 +++++++++++++++++++++++++++++++++---- test/perf_test/spawn.go | 12 +++-- test/perf_test/tle.go | 0 7 files changed, 89 insertions(+), 27 deletions(-) delete mode 100644 test/perf_test/calculate.go delete mode 100644 test/perf_test/memory.go delete mode 100644 test/perf_test/output.go delete mode 100644 test/perf_test/tle.go diff --git a/test/perf_test/calculate.go b/test/perf_test/calculate.go deleted file mode 100644 index e69de29b..00000000 diff --git a/test/perf_test/input.go b/test/perf_test/input.go index b1007089..1f098a85 100644 --- a/test/perf_test/input.go +++ b/test/perf_test/input.go @@ -1,12 +1,8 @@ package perf_test -import ( - "fmt" - "math/rand" -) +import "math/rand" -const bigInputCode = -`#include +const bigInputCode = `#include int main(){ int r = 0; while (char c = get_char()){ @@ -23,17 +19,15 @@ func BigInputProblem() *PerfTestSet { maxSize := 50000000 // 50MB return &PerfTestSet{ Name: "INPUT", + CapTime: 5000, ExpectedTime: 2000, // TODO - TestGenerator: func(r *rand.Rand) *PerfTest { + TestGenerator: func(r *rand.Rand) []byte { strSize := maxSize - r.Intn(10) input := make([]byte, strSize) for i := range input { input[i] = byte(r.Intn(26) + 'A') } - return &PerfTest{ - Input: input, - Output: []byte(fmt.Sprintf("%v", strSize)), - } + return input }, TestCode: []byte(bigInputCode), } diff --git a/test/perf_test/memory.go b/test/perf_test/memory.go deleted file mode 100644 index e69de29b..00000000 diff --git a/test/perf_test/output.go b/test/perf_test/output.go deleted file mode 100644 index e69de29b..00000000 diff --git a/test/perf_test/problem.go b/test/perf_test/problem.go index 62b4f10f..e74a97db 100644 --- a/test/perf_test/problem.go +++ b/test/perf_test/problem.go @@ -1,20 +1,90 @@ // Package perf_test provides performance testing package perf_test -import "math/rand" +import ( + "database/sql" + "fmt" + "math/rand" + "time" -type PerfTest struct { - Input []byte - Output []byte -} + "github.com/natsukagami/kjudge/db" + "github.com/natsukagami/kjudge/models" + "github.com/pkg/errors" +) + +// TODO: Output, Memory, Calculate, TLE type PerfTestSet struct { Name string - ExpectedTime int // Expected running time of each test in ms - TestGenerator func(*rand.Rand) *PerfTest - TestCode []byte // Solution to tested problem + ExpectedTime int // Expected running time of each test in ms + CapTime int // Time limit sent to sandbox + TestGenerator func(*rand.Rand) []byte // Returns input + TestCode []byte // Solution to tested problem } -func GenerateContest() { +// Generates problem and returns problem ID +func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID int, expectedTime int) (int, error) { + // Creates problem + problem := &models.Problem{ + ContestID: contestID, + DisplayName: r.Name, + ID: 0, + MaxSubmissionsCount: 0, + MemoryLimit: 1 << 20, // 1GB + Name: fmt.Sprintf("%v", index), + PenaltyPolicy: models.PenaltyPolicyNone, + ScoringMode: models.ScoringModeLast, + SecondsBetweenSubmissions: 0, + TimeLimit: r.CapTime, + } + if err := problem.Write(db); err != nil { + return 0, errors.Wrapf(err, "problem %v", r.Name) + } + + // Creates test group + testGroup := &models.TestGroup{ + ID: 0, + MemoryLimit: sql.NullInt64{Valid: false}, // nil + Name: "main", + ProblemID: problem.ID, + Score: 100, + ScoringMode: models.TestScoringModeSum, + TimeLimit: sql.NullInt64{Valid: false}, // nil + } + if err := testGroup.Write(db); err != nil { + return 0, errors.Wrapf(err, "test group %v", testGroup.Name) + } + + // Creates tests. + rng := rand.New(rand.NewSource(seed)) + for i := 1; i*r.ExpectedTime < expectedTime; i++ { + test := &models.Test{ + ID: 0, + Input: r.TestGenerator(rng), + Name: fmt.Sprintf("%v", i), + Output: []byte(""), + TestGroupID: testGroup.ID, + } + if err := test.Write(db); err != nil { + return 0, errors.Wrapf(err, "test %v", i) + } + } + + return problem.ID, nil +} +// Generates contest and returns contest ID +func GenerateContest(db db.DBContext) (int, error) { + contest := &models.Contest{ + ContestType: "weighted", + StartTime: time.Now().AddDate(0, 0, -1), + EndTime: time.Now().AddDate(0, 0, 1), + ID: 0, + Name: "Performance Testing", + ScoreboardViewStatus: models.ScoreboardViewStatusPublic, + } + if err := contest.Write(db); err != nil { + return 0, err + } + return contest.ID, nil } diff --git a/test/perf_test/spawn.go b/test/perf_test/spawn.go index 5ad4904c..da24fa72 100644 --- a/test/perf_test/spawn.go +++ b/test/perf_test/spawn.go @@ -4,8 +4,8 @@ import ( "fmt" "math/rand" ) -const spawnTimeCode = -`#include + +const spawnTimeCode = `#include int main(){ int a; scanf("%i", &a); printf("%i", a*2); @@ -20,12 +20,10 @@ func SpawnTimeProblem() *PerfTestSet { return &PerfTestSet{ Name: "SPAWN", ExpectedTime: 1, - TestGenerator: func(r *rand.Rand) *PerfTest { + CapTime: 100, + TestGenerator: func(r *rand.Rand) []byte { value := r.Intn(maxValue) - return &PerfTest{ - Input: []byte(fmt.Sprintf("%v", value)), - Output: []byte(fmt.Sprintf("%v", value*2)), - } + return []byte(fmt.Sprintf("%v", value)) }, TestCode: []byte(spawnTimeCode), // TODO } diff --git a/test/perf_test/tle.go b/test/perf_test/tle.go deleted file mode 100644 index e69de29b..00000000 From 2ca6e9716248d29920734d30b73fdad3739a75f2 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 22:09:06 +0700 Subject: [PATCH 13/30] Structure --- test/perf_test/input.go | 6 ++-- test/perf_test/problem.go | 66 ++++++++++++++++++++++++++++++++++----- test/perf_test/spawn.go | 6 ++-- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/test/perf_test/input.go b/test/perf_test/input.go index 1f098a85..be1f55b9 100644 --- a/test/perf_test/input.go +++ b/test/perf_test/input.go @@ -20,8 +20,8 @@ func BigInputProblem() *PerfTestSet { return &PerfTestSet{ Name: "INPUT", CapTime: 5000, - ExpectedTime: 2000, // TODO - TestGenerator: func(r *rand.Rand) []byte { + Count: 10, + Generator: func(r *rand.Rand) []byte { strSize := maxSize - r.Intn(10) input := make([]byte, strSize) for i := range input { @@ -29,6 +29,6 @@ func BigInputProblem() *PerfTestSet { } return input }, - TestCode: []byte(bigInputCode), + Solution: []byte(bigInputCode), } } diff --git a/test/perf_test/problem.go b/test/perf_test/problem.go index e74a97db..4d8d0223 100644 --- a/test/perf_test/problem.go +++ b/test/perf_test/problem.go @@ -4,7 +4,11 @@ package perf_test import ( "database/sql" "fmt" + "log" "math/rand" + "os" + "path/filepath" + "testing" "time" "github.com/natsukagami/kjudge/db" @@ -16,14 +20,15 @@ import ( type PerfTestSet struct { Name string - ExpectedTime int // Expected running time of each test in ms + Count int CapTime int // Time limit sent to sandbox - TestGenerator func(*rand.Rand) []byte // Returns input - TestCode []byte // Solution to tested problem + Generator func(*rand.Rand) []byte // Returns input + Solution []byte // Solution to tested problem + } -// Generates problem and returns problem ID -func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID int, expectedTime int) (int, error) { +// Generates problem and returns id +func (r *PerfTestSet) addProblem(db db.DBContext, seed int64, index int, contestID int) (int, error) { // Creates problem problem := &models.Problem{ ContestID: contestID, @@ -57,10 +62,10 @@ func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID // Creates tests. rng := rand.New(rand.NewSource(seed)) - for i := 1; i*r.ExpectedTime < expectedTime; i++ { + for i := 1; i < r.Count; i++ { test := &models.Test{ ID: 0, - Input: r.TestGenerator(rng), + Input: r.Generator(rng), Name: fmt.Sprintf("%v", i), Output: []byte(""), TestGroupID: testGroup.ID, @@ -73,8 +78,12 @@ func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID return problem.ID, nil } +func (r *PerfTestSet) addSolution(db db.DBContext, problemID int, N int) error { + +} + // Generates contest and returns contest ID -func GenerateContest(db db.DBContext) (int, error) { +func createContest(db db.DBContext) (int, error) { contest := &models.Contest{ ContestType: "weighted", StartTime: time.Now().AddDate(0, 0, -1), @@ -88,3 +97,44 @@ func GenerateContest(db db.DBContext) (int, error) { } return contest.ID, nil } + +// Generates user and returns user ID +func createUser(db db.DBContext) (int, error) { + +} + +func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { + benchDB, err := db.New(dbFile) + if err != nil { + return errors.Wrap(err, "creating DB") + } + + contestID, err := createContest(benchDB) + if err != nil { + return errors.Wrap(err, "creating contest") + } + + userID, err := createUser(benchDB) + if err != nil { + return errors.Wrap(err, "creating user") + } + + for idx, testset := range testList { + problemID, err := testset.addProblem(benchDB, 2403, idx+1, contestID); + if err != nil { + return errors.Wrapf(err, "creating testset %v's problem", testset.Name) + } + testset.addSolution(benchDB, problemID, N) + } + return nil +} + +func generateTestSuite(b *testing.B){ + tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") + if err != nil { + log.Panic("cannot create temp dir:", err) + } + defer os.RemoveAll(tmpDir) + dbFile := filepath.Join(tmpDir, "kjudge.db") + generateDB(dbFile, b.N, BigInputProblem(), SpawnTimeProblem()) +} diff --git a/test/perf_test/spawn.go b/test/perf_test/spawn.go index da24fa72..682ac6be 100644 --- a/test/perf_test/spawn.go +++ b/test/perf_test/spawn.go @@ -19,12 +19,12 @@ func SpawnTimeProblem() *PerfTestSet { maxValue := 1 << 30 return &PerfTestSet{ Name: "SPAWN", - ExpectedTime: 1, + Count: 10000, CapTime: 100, - TestGenerator: func(r *rand.Rand) []byte { + Generator: func(r *rand.Rand) []byte { value := r.Intn(maxValue) return []byte(fmt.Sprintf("%v", value)) }, - TestCode: []byte(spawnTimeCode), // TODO + Solution: []byte(spawnTimeCode), // TODO } } From bf89a772686b6e49f5306b733d16be7747a7a0a8 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 22:29:45 +0700 Subject: [PATCH 14/30] User and solution --- test/perf_test/input.go | 2 +- test/perf_test/problem.go | 44 ++++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/test/perf_test/input.go b/test/perf_test/input.go index be1f55b9..98bf0027 100644 --- a/test/perf_test/input.go +++ b/test/perf_test/input.go @@ -20,7 +20,7 @@ func BigInputProblem() *PerfTestSet { return &PerfTestSet{ Name: "INPUT", CapTime: 5000, - Count: 10, + Count: 50, Generator: func(r *rand.Rand) []byte { strSize := maxSize - r.Intn(10) input := make([]byte, strSize) diff --git a/test/perf_test/problem.go b/test/perf_test/problem.go index 4d8d0223..83213447 100644 --- a/test/perf_test/problem.go +++ b/test/perf_test/problem.go @@ -13,6 +13,7 @@ import ( "github.com/natsukagami/kjudge/db" "github.com/natsukagami/kjudge/models" + "github.com/natsukagami/kjudge/server/auth" "github.com/pkg/errors" ) @@ -78,8 +79,25 @@ func (r *PerfTestSet) addProblem(db db.DBContext, seed int64, index int, contest return problem.ID, nil } -func (r *PerfTestSet) addSolution(db db.DBContext, problemID int, N int) error { +func (r *PerfTestSet) addSolution(db db.DBContext, problemID int, userID string) error { + sub := models.Submission{ + ProblemID: problemID, + UserID: userID, + Source: r.Solution, + Language: models.LanguageCpp, + SubmittedAt: time.Now(), + Verdict: models.VerdictIsInQueue, + } + if err := sub.Write(db); err != nil { + return err + } + job := models.NewJobScore(sub.ID) + if err := job.Write(db); err != nil { + return err + } + + return nil } // Generates contest and returns contest ID @@ -99,8 +117,21 @@ func createContest(db db.DBContext) (int, error) { } // Generates user and returns user ID -func createUser(db db.DBContext) (int, error) { - +func createUser(db db.DBContext) (string, error) { + password, err := auth.PasswordHash("bigquestions") + if err != nil { + return "", errors.Wrap(err, "while hashing password") + } + user := &models.User{ + ID: "Iroh", + Password: string(password), + DisplayName: "The Dragon of the West", + Organization: "Order of the White Lotus", + } + if err := user.Write(db); err != nil { + return "", errors.Wrap(err, "while creating user") + } + return user.ID, nil } func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { @@ -124,12 +155,14 @@ func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { if err != nil { return errors.Wrapf(err, "creating testset %v's problem", testset.Name) } - testset.addSolution(benchDB, problemID, N) + for i := 0; i < N; i++ { + testset.addSolution(benchDB, problemID, userID) + } } return nil } -func generateTestSuite(b *testing.B){ +func GenerateTestSuite(b *testing.B) { tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") if err != nil { log.Panic("cannot create temp dir:", err) @@ -137,4 +170,5 @@ func generateTestSuite(b *testing.B){ defer os.RemoveAll(tmpDir) dbFile := filepath.Join(tmpDir, "kjudge.db") generateDB(dbFile, b.N, BigInputProblem(), SpawnTimeProblem()) + } From 8f86573b3a5dfc958d6a7f9b25cc4e0379e884d7 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 22:53:18 +0700 Subject: [PATCH 15/30] WIP --- cmd/stress/main.go | 38 -------------------------------------- test/perf_test/problem.go | 32 +++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 43 deletions(-) delete mode 100644 cmd/stress/main.go diff --git a/cmd/stress/main.go b/cmd/stress/main.go deleted file mode 100644 index 27cebf6e..00000000 --- a/cmd/stress/main.go +++ /dev/null @@ -1,38 +0,0 @@ -// Compares provided sandboxes -package main - -import ( - "flag" - "log" - - "github.com/natsukagami/kjudge/db" - "github.com/natsukagami/kjudge/worker" -) - -var ( - dbfile = flag.String("file", "kjudge-test.db", "Path to database file.") - outputfile = flag.String("output", "kjudge-test.json", "Path to output file.") - sandboxImpls = flag.Args() - batchCount = flag.Int("count", 10, "Number of iterations.") - batchSize = flag.Int("count", 30, "Size of testset.") -) - -func main() { - flag.Parse() - - db, err := db.New(*dbfile) - if err != nil { - log.Fatalf("%+v", err) - } - defer db.Close() - - for i := 1; i <= *batchCount; i++ { - log.Printf("Running testset number %v", i) - for _, sbxName := range sandboxImpls { - sandbox, err := worker.NewSandbox(sbxName) - if err != nil { - log.Printf("%v", err) - } - } - } -} diff --git a/test/perf_test/problem.go b/test/perf_test/problem.go index 83213447..bd5ddb85 100644 --- a/test/perf_test/problem.go +++ b/test/perf_test/problem.go @@ -14,6 +14,8 @@ import ( "github.com/natsukagami/kjudge/db" "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server/auth" + "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -137,9 +139,9 @@ func createUser(db db.DBContext) (string, error) { func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { benchDB, err := db.New(dbFile) if err != nil { - return errors.Wrap(err, "creating DB") + return errors.Wrapf(err, "creating db file") } - + defer benchDB.Close() contestID, err := createContest(benchDB) if err != nil { return errors.Wrap(err, "creating contest") @@ -162,13 +164,33 @@ func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { return nil } -func GenerateTestSuite(b *testing.B) { +func runSingleTest() + +func BenchmarkAll(b *testing.B) { tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") if err != nil { - log.Panic("cannot create temp dir:", err) + b.Error(err) } defer os.RemoveAll(tmpDir) + dbFile := filepath.Join(tmpDir, "kjudge.db") - generateDB(dbFile, b.N, BigInputProblem(), SpawnTimeProblem()) + + b.Log("Generating test suite") + if err := generateDB(dbFile, b.N, BigInputProblem(), SpawnTimeProblem()); err != nil { + b.Error(err) + } + + for _, sandboxName := range []string{"raw", "isolate"} { + + sandbox, err := worker.NewSandbox(sandboxName) + if err != nil { + b.Error(err) + } + queue := worker.Queue{Sandbox: sandbox, DB: benchDB} + b.ResetTimer() + queue.Start() + + } + } From 121310eb30fe2ababa019b32c9e08d11f73c2c39 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:01:46 +0700 Subject: [PATCH 16/30] WIP --- embed/embed_dev.go | 17 ++-- .../problem.go => performance/helper.go} | 83 ++++++++++++------- test/{perf_test => performance}/input.go | 2 +- test/performance/perf_test.go | 23 +++++ test/{perf_test => performance}/spawn.go | 2 +- worker/queue.go | 19 +++++ 6 files changed, 110 insertions(+), 36 deletions(-) rename test/{perf_test/problem.go => performance/helper.go} (78%) rename test/{perf_test => performance}/input.go (97%) create mode 100644 test/performance/perf_test.go rename test/{perf_test => performance}/spawn.go (96%) diff --git a/embed/embed_dev.go b/embed/embed_dev.go index 2a9c5c08..da28f608 100644 --- a/embed/embed_dev.go +++ b/embed/embed_dev.go @@ -8,18 +8,25 @@ import ( "log" "os" "path/filepath" + "runtime" ) // Content serves content in the /embed directory. var Content fs.FS +func getEmbedDir() string { + _, path, _, _ := runtime.Caller(0) + return filepath.Dir(path) +} + func init() { - wd, err := os.Getwd() - if err != nil { - log.Panicf("cannot get current directory: %v", err) - } + // wd, err := os.Getwd() + // if err != nil { + // log.Panicf("cannot get current directory: %v", err) + // } - embedDir := filepath.Join(wd, "embed") + // embedDir := filepath.Join(wd, "embed") + embedDir := getEmbedDir() stat, err := os.Stat(embedDir) if err != nil { log.Panicf("cannot stat embed directory: %v", err) diff --git a/test/perf_test/problem.go b/test/performance/helper.go similarity index 78% rename from test/perf_test/problem.go rename to test/performance/helper.go index bd5ddb85..1ae25ab8 100644 --- a/test/perf_test/problem.go +++ b/test/performance/helper.go @@ -1,10 +1,10 @@ // Package perf_test provides performance testing -package perf_test +package performance import ( "database/sql" "fmt" - "log" + "io" "math/rand" "os" "path/filepath" @@ -15,7 +15,6 @@ import ( "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server/auth" "github.com/natsukagami/kjudge/worker" - "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -136,12 +135,7 @@ func createUser(db db.DBContext) (string, error) { return user.ID, nil } -func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { - benchDB, err := db.New(dbFile) - if err != nil { - return errors.Wrapf(err, "creating db file") - } - defer benchDB.Close() +func writeTestDB(benchDB db.DBContext, N int, testList ...*PerfTestSet) error { contestID, err := createContest(benchDB) if err != nil { return errors.Wrap(err, "creating contest") @@ -164,33 +158,64 @@ func generateDB(dbFile string, N int, testList ...*PerfTestSet) error { return nil } -func runSingleTest() - -func BenchmarkAll(b *testing.B) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") +// Copy a file by 4096 bytes chunk +func StreamCopy(src string, dst string) error { + inf, err := os.Open(src) if err != nil { - b.Error(err) + return err } - defer os.RemoveAll(tmpDir) - - dbFile := filepath.Join(tmpDir, "kjudge.db") + defer inf.Close() - b.Log("Generating test suite") - if err := generateDB(dbFile, b.N, BigInputProblem(), SpawnTimeProblem()); err != nil { - b.Error(err) + ouf, err := os.Create(dst) + if err != nil { + return err } - - for _, sandboxName := range []string{"raw", "isolate"} { - - sandbox, err := worker.NewSandbox(sandboxName) - if err != nil { - b.Error(err) + defer ouf.Close() + + buf := make([]byte, 4096) + for { + readLen, err := inf.Read(buf) + lastIter := false + if err == io.EOF { + lastIter = true + } else if err != nil { + return err + } + + if _, err := ouf.Write(buf[:readLen]); err != nil { + return err } - queue := worker.Queue{Sandbox: sandbox, DB: benchDB} - b.ResetTimer() - queue.Start() + if lastIter { + break + } + } + return nil +} +func RunSingleTest(b *testing.B, tmpDir string, testset *PerfTestSet, sandboxName string) { + dbFile := filepath.Join(tmpDir, fmt.Sprintf("%v_%v_%v.db", testset.Name, sandboxName, b.N)) + + benchDB, err := db.New(dbFile) + if err != nil { + b.Error(err) + b.FailNow() } + defer benchDB.Close() + b.Logf("Generating %v test suite", testset.Name) + if err := writeTestDB(benchDB, b.N, testset); err != nil { + b.Error(err) + b.FailNow() + } + defer benchDB.Close() + + sandbox, err := worker.NewSandbox(sandboxName) + if err != nil { + b.Error(err) + b.FailNow() + } + b.ResetTimer() + queue := &worker.Queue{Sandbox: sandbox, DB: benchDB} + queue.Run() } diff --git a/test/perf_test/input.go b/test/performance/input.go similarity index 97% rename from test/perf_test/input.go rename to test/performance/input.go index 98bf0027..7a707ffc 100644 --- a/test/perf_test/input.go +++ b/test/performance/input.go @@ -1,4 +1,4 @@ -package perf_test +package performance import "math/rand" diff --git a/test/performance/perf_test.go b/test/performance/perf_test.go new file mode 100644 index 00000000..419d4417 --- /dev/null +++ b/test/performance/perf_test.go @@ -0,0 +1,23 @@ +package performance + +import ( + "fmt" + "os" + "testing" +) + +func BenchmarkSandboxes(b *testing.B) { + tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") + if err != nil { + b.Error(err) + b.FailNow() + } + defer os.RemoveAll(tmpDir) + + for _, testset := range []*PerfTestSet{BigInputProblem(), SpawnTimeProblem()} { + for _, sandboxName := range []string{"raw", "isolate"} { + b.Run(fmt.Sprintf("%v %v", testset.Name, sandboxName), + func(b *testing.B) {RunSingleTest(b, tmpDir, testset, sandboxName)}) + } + } +} diff --git a/test/perf_test/spawn.go b/test/performance/spawn.go similarity index 96% rename from test/perf_test/spawn.go rename to test/performance/spawn.go index 682ac6be..4f8c2342 100644 --- a/test/perf_test/spawn.go +++ b/test/performance/spawn.go @@ -1,4 +1,4 @@ -package perf_test +package performance import ( "fmt" diff --git a/worker/queue.go b/worker/queue.go index 423aba47..69628d5f 100644 --- a/worker/queue.go +++ b/worker/queue.go @@ -41,6 +41,25 @@ func (q *Queue) Start() { } } +// Run starts the queue, solves all pending jobs, then returns +func (q *Queue) Run() { + for { + job, err := models.FirstJob(q.DB) + if err != nil { + log.Printf("[WORKER] Fetching job failed: %+v\n", err) + continue + } + if job == nil { + return + } + + if err := q.HandleJob(job); err != nil { + log.Printf("[WORKER] Handling job failed: %+v\n", err) + } + _ = job.Delete(q.DB) + } +} + // HandleJob dispatches a job. func (q *Queue) HandleJob(job *models.Job) error { // Start a job with a context and submission From c07c28997f2fc31270a8127a983231c809ac791d Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Thu, 8 Jun 2023 13:59:29 +0700 Subject: [PATCH 17/30] Change names --- worker/queue.go | 2 +- worker/run.go | 24 ++++++++++++------------ worker/sandbox.go | 4 ++-- worker/sandbox/isolate/sandbox.go | 22 +++++++++++----------- worker/sandbox/raw/sandbox.go | 16 ++++++++-------- worker/sandbox/{main.go => sandbox.go} | 16 ++++++++-------- 6 files changed, 42 insertions(+), 42 deletions(-) rename worker/sandbox/{main.go => sandbox.go} (84%) diff --git a/worker/queue.go b/worker/queue.go index 423aba47..d8ba7562 100644 --- a/worker/queue.go +++ b/worker/queue.go @@ -14,7 +14,7 @@ import ( // Queue implements a queue that runs each job one by one. type Queue struct { DB *db.DB - Sandbox sandbox.Sandbox + Sandbox sandbox.Runner } // Start starts the queue. It is blocking, so might wanna "go run" it. diff --git a/worker/run.go b/worker/run.go index 268da662..8f87a534 100644 --- a/worker/run.go +++ b/worker/run.go @@ -68,12 +68,12 @@ func (r *RunContext) CompiledSource() (bool, []byte) { } // RunInput creates a SandboxInput for running the submission's source. -func (r *RunContext) RunInput(source []byte) (*sandbox.SandboxInput, error) { +func (r *RunContext) RunInput(source []byte) (*sandbox.Input, error) { command, args, err := RunCommand(r.Sub.Language) if err != nil { return nil, err } - return &sandbox.SandboxInput{ + return &sandbox.Input{ Command: command, Args: args, Files: nil, @@ -87,11 +87,11 @@ func (r *RunContext) RunInput(source []byte) (*sandbox.SandboxInput, error) { // CompareInput creates a SandboxInput for running the comparator. // Also returns whether we have diff-based or comparator-based input. -func (r *RunContext) CompareInput(submissionOutput []byte) (input *sandbox.SandboxInput, useComparator bool, err error) { +func (r *RunContext) CompareInput(submissionOutput []byte) (input *sandbox.Input, useComparator bool, err error) { file, err := models.GetFileWithName(r.DB, r.Problem.ID, "compare") if errors.Is(err, sql.ErrNoRows) { // Use a simple diff - return &sandbox.SandboxInput{ + return &sandbox.Input{ Command: "/usr/bin/diff", Args: []string{"-wqts", "output", "expected"}, Files: map[string][]byte{"output": submissionOutput, "expected": r.Test.Output}, @@ -103,7 +103,7 @@ func (r *RunContext) CompareInput(submissionOutput []byte) (input *sandbox.Sandb return nil, false, err } // Use the given comparator. - return &sandbox.SandboxInput{ + return &sandbox.Input{ Command: "code", Args: []string{"input", "expected", "output"}, Files: map[string][]byte{"input": r.Test.Input, "expected": r.Test.Output, "output": submissionOutput}, @@ -114,7 +114,7 @@ func (r *RunContext) CompareInput(submissionOutput []byte) (input *sandbox.Sandb }, true, nil } -func RunSingleCommand(s sandbox.Sandbox, r *RunContext, source []byte) (output *sandbox.SandboxOutput, err error) { +func RunSingleCommand(s sandbox.Runner, r *RunContext, source []byte) (output *sandbox.Output, err error) { // First, use the sandbox to run the submission itself. input, err := r.RunInput(source) if err != nil { @@ -127,7 +127,7 @@ func RunSingleCommand(s sandbox.Sandbox, r *RunContext, source []byte) (output * return output, nil } -func RunMultipleCommands(s sandbox.Sandbox, r *RunContext, source []byte, stages []string) (output *sandbox.SandboxOutput, err error) { +func RunMultipleCommands(s sandbox.Runner, r *RunContext, source []byte, stages []string) (output *sandbox.Output, err error) { command, args, err := RunCommand(r.Sub.Language) if err != nil { return nil, err @@ -141,7 +141,7 @@ func RunMultipleCommands(s sandbox.Sandbox, r *RunContext, source []byte, stages } stageArgs := strings.Split(stage, " ") - sandboxInput := &sandbox.SandboxInput{ + sandboxInput := &sandbox.Input{ Command: command, Args: append(stageArgs, args...), Files: nil, @@ -167,7 +167,7 @@ func RunMultipleCommands(s sandbox.Sandbox, r *RunContext, source []byte, stages } // Run runs a RunContext. -func Run(s sandbox.Sandbox, r *RunContext) error { +func Run(s sandbox.Runner, r *RunContext) error { compiled, source := r.CompiledSource() if !compiled { // Add a compilation job and re-add ourselves. @@ -181,7 +181,7 @@ func Run(s sandbox.Sandbox, r *RunContext) error { log.Printf("[WORKER] Running submission %v on [test `%v`, group `%v`]\n", r.Sub.ID, r.Test.Name, r.TestGroup.Name) - var output *sandbox.SandboxOutput + var output *sandbox.Output file, err := models.GetFileWithName(r.DB, r.Problem.ID, ".stages") if errors.Is(err, sql.ErrNoRows) { // Problem type is not Chained Type, run a single command @@ -230,7 +230,7 @@ func Run(s sandbox.Sandbox, r *RunContext) error { } // Parse the comparator's output and reflect it into `result`. -func parseComparatorOutput(s *sandbox.SandboxOutput, result *models.TestResult, useComparator bool) error { +func parseComparatorOutput(s *sandbox.Output, result *models.TestResult, useComparator bool) error { if useComparator { // Paste the comparator's output to result result.Verdict = strings.TrimSpace(string(s.Stderr)) @@ -259,7 +259,7 @@ func parseComparatorOutput(s *sandbox.SandboxOutput, result *models.TestResult, } // Parse the sandbox output into a TestResult. -func parseSandboxOutput(s *sandbox.SandboxOutput, r *RunContext) *models.TestResult { +func parseSandboxOutput(s *sandbox.Output, r *RunContext) *models.TestResult { score := 1.0 if !s.Success { score = 0.0 diff --git a/worker/sandbox.go b/worker/sandbox.go index c79ca1ff..8b90c7ba 100644 --- a/worker/sandbox.go +++ b/worker/sandbox.go @@ -7,10 +7,10 @@ import ( "github.com/pkg/errors" ) -func NewSandbox(name string) (sandbox.Sandbox, error) { +func NewSandbox(name string) (sandbox.Runner, error) { switch name { case "raw": - return &raw.Sandbox{}, nil + return &raw.Runner{}, nil case "isolate": return isolate.New(), nil default: diff --git a/worker/sandbox/isolate/sandbox.go b/worker/sandbox/isolate/sandbox.go index 7aa804c1..e77486cd 100644 --- a/worker/sandbox/isolate/sandbox.go +++ b/worker/sandbox/isolate/sandbox.go @@ -29,12 +29,12 @@ func init() { } } -// Sandbox implements worker.Sandbox. -type Sandbox struct { +// Runner implements worker.Runner. +type Runner struct { private struct{} // Makes the sandbox not simply constructible } -var _ sandbox.Sandbox = (*Sandbox)(nil) +var _ sandbox.Runner = (*Runner)(nil) // Panics on not having "isolate" accessible. func mustHaveIsolate() { @@ -49,13 +49,13 @@ func mustHaveIsolate() { // New returns a new sandbox. // Panics if isolate is not installed. -func New() *Sandbox { +func New() *Runner { mustHaveIsolate() - return &Sandbox{private: struct{}{}} + return &Runner{private: struct{}{}} } -// Run implements Sandbox.Run. -func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { +// Run implements Runner.Run. +func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { // Init the sandbox defer s.cleanup() dirBytes, err := exec.Command(isolateCommand, "--init", "--cg").Output() @@ -86,7 +86,7 @@ func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, erro } // Parse the meta file - output := &sandbox.SandboxOutput{ + output := &sandbox.Output{ Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), } @@ -97,7 +97,7 @@ func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, erro return output, nil } -func parseMetaFile(path string, output *sandbox.SandboxOutput) error { +func parseMetaFile(path string, output *sandbox.Output) error { meta, err := ReadMetaFile(path) if err != nil { return err @@ -114,7 +114,7 @@ func parseMetaFile(path string, output *sandbox.SandboxOutput) error { } // Build the command for isolate --run. -func buildCmd(dir, metaFile string, input *sandbox.SandboxInput) *exec.Cmd { +func buildCmd(dir, metaFile string, input *sandbox.Input) *exec.Cmd { // Calculate stuff timeLimit := float64(input.TimeLimit) / float64(time.Second) @@ -147,6 +147,6 @@ func buildCmd(dir, metaFile string, input *sandbox.SandboxInput) *exec.Cmd { return cmd } -func (s *Sandbox) cleanup() { +func (s *Runner) cleanup() { _ = exec.Command(isolateCommand, "--cleanup", "--cg").Run() } diff --git a/worker/sandbox/raw/sandbox.go b/worker/sandbox/raw/sandbox.go index ee1e24b7..c8b8769e 100644 --- a/worker/sandbox/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -21,13 +21,13 @@ import ( "github.com/natsukagami/kjudge/worker/sandbox" ) -// Sandbox implements worker.Sandbox. -type Sandbox struct{} +// Runner implements worker.Runner. +type Runner struct{} -var _ sandbox.Sandbox = (*Sandbox)(nil) +var _ sandbox.Runner = (*Runner)(nil) -// Run implements Sandbox.Run -func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { +// Run implements Runner.Run +func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { dir := os.TempDir() log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) @@ -41,7 +41,7 @@ func (s *Sandbox) Run(input *sandbox.SandboxInput) (*sandbox.SandboxOutput, erro // - MEMORY LIMITS ARE NOT SET. It always reports a memory usage of 0 (it cannot measure them). // - THE PROGRAM DOES NOT MESS WITH THE COMPUTER. LMAO // - The folder will be thrown away later. -func (s *Sandbox) RunFrom(cwd string, input *sandbox.SandboxInput) (*sandbox.SandboxOutput, error) { +func (s *Runner) RunFrom(cwd string, input *sandbox.Input) (*sandbox.Output, error) { if err := input.CopyTo(cwd); err != nil { return nil, err } @@ -73,7 +73,7 @@ func (s *Sandbox) RunFrom(cwd string, input *sandbox.SandboxInput) (*sandbox.San case <-time.After(input.TimeLimit): cancel() <-done - return &sandbox.SandboxOutput{ + return &sandbox.Output{ Success: false, MemoryUsed: 0, RunningTime: input.TimeLimit, @@ -83,7 +83,7 @@ func (s *Sandbox) RunFrom(cwd string, input *sandbox.SandboxInput) (*sandbox.San }, nil case commandErr := <-done: runningTime := time.Since(startTime) - return &sandbox.SandboxOutput{ + return &sandbox.Output{ Success: commandErr == nil, MemoryUsed: 0, RunningTime: runningTime, diff --git a/worker/sandbox/main.go b/worker/sandbox/sandbox.go similarity index 84% rename from worker/sandbox/main.go rename to worker/sandbox/sandbox.go index 87bc66a7..e97d55fd 100644 --- a/worker/sandbox/main.go +++ b/worker/sandbox/sandbox.go @@ -8,19 +8,19 @@ import ( "github.com/pkg/errors" ) -// Sandbox provides a way to run an arbitary command +// Runner provides a way to run an arbitary command // within a sandbox, with configured input/outputs and // proper time and memory limits. // // kjudge currently implements two sandboxes, "isolate" (which requires "github.com/ioi/isolate" to be available) // and "raw" (NOT RECOMMENDED, RUN AT YOUR OWN RISK). // Which sandbox is used can be set at runtime with a command-line switch. -type Sandbox interface { - Run(*SandboxInput) (*SandboxOutput, error) +type Runner interface { + Run(*Input) (*Output, error) } -// SandboxInput is the input to a sandbox. -type SandboxInput struct { +// Input is the input to a sandbox. +type Input struct { Command string `json:"command"` // The passed command Args []string `json:"args"` // any additional arguments, if needed Files map[string][]byte `json:"files"` // Any additional files needed @@ -31,8 +31,8 @@ type SandboxInput struct { Input []byte `json:"input"` } -// SandboxOutput is the output which the sandbox needs to give back. -type SandboxOutput struct { +// Output is the output which the sandbox needs to give back. +type Output struct { Success bool `json:"success"` // Whether the command exited zero. RunningTime time.Duration `json:"running_time"` // The running time of the command. MemoryUsed int `json:"memory_used"` // in KBs @@ -43,7 +43,7 @@ type SandboxOutput struct { } // CopyTo copies all the files it contains into cwd. -func (input *SandboxInput) CopyTo(cwd string) error { +func (input *Input) CopyTo(cwd string) error { // Copy all the files into "cwd" for name, file := range input.Files { if err := os.WriteFile(filepath.Join(cwd, name), file, 0666); err != nil { From 18e4b0e61414bbcd21cedc4201f95139d3fce893 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:16:31 +0700 Subject: [PATCH 18/30] Put sandbox warning inside NewSandbox --- cmd/kjudge/main.go | 3 --- worker/sandbox.go | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/kjudge/main.go b/cmd/kjudge/main.go index e03f13b2..36fe1101 100644 --- a/cmd/kjudge/main.go +++ b/cmd/kjudge/main.go @@ -32,9 +32,6 @@ func main() { } defer db.Close() - if *sandboxImpl == "raw" { - log.Println("'raw' sandbox selected. WE ARE NOT RESPONSIBLE FOR ANY BREAKAGE CAUSED BY FOREIGN CODE.") - } sandbox, err := worker.NewSandbox(*sandboxImpl) if err != nil { log.Fatalf("%v", err) diff --git a/worker/sandbox.go b/worker/sandbox.go index 8b90c7ba..a52a3e67 100644 --- a/worker/sandbox.go +++ b/worker/sandbox.go @@ -1,15 +1,28 @@ package worker import ( + "log" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/natsukagami/kjudge/worker/sandbox/isolate" "github.com/natsukagami/kjudge/worker/sandbox/raw" "github.com/pkg/errors" ) -func NewSandbox(name string) (sandbox.Runner, error) { +func NewSandbox(name string, ignoreWarnings ...bool) (sandbox.Runner, error) { + if len(ignoreWarnings) > 1 { + return nil, errors.New("Function NewSandbox only takes 1 or 2 arguments") + } + warningsIgnored := false + if len(ignoreWarnings) == 1 && ignoreWarnings[0] { + warningsIgnored = true + } + switch name { case "raw": + if !warningsIgnored { + log.Println("'raw' sandbox selected. WE ARE NOT RESPONSIBLE FOR ANY BREAKAGE CAUSED BY FOREIGN CODE.") + } return &raw.Runner{}, nil case "isolate": return isolate.New(), nil From 8ced4788115cd6431acc8bb6851a083865411cb2 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:16:50 +0700 Subject: [PATCH 19/30] Optimize scoring --- worker/score.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/worker/score.go b/worker/score.go index cb7aafcd..963cc4dd 100644 --- a/worker/score.go +++ b/worker/score.go @@ -180,17 +180,18 @@ func (s *ScoreContext) CompareScores(subs []*models.Submission) *models.ProblemR subs[i], subs[j] = subs[j], subs[i] } + getScoredSub: for _, sub := range subs { score, _, counts := scoreOf(sub) if !counts { continue } + counted++ switch s.Problem.ScoringMode { case models.ScoringModeOnce: - if which == nil { - which = sub - maxScore = score - } + which = sub + maxScore = score + break getScoredSub case models.ScoringModeLast: which = sub maxScore = score @@ -212,7 +213,6 @@ func (s *ScoreContext) CompareScores(subs []*models.Submission) *models.ProblemR default: panic(s) } - counted++ } for _, sub := range subs { From 4f9297fcdd9db208513e3d338daf49f0333ab9ed Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:19:50 +0700 Subject: [PATCH 20/30] gofmt --- worker/score.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/score.go b/worker/score.go index 963cc4dd..fb4aa5db 100644 --- a/worker/score.go +++ b/worker/score.go @@ -180,7 +180,7 @@ func (s *ScoreContext) CompareScores(subs []*models.Submission) *models.ProblemR subs[i], subs[j] = subs[j], subs[i] } - getScoredSub: +getScoredSub: for _, sub := range subs { score, _, counts := scoreOf(sub) if !counts { From 8fc5324d98a1210ae2fbfff8806bea296c468134 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 18:53:34 +0700 Subject: [PATCH 21/30] WIP --- test/performance/helper.go | 224 +++++++++++----------------------- test/performance/input.go | 8 +- test/performance/perf_test.go | 30 +++-- test/performance/spawn.go | 6 +- test/performance/testset.go | 86 +++++++++++++ 5 files changed, 187 insertions(+), 167 deletions(-) create mode 100644 test/performance/testset.go diff --git a/test/performance/helper.go b/test/performance/helper.go index 1ae25ab8..e430b7fe 100644 --- a/test/performance/helper.go +++ b/test/performance/helper.go @@ -1,11 +1,6 @@ -// Package perf_test provides performance testing package performance import ( - "database/sql" - "fmt" - "io" - "math/rand" "os" "path/filepath" "testing" @@ -18,92 +13,49 @@ import ( "github.com/pkg/errors" ) -// TODO: Output, Memory, Calculate, TLE - -type PerfTestSet struct { - Name string - Count int - CapTime int // Time limit sent to sandbox - Generator func(*rand.Rand) []byte // Returns input - Solution []byte // Solution to tested problem - +type BenchmarkContext struct { + tdir string + db *db.DB + user *models.User + contest *models.Contest + problems map[string]*models.Problem } -// Generates problem and returns id -func (r *PerfTestSet) addProblem(db db.DBContext, seed int64, index int, contestID int) (int, error) { - // Creates problem - problem := &models.Problem{ - ContestID: contestID, - DisplayName: r.Name, - ID: 0, - MaxSubmissionsCount: 0, - MemoryLimit: 1 << 20, // 1GB - Name: fmt.Sprintf("%v", index), - PenaltyPolicy: models.PenaltyPolicyNone, - ScoringMode: models.ScoringModeLast, - SecondsBetweenSubmissions: 0, - TimeLimit: r.CapTime, - } - if err := problem.Write(db); err != nil { - return 0, errors.Wrapf(err, "problem %v", r.Name) +func NewBenchmarkContext(tmpDir string) (*BenchmarkContext, error) { + benchDB, err := db.New(filepath.Join(tmpDir, "bench.db")) + if err != nil { + err2 := os.RemoveAll(tmpDir) + if err2 != nil { + return nil, errors.Wrapf(err2, "while handling %v", err) + } + return nil, err } - // Creates test group - testGroup := &models.TestGroup{ - ID: 0, - MemoryLimit: sql.NullInt64{Valid: false}, // nil - Name: "main", - ProblemID: problem.ID, - Score: 100, - ScoringMode: models.TestScoringModeSum, - TimeLimit: sql.NullInt64{Valid: false}, // nil - } - if err := testGroup.Write(db); err != nil { - return 0, errors.Wrapf(err, "test group %v", testGroup.Name) + ctx := &BenchmarkContext{tdir: tmpDir, db: benchDB, problems: make(map[string]*models.Problem)} + + if err := ctx.writeContest(); err != nil { + return nil, err } - // Creates tests. - rng := rand.New(rand.NewSource(seed)) - for i := 1; i < r.Count; i++ { - test := &models.Test{ - ID: 0, - Input: r.Generator(rng), - Name: fmt.Sprintf("%v", i), - Output: []byte(""), - TestGroupID: testGroup.ID, - } - if err := test.Write(db); err != nil { - return 0, errors.Wrapf(err, "test %v", i) - } + if err := ctx.writeUser(); err != nil { + return nil, err } - return problem.ID, nil + return ctx, nil } -func (r *PerfTestSet) addSolution(db db.DBContext, problemID int, userID string) error { - sub := models.Submission{ - ProblemID: problemID, - UserID: userID, - Source: r.Solution, - Language: models.LanguageCpp, - SubmittedAt: time.Now(), - Verdict: models.VerdictIsInQueue, - } - if err := sub.Write(db); err != nil { +func (ctx *BenchmarkContext) Close() error { + if err := os.RemoveAll(ctx.tdir); err != nil { return err } - - job := models.NewJobScore(sub.ID) - if err := job.Write(db); err != nil { + if err := ctx.db.Close(); err != nil { return err } - return nil } -// Generates contest and returns contest ID -func createContest(db db.DBContext) (int, error) { - contest := &models.Contest{ +func (ctx *BenchmarkContext) writeContest() error { + ctx.contest = &models.Contest{ ContestType: "weighted", StartTime: time.Now().AddDate(0, 0, -1), EndTime: time.Now().AddDate(0, 0, 1), @@ -111,111 +63,81 @@ func createContest(db db.DBContext) (int, error) { Name: "Performance Testing", ScoreboardViewStatus: models.ScoreboardViewStatusPublic, } - if err := contest.Write(db); err != nil { - return 0, err - } - return contest.ID, nil + return errors.Wrapf(ctx.contest.Write(ctx.db), "creating contest") } -// Generates user and returns user ID -func createUser(db db.DBContext) (string, error) { +func (ctx *BenchmarkContext) writeUser() error { password, err := auth.PasswordHash("bigquestions") if err != nil { - return "", errors.Wrap(err, "while hashing password") + return errors.Wrap(err, "hashing password") } - user := &models.User{ - ID: "Iroh", - Password: string(password), - DisplayName: "The Dragon of the West", + ctx.user = &models.User{ + ID: "Iroh", + Password: string(password), + DisplayName: "The Dragon of the West", Organization: "Order of the White Lotus", } - if err := user.Write(db); err != nil { - return "", errors.Wrap(err, "while creating user") - } - return user.ID, nil + return errors.Wrap(ctx.user.Write(ctx.db), "creating user") } -func writeTestDB(benchDB db.DBContext, N int, testList ...*PerfTestSet) error { - contestID, err := createContest(benchDB) - if err != nil { - return errors.Wrap(err, "creating contest") - } - - userID, err := createUser(benchDB) +func (ctx *BenchmarkContext) writeProblem(testset *PerfTestSet) error { + problem, err := testset.AddToDB(ctx.db, 2403, len(ctx.problems)+1, ctx.contest.ID) if err != nil { - return errors.Wrap(err, "creating user") + return errors.Wrapf(err, "creating testset %v's problem", testset.Name) } - for idx, testset := range testList { - problemID, err := testset.addProblem(benchDB, 2403, idx+1, contestID); - if err != nil { - return errors.Wrapf(err, "creating testset %v's problem", testset.Name) - } - for i := 0; i < N; i++ { - testset.addSolution(benchDB, problemID, userID) - } - } + ctx.problems[testset.Name] = problem return nil } -// Copy a file by 4096 bytes chunk -func StreamCopy(src string, dst string) error { - inf, err := os.Open(src) - if err != nil { - return err - } - defer inf.Close() - - ouf, err := os.Create(dst) - if err != nil { - return err - } - defer ouf.Close() - - buf := make([]byte, 4096) - for { - readLen, err := inf.Read(buf) - lastIter := false - if err == io.EOF { - lastIter = true - } else if err != nil { - return err +const testSolution = `#include "solution.hpp" +` + +func (ctx *BenchmarkContext) writeSolutions(N int, problemName string) error { + problem := ctx.problems[problemName] + for i := 0; i < N; i++ { + sub := models.Submission{ + ProblemID: problem.ID, + UserID: ctx.user.ID, + Source: []byte(testSolution), + Language: models.LanguageCpp, + SubmittedAt: time.Now(), + Verdict: models.VerdictIsInQueue, } - - if _, err := ouf.Write(buf[:readLen]); err != nil { + if err := sub.Write(ctx.db); err != nil { return err } - if lastIter { - break + + job := models.NewJobScore(sub.ID) + if err := job.Write(ctx.db); err != nil { + return err } } return nil } -func RunSingleTest(b *testing.B, tmpDir string, testset *PerfTestSet, sandboxName string) { - dbFile := filepath.Join(tmpDir, fmt.Sprintf("%v_%v_%v.db", testset.Name, sandboxName, b.N)) - - benchDB, err := db.New(dbFile) +func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sandboxName string) { + sandbox, err := worker.NewSandbox(sandboxName) if err != nil { - b.Error(err) - b.FailNow() + b.Fatal(err) } - defer benchDB.Close() - - b.Logf("Generating %v test suite", testset.Name) - if err := writeTestDB(benchDB, b.N, testset); err != nil { - b.Error(err) - b.FailNow() - } - defer benchDB.Close() - sandbox, err := worker.NewSandbox(sandboxName) - if err != nil { - b.Error(err) - b.FailNow() + for i := 0; i < b.N; i++ { + ctx.writeSolutions(b.N, testset.Name) } + queue := &worker.Queue{Sandbox: sandbox, DB: ctx.db} + b.ResetTimer() - queue := &worker.Queue{Sandbox: sandbox, DB: benchDB} queue.Run() + b.StopTimer() + + if err := ctx.assertRunComplete(testset); err != nil { + b.Fatal(err) + } +} + +// TODO +func (ctx *BenchmarkContext) assertRunComplete(testset *PerfTestSet) error { + return nil } diff --git a/test/performance/input.go b/test/performance/input.go index 7a707ffc..aab446e1 100644 --- a/test/performance/input.go +++ b/test/performance/input.go @@ -5,7 +5,7 @@ import "math/rand" const bigInputCode = `#include int main(){ int r = 0; - while (char c = get_char()){ + while (char c = getchar()){ if ('a' <= c && c <= 'b') r++; else break; } @@ -18,9 +18,9 @@ int main(){ func BigInputProblem() *PerfTestSet { maxSize := 50000000 // 50MB return &PerfTestSet{ - Name: "INPUT", - CapTime: 5000, - Count: 50, + Name: "INPUT", + CapTime: 5000, + Count: 50, Generator: func(r *rand.Rand) []byte { strSize := maxSize - r.Intn(10) input := make([]byte, strSize) diff --git a/test/performance/perf_test.go b/test/performance/perf_test.go index 419d4417..bcd59b40 100644 --- a/test/performance/perf_test.go +++ b/test/performance/perf_test.go @@ -2,22 +2,34 @@ package performance import ( "fmt" - "os" + "log" "testing" ) +var testList = []*PerfTestSet{BigInputProblem(), SpawnTimeProblem()} +var sandboxList = []string{"raw", "isolate"} + func BenchmarkSandboxes(b *testing.B) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "kjudge_bench") + log.Println("creating test DB") + + ctx, err := NewBenchmarkContext(b.TempDir()) if err != nil { - b.Error(err) - b.FailNow() + b.Fatal(err) + } + defer ctx.Close() + + for _, testset := range testList { + log.Printf("creating problem %v", testset.Name) + if err := ctx.writeProblem(testset); err != nil { + b.Fatal(err) + } } - defer os.RemoveAll(tmpDir) - for _, testset := range []*PerfTestSet{BigInputProblem(), SpawnTimeProblem()} { - for _, sandboxName := range []string{"raw", "isolate"} { - b.Run(fmt.Sprintf("%v %v", testset.Name, sandboxName), - func(b *testing.B) {RunSingleTest(b, tmpDir, testset, sandboxName)}) + for _, testset := range testList { + for _, sandboxName := range sandboxList { + testName := fmt.Sprintf("%v %v", testset.Name, sandboxName) + log.Printf("running %v", testName) + b.Run(testName, func(b *testing.B) { RunSingleTest(b, ctx, testset, sandboxName) }) } } } diff --git a/test/performance/spawn.go b/test/performance/spawn.go index 4f8c2342..997a44bb 100644 --- a/test/performance/spawn.go +++ b/test/performance/spawn.go @@ -18,9 +18,9 @@ func SpawnTimeProblem() *PerfTestSet { // maxValue * 2 must not cause integer overflow maxValue := 1 << 30 return &PerfTestSet{ - Name: "SPAWN", - Count: 10000, - CapTime: 100, + Name: "SPAWN", + Count: 10000, + CapTime: 100, Generator: func(r *rand.Rand) []byte { value := r.Intn(maxValue) return []byte(fmt.Sprintf("%v", value)) diff --git a/test/performance/testset.go b/test/performance/testset.go new file mode 100644 index 00000000..68d23e60 --- /dev/null +++ b/test/performance/testset.go @@ -0,0 +1,86 @@ +// Package perf_test provides performance testing +package performance + +import ( + "database/sql" + "fmt" + "math/rand" + + "github.com/natsukagami/kjudge/db" + "github.com/natsukagami/kjudge/models" + "github.com/pkg/errors" +) + +// TODO: Output, Memory, Calculate, TLE + +type PerfTestSet struct { + Name string + Count int + CapTime int // Time limit sent to sandbox + Generator func(*rand.Rand) []byte // Returns input + Solution []byte // Solution to tested problem + +} + +// Generates problem and returns id +func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID int) (*models.Problem, error) { + // Creates problem + problem := &models.Problem{ + ContestID: contestID, + DisplayName: r.Name, + ID: 0, + MaxSubmissionsCount: 0, + MemoryLimit: 1 << 20, // 1GB + Name: fmt.Sprintf("%v", index), + PenaltyPolicy: models.PenaltyPolicyNone, + ScoringMode: models.ScoringModeLast, + SecondsBetweenSubmissions: 0, + TimeLimit: r.CapTime, + } + if err := problem.Write(db); err != nil { + return nil, errors.Wrapf(err, "problem %v", r.Name) + } + + // Creates test group + testGroup := &models.TestGroup{ + ID: 0, + MemoryLimit: sql.NullInt64{Valid: false}, // nil + Name: "main", + ProblemID: problem.ID, + Score: 100, + ScoringMode: models.TestScoringModeSum, + TimeLimit: sql.NullInt64{Valid: false}, // nil + } + if err := testGroup.Write(db); err != nil { + return nil, errors.Wrapf(err, "test group %v", testGroup.Name) + } + + // Creates tests. + rng := rand.New(rand.NewSource(seed)) + for i := 1; i < r.Count; i++ { + test := &models.Test{ + ID: 0, + Input: r.Generator(rng), + Name: fmt.Sprintf("%v", i), + Output: []byte(""), + TestGroupID: testGroup.ID, + } + if err := test.Write(db); err != nil { + return nil, errors.Wrapf(err, "test %v", i) + } + } + + // Uploads solution + solutionFile := &models.File{ + Content: r.Solution, + Filename: "solution.hpp", + ID: 0, + ProblemID: problem.ID, + Public: true, + } + if err := problem.WriteFiles(db, []*models.File{solutionFile}); err != nil { + return nil, errors.Wrap(err, "solution file") + } + + return problem, nil +} From 1a18ef906a77cc51714c3d74c470be4283f8d504 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 22:36:02 +0700 Subject: [PATCH 22/30] New option-choosing for sandbox --- worker/sandbox.go | 21 ++++++--------------- worker/sandbox/isolate/sandbox.go | 11 ++++++++--- worker/sandbox/raw/sandbox.go | 15 ++++++++++++++- worker/sandbox/sandbox.go | 16 ++++++++++++++++ 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/worker/sandbox.go b/worker/sandbox.go index a52a3e67..9b876a0e 100644 --- a/worker/sandbox.go +++ b/worker/sandbox.go @@ -1,31 +1,22 @@ package worker import ( - "log" - "github.com/natsukagami/kjudge/worker/sandbox" "github.com/natsukagami/kjudge/worker/sandbox/isolate" "github.com/natsukagami/kjudge/worker/sandbox/raw" "github.com/pkg/errors" ) -func NewSandbox(name string, ignoreWarnings ...bool) (sandbox.Runner, error) { - if len(ignoreWarnings) > 1 { - return nil, errors.New("Function NewSandbox only takes 1 or 2 arguments") - } - warningsIgnored := false - if len(ignoreWarnings) == 1 && ignoreWarnings[0] { - warningsIgnored = true +func NewSandbox(name string, options ...sandbox.Option) (sandbox.Runner, error) { + setting := sandbox.DefaultSettings + for _, option := range options { + setting = option(setting) } - switch name { case "raw": - if !warningsIgnored { - log.Println("'raw' sandbox selected. WE ARE NOT RESPONSIBLE FOR ANY BREAKAGE CAUSED BY FOREIGN CODE.") - } - return &raw.Runner{}, nil + return raw.New(setting), nil case "isolate": - return isolate.New(), nil + return isolate.New(setting), nil default: return nil, errors.Errorf("Sandbox %s doesn't exists or not yet implemented.", name) } diff --git a/worker/sandbox/isolate/sandbox.go b/worker/sandbox/isolate/sandbox.go index e77486cd..57125e04 100644 --- a/worker/sandbox/isolate/sandbox.go +++ b/worker/sandbox/isolate/sandbox.go @@ -31,7 +31,8 @@ func init() { // Runner implements worker.Runner. type Runner struct { - private struct{} // Makes the sandbox not simply constructible + settings sandbox.Settings + private struct{} // Makes the sandbox not simply constructible } var _ sandbox.Runner = (*Runner)(nil) @@ -49,9 +50,13 @@ func mustHaveIsolate() { // New returns a new sandbox. // Panics if isolate is not installed. -func New() *Runner { +func New(settings sandbox.Settings) *Runner { mustHaveIsolate() - return &Runner{private: struct{}{}} + return &Runner{settings: settings, private: struct{}{}} +} + +func (s *Runner) Settings() *sandbox.Settings { + return &s.settings } // Run implements Runner.Run. diff --git a/worker/sandbox/raw/sandbox.go b/worker/sandbox/raw/sandbox.go index c8b8769e..b35cbd97 100644 --- a/worker/sandbox/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -22,10 +22,23 @@ import ( ) // Runner implements worker.Runner. -type Runner struct{} +type Runner struct { + settings sandbox.Settings +} var _ sandbox.Runner = (*Runner)(nil) +func New(settings sandbox.Settings) *Runner { + if !settings.IgnoreWarning { + log.Println("'raw' sandbox selected. WE ARE NOT RESPONSIBLE FOR ANY BREAKAGE CAUSED BY FOREIGN CODE.") + } + return &Runner{settings: settings} +} + +func (s *Runner) Settings() *sandbox.Settings { + return &s.settings +} + // Run implements Runner.Run func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { dir := os.TempDir() diff --git a/worker/sandbox/sandbox.go b/worker/sandbox/sandbox.go index e97d55fd..b2873043 100644 --- a/worker/sandbox/sandbox.go +++ b/worker/sandbox/sandbox.go @@ -16,6 +16,7 @@ import ( // and "raw" (NOT RECOMMENDED, RUN AT YOUR OWN RISK). // Which sandbox is used can be set at runtime with a command-line switch. type Runner interface { + Settings() *Settings Run(*Input) (*Output, error) } @@ -42,6 +43,21 @@ type Output struct { ErrorMessage string `json:"error_message,omitempty"` } +type Settings struct { + IgnoreWarning bool +} + +var DefaultSettings = Settings{IgnoreWarning: false} + +type Option func(Settings) Settings + +func IgnoreWarnings(ignore bool) Option { + return func(o Settings) Settings { + o.IgnoreWarning = ignore + return o + } +} + // CopyTo copies all the files it contains into cwd. func (input *Input) CopyTo(cwd string) error { // Copy all the files into "cwd" From 8151a73875c9b9a01025105649cc5e11be25d807 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Fri, 9 Jun 2023 23:06:27 +0700 Subject: [PATCH 23/30] Ignore warnings --- test/performance/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/performance/helper.go b/test/performance/helper.go index e430b7fe..e5b68dcb 100644 --- a/test/performance/helper.go +++ b/test/performance/helper.go @@ -10,6 +10,7 @@ import ( "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server/auth" "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -117,7 +118,7 @@ func (ctx *BenchmarkContext) writeSolutions(N int, problemName string) error { } func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sandboxName string) { - sandbox, err := worker.NewSandbox(sandboxName) + sandbox, err := worker.NewSandbox(sandboxName, sandbox.IgnoreWarnings(true)) if err != nil { b.Fatal(err) } From 5fa698e36860d715f4d060debf40cc5c92f74973 Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 15:07:32 +0700 Subject: [PATCH 24/30] Can disable sandbox logs --- test/performance/helper.go | 10 +++++++++- test/performance/input.go | 2 +- test/performance/perf_test.go | 2 +- test/performance/spawn.go | 2 +- worker/sandbox/raw/sandbox.go | 4 +++- worker/sandbox/sandbox.go | 15 --------------- worker/sandbox/settings.go | 24 ++++++++++++++++++++++++ 7 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 worker/sandbox/settings.go diff --git a/test/performance/helper.go b/test/performance/helper.go index e5b68dcb..2d60578b 100644 --- a/test/performance/helper.go +++ b/test/performance/helper.go @@ -1,6 +1,7 @@ package performance import ( + "log" "os" "path/filepath" "testing" @@ -118,14 +119,21 @@ func (ctx *BenchmarkContext) writeSolutions(N int, problemName string) error { } func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sandboxName string) { - sandbox, err := worker.NewSandbox(sandboxName, sandbox.IgnoreWarnings(true)) + log.Printf("running %v %v %v times", testset.Name, sandboxName, b.N) + sandbox, err := worker.NewSandbox( + sandboxName, + sandbox.IgnoreWarnings(true), + sandbox.EnableSandboxLogs(false)) + if err != nil { b.Fatal(err) } + log.Printf("Generating %v solutions", b.N) for i := 0; i < b.N; i++ { ctx.writeSolutions(b.N, testset.Name) } + log.Printf("Generated solutions") queue := &worker.Queue{Sandbox: sandbox, DB: ctx.db} diff --git a/test/performance/input.go b/test/performance/input.go index aab446e1..6312e12e 100644 --- a/test/performance/input.go +++ b/test/performance/input.go @@ -20,7 +20,7 @@ func BigInputProblem() *PerfTestSet { return &PerfTestSet{ Name: "INPUT", CapTime: 5000, - Count: 50, + Count: 10, Generator: func(r *rand.Rand) []byte { strSize := maxSize - r.Intn(10) input := make([]byte, strSize) diff --git a/test/performance/perf_test.go b/test/performance/perf_test.go index bcd59b40..46e2c490 100644 --- a/test/performance/perf_test.go +++ b/test/performance/perf_test.go @@ -7,7 +7,7 @@ import ( ) var testList = []*PerfTestSet{BigInputProblem(), SpawnTimeProblem()} -var sandboxList = []string{"raw", "isolate"} +var sandboxList = []string{"isolate", "raw"} func BenchmarkSandboxes(b *testing.B) { log.Println("creating test DB") diff --git a/test/performance/spawn.go b/test/performance/spawn.go index 997a44bb..46a087bb 100644 --- a/test/performance/spawn.go +++ b/test/performance/spawn.go @@ -19,7 +19,7 @@ func SpawnTimeProblem() *PerfTestSet { maxValue := 1 << 30 return &PerfTestSet{ Name: "SPAWN", - Count: 10000, + Count: 100, CapTime: 100, Generator: func(r *rand.Rand) []byte { value := r.Intn(maxValue) diff --git a/worker/sandbox/raw/sandbox.go b/worker/sandbox/raw/sandbox.go index b35cbd97..9089f074 100644 --- a/worker/sandbox/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -43,7 +43,9 @@ func (s *Runner) Settings() *sandbox.Settings { func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { dir := os.TempDir() - log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) + if s.Settings().LogSandbox { + log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) + } return s.RunFrom(dir, input) } diff --git a/worker/sandbox/sandbox.go b/worker/sandbox/sandbox.go index b2873043..20acb01d 100644 --- a/worker/sandbox/sandbox.go +++ b/worker/sandbox/sandbox.go @@ -43,21 +43,6 @@ type Output struct { ErrorMessage string `json:"error_message,omitempty"` } -type Settings struct { - IgnoreWarning bool -} - -var DefaultSettings = Settings{IgnoreWarning: false} - -type Option func(Settings) Settings - -func IgnoreWarnings(ignore bool) Option { - return func(o Settings) Settings { - o.IgnoreWarning = ignore - return o - } -} - // CopyTo copies all the files it contains into cwd. func (input *Input) CopyTo(cwd string) error { // Copy all the files into "cwd" diff --git a/worker/sandbox/settings.go b/worker/sandbox/settings.go new file mode 100644 index 00000000..ef528ad9 --- /dev/null +++ b/worker/sandbox/settings.go @@ -0,0 +1,24 @@ +package sandbox + +type Settings struct { + LogSandbox bool + IgnoreWarning bool +} + +var DefaultSettings = Settings{LogSandbox: true, IgnoreWarning: false} + +type Option func(Settings) Settings + +func IgnoreWarnings(ignore bool) Option { + return func(o Settings) Settings { + o.IgnoreWarning = ignore + return o + } +} + +func EnableSandboxLogs(enable bool) Option { + return func(o Settings) Settings { + o.LogSandbox = enable + return o + } +} From 7e7822f0d1ed32dbfd4f62e1801a58d2cb14a208 Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 15:51:45 +0700 Subject: [PATCH 25/30] Sandbox options --- worker/sandbox/raw/sandbox.go | 4 +++- worker/sandbox/sandbox.go | 15 --------------- worker/sandbox/settings.go | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 worker/sandbox/settings.go diff --git a/worker/sandbox/raw/sandbox.go b/worker/sandbox/raw/sandbox.go index b35cbd97..9089f074 100644 --- a/worker/sandbox/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -43,7 +43,9 @@ func (s *Runner) Settings() *sandbox.Settings { func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { dir := os.TempDir() - log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) + if s.Settings().LogSandbox { + log.Printf("[SANDBOX] Running %s %v\n", input.Command, input.Args) + } return s.RunFrom(dir, input) } diff --git a/worker/sandbox/sandbox.go b/worker/sandbox/sandbox.go index b2873043..20acb01d 100644 --- a/worker/sandbox/sandbox.go +++ b/worker/sandbox/sandbox.go @@ -43,21 +43,6 @@ type Output struct { ErrorMessage string `json:"error_message,omitempty"` } -type Settings struct { - IgnoreWarning bool -} - -var DefaultSettings = Settings{IgnoreWarning: false} - -type Option func(Settings) Settings - -func IgnoreWarnings(ignore bool) Option { - return func(o Settings) Settings { - o.IgnoreWarning = ignore - return o - } -} - // CopyTo copies all the files it contains into cwd. func (input *Input) CopyTo(cwd string) error { // Copy all the files into "cwd" diff --git a/worker/sandbox/settings.go b/worker/sandbox/settings.go new file mode 100644 index 00000000..ef528ad9 --- /dev/null +++ b/worker/sandbox/settings.go @@ -0,0 +1,24 @@ +package sandbox + +type Settings struct { + LogSandbox bool + IgnoreWarning bool +} + +var DefaultSettings = Settings{LogSandbox: true, IgnoreWarning: false} + +type Option func(Settings) Settings + +func IgnoreWarnings(ignore bool) Option { + return func(o Settings) Settings { + o.IgnoreWarning = ignore + return o + } +} + +func EnableSandboxLogs(enable bool) Option { + return func(o Settings) Settings { + o.LogSandbox = enable + return o + } +} From fb5b0207883166d5c25873b4509e2953945bc2bb Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 16:03:13 +0700 Subject: [PATCH 26/30] Add options for queue --- cmd/kjudge/main.go | 3 ++- worker/{ => queue}/queue.go | 18 ++++++++++++++---- worker/queue/settings.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) rename worker/{ => queue}/queue.go (79%) create mode 100644 worker/queue/settings.go diff --git a/cmd/kjudge/main.go b/cmd/kjudge/main.go index 36fe1101..e4e7a3c6 100644 --- a/cmd/kjudge/main.go +++ b/cmd/kjudge/main.go @@ -12,6 +12,7 @@ import ( _ "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server" "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/queue" ) var ( @@ -43,7 +44,7 @@ func main() { } // Start the queue - queue := worker.Queue{Sandbox: sandbox, DB: db} + queue := queue.NewQueue(db, sandbox) // Build the server server, err := server.New(db, opts...) diff --git a/worker/queue.go b/worker/queue/queue.go similarity index 79% rename from worker/queue.go rename to worker/queue/queue.go index d8ba7562..80f7420e 100644 --- a/worker/queue.go +++ b/worker/queue/queue.go @@ -1,4 +1,4 @@ -package worker +package queue import ( "log" @@ -7,6 +7,7 @@ import ( "github.com/mattn/go-sqlite3" "github.com/natsukagami/kjudge/db" "github.com/natsukagami/kjudge/models" + "github.com/natsukagami/kjudge/worker" "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -15,6 +16,15 @@ import ( type Queue struct { DB *db.DB Sandbox sandbox.Runner + Settings Settings +} + +func NewQueue(db *db.DB, sandbox sandbox.Runner, options ...Option) Queue { + setting := DefaultSettings + for _, option := range options { + setting = option(setting) + } + return Queue{DB: db, Sandbox: sandbox, Settings: setting} } // Start starts the queue. It is blocking, so might wanna "go run" it. @@ -60,7 +70,7 @@ func (q *Queue) HandleJob(job *models.Job) error { } switch job.Type { case models.JobTypeCompile: - if _, err := Compile(&CompileContext{DB: tx, Sub: sub, Problem: problem}); err != nil { + if _, err := worker.Compile(&worker.CompileContext{DB: tx, Sub: sub, Problem: problem}); err != nil { return err } case models.JobTypeRun: @@ -72,7 +82,7 @@ func (q *Queue) HandleJob(job *models.Job) error { if err != nil { return err } - if err := Run(q.Sandbox, &RunContext{ + if err := worker.Run(q.Sandbox, &worker.RunContext{ DB: tx, Sub: sub, Problem: problem, TestGroup: tg, Test: test}); err != nil { return err } @@ -81,7 +91,7 @@ func (q *Queue) HandleJob(job *models.Job) error { if err != nil { return err } - if err := Score(&ScoreContext{DB: tx, Sub: sub, Problem: problem, Contest: contest}); err != nil { + if err := worker.Score(&worker.ScoreContext{DB: tx, Sub: sub, Problem: problem, Contest: contest}); err != nil { return err } } diff --git a/worker/queue/settings.go b/worker/queue/settings.go new file mode 100644 index 00000000..4575b50e --- /dev/null +++ b/worker/queue/settings.go @@ -0,0 +1,32 @@ +package queue + +type Settings struct { + LogCompile bool + LogRun bool + LogScore bool +} + +var DefaultSettings = Settings{LogCompile: true, LogRun: true, LogScore: true} + +type Option func(Settings) Settings + +func CompileLogs(enable bool) Option { + return func(o Settings) Settings { + o.LogCompile = enable + return o + } +} + +func RunLogs(enable bool) Option { + return func(o Settings) Settings { + o.LogRun = enable + return o + } +} + +func ScoreLogs(enable bool) Option { + return func(o Settings) Settings { + o.LogScore = enable + return o + } +} From bfcb85f4ef06a64b58971d37d4e867b69324716b Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 16:22:01 +0700 Subject: [PATCH 27/30] Everything --- worker/compile.go | 19 ++++++++++++++----- worker/queue/queue.go | 12 +++++++----- worker/queue/settings.go | 4 ++-- worker/run.go | 16 ++++++++++++---- worker/sandbox/settings.go | 2 +- worker/score.go | 30 +++++++++++++++++++----------- 6 files changed, 55 insertions(+), 28 deletions(-) diff --git a/worker/compile.go b/worker/compile.go index 7f087a6f..703ed8b0 100644 --- a/worker/compile.go +++ b/worker/compile.go @@ -25,9 +25,17 @@ import ( // CompileContext is the information needed to perform compilation. type CompileContext struct { - DB *sqlx.Tx - Sub *models.Submission - Problem *models.Problem + DB *sqlx.Tx + Sub *models.Submission + Problem *models.Problem + AllowLogs bool +} + +func (c *CompileContext) Log(format string, v ...interface{}) { + if !c.AllowLogs { + return + } + log.Printf(format, v) } // Compile performs compilation. @@ -65,7 +73,7 @@ func Compile(c *CompileContext) (bool, error) { return false, c.Sub.Write(c.DB) } - log.Printf("[WORKER] Compiling submission %v\n", c.Sub.ID) + c.Log("[WORKER] Compiling submission %v\n", c.Sub.ID) // Now, create a temporary directory. dir, err := os.MkdirTemp("", "*") @@ -96,7 +104,8 @@ func Compile(c *CompileContext) (bool, error) { c.Sub.CompiledSource = nil c.Sub.Verdict = models.VerdictCompileError } - log.Printf("[WORKER] Compiling submission %v succeeded (result = %v).", c.Sub.ID, result) + + c.Log("[WORKER] Compiling submission %v succeeded (result = %v).", c.Sub.ID, result) return result, c.Sub.Write(c.DB) } diff --git a/worker/queue/queue.go b/worker/queue/queue.go index 80f7420e..4b55b9f1 100644 --- a/worker/queue/queue.go +++ b/worker/queue/queue.go @@ -14,8 +14,8 @@ import ( // Queue implements a queue that runs each job one by one. type Queue struct { - DB *db.DB - Sandbox sandbox.Runner + DB *db.DB + Sandbox sandbox.Runner Settings Settings } @@ -70,7 +70,8 @@ func (q *Queue) HandleJob(job *models.Job) error { } switch job.Type { case models.JobTypeCompile: - if _, err := worker.Compile(&worker.CompileContext{DB: tx, Sub: sub, Problem: problem}); err != nil { + if _, err := worker.Compile(&worker.CompileContext{ + DB: tx, Sub: sub, Problem: problem, AllowLogs: q.Settings.LogCompile}); err != nil { return err } case models.JobTypeRun: @@ -83,7 +84,7 @@ func (q *Queue) HandleJob(job *models.Job) error { return err } if err := worker.Run(q.Sandbox, &worker.RunContext{ - DB: tx, Sub: sub, Problem: problem, TestGroup: tg, Test: test}); err != nil { + DB: tx, Sub: sub, Problem: problem, TestGroup: tg, Test: test, AllowLogs: q.Settings.LogScore}); err != nil { return err } case models.JobTypeScore: @@ -91,7 +92,8 @@ func (q *Queue) HandleJob(job *models.Job) error { if err != nil { return err } - if err := worker.Score(&worker.ScoreContext{DB: tx, Sub: sub, Problem: problem, Contest: contest}); err != nil { + if err := worker.Score(&worker.ScoreContext{ + DB: tx, Sub: sub, Problem: problem, Contest: contest, AllowLogs: q.Settings.LogScore}); err != nil { return err } } diff --git a/worker/queue/settings.go b/worker/queue/settings.go index 4575b50e..e5ecf81a 100644 --- a/worker/queue/settings.go +++ b/worker/queue/settings.go @@ -2,8 +2,8 @@ package queue type Settings struct { LogCompile bool - LogRun bool - LogScore bool + LogRun bool + LogScore bool } var DefaultSettings = Settings{LogCompile: true, LogRun: true, LogScore: true} diff --git a/worker/run.go b/worker/run.go index 8f87a534..57390d2b 100644 --- a/worker/run.go +++ b/worker/run.go @@ -24,6 +24,14 @@ type RunContext struct { Problem *models.Problem TestGroup *models.TestGroup Test *models.Test + AllowLogs bool +} + +func (r *RunContext) Log(format string, v ...interface{}) { + if !r.AllowLogs { + return + } + log.Printf(format, v) } // TimeLimit returns the time limit of the context, in time.Duration. @@ -171,15 +179,15 @@ func Run(s sandbox.Runner, r *RunContext) error { compiled, source := r.CompiledSource() if !compiled { // Add a compilation job and re-add ourselves. - log.Printf("[WORKER] Submission %v not compiled, creating Compile job.\n", r.Sub.ID) + r.Log("[WORKER] Submission %v not compiled, creating Compile job.\n", r.Sub.ID) return models.BatchInsertJobs(r.DB, models.NewJobCompile(r.Sub.ID), models.NewJobRun(r.Sub.ID, r.Test.ID)) } if source == nil { - log.Printf("[WORKER] Not running a submission that failed to compile.\n") + r.Log("[WORKER] Not running a submission that failed to compile.\n") return nil } - log.Printf("[WORKER] Running submission %v on [test `%v`, group `%v`]\n", r.Sub.ID, r.Test.Name, r.TestGroup.Name) + r.Log("[WORKER] Running submission %v on [test `%v`, group `%v`]\n", r.Sub.ID, r.Test.Name, r.TestGroup.Name) var output *sandbox.Output file, err := models.GetFileWithName(r.DB, r.Problem.ID, ".stages") @@ -223,7 +231,7 @@ func Run(s sandbox.Runner, r *RunContext) error { return err } - log.Printf("[WORKER] Done running submission %v on [test `%v`, group `%v`]: %.1f (t = %v, m = %v)\n", + r.Log("[WORKER] Done running submission %v on [test `%v`, group `%v`]: %.1f (t = %v, m = %v)\n", r.Sub.ID, r.Test.Name, r.TestGroup.Name, result.Score, result.RunningTime, result.MemoryUsed) return result.Write(r.DB) diff --git a/worker/sandbox/settings.go b/worker/sandbox/settings.go index ef528ad9..2febedfc 100644 --- a/worker/sandbox/settings.go +++ b/worker/sandbox/settings.go @@ -1,7 +1,7 @@ package sandbox type Settings struct { - LogSandbox bool + LogSandbox bool IgnoreWarning bool } diff --git a/worker/score.go b/worker/score.go index fb4aa5db..a4095cc5 100644 --- a/worker/score.go +++ b/worker/score.go @@ -13,10 +13,18 @@ import ( // ScoreContext is a context for calculating a submission's score // and update the user's problem scores. type ScoreContext struct { - DB *sqlx.Tx - Sub *models.Submission - Problem *models.Problem - Contest *models.Contest + DB *sqlx.Tx + Sub *models.Submission + Problem *models.Problem + Contest *models.Contest + AllowLogs bool +} + +func (s *ScoreContext) Log(format string, v ...interface{}) { + if !s.AllowLogs { + return + } + log.Printf(format, v) } // Score does scoring on a submission and updates the user's ProblemResult. @@ -32,10 +40,10 @@ func Score(s *ScoreContext) error { } if compiled, source := s.CompiledSource(); !compiled { // Add a compilation job and re-add ourselves. - log.Printf("[WORKER] Submission %v not compiled, creating Compile job.\n", s.Sub.ID) + s.Log("[WORKER] Submission %v not compiled, creating Compile job.\n", s.Sub.ID) return models.BatchInsertJobs(s.DB, models.NewJobCompile(s.Sub.ID), models.NewJobScore(s.Sub.ID)) } else if source == nil { - log.Printf("[WORKER] Not running a submission that failed to compile.\n") + s.Log("[WORKER] Not running a submission that failed to compile.\n") s.Sub.Verdict = models.VerdictCompileError if err := s.Sub.Write(s.DB); err != nil { return err @@ -46,12 +54,12 @@ func Score(s *ScoreContext) error { return err } pr := s.CompareScores(subs) - log.Printf("[WORKER] Problem results updated for user %s, problem %d (score = %.1f, penalty = %d)\n", s.Sub.UserID, s.Problem.ID, pr.Score, pr.Penalty) + s.Log("[WORKER] Problem results updated for user %s, problem %d (score = %.1f, penalty = %d)\n", s.Sub.UserID, s.Problem.ID, pr.Score, pr.Penalty) return pr.Write(s.DB) } if missing := MissingTests(tests, testResults); len(missing) > 0 { - log.Printf("[WORKER] Submission %v needs to run %d tests before being scored.\n", s.Sub.ID, len(missing)) + s.Log("[WORKER] Submission %v needs to run %d tests before being scored.\n", s.Sub.ID, len(missing)) var jobs []*models.Job for _, m := range missing { jobs = append(jobs, models.NewJobRun(s.Sub.ID, m.ID)) @@ -60,7 +68,7 @@ func Score(s *ScoreContext) error { return models.BatchInsertJobs(s.DB, jobs...) } - log.Printf("[WORKER] Scoring submission %d\n", s.Sub.ID) + s.Log("[WORKER] Scoring submission %d\n", s.Sub.ID) // Calculate the score by summing scores on each test group. s.Sub.Score = sql.NullFloat64{Float64: 0.0, Valid: true} for _, tg := range tests { @@ -78,7 +86,7 @@ func Score(s *ScoreContext) error { if err := s.Sub.Write(s.DB); err != nil { return err } - log.Printf("[WORKER] Submission %d scored (verdict = %s, score = %.1f). Updating problem results\n", s.Sub.ID, s.Sub.Verdict, s.Sub.Score.Float64) + s.Log("[WORKER] Submission %d scored (verdict = %s, score = %.1f). Updating problem results\n", s.Sub.ID, s.Sub.Verdict, s.Sub.Score.Float64) // Update the ProblemResult subs, err := models.GetUserProblemSubmissions(s.DB, s.Sub.UserID, s.Problem.ID) @@ -86,7 +94,7 @@ func Score(s *ScoreContext) error { return err } pr := s.CompareScores(subs) - log.Printf("[WORKER] Problem results updated for user %s, problem %d (score = %.1f, penalty = %d)\n", s.Sub.UserID, s.Problem.ID, pr.Score, pr.Penalty) + s.Log("[WORKER] Problem results updated for user %s, problem %d (score = %.1f, penalty = %d)\n", s.Sub.UserID, s.Problem.ID, pr.Score, pr.Penalty) return pr.Write(s.DB) } From 40f88b84e185666380fd3f9f91706379976ba6bb Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 16:32:32 +0700 Subject: [PATCH 28/30] Remove worker logs --- test/performance/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/performance/helper.go b/test/performance/helper.go index 2d60578b..d2a3f026 100644 --- a/test/performance/helper.go +++ b/test/performance/helper.go @@ -11,6 +11,7 @@ import ( "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server/auth" "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/queue" "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -135,7 +136,7 @@ func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sa } log.Printf("Generated solutions") - queue := &worker.Queue{Sandbox: sandbox, DB: ctx.db} + queue := queue.NewQueue(ctx.db, sandbox, queue.CompileLogs(false), queue.RunLogs(false), queue.ScoreLogs(false)) b.ResetTimer() queue.Run() From 8c500b60a9ae368db192efa8574025536d9ad272 Mon Sep 17 00:00:00 2001 From: noe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 16:54:10 +0700 Subject: [PATCH 29/30] Expand logs --- test/performance/helper.go | 3 ++- worker/compile.go | 2 +- worker/queue/queue.go | 2 +- worker/run.go | 2 +- worker/score.go | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/performance/helper.go b/test/performance/helper.go index 2d60578b..f0817b34 100644 --- a/test/performance/helper.go +++ b/test/performance/helper.go @@ -11,6 +11,7 @@ import ( "github.com/natsukagami/kjudge/models" "github.com/natsukagami/kjudge/server/auth" "github.com/natsukagami/kjudge/worker" + "github.com/natsukagami/kjudge/worker/queue" "github.com/natsukagami/kjudge/worker/sandbox" "github.com/pkg/errors" ) @@ -135,7 +136,7 @@ func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sa } log.Printf("Generated solutions") - queue := &worker.Queue{Sandbox: sandbox, DB: ctx.db} + queue := &queue.Queue{Sandbox: sandbox, DB: ctx.db} b.ResetTimer() queue.Run() diff --git a/worker/compile.go b/worker/compile.go index 703ed8b0..5b0a3775 100644 --- a/worker/compile.go +++ b/worker/compile.go @@ -35,7 +35,7 @@ func (c *CompileContext) Log(format string, v ...interface{}) { if !c.AllowLogs { return } - log.Printf(format, v) + log.Printf(format, v...) } // Compile performs compilation. diff --git a/worker/queue/queue.go b/worker/queue/queue.go index b777d9d4..426af61d 100644 --- a/worker/queue/queue.go +++ b/worker/queue/queue.go @@ -103,7 +103,7 @@ func (q *Queue) HandleJob(job *models.Job) error { return err } if err := worker.Run(q.Sandbox, &worker.RunContext{ - DB: tx, Sub: sub, Problem: problem, TestGroup: tg, Test: test, AllowLogs: q.Settings.LogScore}); err != nil { + DB: tx, Sub: sub, Problem: problem, TestGroup: tg, Test: test, AllowLogs: q.Settings.LogRun}); err != nil { return err } case models.JobTypeScore: diff --git a/worker/run.go b/worker/run.go index 57390d2b..5ba2346e 100644 --- a/worker/run.go +++ b/worker/run.go @@ -31,7 +31,7 @@ func (r *RunContext) Log(format string, v ...interface{}) { if !r.AllowLogs { return } - log.Printf(format, v) + log.Printf(format, v...) } // TimeLimit returns the time limit of the context, in time.Duration. diff --git a/worker/score.go b/worker/score.go index a4095cc5..8e4268f4 100644 --- a/worker/score.go +++ b/worker/score.go @@ -24,7 +24,7 @@ func (s *ScoreContext) Log(format string, v ...interface{}) { if !s.AllowLogs { return } - log.Printf(format, v) + log.Printf(format, v...) } // Score does scoring on a submission and updates the user's ProblemResult. From 5afdabe34a8ac222232eacb97aad673fa2d8aee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Minh=20Nh=E1=BA=ADt?= <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 10 Jun 2023 16:57:50 +0700 Subject: [PATCH 30/30] Delete test/performance directory --- test/performance/helper.go | 153 ---------------------------------- test/performance/input.go | 34 -------- test/performance/perf_test.go | 35 -------- test/performance/spawn.go | 30 ------- test/performance/testset.go | 86 ------------------- 5 files changed, 338 deletions(-) delete mode 100644 test/performance/helper.go delete mode 100644 test/performance/input.go delete mode 100644 test/performance/perf_test.go delete mode 100644 test/performance/spawn.go delete mode 100644 test/performance/testset.go diff --git a/test/performance/helper.go b/test/performance/helper.go deleted file mode 100644 index d2a3f026..00000000 --- a/test/performance/helper.go +++ /dev/null @@ -1,153 +0,0 @@ -package performance - -import ( - "log" - "os" - "path/filepath" - "testing" - "time" - - "github.com/natsukagami/kjudge/db" - "github.com/natsukagami/kjudge/models" - "github.com/natsukagami/kjudge/server/auth" - "github.com/natsukagami/kjudge/worker" - "github.com/natsukagami/kjudge/worker/queue" - "github.com/natsukagami/kjudge/worker/sandbox" - "github.com/pkg/errors" -) - -type BenchmarkContext struct { - tdir string - db *db.DB - user *models.User - contest *models.Contest - problems map[string]*models.Problem -} - -func NewBenchmarkContext(tmpDir string) (*BenchmarkContext, error) { - benchDB, err := db.New(filepath.Join(tmpDir, "bench.db")) - if err != nil { - err2 := os.RemoveAll(tmpDir) - if err2 != nil { - return nil, errors.Wrapf(err2, "while handling %v", err) - } - return nil, err - } - - ctx := &BenchmarkContext{tdir: tmpDir, db: benchDB, problems: make(map[string]*models.Problem)} - - if err := ctx.writeContest(); err != nil { - return nil, err - } - - if err := ctx.writeUser(); err != nil { - return nil, err - } - - return ctx, nil -} - -func (ctx *BenchmarkContext) Close() error { - if err := os.RemoveAll(ctx.tdir); err != nil { - return err - } - if err := ctx.db.Close(); err != nil { - return err - } - return nil -} - -func (ctx *BenchmarkContext) writeContest() error { - ctx.contest = &models.Contest{ - ContestType: "weighted", - StartTime: time.Now().AddDate(0, 0, -1), - EndTime: time.Now().AddDate(0, 0, 1), - ID: 0, - Name: "Performance Testing", - ScoreboardViewStatus: models.ScoreboardViewStatusPublic, - } - return errors.Wrapf(ctx.contest.Write(ctx.db), "creating contest") -} - -func (ctx *BenchmarkContext) writeUser() error { - password, err := auth.PasswordHash("bigquestions") - if err != nil { - return errors.Wrap(err, "hashing password") - } - ctx.user = &models.User{ - ID: "Iroh", - Password: string(password), - DisplayName: "The Dragon of the West", - Organization: "Order of the White Lotus", - } - return errors.Wrap(ctx.user.Write(ctx.db), "creating user") -} - -func (ctx *BenchmarkContext) writeProblem(testset *PerfTestSet) error { - problem, err := testset.AddToDB(ctx.db, 2403, len(ctx.problems)+1, ctx.contest.ID) - if err != nil { - return errors.Wrapf(err, "creating testset %v's problem", testset.Name) - } - - ctx.problems[testset.Name] = problem - return nil -} - -const testSolution = `#include "solution.hpp" -` - -func (ctx *BenchmarkContext) writeSolutions(N int, problemName string) error { - problem := ctx.problems[problemName] - for i := 0; i < N; i++ { - sub := models.Submission{ - ProblemID: problem.ID, - UserID: ctx.user.ID, - Source: []byte(testSolution), - Language: models.LanguageCpp, - SubmittedAt: time.Now(), - Verdict: models.VerdictIsInQueue, - } - if err := sub.Write(ctx.db); err != nil { - return err - } - - job := models.NewJobScore(sub.ID) - if err := job.Write(ctx.db); err != nil { - return err - } - } - return nil -} - -func RunSingleTest(b *testing.B, ctx *BenchmarkContext, testset *PerfTestSet, sandboxName string) { - log.Printf("running %v %v %v times", testset.Name, sandboxName, b.N) - sandbox, err := worker.NewSandbox( - sandboxName, - sandbox.IgnoreWarnings(true), - sandbox.EnableSandboxLogs(false)) - - if err != nil { - b.Fatal(err) - } - - log.Printf("Generating %v solutions", b.N) - for i := 0; i < b.N; i++ { - ctx.writeSolutions(b.N, testset.Name) - } - log.Printf("Generated solutions") - - queue := queue.NewQueue(ctx.db, sandbox, queue.CompileLogs(false), queue.RunLogs(false), queue.ScoreLogs(false)) - - b.ResetTimer() - queue.Run() - b.StopTimer() - - if err := ctx.assertRunComplete(testset); err != nil { - b.Fatal(err) - } -} - -// TODO -func (ctx *BenchmarkContext) assertRunComplete(testset *PerfTestSet) error { - return nil -} diff --git a/test/performance/input.go b/test/performance/input.go deleted file mode 100644 index 6312e12e..00000000 --- a/test/performance/input.go +++ /dev/null @@ -1,34 +0,0 @@ -package performance - -import "math/rand" - -const bigInputCode = `#include -int main(){ - int r = 0; - while (char c = getchar()){ - if ('a' <= c && c <= 'b') r++; - else break; - } - printf("%i", r); -} -` - -// 50MB input to compare disk read time. O(1) memory. -// Problem: Given a string, print it's length -func BigInputProblem() *PerfTestSet { - maxSize := 50000000 // 50MB - return &PerfTestSet{ - Name: "INPUT", - CapTime: 5000, - Count: 10, - Generator: func(r *rand.Rand) []byte { - strSize := maxSize - r.Intn(10) - input := make([]byte, strSize) - for i := range input { - input[i] = byte(r.Intn(26) + 'A') - } - return input - }, - Solution: []byte(bigInputCode), - } -} diff --git a/test/performance/perf_test.go b/test/performance/perf_test.go deleted file mode 100644 index 46e2c490..00000000 --- a/test/performance/perf_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package performance - -import ( - "fmt" - "log" - "testing" -) - -var testList = []*PerfTestSet{BigInputProblem(), SpawnTimeProblem()} -var sandboxList = []string{"isolate", "raw"} - -func BenchmarkSandboxes(b *testing.B) { - log.Println("creating test DB") - - ctx, err := NewBenchmarkContext(b.TempDir()) - if err != nil { - b.Fatal(err) - } - defer ctx.Close() - - for _, testset := range testList { - log.Printf("creating problem %v", testset.Name) - if err := ctx.writeProblem(testset); err != nil { - b.Fatal(err) - } - } - - for _, testset := range testList { - for _, sandboxName := range sandboxList { - testName := fmt.Sprintf("%v %v", testset.Name, sandboxName) - log.Printf("running %v", testName) - b.Run(testName, func(b *testing.B) { RunSingleTest(b, ctx, testset, sandboxName) }) - } - } -} diff --git a/test/performance/spawn.go b/test/performance/spawn.go deleted file mode 100644 index 46a087bb..00000000 --- a/test/performance/spawn.go +++ /dev/null @@ -1,30 +0,0 @@ -package performance - -import ( - "fmt" - "math/rand" -) - -const spawnTimeCode = `#include -int main(){ - int a; scanf("%i", &a); - printf("%i", a*2); -} -` - -// O(1) problem to compare sandbox spawn time. -// Problem: Input one number, then output the double of that number -func SpawnTimeProblem() *PerfTestSet { - // maxValue * 2 must not cause integer overflow - maxValue := 1 << 30 - return &PerfTestSet{ - Name: "SPAWN", - Count: 100, - CapTime: 100, - Generator: func(r *rand.Rand) []byte { - value := r.Intn(maxValue) - return []byte(fmt.Sprintf("%v", value)) - }, - Solution: []byte(spawnTimeCode), // TODO - } -} diff --git a/test/performance/testset.go b/test/performance/testset.go deleted file mode 100644 index 68d23e60..00000000 --- a/test/performance/testset.go +++ /dev/null @@ -1,86 +0,0 @@ -// Package perf_test provides performance testing -package performance - -import ( - "database/sql" - "fmt" - "math/rand" - - "github.com/natsukagami/kjudge/db" - "github.com/natsukagami/kjudge/models" - "github.com/pkg/errors" -) - -// TODO: Output, Memory, Calculate, TLE - -type PerfTestSet struct { - Name string - Count int - CapTime int // Time limit sent to sandbox - Generator func(*rand.Rand) []byte // Returns input - Solution []byte // Solution to tested problem - -} - -// Generates problem and returns id -func (r *PerfTestSet) AddToDB(db db.DBContext, seed int64, index int, contestID int) (*models.Problem, error) { - // Creates problem - problem := &models.Problem{ - ContestID: contestID, - DisplayName: r.Name, - ID: 0, - MaxSubmissionsCount: 0, - MemoryLimit: 1 << 20, // 1GB - Name: fmt.Sprintf("%v", index), - PenaltyPolicy: models.PenaltyPolicyNone, - ScoringMode: models.ScoringModeLast, - SecondsBetweenSubmissions: 0, - TimeLimit: r.CapTime, - } - if err := problem.Write(db); err != nil { - return nil, errors.Wrapf(err, "problem %v", r.Name) - } - - // Creates test group - testGroup := &models.TestGroup{ - ID: 0, - MemoryLimit: sql.NullInt64{Valid: false}, // nil - Name: "main", - ProblemID: problem.ID, - Score: 100, - ScoringMode: models.TestScoringModeSum, - TimeLimit: sql.NullInt64{Valid: false}, // nil - } - if err := testGroup.Write(db); err != nil { - return nil, errors.Wrapf(err, "test group %v", testGroup.Name) - } - - // Creates tests. - rng := rand.New(rand.NewSource(seed)) - for i := 1; i < r.Count; i++ { - test := &models.Test{ - ID: 0, - Input: r.Generator(rng), - Name: fmt.Sprintf("%v", i), - Output: []byte(""), - TestGroupID: testGroup.ID, - } - if err := test.Write(db); err != nil { - return nil, errors.Wrapf(err, "test %v", i) - } - } - - // Uploads solution - solutionFile := &models.File{ - Content: r.Solution, - Filename: "solution.hpp", - ID: 0, - ProblemID: problem.ID, - Public: true, - } - if err := problem.WriteFiles(db, []*models.File{solutionFile}); err != nil { - return nil, errors.Wrap(err, "solution file") - } - - return problem, nil -}