From 82720c7580c1cd52b7036dbfce49145fcea2e047 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 14 Mar 2022 08:45:17 +0100 Subject: [PATCH 01/26] Add optional gops and http->https redirect --- go.mod | 3 +-- go.sum | 17 +++++++++++++++++ server.go | 22 +++++++++++++++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 97718ee..109736b 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,14 @@ require ( github.com/go-ldap/ldap/v3 v3.4.1 github.com/go-sql-driver/mysql v1.5.0 github.com/golang-jwt/jwt/v4 v4.1.0 + github.com/google/gops v0.3.22 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/sessions v1.2.1 github.com/iamlouk/lrucache v0.2.1 github.com/jmoiron/sqlx v1.3.1 github.com/mattn/go-sqlite3 v1.14.6 - github.com/stretchr/testify v1.5.1 // indirect github.com/vektah/gqlparser/v2 v2.1.0 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 gopkg.in/yaml.v2 v2.3.0 // indirect ) - diff --git a/go.sum b/go.sum index 7ebe111..4c8f811 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzS github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/squirrel v1.5.1 h1:kWAKlLLJFxZG7N2E0mBMNWVp5AuUX+JUrnhFN74Eg+w= github.com/Masterminds/squirrel v1.5.1/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.0.3 h1:M5ZnqLOoZR8ygVq0FfkXsNOKzMCk0xRiow0R5+5VkQ0= github.com/agnivade/levenshtein v1.0.3/go.mod h1:4SFRZbbXWLF4MU1T9Qg0pGgH3Pjs+t6ie5efyrwRJXs= @@ -26,11 +27,15 @@ github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkPro github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-ldap/ldap/v3 v3.4.1 h1:fU/0xli6HY02ocbMuozHAYsaHLcnkLjvho2r5a34BUU= github.com/go-ldap/ldap/v3 v3.4.1/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/google/gops v0.3.22 h1:lyvhDxfPLHAOR2xIYwjPhN387qHxyU21Sk9sz/GhmhQ= +github.com/google/gops v0.3.22/go.mod h1:7diIdLsqpCihPSX3fQagksT/Ku/y4RL9LHTlKyEUDl8= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= @@ -51,6 +56,7 @@ github.com/iamlouk/lrucache v0.2.1 h1:AtOSeg8ZOmEE0phkzuYsEtH9GdKRrJUz21nVWrYglD github.com/iamlouk/lrucache v0.2.1/go.mod h1:dbHtdSvjMz0Y55CQNkbwkFEbvcWkfHUz9IxUC6wIA9A= github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -82,6 +88,7 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -92,11 +99,15 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= github.com/vektah/gqlparser/v2 v2.1.0 h1:uiKJ+T5HMGGQM2kRKQ8Pxw8+Zq9qhhZhz/lieYvCMns= github.com/vektah/gqlparser/v2 v2.1.0/go.mod h1:SyUiHgLATUR8BiYURfTirrTcGpcE+4XkV2se04Px1Ms= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -111,10 +122,14 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -131,5 +146,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/server.go b/server.go index 56ed59d..c9361c1 100644 --- a/server.go +++ b/server.go @@ -31,6 +31,7 @@ import ( "github.com/ClusterCockpit/cc-backend/repository" "github.com/ClusterCockpit/cc-backend/schema" "github.com/ClusterCockpit/cc-backend/templates" + "github.com/google/gops/agent" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/jmoiron/sqlx" @@ -83,6 +84,10 @@ type ProgramConfig struct { HttpsCertFile string `json:"https-cert-file"` HttpsKeyFile string `json:"https-key-file"` + // If not the empty string and `addr` does not end in ":80", + // redirect every request incoming at port 80 to that url. + RedirectHttpTo string `json:"redirect-http-to"` + // If overwriten, at least all the options in the defaults below must // be provided! Most options here can be overwritten by the user. UiDefaults map[string]interface{} `json:"ui-defaults"` @@ -102,8 +107,6 @@ var programConfig ProgramConfig = ProgramConfig{ LdapConfig: nil, SessionMaxAge: "168h", JwtMaxAge: "0", - HttpsCertFile: "", - HttpsKeyFile: "", UiDefaults: map[string]interface{}{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, @@ -251,12 +254,13 @@ var routes []Route = []Route{ } func main() { - var flagReinitDB, flagStopImmediately, flagSyncLDAP bool + var flagReinitDB, flagStopImmediately, flagSyncLDAP, flagGops bool var flagConfigFile, flagImportJob string var flagNewUser, flagDelUser, flagGenJWT string flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize `job`, `tag`, and `jobtag` tables") flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the `user` table with ldap") flag.BoolVar(&flagStopImmediately, "no-server", false, "Do not start a server, stop right after initialization and argument handling") + flag.BoolVar(&flagGops, "gops", false, "Enable a github.com/google/gops/agent") flag.StringVar(&flagConfigFile, "config", "", "Location of the config file for this server (overwrites the defaults)") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,api,user]:`") flag.StringVar(&flagDelUser, "del-user", "", "Remove user by username") @@ -264,6 +268,12 @@ func main() { flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") flag.Parse() + if flagGops { + if err := agent.Listen(agent.Options{}); err != nil { + log.Fatalf("gops/agent.Listen failed: %s", err.Error()) + } + } + if err := loadEnv("./.env"); err != nil && !os.IsNotExist(err) { log.Fatalf("parsing './.env' file failed: %s", err.Error()) } @@ -537,6 +547,12 @@ func main() { log.Fatal(err) } + if !strings.HasSuffix(programConfig.Addr, ":80") && programConfig.RedirectHttpTo != "" { + go func() { + http.ListenAndServe(":80", http.RedirectHandler(programConfig.RedirectHttpTo, http.StatusMovedPermanently)) + }() + } + if programConfig.HttpsCertFile != "" && programConfig.HttpsKeyFile != "" { cert, err := tls.LoadX509KeyPair(programConfig.HttpsCertFile, programConfig.HttpsKeyFile) if err != nil { From 2651b96499f32e76c509e95697927effdba7f58a Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 14 Mar 2022 09:08:02 +0100 Subject: [PATCH 02/26] Add subcluster and walltime to Job types --- graph/generated/generated.go | 98 ++++++++++++++++++++++++++++++++++++ graph/schema.graphqls | 2 + init-db.go | 2 + repository/import.go | 8 +-- repository/job.go | 27 ++++++++-- schema/job.go | 3 +- 6 files changed, 131 insertions(+), 9 deletions(-) diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 204e623..3ed5804 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -98,8 +98,10 @@ type ComplexityRoot struct { SMT func(childComplexity int) int StartTime func(childComplexity int) int State func(childComplexity int) int + SubCluster func(childComplexity int) int Tags func(childComplexity int) int User func(childComplexity int) int + Walltime func(childComplexity int) int } JobMetric struct { @@ -503,6 +505,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Job.State(childComplexity), true + case "Job.subCluster": + if e.complexity.Job.SubCluster == nil { + break + } + + return e.complexity.Job.SubCluster(childComplexity), true + case "Job.tags": if e.complexity.Job.Tags == nil { break @@ -517,6 +526,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Job.User(childComplexity), true + case "Job.walltime": + if e.complexity.Job.Walltime == nil { + break + } + + return e.complexity.Job.Walltime(childComplexity), true + case "JobMetric.scope": if e.complexity.JobMetric.Scope == nil { break @@ -1212,8 +1228,10 @@ type Job { user: String! project: String! cluster: String! + subCluster: String! startTime: Time! duration: Int! + walltime: Int! numNodes: Int! numHWThreads: Int! numAcc: Int! @@ -2648,6 +2666,41 @@ func (ec *executionContext) _Job_cluster(ctx context.Context, field graphql.Coll return ec.marshalNString2string(ctx, field.Selections, res) } +func (ec *executionContext) _Job_subCluster(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Job", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.SubCluster, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + func (ec *executionContext) _Job_startTime(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -2718,6 +2771,41 @@ func (ec *executionContext) _Job_duration(ctx context.Context, field graphql.Col return ec.marshalNInt2int32(ctx, field.Selections, res) } +func (ec *executionContext) _Job_walltime(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Job", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Walltime, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int64) + fc.Result = res + return ec.marshalNInt2int64(ctx, field.Selections, res) +} + func (ec *executionContext) _Job_numNodes(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -7695,6 +7783,11 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj if out.Values[i] == graphql.Null { atomic.AddUint32(&invalids, 1) } + case "subCluster": + out.Values[i] = ec._Job_subCluster(ctx, field, obj) + if out.Values[i] == graphql.Null { + atomic.AddUint32(&invalids, 1) + } case "startTime": out.Values[i] = ec._Job_startTime(ctx, field, obj) if out.Values[i] == graphql.Null { @@ -7705,6 +7798,11 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj if out.Values[i] == graphql.Null { atomic.AddUint32(&invalids, 1) } + case "walltime": + out.Values[i] = ec._Job_walltime(ctx, field, obj) + if out.Values[i] == graphql.Null { + atomic.AddUint32(&invalids, 1) + } case "numNodes": out.Values[i] = ec._Job_numNodes(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 79553d4..0b85a34 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -11,8 +11,10 @@ type Job { user: String! project: String! cluster: String! + subCluster: String! startTime: Time! duration: Int! + walltime: Int! numNodes: Int! numHWThreads: Int! numAcc: Int! diff --git a/init-db.go b/init-db.go index b9fd9f7..0346080 100644 --- a/init-db.go +++ b/init-db.go @@ -25,6 +25,7 @@ const JOBS_DB_SCHEMA string = ` id INTEGER PRIMARY KEY /*!40101 AUTO_INCREMENT */, job_id BIGINT NOT NULL, cluster VARCHAR(255) NOT NULL, + subcluster VARCHAR(255) NOT NULL, start_time BIGINT NOT NULL, -- Unix timestamp user VARCHAR(255) NOT NULL, @@ -32,6 +33,7 @@ const JOBS_DB_SCHEMA string = ` ` + "`partition`" + ` VARCHAR(255) NOT NULL, -- partition is a keyword in mysql -.- array_job_id BIGINT NOT NULL, duration INT, + walltime INT, job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON diff --git a/repository/import.go b/repository/import.go index ff996a4..94fba51 100644 --- a/repository/import.go +++ b/repository/import.go @@ -16,12 +16,12 @@ import ( ) const NamedJobInsert string = `INSERT INTO job ( - job_id, user, project, cluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc, - exclusive, monitoring_status, smt, job_state, start_time, duration, resources, meta_data, + job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc, + exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data, mem_used_max, flops_any_avg, mem_bw_avg, load_avg, net_bw_avg, net_data_vol_total, file_bw_avg, file_data_vol_total ) VALUES ( - :job_id, :user, :project, :cluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, - :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :resources, :meta_data, + :job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, + :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data, :mem_used_max, :flops_any_avg, :mem_bw_avg, :load_avg, :net_bw_avg, :net_data_vol_total, :file_bw_avg, :file_data_vol_total );` diff --git a/repository/job.go b/repository/job.go index 7f3f9ee..732ef6f 100644 --- a/repository/job.go +++ b/repository/job.go @@ -13,6 +13,7 @@ import ( "github.com/ClusterCockpit/cc-backend/graph/model" "github.com/ClusterCockpit/cc-backend/schema" sq "github.com/Masterminds/squirrel" + "github.com/iamlouk/lrucache" "github.com/jmoiron/sqlx" ) @@ -20,10 +21,12 @@ type JobRepository struct { DB *sqlx.DB stmtCache *sq.StmtCache + cache *lrucache.Cache } func (r *JobRepository) Init() error { r.stmtCache = sq.NewStmtCache(r.DB) + r.cache = lrucache.New(100) return nil } @@ -120,11 +123,11 @@ func (r *JobRepository) Start(job *schema.JobMeta) (id int64, err error) { } res, err := r.DB.NamedExec(`INSERT INTO job ( - job_id, user, project, cluster, `+"`partition`"+`, array_job_id, num_nodes, num_hwthreads, num_acc, - exclusive, monitoring_status, smt, job_state, start_time, duration, resources, meta_data + job_id, user, project, cluster, subcluster, `+"`partition`"+`, array_job_id, num_nodes, num_hwthreads, num_acc, + exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data ) VALUES ( - :job_id, :user, :project, :cluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, - :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :resources, :meta_data + :job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, + :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data );`, job) if err != nil { return -1, err @@ -260,3 +263,19 @@ func (r *JobRepository) FindJobOrUser(ctx context.Context, searchterm string) (j return 0, "", ErrNotFound } + +func (r *JobRepository) Partitions(cluster string) ([]string, error) { + var err error + partitions := r.cache.Get("partitions:"+cluster, func() (interface{}, time.Duration, int) { + parts := []string{} + if err = r.DB.Select(&parts, `SELECT DISTINCT job.partition FROM job WHERE job.cluster = ?;`, cluster); err != nil { + return nil, 0, 1000 + } + + return parts, 1 * time.Hour, 1 + }) + if err != nil { + return nil, err + } + return partitions.([]string), nil +} diff --git a/schema/job.go b/schema/job.go index e6e9b25..db7c707 100644 --- a/schema/job.go +++ b/schema/job.go @@ -14,6 +14,7 @@ type BaseJob struct { User string `json:"user" db:"user"` Project string `json:"project" db:"project"` Cluster string `json:"cluster" db:"cluster"` + SubCluster string `json:"subCluster" db:"subcluster"` Partition string `json:"partition" db:"partition"` ArrayJobId int32 `json:"arrayJobId" db:"array_job_id"` NumNodes int32 `json:"numNodes" db:"num_nodes"` @@ -24,6 +25,7 @@ type BaseJob struct { SMT int32 `json:"smt" db:"smt"` State JobState `json:"jobState" db:"job_state"` Duration int32 `json:"duration" db:"duration"` + Walltime int64 `json:"walltime" db:"walltime"` Tags []*Tag `json:"tags"` RawResources []byte `json:"-" db:"resources"` Resources []*Resource `json:"resources"` @@ -54,7 +56,6 @@ type Job struct { type JobMeta struct { ID *int64 `json:"id,omitempty"` // never used in the job-archive, only available via REST-API BaseJob - Walltime int64 `json:"walltime"` // TODO: Missing in DB StartTime int64 `json:"startTime" db:"start_time"` Statistics map[string]JobStatistics `json:"statistics,omitempty"` } From 85ad6d9543e45b32185c4005d9947ca7effd2d60 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 14 Mar 2022 10:18:56 +0100 Subject: [PATCH 03/26] subclusters instead of slurm partitions --- api_test.go | 16 +- config/config.go | 67 +- config/nodelist.go | 136 ++++ config/nodelist_test.go | 37 ++ graph/generated/generated.go | 1110 +++++++++++++++++---------------- graph/model/models.go | 2 +- graph/model/models_gen.go | 17 +- graph/schema.graphqls | 5 +- graph/schema.resolvers.go | 10 +- graph/stats.go | 6 +- metricdata/archive.go | 2 +- metricdata/cc-metric-store.go | 2 +- metricdata/metricdata.go | 4 +- repository/import.go | 7 +- repository/job.go | 8 +- 15 files changed, 868 insertions(+), 561 deletions(-) create mode 100644 config/nodelist.go create mode 100644 config/nodelist_test.go diff --git a/api_test.go b/api_test.go index 7fd3fa3..11a2454 100644 --- a/api_test.go +++ b/api_test.go @@ -30,9 +30,14 @@ func setup(t *testing.T) *api.RestApi { const testclusterJson = `{ "name": "testcluster", - "partitions": [ + "subClusters": [ { - "name": "default", + "name": "sc0", + "nodes": "host120,host121,host122" + }, + { + "name": "sc1", + "nodes": "host123,host124,host125", "processorType": "Intel Core i7-4770", "socketsPerNode": 1, "coresPerSocket": 4, @@ -141,7 +146,7 @@ func TestRestApi(t *testing.T) { Timestep: 60, Series: []schema.Series{ { - Hostname: "testhost", + Hostname: "host123", Statistics: &schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3}, Data: []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3}, }, @@ -173,7 +178,7 @@ func TestRestApi(t *testing.T) { "tags": [{ "type": "testTagType", "name": "testTagName" }], "resources": [ { - "hostname": "testhost", + "hostname": "host123", "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7] } ], @@ -211,6 +216,7 @@ func TestRestApi(t *testing.T) { job.User != "testuser" || job.Project != "testproj" || job.Cluster != "testcluster" || + job.SubCluster != "sc1" || job.Partition != "default" || job.ArrayJobId != 0 || job.NumNodes != 1 || @@ -219,7 +225,7 @@ func TestRestApi(t *testing.T) { job.Exclusive != 1 || job.MonitoringStatus != 1 || job.SMT != 1 || - !reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "testhost", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) || + !reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) || job.StartTime.Unix() != 123456789 { t.Fatalf("unexpected job properties: %#v", job) } diff --git a/config/config.go b/config/config.go index 816ad8f..ef49a12 100644 --- a/config/config.go +++ b/config/config.go @@ -20,10 +20,14 @@ import ( var db *sqlx.DB var lookupConfigStmt *sqlx.Stmt + var lock sync.RWMutex var uiDefaults map[string]interface{} + var cache *lrucache.Cache = lrucache.New(1024) + var Clusters []*model.Cluster +var nodeLists map[string]map[string]NodeList func Init(usersdb *sqlx.DB, authEnabled bool, uiConfig map[string]interface{}, jobArchive string) error { db = usersdb @@ -34,6 +38,7 @@ func Init(usersdb *sqlx.DB, authEnabled bool, uiConfig map[string]interface{}, j } Clusters = []*model.Cluster{} + nodeLists = map[string]map[string]NodeList{} for _, de := range entries { raw, err := os.ReadFile(filepath.Join(jobArchive, de.Name(), "cluster.json")) if err != nil { @@ -53,8 +58,8 @@ func Init(usersdb *sqlx.DB, authEnabled bool, uiConfig map[string]interface{}, j return err } - if len(cluster.Name) == 0 || len(cluster.MetricConfig) == 0 || len(cluster.Partitions) == 0 { - return errors.New("cluster.name, cluster.metricConfig and cluster.Partitions should not be empty") + if len(cluster.Name) == 0 || len(cluster.MetricConfig) == 0 || len(cluster.SubClusters) == 0 { + return errors.New("cluster.name, cluster.metricConfig and cluster.SubClusters should not be empty") } for _, mc := range cluster.MetricConfig { @@ -83,6 +88,19 @@ func Init(usersdb *sqlx.DB, authEnabled bool, uiConfig map[string]interface{}, j } Clusters = append(Clusters, &cluster) + + nodeLists[cluster.Name] = make(map[string]NodeList) + for _, sc := range cluster.SubClusters { + if sc.Nodes == "" { + continue + } + + nl, err := ParseNodeList(sc.Nodes) + if err != nil { + return fmt.Errorf("in %s/cluster.json: %w", cluster.Name, err) + } + nodeLists[cluster.Name][sc.Name] = nl + } } if authEnabled { @@ -188,7 +206,7 @@ func UpdateConfig(key, value string, ctx context.Context) error { return nil } -func GetClusterConfig(cluster string) *model.Cluster { +func GetCluster(cluster string) *model.Cluster { for _, c := range Clusters { if c.Name == cluster { return c @@ -197,11 +215,11 @@ func GetClusterConfig(cluster string) *model.Cluster { return nil } -func GetPartition(cluster, partition string) *model.Partition { +func GetSubCluster(cluster, subcluster string) *model.SubCluster { for _, c := range Clusters { if c.Name == cluster { - for _, p := range c.Partitions { - if p.Name == partition { + for _, p := range c.SubClusters { + if p.Name == subcluster { return p } } @@ -222,3 +240,40 @@ func GetMetricConfig(cluster, metric string) *model.MetricConfig { } return nil } + +// AssignSubCluster sets the `job.subcluster` property of the job based +// on its cluster and resources. +func AssignSubCluster(job *schema.BaseJob) error { + cluster := GetCluster(job.Cluster) + if cluster == nil { + return fmt.Errorf("unkown cluster: %#v", job.Cluster) + } + + if job.SubCluster != "" { + for _, sc := range cluster.SubClusters { + if sc.Name == job.SubCluster { + return nil + } + } + return fmt.Errorf("already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster) + } + + if len(job.Resources) == 0 { + return fmt.Errorf("job without any resources/hosts") + } + + host0 := job.Resources[0].Hostname + for sc, nl := range nodeLists[job.Cluster] { + if nl != nil && nl.Contains(host0) { + job.SubCluster = sc + return nil + } + } + + if cluster.SubClusters[0].Nodes == "" { + job.SubCluster = cluster.SubClusters[0].Name + return nil + } + + return fmt.Errorf("no subcluster found for cluster %#v and host %#v", job.Cluster, host0) +} diff --git a/config/nodelist.go b/config/nodelist.go new file mode 100644 index 0000000..800e1ba --- /dev/null +++ b/config/nodelist.go @@ -0,0 +1,136 @@ +package config + +import ( + "fmt" + "strconv" + "strings" + + "github.com/ClusterCockpit/cc-backend/log" +) + +type NLExprString string + +func (nle NLExprString) consume(input string) (next string, ok bool) { + str := string(nle) + if strings.HasPrefix(input, str) { + return strings.TrimPrefix(input, str), true + } + return "", false +} + +type NLExprIntRange struct { + start, end int64 + zeroPadded bool + digits int +} + +func (nle NLExprIntRange) consume(input string) (next string, ok bool) { + if !nle.zeroPadded || nle.digits < 1 { + log.Error("node list: only zero-padded ranges are allowed") + return "", false + } + + if len(input) < nle.digits { + return "", false + } + + numerals, rest := input[:nle.digits], input[nle.digits:] + for len(numerals) > 1 && numerals[0] == '0' { + numerals = numerals[1:] + } + + x, err := strconv.ParseInt(numerals, 10, 32) + if err != nil { + return "", false + } + + if nle.start <= x && x <= nle.end { + return rest, true + } + + return "", false +} + +type NodeList [][]interface { + consume(input string) (next string, ok bool) +} + +func (nl *NodeList) Contains(name string) bool { + var ok bool + for _, term := range *nl { + str := name + for _, expr := range term { + str, ok = expr.consume(str) + if !ok { + break + } + } + + if ok && str == "" { + return true + } + } + + return false +} + +func ParseNodeList(raw string) (NodeList, error) { + nl := NodeList{} + + isLetter := func(r byte) bool { return ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') } + isDigit := func(r byte) bool { return '0' <= r && r <= '9' } + + for _, rawterm := range strings.Split(raw, ",") { + exprs := []interface { + consume(input string) (next string, ok bool) + }{} + for i := 0; i < len(rawterm); i++ { + c := rawterm[i] + if isLetter(c) || isDigit(c) { + j := i + for j < len(rawterm) && (isLetter(rawterm[j]) || isDigit(rawterm[j])) { + j++ + } + exprs = append(exprs, NLExprString(rawterm[i:j])) + i = j - 1 + } else if c == '[' { + end := strings.Index(rawterm[i:], "]") + if end == -1 { + return nil, fmt.Errorf("node list: unclosed '['") + } + + minus := strings.Index(rawterm[i:i+end], "-") + if minus == -1 { + return nil, fmt.Errorf("node list: no '-' found inside '[...]'") + } + + s1, s2 := rawterm[i+1:i+minus], rawterm[i+minus+1:i+end] + if len(s1) != len(s2) || len(s1) == 0 { + return nil, fmt.Errorf("node list: %#v and %#v are not of equal length or of length zero", s1, s2) + } + + x1, err := strconv.ParseInt(s1, 10, 32) + if err != nil { + return nil, fmt.Errorf("node list: %w", err) + } + x2, err := strconv.ParseInt(s2, 10, 32) + if err != nil { + return nil, fmt.Errorf("node list: %w", err) + } + + exprs = append(exprs, NLExprIntRange{ + start: x1, + end: x2, + digits: len(s1), + zeroPadded: true, + }) + i += end + } else { + return nil, fmt.Errorf("node list: invalid character: %#v", rune(c)) + } + } + nl = append(nl, exprs) + } + + return nl, nil +} diff --git a/config/nodelist_test.go b/config/nodelist_test.go new file mode 100644 index 0000000..6768d59 --- /dev/null +++ b/config/nodelist_test.go @@ -0,0 +1,37 @@ +package config + +import ( + "testing" +) + +func TestNodeList(t *testing.T) { + nl, err := ParseNodeList("hallo,wel123t,emmy[01-99],fritz[005-500],woody[100-200]") + if err != nil { + t.Fatal(err) + } + + // fmt.Printf("terms\n") + // for i, term := range nl.terms { + // fmt.Printf("term %d: %#v\n", i, term) + // } + + if nl.Contains("hello") || nl.Contains("woody") { + t.Fail() + } + + if nl.Contains("fritz1") || nl.Contains("fritz9") || nl.Contains("fritz004") || nl.Contains("woody201") { + t.Fail() + } + + if !nl.Contains("hallo") || !nl.Contains("wel123t") { + t.Fail() + } + + if !nl.Contains("emmy01") || !nl.Contains("emmy42") || !nl.Contains("emmy99") { + t.Fail() + } + + if !nl.Contains("woody100") || !nl.Contains("woody199") { + t.Fail() + } +} diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 3ed5804..323229c 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -37,6 +37,7 @@ type Config struct { } type ResolverRoot interface { + Cluster() ClusterResolver Job() JobResolver Mutation() MutationResolver Query() QueryResolver @@ -56,7 +57,7 @@ type ComplexityRoot struct { FilterRanges func(childComplexity int) int MetricConfig func(childComplexity int) int Name func(childComplexity int) int - Partitions func(childComplexity int) int + SubClusters func(childComplexity int) int } Count struct { @@ -169,18 +170,6 @@ type ComplexityRoot struct { Metrics func(childComplexity int) int } - Partition struct { - CoresPerSocket func(childComplexity int) int - FlopRateScalar func(childComplexity int) int - FlopRateSimd func(childComplexity int) int - MemoryBandwidth func(childComplexity int) int - Name func(childComplexity int) int - ProcessorType func(childComplexity int) int - SocketsPerNode func(childComplexity int) int - ThreadsPerCore func(childComplexity int) int - Topology func(childComplexity int) int - } - Query struct { Clusters func(childComplexity int) int Job func(childComplexity int, id string) int @@ -214,6 +203,19 @@ type ComplexityRoot struct { Min func(childComplexity int) int } + SubCluster struct { + CoresPerSocket func(childComplexity int) int + FlopRateScalar func(childComplexity int) int + FlopRateSimd func(childComplexity int) int + MemoryBandwidth func(childComplexity int) int + Name func(childComplexity int) int + Nodes func(childComplexity int) int + ProcessorType func(childComplexity int) int + SocketsPerNode func(childComplexity int) int + ThreadsPerCore func(childComplexity int) int + Topology func(childComplexity int) int + } + Tag struct { ID func(childComplexity int) int Name func(childComplexity int) int @@ -235,6 +237,9 @@ type ComplexityRoot struct { } } +type ClusterResolver interface { + SubClusters(ctx context.Context, obj *model.Cluster) ([]*model.SubCluster, error) +} type JobResolver interface { MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) @@ -316,12 +321,12 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Cluster.Name(childComplexity), true - case "Cluster.partitions": - if e.complexity.Cluster.Partitions == nil { + case "Cluster.subClusters": + if e.complexity.Cluster.SubClusters == nil { break } - return e.complexity.Cluster.Partitions(childComplexity), true + return e.complexity.Cluster.SubClusters(childComplexity), true case "Count.count": if e.complexity.Count.Count == nil { @@ -824,69 +829,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.NodeMetrics.Metrics(childComplexity), true - case "Partition.coresPerSocket": - if e.complexity.Partition.CoresPerSocket == nil { - break - } - - return e.complexity.Partition.CoresPerSocket(childComplexity), true - - case "Partition.flopRateScalar": - if e.complexity.Partition.FlopRateScalar == nil { - break - } - - return e.complexity.Partition.FlopRateScalar(childComplexity), true - - case "Partition.flopRateSimd": - if e.complexity.Partition.FlopRateSimd == nil { - break - } - - return e.complexity.Partition.FlopRateSimd(childComplexity), true - - case "Partition.memoryBandwidth": - if e.complexity.Partition.MemoryBandwidth == nil { - break - } - - return e.complexity.Partition.MemoryBandwidth(childComplexity), true - - case "Partition.name": - if e.complexity.Partition.Name == nil { - break - } - - return e.complexity.Partition.Name(childComplexity), true - - case "Partition.processorType": - if e.complexity.Partition.ProcessorType == nil { - break - } - - return e.complexity.Partition.ProcessorType(childComplexity), true - - case "Partition.socketsPerNode": - if e.complexity.Partition.SocketsPerNode == nil { - break - } - - return e.complexity.Partition.SocketsPerNode(childComplexity), true - - case "Partition.threadsPerCore": - if e.complexity.Partition.ThreadsPerCore == nil { - break - } - - return e.complexity.Partition.ThreadsPerCore(childComplexity), true - - case "Partition.topology": - if e.complexity.Partition.Topology == nil { - break - } - - return e.complexity.Partition.Topology(childComplexity), true - case "Query.clusters": if e.complexity.Query.Clusters == nil { break @@ -1074,6 +1016,76 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.StatsSeries.Min(childComplexity), true + case "SubCluster.coresPerSocket": + if e.complexity.SubCluster.CoresPerSocket == nil { + break + } + + return e.complexity.SubCluster.CoresPerSocket(childComplexity), true + + case "SubCluster.flopRateScalar": + if e.complexity.SubCluster.FlopRateScalar == nil { + break + } + + return e.complexity.SubCluster.FlopRateScalar(childComplexity), true + + case "SubCluster.flopRateSimd": + if e.complexity.SubCluster.FlopRateSimd == nil { + break + } + + return e.complexity.SubCluster.FlopRateSimd(childComplexity), true + + case "SubCluster.memoryBandwidth": + if e.complexity.SubCluster.MemoryBandwidth == nil { + break + } + + return e.complexity.SubCluster.MemoryBandwidth(childComplexity), true + + case "SubCluster.name": + if e.complexity.SubCluster.Name == nil { + break + } + + return e.complexity.SubCluster.Name(childComplexity), true + + case "SubCluster.nodes": + if e.complexity.SubCluster.Nodes == nil { + break + } + + return e.complexity.SubCluster.Nodes(childComplexity), true + + case "SubCluster.processorType": + if e.complexity.SubCluster.ProcessorType == nil { + break + } + + return e.complexity.SubCluster.ProcessorType(childComplexity), true + + case "SubCluster.socketsPerNode": + if e.complexity.SubCluster.SocketsPerNode == nil { + break + } + + return e.complexity.SubCluster.SocketsPerNode(childComplexity), true + + case "SubCluster.threadsPerCore": + if e.complexity.SubCluster.ThreadsPerCore == nil { + break + } + + return e.complexity.SubCluster.ThreadsPerCore(childComplexity), true + + case "SubCluster.topology": + if e.complexity.SubCluster.Topology == nil { + break + } + + return e.complexity.SubCluster.Topology(childComplexity), true + case "Tag.id": if e.complexity.Tag.ID == nil { break @@ -1250,11 +1262,12 @@ type Cluster { name: String! metricConfig: [MetricConfig!]! filterRanges: FilterRanges! - partitions: [Partition!]! + subClusters: [SubCluster!]! } -type Partition { +type SubCluster { name: String! + nodes: String! processorType: String! socketsPerNode: Int! coresPerSocket: Int! @@ -2141,7 +2154,7 @@ func (ec *executionContext) _Cluster_filterRanges(ctx context.Context, field gra return ec.marshalNFilterRanges2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐFilterRanges(ctx, field.Selections, res) } -func (ec *executionContext) _Cluster_partitions(ctx context.Context, field graphql.CollectedField, obj *model.Cluster) (ret graphql.Marshaler) { +func (ec *executionContext) _Cluster_subClusters(ctx context.Context, field graphql.CollectedField, obj *model.Cluster) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -2152,14 +2165,14 @@ func (ec *executionContext) _Cluster_partitions(ctx context.Context, field graph Object: "Cluster", Field: field, Args: nil, - IsMethod: false, - IsResolver: false, + IsMethod: true, + IsResolver: true, } ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Partitions, nil + return ec.resolvers.Cluster().SubClusters(rctx, obj) }) if err != nil { ec.Error(ctx, err) @@ -2171,9 +2184,9 @@ func (ec *executionContext) _Cluster_partitions(ctx context.Context, field graph } return graphql.Null } - res := resTmp.([]*model.Partition) + res := resTmp.([]*model.SubCluster) fc.Result = res - return ec.marshalNPartition2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐPartitionᚄ(ctx, field.Selections, res) + return ec.marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐSubClusterᚄ(ctx, field.Selections, res) } func (ec *executionContext) _Count_name(ctx context.Context, field graphql.CollectedField, obj *model.Count) (ret graphql.Marshaler) { @@ -4570,321 +4583,6 @@ func (ec *executionContext) _NodeMetrics_metrics(ctx context.Context, field grap return ec.marshalNJobMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐJobMetricWithNameᚄ(ctx, field.Selections, res) } -func (ec *executionContext) _Partition_name(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.Name, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(string) - fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_processorType(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.ProcessorType, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(string) - fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_socketsPerNode(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.SocketsPerNode, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_coresPerSocket(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.CoresPerSocket, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_threadsPerCore(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.ThreadsPerCore, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_flopRateScalar(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.FlopRateScalar, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_flopRateSimd(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.FlopRateSimd, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_memoryBandwidth(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.MemoryBandwidth, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(int) - fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) -} - -func (ec *executionContext) _Partition_topology(ctx context.Context, field graphql.CollectedField, obj *model.Partition) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Partition", - Field: field, - Args: nil, - IsMethod: false, - IsResolver: false, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return obj.Topology, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(*model.Topology) - fc.Result = res - return ec.marshalNTopology2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐTopology(ctx, field.Selections, res) -} - func (ec *executionContext) _Query_clusters(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -5729,6 +5427,356 @@ func (ec *executionContext) _StatsSeries_max(ctx context.Context, field graphql. return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐFloatᚄ(ctx, field.Selections, res) } +func (ec *executionContext) _SubCluster_name(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_nodes(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Nodes, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_processorType(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.ProcessorType, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_socketsPerNode(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.SocketsPerNode, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_coresPerSocket(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.CoresPerSocket, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_threadsPerCore(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.ThreadsPerCore, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_flopRateScalar(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.FlopRateScalar, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_flopRateSimd(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.FlopRateSimd, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_memoryBandwidth(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.MemoryBandwidth, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) _SubCluster_topology(ctx context.Context, field graphql.CollectedField, obj *model.SubCluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "SubCluster", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Topology, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*model.Topology) + fc.Result = res + return ec.marshalNTopology2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐTopology(ctx, field.Selections, res) +} + func (ec *executionContext) _Tag_id(ctx context.Context, field graphql.CollectedField, obj *schema.Tag) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -7586,23 +7634,32 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, case "name": out.Values[i] = ec._Cluster_name(ctx, field, obj) if out.Values[i] == graphql.Null { - invalids++ + atomic.AddUint32(&invalids, 1) } case "metricConfig": out.Values[i] = ec._Cluster_metricConfig(ctx, field, obj) if out.Values[i] == graphql.Null { - invalids++ + atomic.AddUint32(&invalids, 1) } case "filterRanges": out.Values[i] = ec._Cluster_filterRanges(ctx, field, obj) if out.Values[i] == graphql.Null { - invalids++ - } - case "partitions": - out.Values[i] = ec._Cluster_partitions(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + atomic.AddUint32(&invalids, 1) } + case "subClusters": + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Cluster_subClusters(ctx, field, obj) + if res == graphql.Null { + atomic.AddUint32(&invalids, 1) + } + return res + }) default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -8263,73 +8320,6 @@ func (ec *executionContext) _NodeMetrics(ctx context.Context, sel ast.SelectionS return out } -var partitionImplementors = []string{"Partition"} - -func (ec *executionContext) _Partition(ctx context.Context, sel ast.SelectionSet, obj *model.Partition) graphql.Marshaler { - fields := graphql.CollectFields(ec.OperationContext, sel, partitionImplementors) - - out := graphql.NewFieldSet(fields) - var invalids uint32 - for i, field := range fields { - switch field.Name { - case "__typename": - out.Values[i] = graphql.MarshalString("Partition") - case "name": - out.Values[i] = ec._Partition_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "processorType": - out.Values[i] = ec._Partition_processorType(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "socketsPerNode": - out.Values[i] = ec._Partition_socketsPerNode(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "coresPerSocket": - out.Values[i] = ec._Partition_coresPerSocket(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "threadsPerCore": - out.Values[i] = ec._Partition_threadsPerCore(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "flopRateScalar": - out.Values[i] = ec._Partition_flopRateScalar(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "flopRateSimd": - out.Values[i] = ec._Partition_flopRateSimd(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "memoryBandwidth": - out.Values[i] = ec._Partition_memoryBandwidth(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - case "topology": - out.Values[i] = ec._Partition_topology(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ - } - default: - panic("unknown field " + strconv.Quote(field.Name)) - } - } - out.Dispatch() - if invalids > 0 { - return graphql.Null - } - return out -} - var queryImplementors = []string{"Query"} func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) graphql.Marshaler { @@ -8603,6 +8593,78 @@ func (ec *executionContext) _StatsSeries(ctx context.Context, sel ast.SelectionS return out } +var subClusterImplementors = []string{"SubCluster"} + +func (ec *executionContext) _SubCluster(ctx context.Context, sel ast.SelectionSet, obj *model.SubCluster) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, subClusterImplementors) + + out := graphql.NewFieldSet(fields) + var invalids uint32 + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("SubCluster") + case "name": + out.Values[i] = ec._SubCluster_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "nodes": + out.Values[i] = ec._SubCluster_nodes(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "processorType": + out.Values[i] = ec._SubCluster_processorType(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "socketsPerNode": + out.Values[i] = ec._SubCluster_socketsPerNode(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "coresPerSocket": + out.Values[i] = ec._SubCluster_coresPerSocket(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "threadsPerCore": + out.Values[i] = ec._SubCluster_threadsPerCore(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "flopRateScalar": + out.Values[i] = ec._SubCluster_flopRateScalar(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "flopRateSimd": + out.Values[i] = ec._SubCluster_flopRateSimd(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "memoryBandwidth": + out.Values[i] = ec._SubCluster_memoryBandwidth(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "topology": + out.Values[i] = ec._SubCluster_topology(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch() + if invalids > 0 { + return graphql.Null + } + return out +} + var tagImplementors = []string{"Tag"} func (ec *executionContext) _Tag(ctx context.Context, sel ast.SelectionSet, obj *schema.Tag) graphql.Marshaler { @@ -9760,53 +9822,6 @@ func (ec *executionContext) marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockp return ret } -func (ec *executionContext) marshalNPartition2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐPartitionᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.Partition) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNPartition2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐPartition(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() - return ret -} - -func (ec *executionContext) marshalNPartition2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐPartition(ctx context.Context, sel ast.SelectionSet, v *model.Partition) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - return ec._Partition(ctx, sel, v) -} - func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐResourceᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Resource) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -9913,6 +9928,53 @@ func (ec *executionContext) marshalNString2ᚕstringᚄ(ctx context.Context, sel return ret } +func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐSubClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.SubCluster) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐSubCluster(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + return ret +} + +func (ec *executionContext) marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐSubCluster(ctx context.Context, sel ast.SelectionSet, v *model.SubCluster) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + return ec._SubCluster(ctx, sel, v) +} + func (ec *executionContext) marshalNTag2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐTag(ctx context.Context, sel ast.SelectionSet, v schema.Tag) graphql.Marshaler { return ec._Tag(ctx, sel, &v) } diff --git a/graph/model/models.go b/graph/model/models.go index e67098e..7a2d936 100644 --- a/graph/model/models.go +++ b/graph/model/models.go @@ -6,7 +6,7 @@ type Cluster struct { Name string `json:"name"` MetricConfig []*MetricConfig `json:"metricConfig"` FilterRanges *FilterRanges `json:"filterRanges"` - Partitions []*Partition `json:"partitions"` + SubClusters []*SubCluster `json:"subClusters"` // NOT part of the API: MetricDataRepository *MetricDataRepository `json:"metricDataRepository"` diff --git a/graph/model/models_gen.go b/graph/model/models_gen.go index 174f679..95f58b0 100644 --- a/graph/model/models_gen.go +++ b/graph/model/models_gen.go @@ -122,8 +122,16 @@ type PageRequest struct { Page int `json:"page"` } -type Partition struct { +type StringInput struct { + Eq *string `json:"eq"` + Contains *string `json:"contains"` + StartsWith *string `json:"startsWith"` + EndsWith *string `json:"endsWith"` +} + +type SubCluster struct { Name string `json:"name"` + Nodes string `json:"nodes"` ProcessorType string `json:"processorType"` SocketsPerNode int `json:"socketsPerNode"` CoresPerSocket int `json:"coresPerSocket"` @@ -134,13 +142,6 @@ type Partition struct { Topology *Topology `json:"topology"` } -type StringInput struct { - Eq *string `json:"eq"` - Contains *string `json:"contains"` - StartsWith *string `json:"startsWith"` - EndsWith *string `json:"endsWith"` -} - type TimeRange struct { From *time.Time `json:"from"` To *time.Time `json:"to"` diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 0b85a34..25d44a8 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -33,11 +33,12 @@ type Cluster { name: String! metricConfig: [MetricConfig!]! filterRanges: FilterRanges! - partitions: [Partition!]! + subClusters: [SubCluster!]! } -type Partition { +type SubCluster { name: String! + nodes: String! processorType: String! socketsPerNode: Int! coresPerSocket: Int! diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go index 3fa95b0..8fec60c 100644 --- a/graph/schema.resolvers.go +++ b/graph/schema.resolvers.go @@ -18,6 +18,10 @@ import ( "github.com/ClusterCockpit/cc-backend/schema" ) +func (r *clusterResolver) SubClusters(ctx context.Context, obj *model.Cluster) ([]*model.SubCluster, error) { + panic(fmt.Errorf("not implemented")) +} + func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) { return r.Repo.FetchMetadata(obj) } @@ -204,7 +208,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, partiti } if metrics == nil { - for _, mc := range config.GetClusterConfig(cluster).MetricConfig { + for _, mc := range config.GetCluster(cluster).MetricConfig { metrics = append(metrics, mc.Name) } } @@ -236,6 +240,9 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, partiti return nodeMetrics, nil } +// Cluster returns generated.ClusterResolver implementation. +func (r *Resolver) Cluster() generated.ClusterResolver { return &clusterResolver{r} } + // Job returns generated.JobResolver implementation. func (r *Resolver) Job() generated.JobResolver { return &jobResolver{r} } @@ -245,6 +252,7 @@ func (r *Resolver) Mutation() generated.MutationResolver { return &mutationResol // Query returns generated.QueryResolver implementation. func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } +type clusterResolver struct{ *Resolver } type jobResolver struct{ *Resolver } type mutationResolver struct{ *Resolver } type queryResolver struct{ *Resolver } diff --git a/graph/stats.go b/graph/stats.go index da21995..fb24bab 100644 --- a/graph/stats.go +++ b/graph/stats.go @@ -32,8 +32,8 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF // `socketsPerNode` and `coresPerSocket` can differ from cluster to cluster, so we need to explicitly loop over those. for _, cluster := range config.Clusters { - for _, partition := range cluster.Partitions { - corehoursCol := fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes * %d * %d) / 3600) as int)", partition.SocketsPerNode, partition.CoresPerSocket) + for _, subcluster := range cluster.SubClusters { + corehoursCol := fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes * %d * %d) / 3600) as int)", subcluster.SocketsPerNode, subcluster.CoresPerSocket) var query sq.SelectBuilder if groupBy == nil { query = sq.Select( @@ -54,7 +54,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF query = query. Where("job.cluster = ?", cluster.Name). - Where("job.partition = ?", partition.Name) + Where("job.subcluster = ?", subcluster.Name) query = repository.SecurityCheck(ctx, query) for _, f := range filter { diff --git a/metricdata/archive.go b/metricdata/archive.go index e2aff03..e3cae79 100644 --- a/metricdata/archive.go +++ b/metricdata/archive.go @@ -157,7 +157,7 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { // Writes a running job to the job-archive func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) { allMetrics := make([]string, 0) - metricConfigs := config.GetClusterConfig(job.Cluster).MetricConfig + metricConfigs := config.GetCluster(job.Cluster).MetricConfig for _, mc := range metricConfigs { allMetrics = append(allMetrics, mc.Name) } diff --git a/metricdata/cc-metric-store.go b/metricdata/cc-metric-store.go index 78d8750..c77d43d 100644 --- a/metricdata/cc-metric-store.go +++ b/metricdata/cc-metric-store.go @@ -227,7 +227,7 @@ var ( func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scopes []schema.MetricScope) ([]ApiQuery, []schema.MetricScope, error) { queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) - topology := config.GetPartition(job.Cluster, job.Partition).Topology + topology := config.GetSubCluster(job.Cluster, job.SubCluster).Topology assignedScope := []schema.MetricScope{} for _, metric := range metrics { diff --git a/metricdata/metricdata.go b/metricdata/metricdata.go index d4d9817..8f4122a 100644 --- a/metricdata/metricdata.go +++ b/metricdata/metricdata.go @@ -79,7 +79,7 @@ func LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ct } if metrics == nil { - cluster := config.GetClusterConfig(job.Cluster) + cluster := config.GetCluster(job.Cluster) for _, mc := range cluster.MetricConfig { metrics = append(metrics, mc.Name) } @@ -167,7 +167,7 @@ func LoadNodeData(cluster, partition string, metrics, nodes []string, scopes []s } if metrics == nil { - for _, m := range config.GetClusterConfig(cluster).MetricConfig { + for _, m := range config.GetCluster(cluster).MetricConfig { metrics = append(metrics, m.Name) } } diff --git a/repository/import.go b/repository/import.go index 94fba51..a18c189 100644 --- a/repository/import.go +++ b/repository/import.go @@ -122,12 +122,13 @@ func (r *JobRepository) ImportJob(jobMeta *schema.JobMeta, jobData *schema.JobDa return nil } +// This function also sets the subcluster if necessary! func SanityChecks(job *schema.BaseJob) error { - if c := config.GetClusterConfig(job.Cluster); c == nil { + if c := config.GetCluster(job.Cluster); c == nil { return fmt.Errorf("no such cluster: %#v", job.Cluster) } - if p := config.GetPartition(job.Cluster, job.Partition); p == nil { - return fmt.Errorf("no such partition: %#v (on cluster %#v)", job.Partition, job.Cluster) + if err := config.AssignSubCluster(job); err != nil { + return err } if !job.State.Valid() { return fmt.Errorf("not a valid job state: %#v", job.State) diff --git a/repository/job.go b/repository/job.go index 732ef6f..567b512 100644 --- a/repository/job.go +++ b/repository/job.go @@ -31,17 +31,17 @@ func (r *JobRepository) Init() error { } var jobColumns []string = []string{ - "job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.start_time", "job.partition", "job.array_job_id", + "job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.partition", "job.array_job_id", "job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state", - "job.duration", "job.resources", // "job.meta_data", + "job.duration", "job.walltime", "job.resources", // "job.meta_data", } func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { job := &schema.Job{} if err := row.Scan( - &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, + &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, - &job.Duration, &job.RawResources /*&job.MetaData*/); err != nil { + &job.Duration, &job.Walltime, &job.RawResources /*&job.MetaData*/); err != nil { return nil, err } From 839db9fdae59013e259b73e8b10374056e91df7a Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 14 Mar 2022 10:24:27 +0100 Subject: [PATCH 04/26] List of slurm partitions via GraphQL --- gqlgen.yml | 4 ++ graph/generated/generated.go | 85 ++++++++++++++++++++++++++++-------- graph/schema.graphqls | 3 +- graph/schema.resolvers.go | 4 +- 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/gqlgen.yml b/gqlgen.yml index 576f2bb..f02ce61 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -61,6 +61,10 @@ models: resolver: true metaData: resolver: true + Cluster: + fields: + partitions: + resolver: true NullableFloat: { model: "github.com/ClusterCockpit/cc-backend/schema.Float" } MetricScope: { model: "github.com/ClusterCockpit/cc-backend/schema.MetricScope" } JobStatistics: { model: "github.com/ClusterCockpit/cc-backend/schema.JobStatistics" } diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 323229c..877ec42 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -57,6 +57,7 @@ type ComplexityRoot struct { FilterRanges func(childComplexity int) int MetricConfig func(childComplexity int) int Name func(childComplexity int) int + Partitions func(childComplexity int) int SubClusters func(childComplexity int) int } @@ -238,7 +239,7 @@ type ComplexityRoot struct { } type ClusterResolver interface { - SubClusters(ctx context.Context, obj *model.Cluster) ([]*model.SubCluster, error) + Partitions(ctx context.Context, obj *model.Cluster) ([]string, error) } type JobResolver interface { MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) @@ -321,6 +322,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Cluster.Name(childComplexity), true + case "Cluster.partitions": + if e.complexity.Cluster.Partitions == nil { + break + } + + return e.complexity.Cluster.Partitions(childComplexity), true + case "Cluster.subClusters": if e.complexity.Cluster.SubClusters == nil { break @@ -1260,9 +1268,10 @@ type Job { type Cluster { name: String! + partitions: [String!]! # Slurm partitions metricConfig: [MetricConfig!]! filterRanges: FilterRanges! - subClusters: [SubCluster!]! + subClusters: [SubCluster!]! # Hardware partitions/subclusters } type SubCluster { @@ -2084,6 +2093,41 @@ func (ec *executionContext) _Cluster_name(ctx context.Context, field graphql.Col return ec.marshalNString2string(ctx, field.Selections, res) } +func (ec *executionContext) _Cluster_partitions(ctx context.Context, field graphql.CollectedField, obj *model.Cluster) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Cluster", + Field: field, + Args: nil, + IsMethod: true, + IsResolver: true, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Cluster().Partitions(rctx, obj) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]string) + fc.Result = res + return ec.marshalNString2ᚕstringᚄ(ctx, field.Selections, res) +} + func (ec *executionContext) _Cluster_metricConfig(ctx context.Context, field graphql.CollectedField, obj *model.Cluster) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -2165,14 +2209,14 @@ func (ec *executionContext) _Cluster_subClusters(ctx context.Context, field grap Object: "Cluster", Field: field, Args: nil, - IsMethod: true, - IsResolver: true, + IsMethod: false, + IsResolver: false, } ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Cluster().SubClusters(rctx, obj) + return obj.SubClusters, nil }) if err != nil { ec.Error(ctx, err) @@ -7636,6 +7680,20 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, if out.Values[i] == graphql.Null { atomic.AddUint32(&invalids, 1) } + case "partitions": + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Cluster_partitions(ctx, field, obj) + if res == graphql.Null { + atomic.AddUint32(&invalids, 1) + } + return res + }) case "metricConfig": out.Values[i] = ec._Cluster_metricConfig(ctx, field, obj) if out.Values[i] == graphql.Null { @@ -7647,19 +7705,10 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, atomic.AddUint32(&invalids, 1) } case "subClusters": - field := field - out.Concurrently(i, func() (res graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - } - }() - res = ec._Cluster_subClusters(ctx, field, obj) - if res == graphql.Null { - atomic.AddUint32(&invalids, 1) - } - return res - }) + out.Values[i] = ec._Cluster_subClusters(ctx, field, obj) + if out.Values[i] == graphql.Null { + atomic.AddUint32(&invalids, 1) + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 25d44a8..9c0e341 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -31,9 +31,10 @@ type Job { type Cluster { name: String! + partitions: [String!]! # Slurm partitions metricConfig: [MetricConfig!]! filterRanges: FilterRanges! - subClusters: [SubCluster!]! + subClusters: [SubCluster!]! # Hardware partitions/subclusters } type SubCluster { diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go index 8fec60c..fdd1937 100644 --- a/graph/schema.resolvers.go +++ b/graph/schema.resolvers.go @@ -18,8 +18,8 @@ import ( "github.com/ClusterCockpit/cc-backend/schema" ) -func (r *clusterResolver) SubClusters(ctx context.Context, obj *model.Cluster) ([]*model.SubCluster, error) { - panic(fmt.Errorf("not implemented")) +func (r *clusterResolver) Partitions(ctx context.Context, obj *model.Cluster) ([]string, error) { + return r.Repo.Partitions(obj.Name) } func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) { From e9195aa2e3ee008251663ac334b2a6e0502ab67c Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 14 Mar 2022 10:43:35 +0100 Subject: [PATCH 05/26] fix repository/ tests --- .github/workflows/test.yml | 5 +++-- test/test.db | Bin 131072 -> 131072 bytes 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d6a89d..a631ef2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,5 +14,6 @@ jobs: run: | go build ./... go vet ./... - go test . - env BASEPATH="../" go test ./repository + go test . + env BASEPATH="../" go test ./repository + env BASEPATH="../" go test ./config diff --git a/test/test.db b/test/test.db index 7dcff504474b7c7fdb8f402616a8fe01472c3a11..0d9b290daccf43c1a470997b5eb7b26e452bb4bf 100644 GIT binary patch delta 145 zcmZo@;Am*zm>@0a%D}*&0K_oBJyFM4(Un0j+JF}*#KjuQz;}lKI&UrCMAlHAkKC7e zj&M)pw%^!zm4&N2ot0f&SeS9T>E`?FQ(3un6v`8Ga!N9DQxzt!=N1(-&`~HZO-jxw qEiOqdQV4O4n7ol&m|ZD1F*8p|bMtgoJEqO0yv@0a!oa|w0K_oBIZ?-0(S24+bvQ=?r{F_|Nl}@^!LbW%p#$ zW|v`o$vS Date: Mon, 14 Mar 2022 11:08:01 +0100 Subject: [PATCH 06/26] Update frontend/ --- frontend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend b/frontend index eae185a..7dbabf1 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit eae185a9f6c006b61657df6897447f9a0761b42f +Subproject commit 7dbabf140d704b38a1fe29b8248e4226ce0c1b23 From 7be38277a98edfb6365e1e34150f91a67b387e9e Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 15 Mar 2022 08:29:29 +0100 Subject: [PATCH 07/26] cleanup and comments --- .github/workflows/test.yml | 4 +- init-db.go => repository/init.go | 37 +++--- repository/job_test.go | 9 +- routes.go | 128 +++++++++++++++++++ runtimeSetup.go | 7 ++ server.go | 208 +++++++------------------------ api_test.go => test/api_test.go | 14 +-- test/db.go | 26 ---- 8 files changed, 210 insertions(+), 223 deletions(-) rename init-db.go => repository/init.go (88%) rename api_test.go => test/api_test.go (95%) delete mode 100644 test/db.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a631ef2..1b0590e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,4 @@ jobs: run: | go build ./... go vet ./... - go test . - env BASEPATH="../" go test ./repository - env BASEPATH="../" go test ./config + go test ./... diff --git a/init-db.go b/repository/init.go similarity index 88% rename from init-db.go rename to repository/init.go index 0346080..44b8bd6 100644 --- a/init-db.go +++ b/repository/init.go @@ -1,4 +1,4 @@ -package main +package repository import ( "bufio" @@ -9,14 +9,13 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/log" - "github.com/ClusterCockpit/cc-backend/repository" "github.com/ClusterCockpit/cc-backend/schema" "github.com/jmoiron/sqlx" ) // `AUTO_INCREMENT` is in a comment because of this hack: // https://stackoverflow.com/a/41028314 (sqlite creates unique ids automatically) -const JOBS_DB_SCHEMA string = ` +const JobsDBSchema string = ` DROP TABLE IF EXISTS jobtag; DROP TABLE IF EXISTS job; DROP TABLE IF EXISTS tag; @@ -32,8 +31,8 @@ const JOBS_DB_SCHEMA string = ` project VARCHAR(255) NOT NULL, ` + "`partition`" + ` VARCHAR(255) NOT NULL, -- partition is a keyword in mysql -.- array_job_id BIGINT NOT NULL, - duration INT, - walltime INT, + duration INT NOT NULL DEFAULT 0, + walltime INT NOT NULL DEFAULT 0, job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON @@ -68,7 +67,8 @@ const JOBS_DB_SCHEMA string = ` FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE); ` -const JOBS_DB_INDEXES string = ` +// Indexes are created after the job-archive is traversed for faster inserts. +const JobsDbIndexes string = ` CREATE INDEX job_by_user ON job (user); CREATE INDEX job_by_starttime ON job (start_time); CREATE INDEX job_by_job_id ON job (job_id); @@ -77,12 +77,12 @@ const JOBS_DB_INDEXES string = ` // Delete the tables "job", "tag" and "jobtag" from the database and // repopulate them using the jobs found in `archive`. -func initDB(db *sqlx.DB, archive string) error { +func InitDB(db *sqlx.DB, archive string) error { starttime := time.Now() log.Print("Building job table...") // Basic database structure: - _, err := db.Exec(JOBS_DB_SCHEMA) + _, err := db.Exec(JobsDBSchema) if err != nil { return err } @@ -96,16 +96,21 @@ func initDB(db *sqlx.DB, archive string) error { return err } + // Inserts are bundled into transactions because in sqlite, + // that speeds up inserts A LOT. tx, err := db.Beginx() if err != nil { return err } - stmt, err := tx.PrepareNamed(repository.NamedJobInsert) + stmt, err := tx.PrepareNamed(NamedJobInsert) if err != nil { return err } + // Not using log.Print because we want the line to end with `\r` and + // this function is only ever called when a special command line flag + // is passed anyways. fmt.Printf("%d jobs inserted...\r", 0) i := 0 tags := make(map[string]int64) @@ -159,6 +164,8 @@ func initDB(db *sqlx.DB, archive string) error { return err } + // For compability with the old job-archive directory structure where + // there was no start time directory. for _, startTimeDir := range startTimeDirs { if startTimeDir.Type().IsRegular() && startTimeDir.Name() == "meta.json" { if err := handleDirectory(dirpath); err != nil { @@ -180,7 +187,7 @@ func initDB(db *sqlx.DB, archive string) error { // Create indexes after inserts so that they do not // need to be continually updated. - if _, err := db.Exec(JOBS_DB_INDEXES); err != nil { + if _, err := db.Exec(JobsDbIndexes); err != nil { return err } @@ -226,7 +233,7 @@ func loadJob(tx *sqlx.Tx, stmt *sqlx.NamedStmt, tags map[string]int64, path stri return err } - if err := repository.SanityChecks(&job.BaseJob); err != nil { + if err := SanityChecks(&job.BaseJob); err != nil { return err } @@ -262,11 +269,3 @@ func loadJob(tx *sqlx.Tx, stmt *sqlx.NamedStmt, tags map[string]int64, path stri return nil } - -func loadJobStat(job *schema.JobMeta, metric string) float64 { - if stats, ok := job.Statistics[metric]; ok { - return stats.Avg - } - - return 0.0 -} diff --git a/repository/job_test.go b/repository/job_test.go index 9d8d132..5cf54bb 100644 --- a/repository/job_test.go +++ b/repository/job_test.go @@ -5,14 +5,17 @@ import ( "testing" "github.com/jmoiron/sqlx" - - "github.com/ClusterCockpit/cc-backend/test" + _ "github.com/mattn/go-sqlite3" ) var db *sqlx.DB func init() { - db = test.InitDB() + var err error + db, err = sqlx.Open("sqlite3", "../test/test.db") + if err != nil { + fmt.Println(err) + } } func setup(t *testing.T) *JobRepository { diff --git a/routes.go b/routes.go index 0caeae2..243a4e7 100644 --- a/routes.go +++ b/routes.go @@ -9,6 +9,9 @@ import ( "github.com/ClusterCockpit/cc-backend/auth" "github.com/ClusterCockpit/cc-backend/config" + "github.com/ClusterCockpit/cc-backend/graph" + "github.com/ClusterCockpit/cc-backend/graph/model" + "github.com/ClusterCockpit/cc-backend/log" "github.com/ClusterCockpit/cc-backend/schema" "github.com/ClusterCockpit/cc-backend/templates" "github.com/gorilla/mux" @@ -24,6 +27,131 @@ type Route struct { Setup func(i InfoType, r *http.Request) InfoType } +var routes []Route = []Route{ + {"/", "home.tmpl", "ClusterCockpit", false, setupHomeRoute}, + {"/config", "config.tmpl", "Settings", false, func(i InfoType, r *http.Request) InfoType { return i }}, + {"/monitoring/jobs/", "monitoring/jobs.tmpl", "Jobs - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { return i }}, + {"/monitoring/job/{id:[0-9]+}", "monitoring/job.tmpl", "Job - ClusterCockpit", false, setupJobRoute}, + {"/monitoring/users/", "monitoring/list.tmpl", "Users - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { i["listType"] = "USER"; return i }}, + {"/monitoring/projects/", "monitoring/list.tmpl", "Projects - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { i["listType"] = "PROJECT"; return i }}, + {"/monitoring/tags/", "monitoring/taglist.tmpl", "Tags - ClusterCockpit", false, setupTaglistRoute}, + {"/monitoring/user/{id}", "monitoring/user.tmpl", "User - ClusterCockpit", true, setupUserRoute}, + {"/monitoring/systems/{cluster}", "monitoring/systems.tmpl", "Cluster - ClusterCockpit", false, setupClusterRoute}, + {"/monitoring/node/{cluster}/{hostname}", "monitoring/node.tmpl", "Node - ClusterCockpit", false, setupNodeRoute}, + {"/monitoring/analysis/{cluster}", "monitoring/analysis.tmpl", "Analaysis - ClusterCockpit", true, setupAnalysisRoute}, +} + +func setupHomeRoute(i InfoType, r *http.Request) InfoType { + type cluster struct { + Name string + RunningJobs int + TotalJobs int + RecentShortJobs int + } + + runningJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{ + State: []schema.JobState{schema.JobStateRunning}, + }}, nil) + if err != nil { + log.Errorf("failed to count jobs: %s", err.Error()) + runningJobs = map[string]int{} + } + totalJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, nil, nil) + if err != nil { + log.Errorf("failed to count jobs: %s", err.Error()) + totalJobs = map[string]int{} + } + + from := time.Now().Add(-24 * time.Hour) + recentShortJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{ + StartTime: &model.TimeRange{From: &from, To: nil}, + Duration: &model.IntRange{From: 0, To: graph.ShortJobDuration}, + }}, nil) + if err != nil { + log.Errorf("failed to count jobs: %s", err.Error()) + recentShortJobs = map[string]int{} + } + + clusters := make([]cluster, 0) + for _, c := range config.Clusters { + clusters = append(clusters, cluster{ + Name: c.Name, + RunningJobs: runningJobs[c.Name], + TotalJobs: totalJobs[c.Name], + RecentShortJobs: recentShortJobs[c.Name], + }) + } + + i["clusters"] = clusters + return i +} + +func setupJobRoute(i InfoType, r *http.Request) InfoType { + i["id"] = mux.Vars(r)["id"] + return i +} + +func setupUserRoute(i InfoType, r *http.Request) InfoType { + i["id"] = mux.Vars(r)["id"] + i["username"] = mux.Vars(r)["id"] + return i +} + +func setupClusterRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["id"] = vars["cluster"] + i["cluster"] = vars["cluster"] + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupNodeRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["cluster"] = vars["cluster"] + i["hostname"] = vars["hostname"] + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupAnalysisRoute(i InfoType, r *http.Request) InfoType { + i["cluster"] = mux.Vars(r)["cluster"] + return i +} + +func setupTaglistRoute(i InfoType, r *http.Request) InfoType { + var username *string = nil + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleAdmin) { + username = &user.Username + } + + tags, counts, err := jobRepo.CountTags(username) + tagMap := make(map[string][]map[string]interface{}) + if err != nil { + log.Errorf("GetTags failed: %s", err.Error()) + i["tagmap"] = tagMap + return i + } + + for _, tag := range tags { + tagItem := map[string]interface{}{ + "id": tag.ID, + "name": tag.Name, + "count": counts[tag.Name], + } + tagMap[tag.Type] = append(tagMap[tag.Type], tagItem) + } + i["tagmap"] = tagMap + return i +} + func buildFilterPresets(query url.Values) map[string]interface{} { filterPresets := map[string]interface{}{} diff --git a/runtimeSetup.go b/runtimeSetup.go index 070cf30..f43e569 100644 --- a/runtimeSetup.go +++ b/runtimeSetup.go @@ -12,6 +12,9 @@ import ( "syscall" ) +// Very simple and limited .env file reader. +// All variable definitions found are directly +// added to the processes environment. func loadEnv(file string) error { f, err := os.Open(file) if err != nil { @@ -74,6 +77,10 @@ func loadEnv(file string) error { return s.Err() } +// Changes the processes user and group to that +// specified in the config.json. The go runtime +// takes care of all threads (and not only the calling one) +// executing the underlying systemcall. func dropPrivileges() error { if programConfig.Group != "" { g, err := user.LookupGroup(programConfig.Group) diff --git a/server.go b/server.go index c9361c1..de89430 100644 --- a/server.go +++ b/server.go @@ -25,11 +25,9 @@ import ( "github.com/ClusterCockpit/cc-backend/config" "github.com/ClusterCockpit/cc-backend/graph" "github.com/ClusterCockpit/cc-backend/graph/generated" - "github.com/ClusterCockpit/cc-backend/graph/model" "github.com/ClusterCockpit/cc-backend/log" "github.com/ClusterCockpit/cc-backend/metricdata" "github.com/ClusterCockpit/cc-backend/repository" - "github.com/ClusterCockpit/cc-backend/schema" "github.com/ClusterCockpit/cc-backend/templates" "github.com/google/gops/agent" "github.com/gorilla/handlers" @@ -40,7 +38,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -var db *sqlx.DB var jobRepo *repository.JobRepository // Format of the configurartion (file). See below for the defaults. @@ -127,147 +124,22 @@ var programConfig ProgramConfig = ProgramConfig{ }, } -func setupHomeRoute(i InfoType, r *http.Request) InfoType { - type cluster struct { - Name string - RunningJobs int - TotalJobs int - RecentShortJobs int - } - - runningJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{ - State: []schema.JobState{schema.JobStateRunning}, - }}, nil) - if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) - runningJobs = map[string]int{} - } - totalJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, nil, nil) - if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) - totalJobs = map[string]int{} - } - - from := time.Now().Add(-24 * time.Hour) - recentShortJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{ - StartTime: &model.TimeRange{From: &from, To: nil}, - Duration: &model.IntRange{From: 0, To: graph.ShortJobDuration}, - }}, nil) - if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) - recentShortJobs = map[string]int{} - } - - clusters := make([]cluster, 0) - for _, c := range config.Clusters { - clusters = append(clusters, cluster{ - Name: c.Name, - RunningJobs: runningJobs[c.Name], - TotalJobs: totalJobs[c.Name], - RecentShortJobs: recentShortJobs[c.Name], - }) - } - - i["clusters"] = clusters - return i -} - -func setupJobRoute(i InfoType, r *http.Request) InfoType { - i["id"] = mux.Vars(r)["id"] - return i -} - -func setupUserRoute(i InfoType, r *http.Request) InfoType { - i["id"] = mux.Vars(r)["id"] - i["username"] = mux.Vars(r)["id"] - return i -} - -func setupClusterRoute(i InfoType, r *http.Request) InfoType { - vars := mux.Vars(r) - i["id"] = vars["cluster"] - i["cluster"] = vars["cluster"] - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } - return i -} - -func setupNodeRoute(i InfoType, r *http.Request) InfoType { - vars := mux.Vars(r) - i["cluster"] = vars["cluster"] - i["hostname"] = vars["hostname"] - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } - return i -} - -func setupAnalysisRoute(i InfoType, r *http.Request) InfoType { - i["cluster"] = mux.Vars(r)["cluster"] - return i -} - -func setupTaglistRoute(i InfoType, r *http.Request) InfoType { - var username *string = nil - if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleAdmin) { - username = &user.Username - } - - tags, counts, err := jobRepo.CountTags(username) - tagMap := make(map[string][]map[string]interface{}) - if err != nil { - log.Errorf("GetTags failed: %s", err.Error()) - i["tagmap"] = tagMap - return i - } - - for _, tag := range tags { - tagItem := map[string]interface{}{ - "id": tag.ID, - "name": tag.Name, - "count": counts[tag.Name], - } - tagMap[tag.Type] = append(tagMap[tag.Type], tagItem) - } - log.Infof("TAGS %+v", tags) - i["tagmap"] = tagMap - return i -} - -var routes []Route = []Route{ - {"/", "home.tmpl", "ClusterCockpit", false, setupHomeRoute}, - {"/config", "config.tmpl", "Settings", false, func(i InfoType, r *http.Request) InfoType { return i }}, - {"/monitoring/jobs/", "monitoring/jobs.tmpl", "Jobs - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { return i }}, - {"/monitoring/job/{id:[0-9]+}", "monitoring/job.tmpl", "Job - ClusterCockpit", false, setupJobRoute}, - {"/monitoring/users/", "monitoring/list.tmpl", "Users - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { i["listType"] = "USER"; return i }}, - {"/monitoring/projects/", "monitoring/list.tmpl", "Projects - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { i["listType"] = "PROJECT"; return i }}, - {"/monitoring/tags/", "monitoring/taglist.tmpl", "Tags - ClusterCockpit", false, setupTaglistRoute}, - {"/monitoring/user/{id}", "monitoring/user.tmpl", "User - ClusterCockpit", true, setupUserRoute}, - {"/monitoring/systems/{cluster}", "monitoring/systems.tmpl", "Cluster - ClusterCockpit", false, setupClusterRoute}, - {"/monitoring/node/{cluster}/{hostname}", "monitoring/node.tmpl", "Node - ClusterCockpit", false, setupNodeRoute}, - {"/monitoring/analysis/{cluster}", "monitoring/analysis.tmpl", "Analaysis - ClusterCockpit", true, setupAnalysisRoute}, -} - func main() { var flagReinitDB, flagStopImmediately, flagSyncLDAP, flagGops bool var flagConfigFile, flagImportJob string var flagNewUser, flagDelUser, flagGenJWT string - flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize `job`, `tag`, and `jobtag` tables") - flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the `user` table with ldap") + flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)") + flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'user' table with ldap") flag.BoolVar(&flagStopImmediately, "no-server", false, "Do not start a server, stop right after initialization and argument handling") - flag.BoolVar(&flagGops, "gops", false, "Enable a github.com/google/gops/agent") - flag.StringVar(&flagConfigFile, "config", "", "Location of the config file for this server (overwrites the defaults)") + flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") + flag.StringVar(&flagConfigFile, "config", "", "Overwrite the global config options by those specified in `config.json`") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,api,user]:`") - flag.StringVar(&flagDelUser, "del-user", "", "Remove user by username") - flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by the username") + flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") + flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") flag.Parse() + // See https://github.com/google/gops (Runtime overhead is almost zero) if flagGops { if err := agent.Listen(agent.Options{}); err != nil { log.Fatalf("gops/agent.Listen failed: %s", err.Error()) @@ -291,18 +163,24 @@ func main() { } } + // As a special case for `db`, allow using an environment variable instead of the value + // stored in the config. This can be done for people having security concerns about storing + // the password for their mysql database in the config.json. if strings.HasPrefix(programConfig.DB, "env:") { envvar := strings.TrimPrefix(programConfig.DB, "env:") programConfig.DB = os.Getenv(envvar) } var err error + var db *sqlx.DB if programConfig.DBDriver == "sqlite3" { db, err = sqlx.Open("sqlite3", fmt.Sprintf("%s?_foreign_keys=on", programConfig.DB)) if err != nil { log.Fatal(err) } + // sqlite does not multithread. Having more than one connection open would just mean + // waiting for locks. db.SetMaxOpenConns(1) } else if programConfig.DBDriver == "mysql" { db, err = sqlx.Open("mysql", fmt.Sprintf("%s?multiStatements=true", programConfig.DB)) @@ -317,7 +195,9 @@ func main() { log.Fatalf("unsupported database driver: %s", programConfig.DBDriver) } - // Initialize sub-modules... + // Initialize sub-modules and handle all command line flags. + // The order here is important! For example, the metricdata package + // depends on the config package. var authentication *auth.Authentication if !programConfig.DisableAuthentication { @@ -380,7 +260,7 @@ func main() { } if flagReinitDB { - if err := initDB(db, programConfig.JobArchive); err != nil { + if err := repository.InitDB(db, programConfig.JobArchive); err != nil { log.Fatal(err) } } @@ -400,11 +280,13 @@ func main() { return } - // Build routes... + // Setup the http.Handler/Router used by the server resolver := &graph.Resolver{DB: db, Repo: jobRepo} graphQLEndpoint := handler.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: resolver})) if os.Getenv("DEBUG") != "1" { + // Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed. + // The problem with this is that then, no more stacktrace is printed to stderr. graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error { switch e := err.(type) { case string: @@ -417,7 +299,6 @@ func main() { }) } - graphQLPlayground := playground.Handler("GraphQL playground", "/query") api := &api.RestApi{ JobRepository: jobRepo, Resolver: resolver, @@ -425,33 +306,21 @@ func main() { Authentication: authentication, } - handleGetLogin := func(rw http.ResponseWriter, r *http.Request) { - templates.Render(rw, r, "login.tmpl", &templates.Page{ - Title: "Login", - }) - } - r := mux.NewRouter() - r.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - templates.Render(rw, r, "404.tmpl", &templates.Page{ - Title: "Not found", - }) - }) - r.Handle("/playground", graphQLPlayground) - - r.HandleFunc("/login", handleGetLogin).Methods(http.MethodGet) + r.HandleFunc("/login", func(rw http.ResponseWriter, r *http.Request) { + templates.Render(rw, r, "login.tmpl", &templates.Page{Title: "Login"}) + }).Methods(http.MethodGet) r.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) { - templates.Render(rw, r, "imprint.tmpl", &templates.Page{ - Title: "Imprint", - }) + templates.Render(rw, r, "imprint.tmpl", &templates.Page{Title: "Imprint"}) }) r.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) { - templates.Render(rw, r, "privacy.tmpl", &templates.Page{ - Title: "Privacy", - }) + templates.Render(rw, r, "privacy.tmpl", &templates.Page{Title: "Privacy"}) }) + // Some routes, such as /login or /query, should only be accessible to a user that is logged in. + // Those should be mounted to this subrouter. If authentication is enabled, a middleware will prevent + // any unauthenticated accesses. secured := r.PathPrefix("/").Subrouter() if !programConfig.DisableAuthentication { r.Handle("/login", authentication.Login( @@ -490,8 +359,11 @@ func main() { }) }) } + + r.Handle("/playground", playground.Handler("GraphQL playground", "/query")) secured.Handle("/query", graphQLEndpoint) + // Send a searchId and then reply with a redirect to a user or job. secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) { if search := r.URL.Query().Get("searchId"); search != "" { job, username, err := api.JobRepository.FindJobOrUser(r.Context(), search) @@ -515,6 +387,7 @@ func main() { } }) + // Mount all /monitoring/... and /api/... routes. setupRoutes(secured, routes) api.MountRoutes(secured) @@ -525,11 +398,18 @@ func main() { handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization"}), handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}), handlers.AllowedOrigins([]string{"*"}))) - handler := handlers.CustomLoggingHandler(log.InfoWriter, r, func(w io.Writer, params handlers.LogFormatterParams) { - log.Finfof(w, "%s %s (%d, %.02fkb, %dms)", - params.Request.Method, params.URL.RequestURI(), - params.StatusCode, float32(params.Size)/1024, - time.Since(params.TimeStamp).Milliseconds()) + handler := handlers.CustomLoggingHandler(io.Discard, r, func(_ io.Writer, params handlers.LogFormatterParams) { + if strings.HasPrefix(params.Request.RequestURI, "/api/") { + log.Infof("%s %s (%d, %.02fkb, %dms)", + params.Request.Method, params.URL.RequestURI(), + params.StatusCode, float32(params.Size)/1024, + time.Since(params.TimeStamp).Milliseconds()) + } else { + log.Debugf("%s %s (%d, %.02fkb, %dms)", + params.Request.Method, params.URL.RequestURI(), + params.StatusCode, float32(params.Size)/1024, + time.Since(params.TimeStamp).Milliseconds()) + } }) var wg sync.WaitGroup diff --git a/api_test.go b/test/api_test.go similarity index 95% rename from api_test.go rename to test/api_test.go index 11a2454..b4427f7 100644 --- a/api_test.go +++ b/test/api_test.go @@ -1,4 +1,4 @@ -package main +package test import ( "bytes" @@ -21,13 +21,11 @@ import ( "github.com/ClusterCockpit/cc-backend/schema" "github.com/gorilla/mux" "github.com/jmoiron/sqlx" + + _ "github.com/mattn/go-sqlite3" ) func setup(t *testing.T) *api.RestApi { - if db != nil { - panic("prefer using sub-tests (`t.Run`) or implement `cleanup` before calling setup twice.") - } - const testclusterJson = `{ "name": "testcluster", "subClusters": [ @@ -96,17 +94,17 @@ func setup(t *testing.T) *api.RestApi { } f.Close() - db, err = sqlx.Open("sqlite3", fmt.Sprintf("%s?_foreign_keys=on", dbfilepath)) + db, err := sqlx.Open("sqlite3", fmt.Sprintf("%s?_foreign_keys=on", dbfilepath)) if err != nil { t.Fatal(err) } db.SetMaxOpenConns(1) - if _, err := db.Exec(JOBS_DB_SCHEMA); err != nil { + if _, err := db.Exec(repository.JobsDBSchema); err != nil { t.Fatal(err) } - if err := config.Init(db, false, programConfig.UiDefaults, jobarchive); err != nil { + if err := config.Init(db, false, map[string]interface{}{}, jobarchive); err != nil { t.Fatal(err) } diff --git a/test/db.go b/test/db.go deleted file mode 100644 index 8553ef6..0000000 --- a/test/db.go +++ /dev/null @@ -1,26 +0,0 @@ -package test - -import ( - "fmt" - "os" - - "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" -) - -func InitDB() *sqlx.DB { - - bp := "./" - ebp := os.Getenv("BASEPATH") - - if ebp != "" { - bp = ebp + "test/" - } - - db, err := sqlx.Open("sqlite3", bp+"test.db") - if err != nil { - fmt.Println(err) - } - - return db -} From dc6234088b693cf25d4816ac3a602dc72242eede Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 15 Mar 2022 08:36:45 +0100 Subject: [PATCH 08/26] Fix #5: /api/jwt/ request method --- templates/config.tmpl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/config.tmpl b/templates/config.tmpl index 9e587c2..9e44f38 100644 --- a/templates/config.tmpl +++ b/templates/config.tmpl @@ -79,9 +79,7 @@ listElement.querySelectorAll('button.get-jwt').forEach(e => e.addEventListener('click', event => { let row = event.target.parentElement.parentElement let username = row.children[0].innerText - let formData = new FormData() - formData.append('username', username) - fetch('/api/jwt/', { method: 'POST', body: formData }) + fetch(`/api/jwt/?username=${username}`) .then(res => res.text()) .then(text => alert(text)) })) From 641959101ccc22ff9be030dd37668946f8b2a59b Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 15 Mar 2022 09:49:41 +0100 Subject: [PATCH 09/26] Add metadata to REST-API --- api/openapi.yaml | 3 +++ api/rest.go | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/api/openapi.yaml b/api/openapi.yaml index 6249f04..2babbf5 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -38,6 +38,9 @@ paths: - name: items-per-page in: query schema: { type: integer } + - name: with-metadata + in: query + schema: { type: boolean } responses: 200: description: 'Array of jobs' diff --git a/api/rest.go b/api/rest.go index eb031e6..ee83b6c 100644 --- a/api/rest.go +++ b/api/rest.go @@ -108,6 +108,7 @@ type TagJobApiRequest []*struct { // Return a list of jobs func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { + withMetadata := false filter := &model.JobFilter{} page := &model.PageRequest{ItemsPerPage: -1, Page: 1} order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} @@ -156,6 +157,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { return } page.ItemsPerPage = x + case "with-metadata": + withMetadata = true default: http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest) return @@ -170,6 +173,13 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { results := make([]*schema.JobMeta, 0, len(jobs)) for _, job := range jobs { + if withMetadata { + if _, err := api.JobRepository.FetchMetadata(job); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + } + res := &schema.JobMeta{ ID: &job.ID, BaseJob: job.BaseJob, From f5f1869b5a917b24c2996553567dc903ce677ef6 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 15 Mar 2022 11:04:54 +0100 Subject: [PATCH 10/26] Add user name/email to GraphQL API --- auth/auth.go | 19 ++ frontend | 2 +- graph/generated/generated.go | 408 +++++++++++++++++++++++++++++++---- graph/model/models_gen.go | 6 + graph/schema.graphqls | 12 +- graph/schema.resolvers.go | 12 +- routes.go | 9 +- 7 files changed, 417 insertions(+), 51 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 9ec40c8..fd6df5d 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/ClusterCockpit/cc-backend/graph/model" "github.com/ClusterCockpit/cc-backend/log" sq "github.com/Masterminds/squirrel" "github.com/golang-jwt/jwt/v4" @@ -233,6 +234,24 @@ func (auth *Authentication) FetchUser(username string) (*User, error) { return user, nil } +func FetchUser(ctx context.Context, db *sqlx.DB, username string) (*model.User, error) { + me := GetUser(ctx) + if me != nil && !me.HasRole(RoleAdmin) && me.Username != username { + return nil, errors.New("forbidden") + } + + user := &model.User{Username: username} + if err := sq.Select("name", "email").From("user").Where("user.username = ?", username). + RunWith(db).QueryRow().Scan(&user.Name, &user.Email); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + + return nil, err + } + return user, nil +} + // Handle a POST request that should log the user in, starting a new session. func (auth *Authentication) Login(onsuccess http.Handler, onfailure func(rw http.ResponseWriter, r *http.Request, loginErr error)) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { diff --git a/frontend b/frontend index 7dbabf1..c0d1eac 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 7dbabf140d704b38a1fe29b8248e4226ce0c1b23 +Subproject commit c0d1eac12517fb4ab995324584ea9bc1eee39c94 diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 877ec42..e891d46 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -103,6 +103,7 @@ type ComplexityRoot struct { SubCluster func(childComplexity int) int Tags func(childComplexity int) int User func(childComplexity int) int + UserData func(childComplexity int) int Walltime func(childComplexity int) int } @@ -182,6 +183,7 @@ type ComplexityRoot struct { NodeMetrics func(childComplexity int, cluster string, partition *string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int RooflineHeatmap func(childComplexity int, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) int Tags func(childComplexity int) int + User func(childComplexity int, username string) int } Resource struct { @@ -236,14 +238,22 @@ type ComplexityRoot struct { Node func(childComplexity int) int Socket func(childComplexity int) int } + + User struct { + Email func(childComplexity int) int + Name func(childComplexity int) int + Username func(childComplexity int) int + } } type ClusterResolver interface { Partitions(ctx context.Context, obj *model.Cluster) ([]string, error) } type JobResolver interface { - MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) + + MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) + UserData(ctx context.Context, obj *schema.Job) (*model.User, error) } type MutationResolver interface { CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) @@ -255,6 +265,7 @@ type MutationResolver interface { type QueryResolver interface { Clusters(ctx context.Context) ([]*model.Cluster, error) Tags(ctx context.Context) ([]*schema.Tag, error) + User(ctx context.Context, username string) (*model.User, error) Job(ctx context.Context, id string) (*schema.Job, error) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.MetricFootprints, error) @@ -539,6 +550,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Job.User(childComplexity), true + case "Job.userData": + if e.complexity.Job.UserData == nil { + break + } + + return e.complexity.Job.UserData(childComplexity), true + case "Job.walltime": if e.complexity.Job.Walltime == nil { break @@ -947,6 +965,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.Tags(childComplexity), true + case "Query.user": + if e.complexity.Query.User == nil { + break + } + + args, err := ec.field_Query_user_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.User(childComplexity, args["username"].(string)), true + case "Resource.accelerators": if e.complexity.Resource.Accelerators == nil { break @@ -1171,6 +1201,27 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Topology.Socket(childComplexity), true + case "User.email": + if e.complexity.User.Email == nil { + break + } + + return e.complexity.User.Email(childComplexity), true + + case "User.name": + if e.complexity.User.Name == nil { + break + } + + return e.complexity.User.Name(childComplexity), true + + case "User.username": + if e.complexity.User.Username == nil { + break + } + + return e.complexity.User.Username(childComplexity), true + } return 0, false } @@ -1261,9 +1312,11 @@ type Job { arrayJobId: Int! monitoringStatus: Int! state: JobState! - metaData: Any tags: [Tag!]! resources: [Resource!]! + + metaData: Any + userData: User } type Cluster { @@ -1375,10 +1428,18 @@ type Count { count: Int! } +type User { + username: String! + name: String! + email: String! +} + type Query { clusters: [Cluster!]! # List of all clusters tags: [Tag!]! # List of all tags + user(username: String!): User + job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobMetricWithName!]! jobsFootprints(filter: [JobFilter!], metrics: [String!]!): [MetricFootprints]! @@ -1915,6 +1976,21 @@ func (ec *executionContext) field_Query_rooflineHeatmap_args(ctx context.Context return args, nil } +func (ec *executionContext) field_Query_user_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 string + if tmp, ok := rawArgs["username"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("username")) + arg0, err = ec.unmarshalNString2string(ctx, tmp) + if err != nil { + return nil, err + } + } + args["username"] = arg0 + return args, nil +} + func (ec *executionContext) field___Type_enumValues_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -3178,38 +3254,6 @@ func (ec *executionContext) _Job_state(ctx context.Context, field graphql.Collec return ec.marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐJobState(ctx, field.Selections, res) } -func (ec *executionContext) _Job_metaData(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - fc := &graphql.FieldContext{ - Object: "Job", - Field: field, - Args: nil, - IsMethod: true, - IsResolver: true, - } - - ctx = graphql.WithFieldContext(ctx, fc) - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return ec.resolvers.Job().MetaData(rctx, obj) - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - return graphql.Null - } - res := resTmp.(interface{}) - fc.Result = res - return ec.marshalOAny2interface(ctx, field.Selections, res) -} - func (ec *executionContext) _Job_tags(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -3280,6 +3324,70 @@ func (ec *executionContext) _Job_resources(ctx context.Context, field graphql.Co return ec.marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐResourceᚄ(ctx, field.Selections, res) } +func (ec *executionContext) _Job_metaData(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Job", + Field: field, + Args: nil, + IsMethod: true, + IsResolver: true, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Job().MetaData(rctx, obj) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(interface{}) + fc.Result = res + return ec.marshalOAny2interface(ctx, field.Selections, res) +} + +func (ec *executionContext) _Job_userData(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Job", + Field: field, + Args: nil, + IsMethod: true, + IsResolver: true, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Job().UserData(rctx, obj) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*model.User) + fc.Result = res + return ec.marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐUser(ctx, field.Selections, res) +} + func (ec *executionContext) _JobMetric_unit(ctx context.Context, field graphql.CollectedField, obj *schema.JobMetric) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -4697,6 +4805,45 @@ func (ec *executionContext) _Query_tags(ctx context.Context, field graphql.Colle return ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐTagᚄ(ctx, field.Selections, res) } +func (ec *executionContext) _Query_user(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Query", + Field: field, + Args: nil, + IsMethod: true, + IsResolver: true, + } + + ctx = graphql.WithFieldContext(ctx, fc) + rawArgs := field.ArgumentMap(ec.Variables) + args, err := ec.field_Query_user_args(ctx, rawArgs) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + fc.Args = args + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().User(rctx, args["username"].(string)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*model.User) + fc.Result = res + return ec.marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐUser(ctx, field.Selections, res) +} + func (ec *executionContext) _Query_job(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -6188,6 +6335,111 @@ func (ec *executionContext) _Topology_accelerators(ctx context.Context, field gr return ec.marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐAcceleratorᚄ(ctx, field.Selections, res) } +func (ec *executionContext) _User_username(ctx context.Context, field graphql.CollectedField, obj *model.User) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "User", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Username, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) _User_name(ctx context.Context, field graphql.CollectedField, obj *model.User) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "User", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) _User_email(ctx context.Context, field graphql.CollectedField, obj *model.User) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "User", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Email, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + func (ec *executionContext) ___Directive_name(ctx context.Context, field graphql.CollectedField, obj *introspection.Directive) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -7954,17 +8206,6 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj if out.Values[i] == graphql.Null { atomic.AddUint32(&invalids, 1) } - case "metaData": - field := field - out.Concurrently(i, func() (res graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - } - }() - res = ec._Job_metaData(ctx, field, obj) - return res - }) case "tags": field := field out.Concurrently(i, func() (res graphql.Marshaler) { @@ -7984,6 +8225,28 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj if out.Values[i] == graphql.Null { atomic.AddUint32(&invalids, 1) } + case "metaData": + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Job_metaData(ctx, field, obj) + return res + }) + case "userData": + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Job_userData(ctx, field, obj) + return res + }) default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -8412,6 +8675,17 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } return res }) + case "user": + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_user(ctx, field) + return res + }) case "job": field := field out.Concurrently(i, func() (res graphql.Marshaler) { @@ -8817,6 +9091,43 @@ func (ec *executionContext) _Topology(ctx context.Context, sel ast.SelectionSet, return out } +var userImplementors = []string{"User"} + +func (ec *executionContext) _User(ctx context.Context, sel ast.SelectionSet, obj *model.User) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, userImplementors) + + out := graphql.NewFieldSet(fields) + var invalids uint32 + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("User") + case "username": + out.Values[i] = ec._User_username(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "name": + out.Values[i] = ec._User_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "email": + out.Values[i] = ec._User_email(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch() + if invalids > 0 { + return graphql.Null + } + return out +} + var __DirectiveImplementors = []string{"__Directive"} func (ec *executionContext) ___Directive(ctx context.Context, sel ast.SelectionSet, obj *introspection.Directive) graphql.Marshaler { @@ -10852,6 +11163,13 @@ func (ec *executionContext) unmarshalOTimeRange2ᚖgithubᚗcomᚋClusterCockpit return &res, graphql.ErrorOnPath(ctx, err) } +func (ec *executionContext) marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐUser(ctx context.Context, sel ast.SelectionSet, v *model.User) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return ec._User(ctx, sel, v) +} + func (ec *executionContext) marshalO__EnumValue2ᚕgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐEnumValueᚄ(ctx context.Context, sel ast.SelectionSet, v []introspection.EnumValue) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/graph/model/models_gen.go b/graph/model/models_gen.go index 95f58b0..c94b908 100644 --- a/graph/model/models_gen.go +++ b/graph/model/models_gen.go @@ -161,6 +161,12 @@ type Topology struct { Accelerators []*Accelerator `json:"accelerators"` } +type User struct { + Username string `json:"username"` + Name string `json:"name"` + Email string `json:"email"` +} + type Aggregate string const ( diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 9c0e341..7f7f60a 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -24,9 +24,11 @@ type Job { arrayJobId: Int! monitoringStatus: Int! state: JobState! - metaData: Any tags: [Tag!]! resources: [Resource!]! + + metaData: Any + userData: User } type Cluster { @@ -138,10 +140,18 @@ type Count { count: Int! } +type User { + username: String! + name: String! + email: String! +} + type Query { clusters: [Cluster!]! # List of all clusters tags: [Tag!]! # List of all tags + user(username: String!): User + job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobMetricWithName!]! jobsFootprints(filter: [JobFilter!], metrics: [String!]!): [MetricFootprints]! diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go index fdd1937..c878950 100644 --- a/graph/schema.resolvers.go +++ b/graph/schema.resolvers.go @@ -22,12 +22,16 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *model.Cluster) ([ return r.Repo.Partitions(obj.Name) } +func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) { + return r.Repo.GetTags(&obj.ID) +} + func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) { return r.Repo.FetchMetadata(obj) } -func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) { - return r.Repo.GetTags(&obj.ID) +func (r *jobResolver) UserData(ctx context.Context, obj *schema.Job) (*model.User, error) { + return auth.FetchUser(ctx, r.DB, obj.User) } func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) { @@ -102,6 +106,10 @@ func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) { return r.Repo.GetTags(nil) } +func (r *queryResolver) User(ctx context.Context, username string) (*model.User, error) { + return auth.FetchUser(ctx, r.DB, username) +} + func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) { numericId, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/routes.go b/routes.go index 243a4e7..81669d4 100644 --- a/routes.go +++ b/routes.go @@ -92,8 +92,13 @@ func setupJobRoute(i InfoType, r *http.Request) InfoType { } func setupUserRoute(i InfoType, r *http.Request) InfoType { - i["id"] = mux.Vars(r)["id"] - i["username"] = mux.Vars(r)["id"] + username := mux.Vars(r)["id"] + i["id"] = username + i["username"] = username + if user, _ := auth.FetchUser(r.Context(), jobRepo.DB, username); user != nil { + i["name"] = user.Name + i["email"] = user.Email + } return i } From f8660fe323972653b78e7d7830c7fc88b15ece24 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 15 Mar 2022 17:31:14 +0100 Subject: [PATCH 11/26] Update README.md --- README.md | 65 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a193f3a..0bdd333 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,25 @@ -# ClusterCockpit with a Golang backend +# ClusterCockpit REST and GraphQL API backend [![Build](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml) -Create your job-archive accoring to [this specification](https://github.com/ClusterCockpit/cc-specifications). At least one cluster with a valid `cluster.json` file is required. Having no jobs in the job-archive at all is fine. You may use the sample job-archive available for download [in cc-docker/develop](https://github.com/ClusterCockpit/cc-docker/tree/develop). +This is a Golang backend implementation for a REST and GraphQL API according to the [ClusterCockpit specifications](https://github.com/ClusterCockpit/cc-specifications). +It also includes a web interface for ClusterCockpit based on the components implemented in +[cc-frontend](https://github.com/ClusterCockpit/cc-frontend), which is included as a git submodule. +This implementation replaces the previous PHP Symfony based ClusterCockpit web-interface. -### Run server +## Overview + +This is a golang web backend for the ClusterCockpit job-specific performance monitoring framework. +It provides a REST API for integrating ClusterCockpit with a HPC cluster batch system and external analysis scripts. +Data exchange between the web frontend and backend is based on a GraphQL API. +The web frontend is also served by the backend using [Svelte](https://svelte.dev/) components implemented in [cc-frontend](https://github.com/ClusterCockpit/cc-frontend). +Layout and styling is based on [Bootstrap 5](https://getbootstrap.com/) using [Bootstrap Icons](https://icons.getbootstrap.com/). +The backend uses [SQLite 3](https://sqlite.org/) as relational SQL database by default. It can optionally use a MySQL/MariaDB database server. +Finished batch jobs are stored in a so called job archive following [this specification](https://github.com/ClusterCockpit/cc-backend/wiki). +The backend supports authentication using local accounts or an external LDAP directory. +Authorization for APIs is implemented using [JWT](https://jwt.io/) tokens created with public/private key encryption. + +## Howto Build and Run ```sh # The frontend is a submodule, so use `--recursive` @@ -41,22 +56,39 @@ vim ./.env # Show other options: ./cc-backend --help ``` +### Run as systemd daemon -In order to run this program as a deamon, look at [utils/systemd/README.md](./utils/systemd/README.md) where a systemd unit file and more explanation is provided. +In order to run this program as a daemon, look at [utils/systemd/README.md](./utils/systemd/README.md) where a systemd unit file and more explanation is provided. + +## Configuration and Setup + +cc-backend can be used as a local web-interface for an existing job archive or +as a general web-interface server for a live ClusterCockpit Monitoring +framework. + +Create your job-archive according to [this specification](https://github.com/ClusterCockpit/cc-specifications). At least +one cluster with a valid `cluster.json` file is required. Having no jobs in the +job-archive at all is fine. You may use the sample job-archive available for +download [in cc-docker/develop](https://github.com/ClusterCockpit/cc-docker/tree/develop). ### Configuration -A config file in the JSON format can be provided using `--config` to override the defaults. Look at the beginning of `server.go` for the defaults and consequently the format of the configuration file. +A config file in the JSON format can be provided using `--config` to override the defaults. +Look at the beginning of `server.go` for the defaults and consequently the format of the configuration file. ### Update GraphQL schema -This project uses [gqlgen](https://github.com/99designs/gqlgen) for the GraphQL API. The schema can be found in `./graph/schema.graphqls`. After changing it, you need to run `go run github.com/99designs/gqlgen` which will update `graph/model`. In case new resolvers are needed, they will be inserted into `graph/schema.resolvers.go`, where you will need to implement them. +This project uses [gqlgen](https://github.com/99designs/gqlgen) for the GraphQL +API. The schema can be found in `./graph/schema.graphqls`. After changing it, +you need to run `go run github.com/99designs/gqlgen` which will update +`graph/model`. In case new resolvers are needed, they will be inserted into +`graph/schema.resolvers.go`, where you will need to implement them. -### Project Structure +## Project Structure - `api/` contains the REST API. The routes defined there should be called whenever a job starts/stops. The API is documented in the OpenAPI 3.0 format in [./api/openapi.yaml](./api/openapi.yaml). - `auth/` is where the (optional) authentication middleware can be found, which adds the currently authenticated user to the request context. The `user` table is created and managed here as well. - - `auth/ldap.go` contains everything to do with automatically syncing and authenticating users form an LDAP server. + - `auth/ldap.go` contains everything to do with automatically syncing and authenticating users form an LDAP server. - `config` handles the `cluster.json` files and the user-specific configurations (changeable via GraphQL) for the Web-UI such as the selected metrics etc. - `frontend` is a submodule, this is where the Svelte based frontend resides. - `graph/generated` should *not* be touched. @@ -68,24 +100,15 @@ This project uses [gqlgen](https://github.com/99designs/gqlgen) for the GraphQL - `metricdata/archive.go` provides functions for fetching metrics from the job-archive and archiving a job to the job-archive. - `metricdata/cc-metric-store.go` contains an implementation of the `MetricDataRepository` interface which can fetch data from an [cc-metric-store](https://github.com/ClusterCockpit/cc-metric-store) - `metricdata/influxdb-v2` contains an implementation of the `MetricDataRepository` interface which can fetch data from an InfluxDBv2 database. It is currently disabled and out of date and can not be used as of writing. +- `repository/` all SQL related stuff. +- `repository/init.go` initializes the `job` (and `tag` and `jobtag`) table if the `--init-db` flag is provided. Not only is the table created in the correct schema, but the job-archive is traversed as well. - `schema/` contains type definitions used all over this project extracted in this package as Go disallows cyclic dependencies between packages. - `schema/float.go` contains a custom `float64` type which overwrites JSON and GraphQL Marshaling/Unmarshalling. This is needed because a regular optional `Float` in GraphQL will map to `*float64` types in Go. Wrapping every single metric value in an allocation would be a lot of overhead. - `schema/job.go` provides the types representing a job and its resources. Those can be used as type for a `meta.json` file and/or a row in the `job` table. - `templates/` is mostly full of HTML templates and a small helper go module. - `utils/systemd` describes how to deploy/install this as a systemd service -- `utils/` is mostly outdated. Look at the [cc-util repo](https://github.com/ClusterCockpit/cc-util) for more up-to-date scripts. +- `test/` rudimentery tests. +- `utils/` - `.env` *must* be changed before you deploy this. It contains a Base64 encoded [Ed25519](https://en.wikipedia.org/wiki/EdDSA) key-pair, the secret used for sessions and the password to the LDAP server if LDAP authentication is enabled. - `gqlgen.yml` configures the behaviour and generation of [gqlgen](https://github.com/99designs/gqlgen). -- `init-db.go` initializes the `job` (and `tag` and `jobtag`) table if the `--init-db` flag is provided. Not only is the table created in the correct schema, but the job-archive is traversed as well. - `server.go` contains the main function and starts the actual http server. - -### TODO - -- [ ] write (unit) tests -- [ ] fix `LoadNodeData` in cc-metric-store MetricDataRepository. Currently does not work for non-node scoped metrics because partition is unkown for a node -- [ ] make tokens and sessions (currently based on cookies) expire after some configurable time -- [ ] when authenticating using a JWT, check if that user still exists -- [ ] fix InfluxDB MetricDataRepository (new or old line-protocol format? Support node-level metrics only?) -- [ ] documentation, comments in the code base -- [ ] write more TODOs -- [ ] use more prepared statements and [sqrl](https://github.com/elgris/sqrl) instead of *squirrel* From 51b9a40a33a46f308b0dcfeffc6a678e9c4a9f16 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 16 Mar 2022 10:09:00 +0100 Subject: [PATCH 12/26] Increase line width in default config --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index de89430..197ed6e 100644 --- a/server.go +++ b/server.go @@ -112,7 +112,7 @@ var programConfig ProgramConfig = ProgramConfig{ "job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "plot_general_colorBackground": true, "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, - "plot_general_lineWidth": 1, + "plot_general_lineWidth": 3, "plot_list_hideShortRunningJobs": 5 * 60, "plot_list_jobsPerPage": 10, "plot_list_selectedMetrics": []string{"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"}, From 95cf4ba053a32550ea77db90a9fdc00885fcd6c3 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 16 Mar 2022 10:09:28 +0100 Subject: [PATCH 13/26] Add shell skript for easy Demo setup --- startDemo.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 startDemo.sh diff --git a/startDemo.sh b/startDemo.sh new file mode 100755 index 0000000..aa82bc7 --- /dev/null +++ b/startDemo.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +mkdir ./var +cd ./var + +wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive.tar.xz +tar xJf job-archive.tar.xz +rm ./job-archive.tar.xz + +touch ./job.db +cd ../frontend +yarn install +yarn build + +cd .. +go get +go build + +./cc-backend --init-db --add-user demo:admin:AdminDev --no-server +./cc-backend From 465b0fa21b5bc607db760d715e28af092631c0f1 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 16 Mar 2022 10:14:52 +0100 Subject: [PATCH 14/26] Update README.md --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 0bdd333..223304c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,19 @@ Finished batch jobs are stored in a so called job archive following [this specif The backend supports authentication using local accounts or an external LDAP directory. Authorization for APIs is implemented using [JWT](https://jwt.io/) tokens created with public/private key encryption. +## Demo Setup + +We provide a shell skript that downloads demo data and automatically builds and starts cc-backend. +You need `wget`, `go`, and `yarn` in your path to start the demo. The demo will download 32MB of data (223MB on disk). + +```sh +# The frontend is a submodule, so use `--recursive` +git clone --recursive git@github.com:ClusterCockpit/cc-backend.git + +./startDemo.sh +``` +You can access the web interface at http://localhost:8080. Please note that some views do not work without a metric backend (Systems view). + ## Howto Build and Run ```sh From 80978319e2b6029dcdf660bdf3226685e8efe1b2 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 16 Mar 2022 10:15:46 +0100 Subject: [PATCH 15/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 223304c..df65b30 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ git clone --recursive git@github.com:ClusterCockpit/cc-backend.git ./startDemo.sh ``` -You can access the web interface at http://localhost:8080. Please note that some views do not work without a metric backend (Systems view). +You can access the web interface at http://localhost:8080. Credentials for login: `demo:AdminDev`. Please note that some views do not work without a metric backend (Systems view). ## Howto Build and Run From 1b66030525393b012aefc8aecec974dacc20056b Mon Sep 17 00:00:00 2001 From: Christoph Kluge <83573843+spacehamster87@users.noreply.github.com> Date: Wed, 16 Mar 2022 12:06:56 +0100 Subject: [PATCH 16/26] Update test.yml go-version to 1.17.x for influxdb-client --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b0590e..e1f1b7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.16.x + go-version: 1.17.x - name: Checkout code uses: actions/checkout@v2 - name: Build, Vet & Test From 5235feb1d6a07c6ef8eec4a0e246cda53d91a489 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Wed, 16 Mar 2022 16:11:28 +0100 Subject: [PATCH 17/26] GraphQL: Add nodehours info to footprints query --- frontend | 2 +- graph/generated/generated.go | 204 ++++++++++++++++++++++++++++------- graph/model/models_gen.go | 9 +- graph/schema.graphqls | 11 +- graph/schema.resolvers.go | 2 +- graph/stats.go | 14 ++- 6 files changed, 194 insertions(+), 48 deletions(-) diff --git a/frontend b/frontend index c0d1eac..1000e51 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit c0d1eac12517fb4ab995324584ea9bc1eee39c94 +Subproject commit 1000e51693662be6c7c4faf19302f76279c6e0c1 diff --git a/graph/generated/generated.go b/graph/generated/generated.go index e891d46..03a6a75 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -72,6 +72,11 @@ type ComplexityRoot struct { StartTime func(childComplexity int) int } + Footprints struct { + Metrics func(childComplexity int) int + Nodehours func(childComplexity int) int + } + HistoPoint struct { Count func(childComplexity int) int Value func(childComplexity int) int @@ -149,8 +154,8 @@ type ComplexityRoot struct { } MetricFootprints struct { - Footprints func(childComplexity int) int - Name func(childComplexity int) int + Data func(childComplexity int) int + Metric func(childComplexity int) int } MetricStatistics struct { @@ -268,7 +273,7 @@ type QueryResolver interface { User(ctx context.Context, username string) (*model.User, error) Job(ctx context.Context, id string) (*schema.Job, error) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) - JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.MetricFootprints, error) + JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) JobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) JobsCount(ctx context.Context, filter []*model.JobFilter, groupBy model.Aggregate, limit *int) ([]*model.Count, error) @@ -382,6 +387,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.FilterRanges.StartTime(childComplexity), true + case "Footprints.metrics": + if e.complexity.Footprints.Metrics == nil { + break + } + + return e.complexity.Footprints.Metrics(childComplexity), true + + case "Footprints.nodehours": + if e.complexity.Footprints.Nodehours == nil { + break + } + + return e.complexity.Footprints.Nodehours(childComplexity), true + case "HistoPoint.count": if e.complexity.HistoPoint.Count == nil { break @@ -746,19 +765,19 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.MetricConfig.Unit(childComplexity), true - case "MetricFootprints.footprints": - if e.complexity.MetricFootprints.Footprints == nil { + case "MetricFootprints.data": + if e.complexity.MetricFootprints.Data == nil { break } - return e.complexity.MetricFootprints.Footprints(childComplexity), true + return e.complexity.MetricFootprints.Data(childComplexity), true - case "MetricFootprints.name": - if e.complexity.MetricFootprints.Name == nil { + case "MetricFootprints.metric": + if e.complexity.MetricFootprints.Metric == nil { break } - return e.complexity.MetricFootprints.Name(childComplexity), true + return e.complexity.MetricFootprints.Metric(childComplexity), true case "MetricStatistics.avg": if e.complexity.MetricStatistics.Avg == nil { @@ -1412,8 +1431,13 @@ type StatsSeries { } type MetricFootprints { - name: String! - footprints: [NullableFloat!]! + metric: String! + data: [NullableFloat!]! +} + +type Footprints { + nodehours: [NullableFloat!]! + metrics: [MetricFootprints!]! } enum Aggregate { USER, PROJECT, CLUSTER } @@ -1442,7 +1466,7 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobMetricWithName!]! - jobsFootprints(filter: [JobFilter!], metrics: [String!]!): [MetricFootprints]! + jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! @@ -2484,6 +2508,76 @@ func (ec *executionContext) _FilterRanges_startTime(ctx context.Context, field g return ec.marshalNTimeRangeOutput2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐTimeRangeOutput(ctx, field.Selections, res) } +func (ec *executionContext) _Footprints_nodehours(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Footprints", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Nodehours, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) _Footprints_metrics(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Footprints", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Metrics, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.MetricFootprints) + fc.Result = res + return ec.marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprintsᚄ(ctx, field.Selections, res) +} + func (ec *executionContext) _HistoPoint_count(ctx context.Context, field graphql.CollectedField, obj *model.HistoPoint) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -4283,7 +4377,7 @@ func (ec *executionContext) _MetricConfig_alert(ctx context.Context, field graph return ec.marshalNFloat2float64(ctx, field.Selections, res) } -func (ec *executionContext) _MetricFootprints_name(ctx context.Context, field graphql.CollectedField, obj *model.MetricFootprints) (ret graphql.Marshaler) { +func (ec *executionContext) _MetricFootprints_metric(ctx context.Context, field graphql.CollectedField, obj *model.MetricFootprints) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -4301,7 +4395,7 @@ func (ec *executionContext) _MetricFootprints_name(ctx context.Context, field gr ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Name, nil + return obj.Metric, nil }) if err != nil { ec.Error(ctx, err) @@ -4318,7 +4412,7 @@ func (ec *executionContext) _MetricFootprints_name(ctx context.Context, field gr return ec.marshalNString2string(ctx, field.Selections, res) } -func (ec *executionContext) _MetricFootprints_footprints(ctx context.Context, field graphql.CollectedField, obj *model.MetricFootprints) (ret graphql.Marshaler) { +func (ec *executionContext) _MetricFootprints_data(ctx context.Context, field graphql.CollectedField, obj *model.MetricFootprints) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -4336,7 +4430,7 @@ func (ec *executionContext) _MetricFootprints_footprints(ctx context.Context, fi ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Footprints, nil + return obj.Data, nil }) if err != nil { ec.Error(ctx, err) @@ -4957,14 +5051,11 @@ func (ec *executionContext) _Query_jobsFootprints(ctx context.Context, field gra return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.([]*model.MetricFootprints) + res := resTmp.(*model.Footprints) fc.Result = res - return ec.marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx, field.Selections, res) + return ec.marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐFootprints(ctx, field.Selections, res) } func (ec *executionContext) _Query_jobs(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { @@ -8041,6 +8132,38 @@ func (ec *executionContext) _FilterRanges(ctx context.Context, sel ast.Selection return out } +var footprintsImplementors = []string{"Footprints"} + +func (ec *executionContext) _Footprints(ctx context.Context, sel ast.SelectionSet, obj *model.Footprints) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, footprintsImplementors) + + out := graphql.NewFieldSet(fields) + var invalids uint32 + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("Footprints") + case "nodehours": + out.Values[i] = ec._Footprints_nodehours(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + case "metrics": + out.Values[i] = ec._Footprints_metrics(ctx, field, obj) + if out.Values[i] == graphql.Null { + invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch() + if invalids > 0 { + return graphql.Null + } + return out +} + var histoPointImplementors = []string{"HistoPoint"} func (ec *executionContext) _HistoPoint(ctx context.Context, sel ast.SelectionSet, obj *model.HistoPoint) graphql.Marshaler { @@ -8494,13 +8617,13 @@ func (ec *executionContext) _MetricFootprints(ctx context.Context, sel ast.Selec switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("MetricFootprints") - case "name": - out.Values[i] = ec._MetricFootprints_name(ctx, field, obj) + case "metric": + out.Values[i] = ec._MetricFootprints_metric(ctx, field, obj) if out.Values[i] == graphql.Null { invalids++ } - case "footprints": - out.Values[i] = ec._MetricFootprints_footprints(ctx, field, obj) + case "data": + out.Values[i] = ec._MetricFootprints_data(ctx, field, obj) if out.Values[i] == graphql.Null { invalids++ } @@ -8720,9 +8843,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } }() res = ec._Query_jobsFootprints(ctx, field) - if res == graphql.Null { - atomic.AddUint32(&invalids, 1) - } return res }) case "jobs": @@ -10048,7 +10168,7 @@ func (ec *executionContext) marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpi return ec._MetricConfig(ctx, sel, v) } -func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx context.Context, sel ast.SelectionSet, v []*model.MetricFootprints) graphql.Marshaler { +func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprintsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.MetricFootprints) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -10072,7 +10192,7 @@ func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋCluste if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalOMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx, sel, v[i]) + ret[i] = ec.marshalNMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx, sel, v[i]) } if isLen1 { f(i) @@ -10085,6 +10205,16 @@ func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋCluste return ret } +func (ec *executionContext) marshalNMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx context.Context, sel ast.SelectionSet, v *model.MetricFootprints) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + return ec._MetricFootprints(ctx, sel, v) +} + func (ec *executionContext) unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐMetricScope(ctx context.Context, v interface{}) (schema.MetricScope, error) { var res schema.MetricScope err := res.UnmarshalGQL(v) @@ -10753,6 +10883,13 @@ func (ec *executionContext) unmarshalOFloatRange2ᚖgithubᚗcomᚋClusterCockpi return &res, graphql.ErrorOnPath(ctx, err) } +func (ec *executionContext) marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐFootprints(ctx context.Context, sel ast.SelectionSet, v *model.Footprints) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return ec._Footprints(ctx, sel, v) +} + func (ec *executionContext) unmarshalOID2ᚕstringᚄ(ctx context.Context, v interface{}) ([]string, error) { if v == nil { return nil, nil @@ -10959,13 +11096,6 @@ func (ec *executionContext) marshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋ return ret } -func (ec *executionContext) marshalOMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋgraphᚋmodelᚐMetricFootprints(ctx context.Context, sel ast.SelectionSet, v *model.MetricFootprints) graphql.Marshaler { - if v == nil { - return graphql.Null - } - return ec._MetricFootprints(ctx, sel, v) -} - func (ec *executionContext) unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐMetricScopeᚄ(ctx context.Context, v interface{}) ([]schema.MetricScope, error) { if v == nil { return nil, nil diff --git a/graph/model/models_gen.go b/graph/model/models_gen.go index c94b908..7022d80 100644 --- a/graph/model/models_gen.go +++ b/graph/model/models_gen.go @@ -33,6 +33,11 @@ type FloatRange struct { To float64 `json:"to"` } +type Footprints struct { + Nodehours []schema.Float `json:"nodehours"` + Metrics []*MetricFootprints `json:"metrics"` +} + type HistoPoint struct { Count int `json:"count"` Value int `json:"value"` @@ -103,8 +108,8 @@ type MetricConfig struct { } type MetricFootprints struct { - Name string `json:"name"` - Footprints []schema.Float `json:"footprints"` + Metric string `json:"metric"` + Data []schema.Float `json:"data"` } type NodeMetrics struct { diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 7f7f60a..b3fbe29 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -124,8 +124,13 @@ type StatsSeries { } type MetricFootprints { - name: String! - footprints: [NullableFloat!]! + metric: String! + data: [NullableFloat!]! +} + +type Footprints { + nodehours: [NullableFloat!]! + metrics: [MetricFootprints!]! } enum Aggregate { USER, PROJECT, CLUSTER } @@ -154,7 +159,7 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobMetricWithName!]! - jobsFootprints(filter: [JobFilter!], metrics: [String!]!): [MetricFootprints]! + jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go index c878950..58fd99d 100644 --- a/graph/schema.resolvers.go +++ b/graph/schema.resolvers.go @@ -156,7 +156,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str return res, err } -func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.MetricFootprints, error) { +func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { return r.jobsFootprints(ctx, filter, metrics) } diff --git a/graph/stats.go b/graph/stats.go index fb24bab..487d420 100644 --- a/graph/stats.go +++ b/graph/stats.go @@ -254,7 +254,7 @@ func (r *Resolver) rooflineHeatmap(ctx context.Context, filter []*model.JobFilte } // Helper function for the jobsFootprints GraphQL query placed here so that schema.resolvers.go is not too full. -func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.MetricFootprints, error) { +func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil) if err != nil { return nil, err @@ -268,19 +268,25 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF avgs[i] = make([]schema.Float, 0, len(jobs)) } + nodehours := make([]schema.Float, 0, len(jobs)) for _, job := range jobs { if err := metricdata.LoadAverages(job, metrics, avgs, ctx); err != nil { return nil, err } + + nodehours = append(nodehours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes))) } res := make([]*model.MetricFootprints, len(avgs)) for i, arr := range avgs { res[i] = &model.MetricFootprints{ - Name: metrics[i], - Footprints: arr, + Metric: metrics[i], + Data: arr, } } - return res, nil + return &model.Footprints{ + Nodehours: nodehours, + Metrics: res, + }, nil } From 33f6792fbf47d7fdbd5c60ec39965a743611d780 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Thu, 17 Mar 2022 10:54:17 +0100 Subject: [PATCH 18/26] LDAP users have no email address --- auth/auth.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index fd6df5d..b49bddb 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -72,11 +72,11 @@ func (auth *Authentication) Init(db *sqlx.DB, ldapConfig *LdapConfig) error { auth.db = db _, err := db.Exec(` CREATE TABLE IF NOT EXISTS user ( - username varchar(255) PRIMARY KEY, + username varchar(255) PRIMARY KEY NOT NULL, password varchar(255) DEFAULT NULL, - ldap tinyint DEFAULT 0, + ldap tinyint NOT NULL DEFAULT 0, name varchar(255) DEFAULT NULL, - roles varchar(255) DEFAULT NULL, + roles varchar(255) NOT NULL DEFAULT "[]", email varchar(255) DEFAULT NULL);`) if err != nil { return err @@ -241,14 +241,18 @@ func FetchUser(ctx context.Context, db *sqlx.DB, username string) (*model.User, } user := &model.User{Username: username} + var name, email sql.NullString if err := sq.Select("name", "email").From("user").Where("user.username = ?", username). - RunWith(db).QueryRow().Scan(&user.Name, &user.Email); err != nil { + RunWith(db).QueryRow().Scan(&name, &email); err != nil { if err == sql.ErrNoRows { return nil, nil } return nil, err } + + user.Name = name.String + user.Email = email.String return user, nil } From 728e119a0b002ba7e851dfff27819833ba8c371f Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Thu, 17 Mar 2022 11:18:22 +0100 Subject: [PATCH 19/26] Update frontend; Cache MetaData --- frontend | 2 +- repository/job.go | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/frontend b/frontend index 1000e51..184a507 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 1000e51693662be6c7c4faf19302f76279c6e0c1 +Subproject commit 184a5079bd53a89442664c28c5cebb733557f1f1 diff --git a/repository/job.go b/repository/job.go index 567b512..f80672a 100644 --- a/repository/job.go +++ b/repository/job.go @@ -26,7 +26,7 @@ type JobRepository struct { func (r *JobRepository) Init() error { r.stmtCache = sq.NewStmtCache(r.DB) - r.cache = lrucache.New(100) + r.cache = lrucache.New(1024 * 1024) return nil } @@ -59,6 +59,12 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { } func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) { + cachekey := fmt.Sprintf("metadata:%d", job.ID) + if cached := r.cache.Get(cachekey, nil); cached != nil { + job.MetaData = cached.(map[string]string) + return job.MetaData, nil + } + if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil { return nil, err @@ -72,9 +78,42 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error return nil, err } + r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour) return job.MetaData, nil } +func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err error) { + cachekey := fmt.Sprintf("metadata:%d", job.ID) + r.cache.Del(cachekey) + if job.MetaData == nil { + if _, err = r.FetchMetadata(job); err != nil { + return err + } + } + + if job.MetaData != nil { + cpy := make(map[string]string, len(job.MetaData)+1) + for k, v := range job.MetaData { + cpy[k] = v + } + cpy[key] = val + job.MetaData = cpy + } else { + job.MetaData = map[string]string{key: val} + } + + if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil { + return err + } + + if _, err = sq.Update("job").Set("meta_data", job.RawMetaData).Where("job.id = ?", job.ID).RunWith(r.stmtCache).Exec(); err != nil { + return err + } + + r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour) + return nil +} + // Find executes a SQL query to find a specific batch job. // The job is queried using the batch job id, the cluster name, // and the start time of the job in UNIX epoch time seconds. From 94eb30cfcf9859d4ed89874c85a3968ef7ff8a13 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 21 Mar 2022 09:46:41 +0100 Subject: [PATCH 20/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index df65b30..bf66c42 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Data exchange between the web frontend and backend is based on a GraphQL API. The web frontend is also served by the backend using [Svelte](https://svelte.dev/) components implemented in [cc-frontend](https://github.com/ClusterCockpit/cc-frontend). Layout and styling is based on [Bootstrap 5](https://getbootstrap.com/) using [Bootstrap Icons](https://icons.getbootstrap.com/). The backend uses [SQLite 3](https://sqlite.org/) as relational SQL database by default. It can optionally use a MySQL/MariaDB database server. -Finished batch jobs are stored in a so called job archive following [this specification](https://github.com/ClusterCockpit/cc-backend/wiki). +Finished batch jobs are stored in a so called job archive following [this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). The backend supports authentication using local accounts or an external LDAP directory. Authorization for APIs is implemented using [JWT](https://jwt.io/) tokens created with public/private key encryption. From 8b69146a10d5443b589c07f53f4a983613d57fd8 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 21 Mar 2022 09:47:22 +0100 Subject: [PATCH 21/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bf66c42..75b5f26 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ git clone --recursive git@github.com:ClusterCockpit/cc-backend.git ./startDemo.sh ``` -You can access the web interface at http://localhost:8080. Credentials for login: `demo:AdminDev`. Please note that some views do not work without a metric backend (Systems view). +You can access the web interface at http://localhost:8080. Credentials for login: `demo:AdminDev`. Please note that some views do not work without a metric backend (e.g., the Systems view). ## Howto Build and Run From 0206303d982e0b6ed209a940674ce43123e968d1 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 21 Mar 2022 11:28:59 +0100 Subject: [PATCH 22/26] Reject jobs if start time is within 24h --- api/rest.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/api/rest.go b/api/rest.go index ee83b6c..240f41b 100644 --- a/api/rest.go +++ b/api/rest.go @@ -284,15 +284,18 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { } // Check if combination of (job_id, cluster_id, start_time) already exists: - job, err := api.JobRepository.Find(&req.JobID, &req.Cluster, &req.StartTime) - if err != nil && err != sql.ErrNoRows { + job, err := api.JobRepository.Find(&req.JobID, &req.Cluster, nil) + + if err != nil { handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) return } if err != sql.ErrNoRows { - handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) - return + if (req.StartTime - job.StartTimeUnix) < 86400 { + handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) + return + } } id, err := api.JobRepository.Start(&req) From 220758dae1f6936d7c986b2868a661869a38f37d Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 21 Mar 2022 13:04:57 +0100 Subject: [PATCH 23/26] Fix /api/jobs/start_job duplicate check --- api/rest.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/rest.go b/api/rest.go index 240f41b..2478d53 100644 --- a/api/rest.go +++ b/api/rest.go @@ -285,13 +285,10 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // Check if combination of (job_id, cluster_id, start_time) already exists: job, err := api.JobRepository.Find(&req.JobID, &req.Cluster, nil) - - if err != nil { + if err != nil && err != sql.ErrNoRows { handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) return - } - - if err != sql.ErrNoRows { + } else if err == nil { if (req.StartTime - job.StartTimeUnix) < 86400 { handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) return From 8ebf00d980c105ae312cbb4d41ef3710de005634 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 21 Mar 2022 13:28:21 +0100 Subject: [PATCH 24/26] Add aggregation to GraphQL API; Update frontend --- frontend | 2 +- graph/generated/generated.go | 75 ++++++++++++++++++++++++++++-------- graph/model/models_gen.go | 17 ++++---- graph/schema.graphqls | 17 ++++---- 4 files changed, 78 insertions(+), 33 deletions(-) diff --git a/frontend b/frontend index 184a507..03818be 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 184a5079bd53a89442664c28c5cebb733557f1f1 +Subproject commit 03818be47032194f1f450ab242eed56a94a3c5d1 diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 03a6a75..cc179cc 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -143,14 +143,15 @@ type ComplexityRoot struct { } MetricConfig struct { - Alert func(childComplexity int) int - Caution func(childComplexity int) int - Name func(childComplexity int) int - Normal func(childComplexity int) int - Peak func(childComplexity int) int - Scope func(childComplexity int) int - Timestep func(childComplexity int) int - Unit func(childComplexity int) int + Aggregation func(childComplexity int) int + Alert func(childComplexity int) int + Caution func(childComplexity int) int + Name func(childComplexity int) int + Normal func(childComplexity int) int + Peak func(childComplexity int) int + Scope func(childComplexity int) int + Timestep func(childComplexity int) int + Unit func(childComplexity int) int } MetricFootprints struct { @@ -709,6 +710,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.TotalWalltime(childComplexity), true + case "MetricConfig.aggregation": + if e.complexity.MetricConfig.Aggregation == nil { + break + } + + return e.complexity.MetricConfig.Aggregation(childComplexity), true + case "MetricConfig.alert": if e.complexity.MetricConfig.Alert == nil { break @@ -1375,14 +1383,15 @@ type Accelerator { } type MetricConfig { - name: String! - unit: String! - scope: MetricScope! - timestep: Int! - peak: Float! - normal: Float! - caution: Float! - alert: Float! + name: String! + unit: String! + scope: MetricScope! + aggregation: String + timestep: Int! + peak: Float! + normal: Float! + caution: Float! + alert: Float! } type Tag { @@ -4202,6 +4211,38 @@ func (ec *executionContext) _MetricConfig_scope(ctx context.Context, field graph return ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋschemaᚐMetricScope(ctx, field.Selections, res) } +func (ec *executionContext) _MetricConfig_aggregation(ctx context.Context, field graphql.CollectedField, obj *model.MetricConfig) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "MetricConfig", + Field: field, + Args: nil, + IsMethod: false, + IsResolver: false, + } + + ctx = graphql.WithFieldContext(ctx, fc) + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Aggregation, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*string) + fc.Result = res + return ec.marshalOString2ᚖstring(ctx, field.Selections, res) +} + func (ec *executionContext) _MetricConfig_timestep(ctx context.Context, field graphql.CollectedField, obj *model.MetricConfig) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -8570,6 +8611,8 @@ func (ec *executionContext) _MetricConfig(ctx context.Context, sel ast.Selection if out.Values[i] == graphql.Null { invalids++ } + case "aggregation": + out.Values[i] = ec._MetricConfig_aggregation(ctx, field, obj) case "timestep": out.Values[i] = ec._MetricConfig_timestep(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/graph/model/models_gen.go b/graph/model/models_gen.go index 7022d80..acd50bb 100644 --- a/graph/model/models_gen.go +++ b/graph/model/models_gen.go @@ -97,14 +97,15 @@ type JobsStatistics struct { } type MetricConfig struct { - Name string `json:"name"` - Unit string `json:"unit"` - Scope schema.MetricScope `json:"scope"` - Timestep int `json:"timestep"` - Peak float64 `json:"peak"` - Normal float64 `json:"normal"` - Caution float64 `json:"caution"` - Alert float64 `json:"alert"` + Name string `json:"name"` + Unit string `json:"unit"` + Scope schema.MetricScope `json:"scope"` + Aggregation *string `json:"aggregation"` + Timestep int `json:"timestep"` + Peak float64 `json:"peak"` + Normal float64 `json:"normal"` + Caution float64 `json:"caution"` + Alert float64 `json:"alert"` } type MetricFootprints struct { diff --git a/graph/schema.graphqls b/graph/schema.graphqls index b3fbe29..f3209e3 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -68,14 +68,15 @@ type Accelerator { } type MetricConfig { - name: String! - unit: String! - scope: MetricScope! - timestep: Int! - peak: Float! - normal: Float! - caution: Float! - alert: Float! + name: String! + unit: String! + scope: MetricScope! + aggregation: String + timestep: Int! + peak: Float! + normal: Float! + caution: Float! + alert: Float! } type Tag { From 92349708ae87b1af5f69357ea020f2da87b71dc7 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 21 Mar 2022 13:30:19 +0100 Subject: [PATCH 25/26] More URL filter presets; Some tweaks --- metricdata/archive.go | 4 ++-- metricdata/metricdata.go | 23 ++++++++++++++++++----- routes.go | 24 ++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/metricdata/archive.go b/metricdata/archive.go index e3cae79..80271f0 100644 --- a/metricdata/archive.go +++ b/metricdata/archive.go @@ -162,9 +162,9 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) { allMetrics = append(allMetrics, mc.Name) } - // TODO: For now: Only single-node-jobs get archived in full resolution + // TODO: Talk about this! What resolutions to store data at... scopes := []schema.MetricScope{schema.MetricScopeNode} - if job.NumNodes == 1 { + if job.NumNodes <= 8 { scopes = append(scopes, schema.MetricScopeCore) } diff --git a/metricdata/metricdata.go b/metricdata/metricdata.go index 8f4122a..76fdcbe 100644 --- a/metricdata/metricdata.go +++ b/metricdata/metricdata.go @@ -63,7 +63,7 @@ var cache *lrucache.Cache = lrucache.New(512 * 1024 * 1024) // Fetches the metric data for a job. func LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) { - data := cache.Get(cacheKey(job, metrics, scopes), func() (interface{}, time.Duration, int) { + data := cache.Get(cacheKey(job, metrics, scopes), func() (_ interface{}, ttl time.Duration, size int) { var jd schema.JobData var err error if job.State == schema.JobStateRunning || @@ -93,30 +93,43 @@ func LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ct return err, 0, 0 } } + size = jd.Size() } else { jd, err = loadFromArchive(job) if err != nil { return err, 0, 0 } + // Avoid sending unrequested data to the client: if metrics != nil { res := schema.JobData{} for _, metric := range metrics { - if metricdata, ok := jd[metric]; ok { - res[metric] = metricdata + if perscope, ok := jd[metric]; ok { + if len(scopes) > 1 { + subset := make(map[schema.MetricScope]*schema.JobMetric) + for _, scope := range scopes { + if jm, ok := perscope[scope]; ok { + subset[scope] = jm + } + } + perscope = subset + } + + res[metric] = perscope } } jd = res } + size = 1 // loadFromArchive() caches in the same cache. } - ttl := 5 * time.Hour + ttl = 5 * time.Hour if job.State == schema.JobStateRunning { ttl = 2 * time.Minute } prepareJobData(job, jd, scopes) - return jd, ttl, jd.Size() + return jd, ttl, size }) if err, ok := data.(error); ok { diff --git a/routes.go b/routes.go index 81669d4..aa51895 100644 --- a/routes.go +++ b/routes.go @@ -174,8 +174,8 @@ func buildFilterPresets(query url.Values) map[string]interface{} { filterPresets["user"] = query.Get("user") filterPresets["userMatch"] = "eq" } - if query.Get("state") != "" && schema.JobState(query.Get("state")).Valid() { - filterPresets["state"] = query.Get("state") + if len(query["state"]) != 0 { + filterPresets["state"] = query["state"] } if rawtags, ok := query["tag"]; ok { tags := make([]int, len(rawtags)) @@ -188,6 +188,16 @@ func buildFilterPresets(query url.Values) map[string]interface{} { } filterPresets["tags"] = tags } + if query.Get("duration") != "" { + parts := strings.Split(query.Get("duration"), "-") + if len(parts) == 2 { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["duration"] = map[string]int{"from": a, "to": b} + } + } + } if query.Get("numNodes") != "" { parts := strings.Split(query.Get("numNodes"), "-") if len(parts) == 2 { @@ -198,6 +208,16 @@ func buildFilterPresets(query url.Values) map[string]interface{} { } } } + if query.Get("numAccelerators") != "" { + parts := strings.Split(query.Get("numAccelerators"), "-") + if len(parts) == 2 { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["numAccelerators"] = map[string]int{"from": a, "to": b} + } + } + } if query.Get("jobId") != "" { filterPresets["jobId"] = query.Get("jobId") } From 31f31fcecebe5e402ce0c220f0be4cdf556ad9ab Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Mon, 21 Mar 2022 14:21:44 +0100 Subject: [PATCH 26/26] Add test: starting the same job twice is bad --- test/api_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/api_test.go b/test/api_test.go index b4427f7..3f63bd4 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "reflect" "strconv" + "strings" "testing" "github.com/ClusterCockpit/cc-backend/api" @@ -295,4 +296,18 @@ func TestRestApi(t *testing.T) { t.Fatal("unexpected data fetched from archive") } }) + + t.Run("CheckDoubleStart", func(t *testing.T) { + // Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart! + body := strings.Replace(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`, -1) + + req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(body))) + recorder := httptest.NewRecorder() + + r.ServeHTTP(recorder, req) + response := recorder.Result() + if response.StatusCode != http.StatusUnprocessableEntity { + t.Fatal(response.Status, recorder.Body.String()) + } + }) }